--- old/agent/src/share/classes/sun/jvm/hotspot/oops/Array.java 2008-07-12 17:46:55.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/Array.java 2008-07-12 17:46:55.000000000 -0700 @@ -99,7 +99,7 @@ // We have to fetch the length of the array, shift (multiply) it // appropriately, up to wordSize, add the header, and align to // object size. - long s = getLength() << klass.getLog2ElementSize(); + long s = getLength() * klass.getElementSize(); s += klass.getArrayHeaderInBytes(); s = Oop.alignObjectSize(s); return s; --- old/agent/src/share/classes/sun/jvm/hotspot/oops/ArrayKlass.java 2008-07-12 17:46:58.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/ArrayKlass.java 2008-07-12 17:46:57.000000000 -0700 @@ -114,15 +114,7 @@ } public long getArrayHeaderInBytes() { - return Bits.maskBits(getLayoutHelper() >> LH_HEADER_SIZE_SHIFT, 0xFF); - } - - public int getLog2ElementSize() { - return Bits.maskBits(getLayoutHelper() >> LH_LOG2_ELEMENT_SIZE_SHIFT, 0xFF); - } - - public int getElementType() { - return Bits.maskBits(getLayoutHelper() >> LH_ELEMENT_TYPE_SHIFT, 0xFF); + return getHeaderSizeInBytes(); } boolean computeSubtypeOf(Klass k) { --- old/agent/src/share/classes/sun/jvm/hotspot/oops/InstanceKlass.java 2008-07-12 17:47:00.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/InstanceKlass.java 2008-07-12 17:47:00.000000000 -0700 @@ -88,7 +88,8 @@ nonstaticFieldSize = new CIntField(type.getCIntegerField("_nonstatic_field_size"), Oop.getHeaderSize()); staticFieldSize = new CIntField(type.getCIntegerField("_static_field_size"), Oop.getHeaderSize()); staticOopFieldSize = new CIntField(type.getCIntegerField("_static_oop_field_size"), Oop.getHeaderSize()); - nonstaticOopMapSize = new CIntField(type.getCIntegerField("_nonstatic_oop_map_size"), Oop.getHeaderSize()); + mapOffsetInBytes = new CIntField(type.getCIntegerField("_map_offset_in_bytes"), Oop.getHeaderSize()); + objectSize = new CIntField(type.getCIntegerField("_object_size"), Oop.getHeaderSize()); isMarkedDependent = new CIntField(type.getCIntegerField("_is_marked_dependent"), Oop.getHeaderSize()); initState = new CIntField(type.getCIntegerField("_init_state"), Oop.getHeaderSize()); vtableLen = new CIntField(type.getCIntegerField("_vtable_len"), Oop.getHeaderSize()); @@ -141,7 +142,8 @@ private static CIntField nonstaticFieldSize; private static CIntField staticFieldSize; private static CIntField staticOopFieldSize; - private static CIntField nonstaticOopMapSize; + private static CIntField mapOffsetInBytes; + private static CIntField objectSize; private static CIntField isMarkedDependent; private static CIntField initState; private static CIntField vtableLen; @@ -263,7 +265,8 @@ public long getNonstaticFieldSize() { return nonstaticFieldSize.getValue(this); } public long getStaticFieldSize() { return staticFieldSize.getValue(this); } public long getStaticOopFieldSize() { return staticOopFieldSize.getValue(this); } - public long getNonstaticOopMapSize() { return nonstaticOopMapSize.getValue(this); } + public long getMapOffsetInBytes() { return mapOffsetInBytes.getValue(this); } + public long objectSize() { return objectSize.getValue(this); } public boolean getIsMarkedDependent() { return isMarkedDependent.getValue(this) != 0; } public long getVtableLen() { return vtableLen.getValue(this); } public long getItableLen() { return itableLen.getValue(this); } @@ -275,8 +278,9 @@ public long getSizeHelper() { int lh = getLayoutHelper(); if (Assert.ASSERTS_ENABLED) { - Assert.that(lh > 0, "layout helper initialized for instance class"); + Assert.that(isFixedInstance(), "layout helper initialized for instance class"); } + lh = Bits.maskBits(lh, ~Bits.rightNBits(LH_SIZE_LOW_BITS)); return lh / VM.getVM().getAddressSize(); } @@ -455,7 +459,8 @@ visitor.doCInt(nonstaticFieldSize, true); visitor.doCInt(staticFieldSize, true); visitor.doCInt(staticOopFieldSize, true); - visitor.doCInt(nonstaticOopMapSize, true); + visitor.doCInt(mapOffsetInBytes, true); + visitor.doCInt(objectSize, true); visitor.doCInt(isMarkedDependent, true); visitor.doCInt(initState, true); visitor.doCInt(vtableLen, true); @@ -690,13 +695,6 @@ } - public long getObjectSize() { - long bodySize = alignObjectOffset(getVtableLen() * getHeap().getOopSize()) - + alignObjectOffset(getItableLen() * getHeap().getOopSize()) - + (getStaticFieldSize() + getNonstaticOopMapSize()) * getHeap().getOopSize(); - return alignObjectSize(headerSize + bodySize); - } - public Klass arrayKlassImpl(boolean orNull, int n) { // FIXME: in reflective system this would need to change to // actually allocate --- old/agent/src/share/classes/sun/jvm/hotspot/oops/Klass.java 2008-07-12 17:47:03.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/Klass.java 2008-07-12 17:47:02.000000000 -0700 @@ -29,6 +29,7 @@ import sun.jvm.hotspot.debugger.*; import sun.jvm.hotspot.runtime.*; import sun.jvm.hotspot.types.*; +import sun.jvm.hotspot.utilities.*; public class Klass extends Oop implements ClassConstants { static { @@ -40,13 +41,24 @@ } // anon-enum constants for _layout_helper. - public static int LH_INSTANCE_SLOW_PATH_BIT; - public static int LH_LOG2_ELEMENT_SIZE_SHIFT; - public static int LH_ELEMENT_TYPE_SHIFT; + public static int LH_FLAGS_BITS; + public static int LH_FLAGS_SHIFT; + public static int LH_FLAGS_LOW_BITS; + public static int LH_VARIABLE_FLAG; + public static int LH_ARRAY_FLAG; + public static int LH_FLAGS_EBT_MASK; + public static int LH_ELEMENT_SIZE_BITS; + public static int LH_ELEMENT_SIZE_SHIFT; + public static int LH_ELEMENT_SCALE_BITS; + public static int LH_ELEMENT_SIZEM_BITS; + public static int LH_ELEMENT_SIZEM_MASK_IP; + public static int LH_SIZE_LOW_BITS; + public static int LH_SLOW_PATH_LOW_BIT; + public static int LH_HEADER_SIZE_BITS; public static int LH_HEADER_SIZE_SHIFT; - public static int LH_ARRAY_TAG_SHIFT; - public static int LH_ARRAY_TAG_TYPE_VALUE; - public static int LH_ARRAY_TAG_OBJ_VALUE; + public static int LH_FIXED_SIZE_BITS; + public static int LH_FIXED_SIZE_SHIFT; + private static synchronized void initialize(TypeDataBase db) throws WrongTypeException { Type type = db.lookupType("Klass"); @@ -59,13 +71,23 @@ nextSibling = new OopField(type.getOopField("_next_sibling"), Oop.getHeaderSize()); allocCount = new CIntField(type.getCIntegerField("_alloc_count"), Oop.getHeaderSize()); - LH_INSTANCE_SLOW_PATH_BIT = db.lookupIntConstant("Klass::_lh_instance_slow_path_bit").intValue(); - LH_LOG2_ELEMENT_SIZE_SHIFT = db.lookupIntConstant("Klass::_lh_log2_element_size_shift").intValue(); - LH_ELEMENT_TYPE_SHIFT = db.lookupIntConstant("Klass::_lh_element_type_shift").intValue(); - LH_HEADER_SIZE_SHIFT = db.lookupIntConstant("Klass::_lh_header_size_shift").intValue(); - LH_ARRAY_TAG_SHIFT = db.lookupIntConstant("Klass::_lh_array_tag_shift").intValue(); - LH_ARRAY_TAG_TYPE_VALUE = db.lookupIntConstant("Klass::_lh_array_tag_type_value").intValue(); - LH_ARRAY_TAG_OBJ_VALUE = db.lookupIntConstant("Klass::_lh_array_tag_obj_value").intValue(); + LH_FLAGS_BITS = db.lookupIntConstant("LayoutHelper::_flags_bits").intValue(); + LH_FLAGS_SHIFT = db.lookupIntConstant("LayoutHelper::_flags_shift").intValue(); + LH_FLAGS_LOW_BITS = db.lookupIntConstant("LayoutHelper::_flags_low_bits").intValue(); + LH_VARIABLE_FLAG = db.lookupIntConstant("LayoutHelper::_variable_flag").intValue(); + LH_ARRAY_FLAG = db.lookupIntConstant("LayoutHelper::_array_flag").intValue(); + LH_FLAGS_EBT_MASK = db.lookupIntConstant("LayoutHelper::_flags_ebt_mask").intValue(); + LH_ELEMENT_SIZE_BITS = db.lookupIntConstant("LayoutHelper::_element_size_bits").intValue(); + LH_ELEMENT_SIZE_SHIFT = db.lookupIntConstant("LayoutHelper::_element_size_shift").intValue(); + LH_ELEMENT_SCALE_BITS = db.lookupIntConstant("LayoutHelper::_element_scale_bits").intValue(); + LH_ELEMENT_SIZEM_BITS = db.lookupIntConstant("LayoutHelper::_element_sizem_bits").intValue(); + LH_ELEMENT_SIZEM_MASK_IP = db.lookupIntConstant("LayoutHelper::_element_sizem_mask_ip").intValue(); + LH_SIZE_LOW_BITS = db.lookupIntConstant("LayoutHelper::_fixed_size_low_bits").intValue(); + LH_SLOW_PATH_LOW_BIT = db.lookupIntConstant("LayoutHelper::_slow_path_low_bit").intValue(); + LH_HEADER_SIZE_BITS = db.lookupIntConstant("LayoutHelper::_header_size_bits").intValue(); + LH_HEADER_SIZE_SHIFT = db.lookupIntConstant("LayoutHelper::_header_size_shift").intValue(); + LH_FIXED_SIZE_BITS = db.lookupIntConstant("LayoutHelper::_fixed_size_bits").intValue(); + LH_FIXED_SIZE_SHIFT = db.lookupIntConstant("LayoutHelper::_fixed_size_shift").intValue(); } Klass(OopHandle handle, ObjectHeap heap) { @@ -102,6 +124,38 @@ public Klass getNextSiblingKlass() { return (Klass) nextSibling.getValue(this); } public long getAllocCount() { return allocCount.getValue(this); } + public boolean isVariableObject() { return getLayoutHelper() < 0; } + public boolean isFixedInstance() { return getLayoutHelper() > 0; } + + public long getHeaderSizeInBytes() { + if (Assert.ASSERTS_ENABLED) { + Assert.that(isVariableObject(), "layout helper initialized for variable class"); + } + return Bits.maskBits(getLayoutHelper() >> LH_HEADER_SIZE_SHIFT, + (Bits.rightNBits(LH_HEADER_SIZE_BITS) + - Bits.rightNBits(LH_SIZE_LOW_BITS))); + } + + public int getElementSize() { + if (Assert.ASSERTS_ENABLED) { + Assert.that(isVariableObject(), "layout helper initialized for variable class"); + } + int lh = getLayoutHelper(); + int scale = Bits.maskBits(lh >> LH_ELEMENT_SIZE_SHIFT, + Bits.rightNBits(LH_ELEMENT_SCALE_BITS)); + int sizem = Bits.maskBits(lh >> (LH_ELEMENT_SIZE_SHIFT + LH_ELEMENT_SCALE_BITS), + Bits.rightNBits(LH_ELEMENT_SIZEM_BITS)); + return (sizem + 1) << scale; + } + + public int getElementType() { + if (Assert.ASSERTS_ENABLED) { + Assert.that(isVariableObject(), "layout helper initialized for variable class"); + } + return Bits.maskBits(getLayoutHelper() >> LH_FLAGS_SHIFT, + Bits.rightNBits(LH_FLAGS_EBT_MASK)); + } + // computed access flags - takes care of inner classes etc. // This is closer to actual source level than getAccessFlags() etc. public long computeModifierFlags() { --- old/agent/src/share/classes/sun/jvm/hotspot/utilities/HeapGXLWriter.java 2008-07-12 17:47:05.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/utilities/HeapGXLWriter.java 2008-07-12 17:47:04.000000000 -0700 @@ -176,6 +176,7 @@ if (isInstanceKlass) { // write object-size as an attribute long sizeInBytes = reflectedType.getLayoutHelper(); + sizeInBytes &= ~Bits.rightNBits(Klass.LH_SIZE_LOW_BITS); writeAttribute("object-size", "int", Long.toString(sizeInBytes)); // write static fields of this class. --- old/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp 2008-07-12 17:47:07.000000000 -0700 +++ new/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp 2008-07-12 17:47:07.000000000 -0700 @@ -343,6 +343,94 @@ return oop_maps; } +inline Address layout_helper_addr(Register klass) { + return Address(klass, 0, (klassOopDesc::header_size() * HeapWordSize + + Klass::layout_helper_offset_in_bytes())); +} + +static void compute_instance_size(Register obj_size, Register klass, + Label& have_obj_size, Label& size_is_variable, + Register t1, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(obj_size, klass, t1); + + if (hot_part) { + __ lduw(layout_helper_addr(klass), obj_size); + __ andcc(obj_size, ~LayoutHelper::_size_low_mask, obj_size); + if (mixed_arrays) { + __ jcc(Assembler::negative, size_is_variable); + __ bind(have_obj_size); + } + } else { + if (MixedArrays) { + // Side path for fixing up the initial size of a variable object. + // Must round from int to object alignment, and clear high bits. + __ bind(size_is_variable); + __ add(obj_size, LayoutHelper::_header_size_odd_mask, obj_size); + __ set((LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask), t1); + __ and3(obj_size, t1, obj_size); + __ jmp(have_obj_size); + } + } +} + +static void compute_array_size(Register arr_size, Register klass, Register length, + Label& have_scaled_length, Label& need_multiply, + Register t1, Register t2, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(arr_size, klass, length, t1, t2); + + if (hot_part) { + // get the allocation size: round_up(hdr + length << (lh>>16 & 0x1F)) + __ lduw(layout_helper_addr(klass), t1); + // int scale = (lh >> LayoutHelper::_element_size_shift); + __ mov(t1, t2); // spill layout helper + __ srl(t1, LayoutHelper::_element_size_shift); // lh>>16, no mask needed + // size_t arr_size = (array_length << scale); + __ sllx(length, t1, arr_size); + if (MixedArrays) { + // int sizem_ip = (scale & (LayoutHelper::_element_sizem_mask_ip)); + __ andcc(t1, LayoutHelper::_element_sizem_mask_ip, t1); + __ jcc(Assembler::notZero, need_multiply); + __ bind(have_scaled_length); + } + // arr_size += Klass::layout_helper_header_size_in_bytes(lh); + assert(LayoutHelper::_header_size_shift == 0, ""); + // sll(t2, LayoutHelper::_header_size_shift, t2); + __ set((LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask), t1); + __ and(t2, t1, t2); // t2 = lh & _header_size_mask + __ add(arr_size, t2, arr_size); + __ add(arr_size, MinObjAlignmentInBytesMask, arr_size); // align up + __ and3(arr_size, ~MinObjAlignmentInBytesMask, arr_size); + } else { + if (MixedArrays) { + // Side path for scaling by a non-power-of-two array element size. + __ bind(need_multiply); + // int sizem = (sizem_ip >> LayoutHelper::_element_scale_bits); + __ srl(t1, LayoutHelper::_element_scale_bits, t1); + // arr_size = (array_length * (1+sizem)) << scale; + __ add(t1, 1, t1); + // previous value of arr_size is junk; start from length again + assert(wordSize == jintSize, "else use mulx"); + __ smul(length, t1, arr_size); + __ srl(t2, LayoutHelper::_element_size_shift, t1); // lh>>16, mask needed + __ and3(t1, LayoutHelper::_element_scale_mask, t1); + __ sllx(arr_size, t1, arr_size); + __ jmp(have_scaled_length); + } + } +} + +static void initialize_array_body(Register obj, Register arr_size, Register t1, Register t2, StubAssembler* sasm) { + assert_different_registers(obj, arr_size, t1, t2); + + int min_header_size = arrayOopDesc::header_size(T_BYTE); + __ sub(arr_size, min_header_size, arr_size); // body length + __ add(obj, min_header_size, t1); // body start + __ initialize_body(t1, arr_size, 0, t2); +} + + OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { OopMapSet* oop_maps = NULL; @@ -413,15 +501,11 @@ #ifdef ASSERT // assert object can be fast path allocated { - Label ok, not_ok; + Label ok; __ ld(G5_klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc), G1_obj_size); - __ cmp(G1_obj_size, 0); // make sure it's an instance (LH > 0) - __ br(Assembler::lessEqual, false, Assembler::pn, not_ok); - __ delayed()->nop(); - __ btst(Klass::_lh_instance_slow_path_bit, G1_obj_size); + __ btst(LayoutHelper::_slow_path_low_bit, G1_obj_size); __ br(Assembler::zero, false, Assembler::pn, ok); __ delayed()->nop(); - __ bind(not_ok); __ stop("assert(can be fast path allocated)"); __ should_not_reach_here(); __ bind(ok); @@ -435,7 +519,10 @@ __ bind(retry_tlab); // get the instance size - __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); + Label have_obj_size, size_is_variable; + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size, size_is_variable, + G3_t1, sasm, true); __ tlab_allocate(O0_obj, G1_obj_size, 0, G3_t1, slow_path); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); @@ -443,9 +530,17 @@ __ ret(); __ delayed()->restore(); + // generate side path, if any + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size, size_is_variable, + G3_t1, sasm, false); + __ bind(try_eden); // get the instance size - __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); + Label have_obj_size_2, size_is_variable_2; + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size_2, size_is_variable_2, + G3_t1, sasm, true); __ eden_allocate(O0_obj, G1_obj_size, 0, G3_t1, G4_t2, slow_path); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); @@ -453,6 +548,11 @@ __ ret(); __ delayed()->restore(); + // generate side path, if any + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size_2, size_is_variable_2, + G3_t1, sasm, false); + __ bind(slow_path); // pop this frame so generate_stub_call can push it's own @@ -479,14 +579,6 @@ Register G4_length = G4; // Incoming Register O0_obj = O0; // Outgoing - Address klass_lh(G5_klass, 0, ((klassOopDesc::header_size() * HeapWordSize) - + Klass::layout_helper_offset_in_bytes())); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask == 0xFF, "bytewise"); - // Use this offset to pick out an individual byte of the layout_helper: - const int klass_lh_header_size_offset = ((BytesPerInt - 1) // 3 - 2 selects byte {0,1,0,0} - - Klass::_lh_header_size_shift / BitsPerByte); - if (id == new_type_array_id) { __ set_info("new_type_array", dont_gc_arguments); } else { @@ -498,11 +590,15 @@ { Label ok; Register G3_t1 = G3; - __ ld(klass_lh, G3_t1); - __ sra(G3_t1, Klass::_lh_array_tag_shift, G3_t1); - int tag = ((id == new_type_array_id) - ? Klass::_lh_array_tag_type_value - : Klass::_lh_array_tag_obj_value); + __ lduw(layout_helper_addr(G5_klass), G3_t1); + int flag_shift = LayoutHelper::_flags_shift; + flag_shift += LayoutHelper::_flags_low_bits; // shift out BasicType also + int tag = LayoutHelper::for_array((id == new_type_array_id) ? T_BYTE : T_OBJECT).as_int(); + tag >>= flag_shift; + __ sra(G3_t1, flag_shift, G3_t1); + assert(id == new_object_array_id || + tag == (LayoutHelper::for_array(T_DOUBLE).as_int() >> flag_shift), + "same tag for all type arrays"); __ cmp(G3_t1, tag); __ brx(Assembler::equal, false, Assembler::pt, ok); __ delayed()->nop(); @@ -533,46 +629,42 @@ __ bind(retry_tlab); // get the allocation size: (length << (layout_helper & 0x1F)) + header_size - __ ld(klass_lh, G3_t1); - __ sll(G4_length, G3_t1, G1_arr_size); - __ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1); - __ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1); - __ add(G1_arr_size, G3_t1, G1_arr_size); - __ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size); // align up - __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); - + Label have_scaled_length, need_multiply; + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length, need_multiply, + G3_t1, O1_t2, sasm, true); __ tlab_allocate(O0_obj, G1_arr_size, 0, G3_t1, slow_path); // preserves G1_arr_size __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); - __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); - __ sub(G1_arr_size, G3_t1, O1_t2); // body length - __ add(O0_obj, G3_t1, G3_t1); // body start - __ initialize_body(G3_t1, O1_t2); + initialize_array_body(O0_obj, G1_arr_size, G3_t1, O1_t2, sasm); __ verify_oop(O0_obj); __ retl(); __ delayed()->nop(); + // generate side path, if any + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length, need_multiply, + G3_t1, O1_t2, sasm, false); + __ bind(try_eden); + Label have_scaled_length_2, need_multiply_2; // get the allocation size: (length << (layout_helper & 0x1F)) + header_size - __ ld(klass_lh, G3_t1); - __ sll(G4_length, G3_t1, G1_arr_size); - __ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1); - __ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1); - __ add(G1_arr_size, G3_t1, G1_arr_size); - __ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size); - __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); - + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length_2, need_multiply_2, + G3_t1, O1_t2, sasm, true); __ eden_allocate(O0_obj, G1_arr_size, 0, G3_t1, O1_t2, slow_path); // preserves G1_arr_size __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); - __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); - __ sub(G1_arr_size, G3_t1, O1_t2); // body length - __ add(O0_obj, G3_t1, G3_t1); // body start - __ initialize_body(G3_t1, O1_t2); + initialize_array_body(O0_obj, G1_arr_size, G3_t1, O1_t2, sasm); __ verify_oop(O0_obj); __ retl(); __ delayed()->nop(); + // generate side path, if any + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length_2, need_multiply_2, + G3_t1, O1_t2, sasm, false); + __ bind(slow_path); } --- old/src/cpu/sparc/vm/stubGenerator_sparc.cpp 2008-07-12 17:47:10.000000000 -0700 +++ new/src/cpu/sparc/vm/stubGenerator_sparc.cpp 2008-07-12 17:47:09.000000000 -0700 @@ -2700,10 +2700,10 @@ // Load layout helper // - // |array_tag| | header_size | element_type | |log2_element_size| - // 32 30 24 16 8 2 0 + // | array_tag | element_type | log2_element_size | header_size | + // 32 28 24 16 0 // - // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 + // array_tag: typeArray = 0xF, objArray = 0xD, normal instance = 0x0 // int lh_offset = klassOopDesc::header_size() * HeapWordSize + @@ -2716,7 +2716,10 @@ __ load_klass(dst, G4_dst_klass); } // Handle objArrays completely differently... - juint objArray_lh = Klass::array_layout_helper(T_OBJECT); + // This is necessary because they have covariant element types. + // The next type check (for src.klass == dst.klass), can get a + // false negative in the case of object arrays. + jint objArray_lh = LayoutHelper::for_array(T_OBJECT).as_int(); __ set(objArray_lh, O5_temp); __ cmp(G5_lh, O5_temp); __ br(Assembler::equal, false, Assembler::pt, L_objArray); @@ -2731,25 +2734,11 @@ __ brx(Assembler::notEqual, false, Assembler::pn, L_failed); __ delayed()->nop(); - // if (!src->is_Array()) return -1; - __ cmp(G5_lh, Klass::_lh_neutral_value); // < 0 + // if (!src->has_length_field()) return -1; + __ cmp(G5_lh, LayoutHelper::_neutral_value); // < 0 __ br(Assembler::greaterEqual, false, Assembler::pn, L_failed); - // At this point, it is known to be a typeArray (array_tag 0x3). -#ifdef ASSERT - __ delayed()->nop(); - { Label L; - jint lh_prim_tag_in_place = (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift); - __ set(lh_prim_tag_in_place, O5_temp); - __ cmp(G5_lh, O5_temp); - __ br(Assembler::greaterEqual, false, Assembler::pt, L); - __ delayed()->nop(); - __ stop("must be a primitive array"); - __ bind(L); - } -#else __ delayed(); // match next insn to prev branch -#endif arraycopy_range_checks(src, src_pos, dst, dst_pos, length, O5_temp, G4_dst_klass, L_failed); @@ -2763,11 +2752,13 @@ const Register G4_offset = G4_dst_klass; // array offset const Register G3_elsize = G3_src_klass; // log2 element size - __ srl(G5_lh, Klass::_lh_header_size_shift, G4_offset); - __ and3(G4_offset, Klass::_lh_header_size_mask, G4_offset); // array_offset + assert(LayoutHelper::_header_size_shift == 0, ""); + __ set(LayoutHelper::_header_size_mask, G4_offset); + __ and3(G5_lh, G4_offset, G4_offset); // array_offset __ add(src, G4_offset, src); // src array offset __ add(dst, G4_offset, dst); // dst array offset - __ and3(G5_lh, Klass::_lh_log2_element_size_mask, G3_elsize); // log2 element size + __ srl( G5_lh, LayoutHelper::_element_size_shift, G3_elsize); + __ and3(G3_elsize, LayoutHelper::_element_size_mask, G3_elsize); // log2 element size // next registers should be set before the jump to corresponding stub const Register from = O0; // source array address @@ -2807,14 +2798,15 @@ __ br(Assembler::always,false,Assembler::pt,StubRoutines::_jlong_arraycopy); __ delayed()->signx(length, count); // length + Label L_different_objArray, L_plain_copy, L_checkcast_copy; + // objArrayKlass __ BIND(L_objArray); // live at this point: G3_src_klass, G4_dst_klass, src[_pos], dst[_pos], length - Label L_plain_copy, L_checkcast_copy; // test array classes for subtyping __ cmp(G3_src_klass, G4_dst_klass); // usual case is exact equality - __ brx(Assembler::notEqual, true, Assembler::pn, L_checkcast_copy); + __ brx(Assembler::notEqual, true, Assembler::pn, L_different_objArray); __ delayed()->lduw(G4_dst_klass, lh_offset, O5_temp); // hoisted from below // Identically typed arrays can be copied without element-wise checks. @@ -2831,7 +2823,7 @@ __ br(Assembler::always, false, Assembler::pt,StubRoutines::_oop_arraycopy); __ delayed()->signx(length, count); // length - __ BIND(L_checkcast_copy); + __ BIND(L_different_objArray); // live at this point: G3_src_klass, G4_dst_klass { // Before looking at dst.length, make sure dst is also an objArray. --- old/src/cpu/sparc/vm/templateTable_sparc.cpp 2008-07-12 17:47:12.000000000 -0700 +++ new/src/cpu/sparc/vm/templateTable_sparc.cpp 2008-07-12 17:47:12.000000000 -0700 @@ -25,6 +25,8 @@ #include "incls/_precompiled.incl" #include "incls/_templateTable_sparc.cpp.incl" +#error "NYI: mixa.patch not yet implemented for this platform" + #ifndef CC_INTERP #define __ _masm-> @@ -3092,9 +3094,9 @@ //__ ld(RinstanceKlass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc), Roffset); // make sure klass does not have has_finalizer, or is abstract, or interface or java/lang/Class - __ btst(Klass::_lh_instance_slow_path_bit, Roffset); + __ btst(LayoutHelper::_slow_path_low_bit, Roffset); __ br(Assembler::notZero, false, Assembler::pn, slow_case); - __ delayed()->nop(); + __ delayed()->and3(Roffset, ~LayoutHelper::_size_low_mask, Roffset); // allocate the instance // 1) Try to allocate in the TLAB --- old/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2008-07-12 17:47:15.000000000 -0700 +++ new/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2008-07-12 17:47:14.000000000 -0700 @@ -786,6 +786,97 @@ } +inline Address layout_helper_addr(Register klass) { + return Address(klass, (klassOopDesc::header_size() * HeapWordSize + + Klass::layout_helper_offset_in_bytes())); +} + +static void compute_instance_size(Register obj_size, Register klass, + Label& have_obj_size, Label& size_is_variable, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(obj_size, klass); + + if (hot_part) { + __ movl(obj_size, layout_helper_addr(klass)); + __ andl(obj_size, ~LayoutHelper::_size_low_mask); + if (MixedArrays) { + __ jcc(Assembler::negative, size_is_variable); + __ bind(have_obj_size); + } + } else { + if (MixedArrays) { + // Side path for fixing up the initial size of a variable object. + // Must round from int to object alignment, and clear high bits. + __ bind(size_is_variable); + __ addl(obj_size, LayoutHelper::_header_size_odd_mask); + __ andl(obj_size, (LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask)); + __ jmp(have_obj_size); + } + } +} + + +static void compute_array_size(Register arr_size, Register klass, Register length, + Label& have_scaled_length, Label& need_multiply, + Register t1, Register t2, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(arr_size, klass, length, t1, t2); + + if (hot_part) { + // get the allocation size: round_up(hdr + length << (lh>>16 & 0x1F)) + __ movl(t1, layout_helper_addr(klass)); + __ movl(arr_size, length); + // int scale = (lh >> LayoutHelper::_element_size_shift); + __ movl(t1, t2); // spill layout helper + __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, no mask needed + // size_t arr_size = (array_length << scale); + assert(t1 == rcx, "fixed register usage"); + __ shll(arr_size /* by t1=rcx, mod 32 */); + if (MixedArrays) { + // int sizem_ip = (scale & (LayoutHelper::_element_sizem_mask_ip)); + __ andl(t1, LayoutHelper::_element_sizem_mask_ip); + __ jcc(Assembler::notZero, need_multiply); + __ bind(have_scaled_length); + } + // arr_size += Klass::layout_helper_header_size_in_bytes(lh); + assert(LayoutHelper::_header_size_shift == 0, ""); + // shrl(t2, LayoutHelper::_header_size_shift); + __ andl(t2, LayoutHelper::_header_size_mask); + __ addl(arr_size, t2); + __ addl(arr_size, MinObjAlignmentInBytesMask); // align up + __ andl(arr_size, ~MinObjAlignmentInBytesMask); + } else { + if (MixedArrays) { + // Side path for scaling by a non-power-of-two array element size. + __ bind(need_multiply); + __ movl(arr_size, length); // reload; previous value was trash + // int sizem = (sizem_ip >> LayoutHelper::_element_scale_bits); + __ shrl(t1, LayoutHelper::_element_scale_bits); + // arr_size = (array_length * (1+sizem)) << scale; + __ addl(t1, 1); + assert(wordSize == jintSize, "else use imulq, shlq, etc."); + __ imull(arr_size, t1); + __ movl(t1, t2); // reload layout helper + __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, mask needed + __ andl(t1, LayoutHelper::_element_scale_mask); + assert(t1 == rcx, "fixed register usage"); + __ shll(arr_size /* by t1=rcx, mod 32 */); + __ jmp(have_scaled_length); + } + } +} + +static void initialize_array_body(Register obj, Register arr_size, Register t1, Register t2, StubAssembler* sasm) { + assert_different_registers(obj, arr_size, t1, t2); + + int min_header_size = arrayOopDesc::header_size(T_BYTE); + __ subl(arr_size, min_header_size); // body length + __ movl(t1, obj); + __ addl(t1, min_header_size); // body start + __ initialize_body(t1, arr_size, 0, t2); +} + + OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { // for better readability @@ -869,13 +960,10 @@ #ifdef ASSERT // assert object can be fast path allocated { - Label ok, not_ok; - __ movl(obj_size, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); - __ cmpl(obj_size, 0); // make sure it's an instance (LH > 0) - __ jcc(Assembler::lessEqual, not_ok); - __ testl(obj_size, Klass::_lh_instance_slow_path_bit); + Label ok; + __ movl(obj_size, layout_helper_addr(klass)); + __ testl(obj_size, LayoutHelper::_slow_path_low_bit); __ jcc(Assembler::zero, ok); - __ bind(not_ok); __ stop("assert(can be fast path allocated)"); __ should_not_reach_here(); __ bind(ok); @@ -890,7 +978,10 @@ __ bind(retry_tlab); // get the instance size - __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + Label have_obj_size, size_is_variable; + compute_instance_size(obj_size, klass, + have_obj_size, size_is_variable, + sasm, true); __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path); __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); @@ -898,9 +989,17 @@ __ popl(rdi); __ ret(0); + // generate side path, if any + compute_instance_size(obj_size, klass, + have_obj_size, size_is_variable, + sasm, false); + __ bind(try_eden); // get the instance size - __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + Label have_obj_size_2, size_is_variable_2; + compute_instance_size(obj_size, klass, + have_obj_size_2, size_is_variable_2, + sasm, true); __ eden_allocate(obj, obj_size, 0, t1, slow_path); __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); @@ -908,6 +1007,11 @@ __ popl(rdi); __ ret(0); + // generate side path, if any + compute_instance_size(obj_size, klass, + have_obj_size_2, size_is_variable_2, + sasm, false); + __ bind(slow_path); __ popl(rbx); __ popl(rdi); @@ -964,11 +1068,15 @@ { Label ok; Register t0 = obj; - __ movl(t0, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); - __ sarl(t0, Klass::_lh_array_tag_shift); - int tag = ((id == new_type_array_id) - ? Klass::_lh_array_tag_type_value - : Klass::_lh_array_tag_obj_value); + __ movl(t0, layout_helper_addr(klass)); + int flag_shift = LayoutHelper::_flags_shift; + flag_shift += LayoutHelper::_flags_low_bits; // shift out BasicType also + int tag = LayoutHelper::for_array((id == new_type_array_id) ? T_BYTE : T_OBJECT).as_int(); + tag >>= flag_shift; + __ sarl(t0, flag_shift); + assert(id == new_object_array_id || + tag == (LayoutHelper::for_array(T_DOUBLE).as_int() >> flag_shift), + "same tag for all type arrays"); __ cmpl(t0, tag); __ jcc(Assembler::equal, ok); __ stop("assert(is an array klass)"); @@ -995,55 +1103,39 @@ __ bind(retry_tlab); - // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); - __ movl(arr_size, length); - assert(t1 == rcx, "fixed register usage"); - __ shll(arr_size /* by t1=rcx, mod 32 */); - __ shrl(t1, Klass::_lh_header_size_shift); - __ andl(t1, Klass::_lh_header_size_mask); - __ addl(arr_size, t1); - __ addl(arr_size, MinObjAlignmentInBytesMask); // align up - __ andl(arr_size, ~MinObjAlignmentInBytesMask); - + Label have_scaled_length, need_multiply; + compute_array_size(arr_size, klass, length, + have_scaled_length, need_multiply, + t1, t2, sasm, true); __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path); // preserves arr_size __ initialize_header(obj, klass, length, t1, t2); - __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte))); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); - __ andl(t1, Klass::_lh_header_size_mask); - __ subl(arr_size, t1); // body length - __ addl(t1, obj); // body start - __ initialize_body(t1, arr_size, 0, t2); + initialize_array_body(obj, arr_size, t1, t2, sasm); __ verify_oop(obj); __ ret(0); - __ bind(try_eden); - // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); - __ movl(arr_size, length); - assert(t1 == rcx, "fixed register usage"); - __ shll(arr_size /* by t1=rcx, mod 32 */); - __ shrl(t1, Klass::_lh_header_size_shift); - __ andl(t1, Klass::_lh_header_size_mask); - __ addl(arr_size, t1); - __ addl(arr_size, MinObjAlignmentInBytesMask); // align up - __ andl(arr_size, ~MinObjAlignmentInBytesMask); + // generate side path, if any + compute_array_size(arr_size, klass, length, + have_scaled_length, need_multiply, + t1, t2, sasm, false); + __ bind(try_eden); + Label have_scaled_length_2, need_multiply_2; + compute_array_size(arr_size, klass, length, + have_scaled_length_2, need_multiply_2, + t1, t2, sasm, true); __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size __ initialize_header(obj, klass, length, t1, t2); - __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte))); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); - __ andl(t1, Klass::_lh_header_size_mask); - __ subl(arr_size, t1); // body length - __ addl(t1, obj); // body start - __ initialize_body(t1, arr_size, 0, t2); + initialize_array_body(obj, arr_size, t1, t2, sasm); __ verify_oop(obj); __ ret(0); + // generate side path, if any + compute_array_size(arr_size, klass, length, + have_scaled_length_2, need_multiply_2, + t1, t2, sasm, false); + __ bind(slow_path); } --- old/src/cpu/x86/vm/stubGenerator_x86_32.cpp 2008-07-12 17:47:18.000000000 -0700 +++ new/src/cpu/x86/vm/stubGenerator_x86_32.cpp 2008-07-12 17:47:17.000000000 -0700 @@ -1464,6 +1464,7 @@ // to a long, int, short, or byte copy loop. // address generate_unsafe_copy(const char *name, + address* entry, address byte_copy_entry, address short_copy_entry, address int_copy_entry, @@ -1494,6 +1495,9 @@ // bump this on entry, not on exit: inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); + *entry = __ pc(); // Entry point from generic arraycopy stub. + BLOCK_COMMENT("Entry:"); + const Register bits = rsi; __ movl(bits, from); __ orl(bits, to); @@ -1575,6 +1579,7 @@ address entry_jint_arraycopy, address entry_oop_arraycopy, address entry_jlong_arraycopy, + address entry_unsafe_arraycopy, address entry_checkcast_arraycopy) { Label L_failed, L_failed_0, L_objArray; @@ -1676,10 +1681,10 @@ // Load layout helper (32-bits) // - // |array_tag| | header_size | element_type | |log2_element_size| - // 32 30 24 16 8 2 0 + // | array_tag | element_type | log2_element_size | header_size | + // 32 28 24 16 0 // - // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 + // array_tag: typeArray = 0xF, objArray = 0xD, normal instance = 0x0 // int lh_offset = klassOopDesc::header_size() * HeapWordSize + @@ -1687,32 +1692,25 @@ Address src_klass_lh_addr(rcx_src_klass, lh_offset); // Handle objArrays completely differently... - jint objArray_lh = Klass::array_layout_helper(T_OBJECT); + // This is necessary because they have covariant element types. + // The next type check (for src.klass == dst.klass), can get a + // false negative in the case of object arrays. + jint objArray_lh = LayoutHelper::for_array(T_OBJECT).as_int(); __ cmpl(src_klass_lh_addr, objArray_lh); __ jcc(Assembler::equal, L_objArray); // if (src->klass() != dst->klass()) return -1; __ cmpl(rcx_src_klass, dst_klass_addr); - __ jccb(Assembler::notEqual, L_failed_0); + __ jcc(Assembler::notEqual, L_failed_0); const Register rcx_lh = rcx; // layout helper assert(rcx_lh == rcx_src_klass, "known alias"); __ movl(rcx_lh, src_klass_lh_addr); - // if (!src->is_Array()) return -1; - __ cmpl(rcx_lh, Klass::_lh_neutral_value); + // if (!src->has_length_field()) return -1; + __ cmpl(rcx_lh, LayoutHelper::_neutral_value); __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp - // At this point, it is known to be a typeArray (array_tag 0x3). -#ifdef ASSERT - { Label L; - __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); - __ jcc(Assembler::greaterEqual, L); // signed cmp - __ stop("must be a primitive array"); - __ bind(L); - } -#endif - assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh); arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed); @@ -1727,11 +1725,12 @@ const Register rdi_elsize = rdi; // log2 element size __ movl(rsi_offset, rcx_lh); - __ shrl(rsi_offset, Klass::_lh_header_size_shift); - __ andl(rsi_offset, Klass::_lh_header_size_mask); // array_offset + assert(LayoutHelper::_header_size_shift == 0, ""); + __ andl(rsi_offset, LayoutHelper::_header_size_mask); // array_offset __ addl(src_array, rsi_offset); // src array offset __ addl(dst_array, rsi_offset); // dst array offset - __ andl(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize + __ shrl(rcx_lh, LayoutHelper::_element_size_shift); + __ andl(rcx_lh, LayoutHelper::_element_size_mask); // log2 elsize // next registers should be set before the jump to corresponding stub const Register from = src; // source array address @@ -1747,12 +1746,11 @@ __ shll(rsi); // src_pos << rcx (log2 elsize) assert(src_array == from, ""); __ addl(from, rsi); // from = src_array + SRC_POS << log2 elsize - __ movl(rdi, DST_POS); // dst_pos - __ shll(rdi); // dst_pos << rcx (log2 elsize) + __ movl(rsi, DST_POS); // dst_pos (reuse rsi for a temp) + __ shll(rsi); // dst_pos << rcx (log2 elsize) assert(dst_array == to, ""); - __ addl(to, rdi); // to = dst_array + DST_POS << log2 elsize + __ addl(to, rsi); // to = dst_array + DST_POS << log2 elsize __ movl(FROM, from); // src_addr - __ movl(rdi_elsize, rcx_lh); // log2 elsize __ movl(count, LENGTH); // elements count BLOCK_COMMENT("choose copy loop based on element size"); @@ -1779,14 +1777,15 @@ __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + Label L_different_objArray, L_plain_copy, L_checkcast_copy; + // objArrayKlass __ BIND(L_objArray); // live at this point: rcx_src_klass, src[_pos], dst[_pos] - Label L_plain_copy, L_checkcast_copy; // test array classes for subtyping __ cmpl(rcx_src_klass, dst_klass_addr); // usual case is exact equality - __ jccb(Assembler::notEqual, L_checkcast_copy); + __ jccb(Assembler::notEqual, L_different_objArray); // Identically typed arrays can be copied without element-wise checks. assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass); @@ -1805,7 +1804,7 @@ __ movl(COUNT, count); // count __ jump(RuntimeAddress(entry_oop_arraycopy)); - __ BIND(L_checkcast_copy); + __ BIND(L_different_objArray); // live at this point: rcx_src_klass, dst[_pos], src[_pos] { // Handy offsets: @@ -1823,7 +1822,7 @@ // Before looking at dst.length, make sure dst is also an objArray. __ movl(rsi_dst_klass, dst_klass_addr); __ cmpl(dst_klass_lh_addr, objArray_lh); - __ jccb(Assembler::notEqual, L_failed); + __ jcc(Assembler::notEqual, L_failed); // It is safe to examine both src.length and dst.length. __ movl(src_pos, SRC_POS); // reload rsi @@ -1897,6 +1896,7 @@ address entry_jint_arraycopy; address entry_oop_arraycopy; address entry_jlong_arraycopy; + address entry_unsafe_arraycopy; address entry_checkcast_arraycopy; StubRoutines::_arrayof_jbyte_disjoint_arraycopy = @@ -1963,6 +1963,7 @@ StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", + &entry_unsafe_arraycopy, entry_jbyte_arraycopy, entry_jshort_arraycopy, entry_jint_arraycopy, @@ -1975,6 +1976,7 @@ entry_jint_arraycopy, entry_oop_arraycopy, entry_jlong_arraycopy, + entry_unsafe_arraycopy, entry_checkcast_arraycopy); } --- old/src/cpu/x86/vm/stubGenerator_x86_64.cpp 2008-07-12 17:47:21.000000000 -0700 +++ new/src/cpu/x86/vm/stubGenerator_x86_64.cpp 2008-07-12 17:47:20.000000000 -0700 @@ -1044,6 +1044,7 @@ static address oop_copy_entry; static address checkcast_copy_entry; + static address unsafe_copy_entry; // // Verify that a register contains clean 32-bits positive value @@ -2328,6 +2329,9 @@ __ enter(); // required for proper stackwalking of RuntimeStub frame + checkcast_copy_entry = __ pc(); + BLOCK_COMMENT("Entry:"); + // bump this on entry, not on exit: inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); @@ -2515,10 +2519,10 @@ // Load layout helper (32-bits) // - // |array_tag| | header_size | element_type | |log2_element_size| - // 32 30 24 16 8 2 0 + // | array_tag | element_type | log2_element_size | header_size | + // 32 28 24 16 0 // - // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 + // array_tag: typeArray = 0xF, objArray = 0xD, normal instance = 0x0 // int lh_offset = klassOopDesc::header_size() * HeapWordSize + @@ -2529,7 +2533,10 @@ __ movl(rax_lh, Address(r10_src_klass, lh_offset)); // Handle objArrays completely differently... - jint objArray_lh = Klass::array_layout_helper(T_OBJECT); + // This is necessary because they have covariant element types. + // The next type check (for src.klass == dst.klass), can get a + // false negative in the case of object arrays. + jint objArray_lh = LayoutHelper::for_array(T_OBJECT).as_int(); __ cmpl(rax_lh, objArray_lh); __ jcc(Assembler::equal, L_objArray); @@ -2538,20 +2545,10 @@ __ cmpq(r10_src_klass, r9_dst_klass); __ jcc(Assembler::notEqual, L_failed); - // if (!src->is_Array()) return -1; - __ cmpl(rax_lh, Klass::_lh_neutral_value); + // if (!src->has_length_field()) return -1; + __ cmpl(rax_lh, LayoutHelper::_neutral_value); __ jcc(Assembler::greaterEqual, L_failed); - // At this point, it is known to be a typeArray (array_tag 0x3). -#ifdef ASSERT - { Label L; - __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); - __ jcc(Assembler::greaterEqual, L); - __ stop("must be a primitive array"); - __ bind(L); - } -#endif - arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, r10, L_failed); @@ -2565,12 +2562,13 @@ const Register rax_elsize = rax_lh; // element size __ movl(r10_offset, rax_lh); - __ shrl(r10_offset, Klass::_lh_header_size_shift); - __ andq(r10_offset, Klass::_lh_header_size_mask); // array_offset + assert(LayoutHelper::_header_size_shift == 0, ""); + __ andl(r10_offset, LayoutHelper::_header_size_mask); // array_offset __ addq(src, r10_offset); // src array offset __ addq(dst, r10_offset); // dst array offset BLOCK_COMMENT("choose copy loop based on element size"); - __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize + __ shrl(rax_lh, LayoutHelper::_element_size_shift); + __ andl(rax_lh, LayoutHelper::_element_size_mask); // rax_lh -> rax_elsize // next registers should be set before the jump to corresponding stub const Register from = c_rarg0; // source array address @@ -2618,15 +2616,16 @@ __ movslq(count, r11_length); // length __ jump(RuntimeAddress(long_copy_entry)); + Label L_different_objArray, L_plain_copy, L_checkcast_copy; + // objArrayKlass __ BIND(L_objArray); // live at this point: r10_src_klass, src[_pos], dst[_pos] - Label L_plain_copy, L_checkcast_copy; // test array classes for subtyping __ load_klass(r9_dst_klass, dst); __ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality - __ jcc(Assembler::notEqual, L_checkcast_copy); + __ jcc(Assembler::notEqual, L_different_objArray); // Identically typed arrays can be copied without element-wise checks. arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, @@ -2640,7 +2639,7 @@ __ BIND(L_plain_copy); __ jump(RuntimeAddress(oop_copy_entry)); - __ BIND(L_checkcast_copy); + __ BIND(L_different_objArray); // live at this point: r10_src_klass, !r11_length { // assert(r11_length == C_RARG4); // will reload from here --- old/src/cpu/x86/vm/templateTable_x86_32.cpp 2008-07-12 17:47:23.000000000 -0700 +++ new/src/cpu/x86/vm/templateTable_x86_32.cpp 2008-07-12 17:47:23.000000000 -0700 @@ -3040,8 +3040,9 @@ // get instance_size in instanceKlass (scaled to a count of bytes) __ movl(rdx, Address(rcx, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); // test to see if it has a finalizer or is malformed in some way - __ testl(rdx, Klass::_lh_instance_slow_path_bit); + __ testl(rdx, LayoutHelper::_slow_path_low_bit); __ jcc(Assembler::notZero, slow_case); + __ andl(rdx, ~LayoutHelper::_size_low_mask); // // Allocate the instance --- old/src/cpu/x86/vm/templateTable_x86_64.cpp 2008-07-12 17:47:26.000000000 -0700 +++ new/src/cpu/x86/vm/templateTable_x86_64.cpp 2008-07-12 17:47:25.000000000 -0700 @@ -3071,8 +3071,9 @@ Address(rsi, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); // test to see if it has a finalizer or is malformed in some way - __ testl(rdx, Klass::_lh_instance_slow_path_bit); + __ testl(rdx, LayoutHelper::_slow_path_low_bit); __ jcc(Assembler::notZero, slow_case); + __ andl(rdx, ~LayoutHelper::_size_low_mask); // Allocate the instance // 1) Try to allocate in the TLAB --- old/src/share/vm/c1/c1_LIRGenerator.cpp 2008-07-12 17:47:28.000000000 -0700 +++ new/src/share/vm/c1/c1_LIRGenerator.cpp 2008-07-12 17:47:28.000000000 -0700 @@ -652,7 +652,7 @@ jobject2reg_with_patching(klass_reg, klass, info); // If klass is not loaded we do not know if the klass has finalizers: if (UseFastNewInstance && klass->is_loaded() - && !Klass::layout_helper_needs_slow_path(klass->layout_helper())) { + && !klass->layout_helper().needs_slow_path()) { Runtime1::StubID stub_id = klass->is_initialized() ? Runtime1::fast_new_instance_id : Runtime1::fast_new_instance_init_check_id; @@ -660,8 +660,9 @@ assert(klass->is_loaded(), "must be loaded"); // allocate space for instance - assert(klass->size_helper() >= 0, "illegal instance size"); - const int instance_size = align_object_size(klass->size_helper()); + const int instance_size = klass->new_instance_size(); + assert(instance_size == align_object_size(instance_size), "pre-aligned"); + assert(instance_size > 0, "illegal instance size"); __ allocate_object(dst, scratch1, scratch2, scratch3, scratch4, oopDesc::header_size(), instance_size, klass_reg, !klass->is_initialized(), slow_path); } else { --- old/src/share/vm/ci/ciArray.hpp 2008-07-12 17:47:31.000000000 -0700 +++ new/src/share/vm/ci/ciArray.hpp 2008-07-12 17:47:31.000000000 -0700 @@ -47,6 +47,7 @@ int length() { return _length; } // What kind of ciObject is this? - bool is_array() { return true; } + bool is_java_array() { return true; } bool is_java_object() { return true; } + bool has_length_field() { return true; } }; --- old/src/share/vm/ci/ciInstance.cpp 2008-07-12 17:47:33.000000000 -0700 +++ new/src/share/vm/ci/ciInstance.cpp 2008-07-12 17:47:33.000000000 -0700 @@ -31,6 +31,12 @@ // machine. // ------------------------------------------------------------------ +// ciInstance::has_length_field +bool ciInstance::has_length_field() { + return klass()->layout_helper().has_length_field(); +} + +// ------------------------------------------------------------------ // ciObject::java_mirror_type ciType* ciInstance::java_mirror_type() { VM_ENTRY_MARK; --- old/src/share/vm/ci/ciInstance.hpp 2008-07-12 17:47:36.000000000 -0700 +++ new/src/share/vm/ci/ciInstance.hpp 2008-07-12 17:47:35.000000000 -0700 @@ -53,6 +53,8 @@ bool is_instance() { return true; } bool is_java_object() { return true; } + bool has_length_field(); // klass()->layout_helper().has_length_field() + // Constant value of a field. ciConstant field_value(ciField* field); --- old/src/share/vm/ci/ciInstanceKlass.cpp 2008-07-12 17:47:38.000000000 -0700 +++ new/src/share/vm/ci/ciInstanceKlass.cpp 2008-07-12 17:47:38.000000000 -0700 @@ -195,12 +195,13 @@ // ciInstanceKlass* ciInstanceKlass::get_canonical_holder(int offset) { #ifdef ASSERT - if (!(offset >= 0 && offset < layout_helper())) { + int inst_size = layout_helper().new_instance_size_in_bytes(); + if (!(offset >= 0 && offset < inst_size)) { tty->print("*** get_canonical_holder(%d) on ", offset); this->print(); tty->print_cr(" ***"); }; - assert(offset >= 0 && offset < layout_helper(), "offset must be tame"); + assert(offset >= 0 && offset < inst_size, "offset must be tame"); #endif if (offset < instanceOopDesc::base_offset_in_bytes()) { @@ -244,12 +245,12 @@ ciKlass::print_impl(st); GUARDED_VM_ENTRY(st->print(" loader=0x%x", (address)loader());) if (is_loaded()) { - st->print(" loaded=true initialized=%s finalized=%s subklass=%s size=%d flags=", + st->print(" loaded=true initialized=%s finalized=%s subklass=%s ", bool_to_str(is_initialized()), bool_to_str(has_finalizer()), - bool_to_str(has_subklass()), - layout_helper()); - + bool_to_str(has_subklass())); + layout_helper().print_on(st); + st->print(" flags="); _flags.print_klass_flags(); if (_super) { --- old/src/share/vm/ci/ciInstanceKlass.hpp 2008-07-12 17:47:41.000000000 -0700 +++ new/src/share/vm/ci/ciInstanceKlass.hpp 2008-07-12 17:47:40.000000000 -0700 @@ -47,7 +47,7 @@ ciFlags _flags; jint _nonstatic_field_size; - jint _nonstatic_oop_map_size; + jint _nonstatic_oop_map_count; // Lazy fields get filled in only upon request. ciInstanceKlass* _super; @@ -126,9 +126,8 @@ } return _has_subklass; } - jint size_helper() { - return (Klass::layout_helper_size_in_bytes(layout_helper()) - >> LogHeapWordSize); + jint new_instance_size() { + return layout_helper().new_instance_size_in_bytes() >> LogHeapWordSize; } jint nonstatic_field_size() { assert(is_loaded(), "must be loaded"); @@ -136,9 +135,9 @@ jint has_nonstatic_fields() { assert(is_loaded(), "must be loaded"); return _has_nonstatic_fields; } - jint nonstatic_oop_map_size() { + jint nonstatic_oop_map_count() { assert(is_loaded(), "must be loaded"); - return _nonstatic_oop_map_size; } + return _nonstatic_oop_map_count; } ciInstanceKlass* super(); jint nof_implementors() { assert(is_loaded(), "must be loaded"); --- old/src/share/vm/ci/ciKlass.cpp 2008-07-12 17:47:43.000000000 -0700 +++ new/src/share/vm/ci/ciKlass.cpp 2008-07-12 17:47:43.000000000 -0700 @@ -48,7 +48,7 @@ ciKlass::ciKlass(KlassHandle h_k, ciSymbol* name) : ciType(h_k) { assert(get_oop()->is_klass(), "wrong type"); _name = name; - _layout_helper = Klass::_lh_neutral_value; + _layout_helper = LayoutHelper::_neutral_value; } // ------------------------------------------------------------------ @@ -57,7 +57,7 @@ // Unloaded klass variant. ciKlass::ciKlass(ciSymbol* name, ciKlass* klass) : ciType(klass) { _name = name; - _layout_helper = Klass::_lh_neutral_value; + _layout_helper = LayoutHelper::_neutral_value; } // ------------------------------------------------------------------ --- old/src/share/vm/ci/ciKlass.hpp 2008-07-12 17:47:45.000000000 -0700 +++ new/src/share/vm/ci/ciKlass.hpp 2008-07-12 17:47:45.000000000 -0700 @@ -40,7 +40,7 @@ private: ciSymbol* _name; - jint _layout_helper; + LayoutHelper _layout_helper; protected: ciKlass(KlassHandle k_h, ciSymbol* name); @@ -72,7 +72,7 @@ ciSymbol* name() { return _name; } // What is its layout helper value? - jint layout_helper() { return _layout_helper; } + LayoutHelper layout_helper() { return _layout_helper; } bool is_subtype_of(ciKlass* klass); bool is_subclass_of(ciKlass* klass); --- old/src/share/vm/ci/ciObject.hpp 2008-07-12 17:47:48.000000000 -0700 +++ new/src/share/vm/ci/ciObject.hpp 2008-07-12 17:47:47.000000000 -0700 @@ -116,9 +116,11 @@ virtual bool is_instance() { return false; } virtual bool is_method() { return false; } virtual bool is_method_data() { return false; } - virtual bool is_array() { return false; } + virtual bool has_length_field() { return false; } + virtual bool is_allprim() { return false; } virtual bool is_obj_array() { return false; } virtual bool is_type_array() { return false; } + virtual bool is_java_array() { return false; } virtual bool is_symbol() { return false; } virtual bool is_type() { return false; } virtual bool is_return_address() { return false; } @@ -135,6 +137,8 @@ virtual bool is_obj_array_klass_klass() { return false; } virtual bool is_type_array_klass_klass() { return false; } + bool is_array() { return is_java_array(); } + // Is this a type or value which has no associated class? // It is true of primitive types and null objects. virtual bool is_classless() const { return false; } --- old/src/share/vm/ci/ciTypeArray.hpp 2008-07-12 17:47:51.000000000 -0700 +++ new/src/share/vm/ci/ciTypeArray.hpp 2008-07-12 17:47:50.000000000 -0700 @@ -42,6 +42,7 @@ public: // What kind of ciObject is this? + bool is_allprim() { return true; } bool is_type_array() { return true; } // Return character at index. This is only useful if the --- old/src/share/vm/ci/ciTypeArrayKlass.hpp 2008-07-12 17:47:53.000000000 -0700 +++ new/src/share/vm/ci/ciTypeArrayKlass.hpp 2008-07-12 17:47:53.000000000 -0700 @@ -44,7 +44,7 @@ public: // The type of the array elements. BasicType element_type() { - return Klass::layout_helper_element_type(layout_helper()); + return layout_helper().element_type(); } // What kind of ciObject is this? --- old/src/share/vm/classfile/classFileParser.cpp 2008-07-12 17:47:55.000000000 -0700 +++ new/src/share/vm/classfile/classFileParser.cpp 2008-07-12 17:47:55.000000000 -0700 @@ -2960,14 +2960,20 @@ } // Size of instances - int instance_size; - - instance_size = align_object_size(next_nonstatic_type_offset / wordSize); + assert(next_nonstatic_type_offset % wordSize == 0, "already aligned for instance size"); + int instance_size = align_object_size(next_nonstatic_type_offset / wordSize); assert(instance_size == align_object_size(instanceOopDesc::header_size() + nonstatic_field_size), "consistent layout helper value"); - // Size of non-static oop map blocks (in words) allocated at end of klass - int nonstatic_oop_map_size = compute_oop_map_size(super_klass, nonstatic_oop_map_count, first_nonstatic_oop_offset); + // Count of non-static oop map blocks allocated at end of klass. + int total_map_count = compute_oop_map_size(super_klass, nonstatic_oop_map_count, first_nonstatic_oop_offset); + + LayoutHelper layout_helper = LayoutHelper::for_instance(instance_size); + + if (total_map_count == 0) { + // mark this class as having no oops + layout_helper.set_low_bit(LayoutHelper::_obj_allprim_low_bit); + } // Compute reference type ReferenceType rt; @@ -2980,19 +2986,18 @@ // We can now create the basic klassOop for this klass klassOop ik = oopFactory::new_instanceKlass( vtable_size, itable_size, - static_field_size, nonstatic_oop_map_size, + static_field_size, total_map_count, rt, CHECK_(nullHandle)); instanceKlassHandle this_klass (THREAD, ik); assert(this_klass->static_field_size() == static_field_size && - this_klass->nonstatic_oop_map_size() == nonstatic_oop_map_size, "sanity check"); + this_klass->nonstatic_oop_map_count() == total_map_count, "sanity check"); // Fill in information already parsed this_klass->set_access_flags(access_flags); - jint lh = Klass::instance_layout_helper(instance_size, false); - this_klass->set_layout_helper(lh); + this_klass->set_layout_helper(layout_helper); assert(this_klass->oop_is_instance(), "layout is correct"); - assert(this_klass->size_helper() == instance_size, "correct size_helper"); + assert(this_klass->new_instance_size() == instance_size, "correct size_helper"); // Not yet: supers are done below to support the new subtype-checking fields //this_klass->set_super(super_klass()); this_klass->set_class_loader(class_loader()); @@ -3146,7 +3151,7 @@ int ClassFileParser::compute_oop_map_size(instanceKlassHandle super, int nonstatic_oop_map_count, int first_nonstatic_oop_offset) { - int map_size = super.is_null() ? 0 : super->nonstatic_oop_map_size(); + int map_size = super.is_null() ? 0 : super->nonstatic_oop_map_count(); if (nonstatic_oop_map_count > 0) { // We have oops to add to map if (map_size == 0) { @@ -3156,7 +3161,7 @@ OopMapBlock* first_map = super->start_of_nonstatic_oop_maps(); OopMapBlock* last_map = first_map + map_size - 1; - int next_offset = last_map->offset() + (last_map->length() * heapOopSize); + int next_offset = last_map->offset() + last_map->size(); if (next_offset == first_nonstatic_oop_offset) { // There is no gap bettwen superklass's last oop field and first // local oop field, merge maps. @@ -3176,10 +3181,10 @@ int nonstatic_oop_map_count, u2* nonstatic_oop_offsets, u2* nonstatic_oop_length) { OopMapBlock* this_oop_map = k->start_of_nonstatic_oop_maps(); - OopMapBlock* last_oop_map = this_oop_map + k->nonstatic_oop_map_size(); + OopMapBlock* last_oop_map = this_oop_map + k->nonstatic_oop_map_count(); instanceKlass* super = k->superklass(); if (super != NULL) { - int super_oop_map_size = super->nonstatic_oop_map_size(); + int super_oop_map_size = super->nonstatic_oop_map_count(); OopMapBlock* super_oop_map = super->start_of_nonstatic_oop_maps(); // Copy maps from superklass while (super_oop_map_size-- > 0) { @@ -3195,14 +3200,15 @@ nonstatic_oop_map_count--; nonstatic_oop_offsets++; this_oop_map--; - this_oop_map->set_length(this_oop_map->length() + *nonstatic_oop_length++); + int more_size = *nonstatic_oop_length++ << LogBytesPerHeapOop; + this_oop_map->set_size(this_oop_map->size() + more_size); this_oop_map++; } assert((this_oop_map + nonstatic_oop_map_count) == last_oop_map, "just checking"); // Add new map blocks, fill them while (nonstatic_oop_map_count-- > 0) { this_oop_map->set_offset(*nonstatic_oop_offsets++); - this_oop_map->set_length(*nonstatic_oop_length++); + this_oop_map->set_size(*nonstatic_oop_length++ << LogBytesPerHeapOop); this_oop_map++; } } @@ -3262,14 +3268,15 @@ // If it cannot be fast-path allocated, set a bit in the layout helper. // See documentation of instanceKlass::can_be_fastpath_allocated(). - assert(k->size_helper() > 0, "layout_helper is initialized"); + assert(k->layout_helper().is_instance(), "layout_helper is initialized"); if ((!RegisterFinalizersAtInit && k->has_finalizer()) || k->is_abstract() || k->is_interface() || (k->name() == vmSymbols::java_lang_Class() && k->class_loader() == NULL) - || k->size_helper() >= FastAllocateSizeLimit) { + || k->new_instance_size() >= FastAllocateSizeLimit) { // Forbid fast-path allocation. - jint lh = Klass::instance_layout_helper(k->size_helper(), true); + LayoutHelper lh = k->layout_helper(); + lh.set_low_bit(LayoutHelper::_slow_path_low_bit); k->set_layout_helper(lh); } } --- old/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep 2008-07-12 17:47:58.000000000 -0700 +++ new/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep 2008-07-12 17:47:58.000000000 -0700 @@ -108,11 +108,13 @@ compactibleFreeListSpace.cpp concurrentMarkSweepThread.hpp compactibleFreeListSpace.cpp copy.hpp compactibleFreeListSpace.cpp globals.hpp +compactibleFreeListSpace.cpp genOopClosures.inline.hpp compactibleFreeListSpace.cpp handles.inline.hpp compactibleFreeListSpace.cpp init.hpp compactibleFreeListSpace.cpp java.hpp compactibleFreeListSpace.cpp liveRange.hpp compactibleFreeListSpace.cpp oop.inline.hpp +compactibleFreeListSpace.cpp parOopClosures.inline.hpp compactibleFreeListSpace.cpp resourceArea.hpp compactibleFreeListSpace.cpp universe.inline.hpp compactibleFreeListSpace.cpp vmThread.hpp --- old/src/share/vm/gc_implementation/includeDB_gc_parNew 2008-07-12 17:48:01.000000000 -0700 +++ new/src/share/vm/gc_implementation/includeDB_gc_parNew 2008-07-12 17:48:00.000000000 -0700 @@ -79,6 +79,7 @@ parNewGeneration.hpp defNewGeneration.hpp parNewGeneration.hpp parGCAllocBuffer.hpp +parNewGeneration.hpp objArrayOop.inline.hpp parNewGeneration.hpp taskqueue.hpp parOopClosures.hpp genOopClosures.hpp --- old/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp 2008-07-12 17:48:03.000000000 -0700 +++ new/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp 2008-07-12 17:48:03.000000000 -0700 @@ -72,6 +72,7 @@ } bool ParScanThreadState::should_be_partially_scanned(oop new_obj, oop old_obj) const { + // %%% TO DO: generalize objAarry; check Klass::LayoutHelper::has_oops_in_tail return new_obj->is_objArray() && arrayOop(new_obj)->length() > ParGCArrayScanChunk && new_obj != old_obj; @@ -89,6 +90,8 @@ int start = arrayOop(old)->length(); int end = obj->length(); int remainder = end - start; + assert(objArrayOopDesc::before_first_element == -1, "else change range logic"); + assert(start >= -1, ""); assert(start <= end, "just checking"); if (remainder > 2 * ParGCArrayScanChunk) { // Test above combines last partial chunk with a full chunk @@ -106,7 +109,6 @@ } // process our set of indices (include header in first chunk) - // should make sure end is even (aligned to HeapWord in case of compressed oops) if ((HeapWord *)obj < young_old_boundary()) { // object is in to_space obj->oop_iterate_range(&_to_space_closure, start, end); @@ -125,6 +127,7 @@ note_pop(); if ((HeapWord *)obj_to_scan < young_old_boundary()) { + // %%% TO DO: generalize objAarry; check Klass::LayoutHelper::has_oops_in_tail if (obj_to_scan->is_objArray() && obj_to_scan->is_forwarded() && obj_to_scan->forwardee() != obj_to_scan) { @@ -979,11 +982,16 @@ assert(new_obj != NULL, "just checking"); if (forward_ptr == NULL) { + if (new_obj->is_allprim()) { + // Header is OK. Nothing more to scan. + assert((HeapWord*) new_obj->klass() > reserved().end_word(), "klass is in perm"); + return new_obj; + } oop obj_to_push = new_obj; if (par_scan_state->should_be_partially_scanned(obj_to_push, old)) { // Length field used as index of next element to be scanned. // Real length can be obtained from real_forwardee() - arrayOop(old)->set_length(0); + arrayOop(old)->set_length(objArrayOopDesc::before_first_element); obj_to_push = old; assert(obj_to_push->is_forwarded() && obj_to_push->forwardee() != obj_to_push, "push forwarded object"); @@ -1096,11 +1104,16 @@ } if (forward_ptr == NULL) { + if (new_obj->is_allprim()) { + // Header is OK. Nothing more to scan. + assert((HeapWord*) new_obj->klass() > reserved().end_word(), "klass is in perm"); + return new_obj; + } oop obj_to_push = new_obj; if (par_scan_state->should_be_partially_scanned(obj_to_push, old)) { // Length field used as index of next element to be scanned. // Real length can be obtained from real_forwardee() - arrayOop(old)->set_length(0); + arrayOop(old)->set_length(objArrayOopDesc::before_first_element); obj_to_push = old; assert(obj_to_push->is_forwarded() && obj_to_push->forwardee() != obj_to_push, "push forwarded object"); --- old/src/share/vm/includeDB_core 2008-07-12 17:48:06.000000000 -0700 +++ new/src/share/vm/includeDB_core 2008-07-12 17:48:05.000000000 -0700 @@ -2512,6 +2512,7 @@ klass.hpp iterator.hpp klass.hpp klassOop.hpp klass.hpp klassPS.hpp +klass.hpp layoutHelper.hpp klass.hpp memRegion.hpp klass.hpp oop.hpp klass.hpp specialized_oop_closures.hpp @@ -2568,6 +2569,14 @@ klassVtable.hpp handles.hpp klassVtable.hpp oopsHierarchy.hpp +layoutHelper.cpp arrayOop.hpp +layoutHelper.cpp instanceKlass.hpp +layoutHelper.cpp javaClasses.hpp +layoutHelper.cpp layoutHelper.hpp +layoutHelper.cpp systemDictionary.hpp + +layoutHelper.hpp top.hpp + linkResolver.cpp bytecode.hpp linkResolver.cpp collectedHeap.inline.hpp linkResolver.cpp compilationPolicy.hpp @@ -2996,11 +3005,14 @@ objArrayKlassKlass.hpp objArrayKlass.hpp objArrayOop.cpp objArrayKlass.hpp -objArrayOop.cpp objArrayOop.hpp +objArrayOop.cpp objArrayOop.inline.hpp objArrayOop.cpp oop.inline.hpp objArrayOop.hpp arrayOop.hpp +objArrayOop.inline.hpp objArrayKlass.hpp +objArrayOop.inline.hpp objArrayOop.hpp + objectMonitor.hpp os.hpp objectMonitor_.cpp dtrace.hpp @@ -3712,12 +3724,14 @@ space.cpp copy.hpp space.cpp defNewGeneration.hpp space.cpp genCollectedHeap.hpp +space.cpp genOopClosures.inline.hpp space.cpp globalDefinitions.hpp space.cpp java.hpp space.cpp liveRange.hpp space.cpp markSweep.hpp space.cpp oop.inline.hpp space.cpp oop.inline2.hpp +space.cpp parOopClosures.inline.hpp space.cpp safepoint.hpp space.cpp space.hpp space.cpp space.inline.hpp --- old/src/share/vm/memory/iterator.hpp 2008-07-12 17:48:09.000000000 -0700 +++ new/src/share/vm/memory/iterator.hpp 2008-07-12 17:48:08.000000000 -0700 @@ -25,6 +25,7 @@ // The following classes are C++ `closures` for iterating over objects, roots and spaces class ReferenceProcessor; +class OopClosure_nv2v; // OopClosure is used for iterating through roots (oop*) @@ -34,9 +35,10 @@ OopClosure(ReferenceProcessor* rp) : _ref_processor(rp) { } OopClosure() : _ref_processor(NULL) { } virtual void do_oop(oop* o) = 0; - virtual void do_oop_v(oop* o) { do_oop(o); } virtual void do_oop(narrowOop* o) = 0; - virtual void do_oop_v(narrowOop* o) { do_oop(o); } + // The _v variant is just a macro-friendly alias for the abstract virtual. + void do_oop_v(oop* o) { do_oop(o); } + void do_oop_v(narrowOop* o) { do_oop(o); } // In support of post-processing of weak links of KlassKlass objects; // see KlassKlass::oop_oop_iterate(). @@ -55,7 +57,24 @@ Prefetch::style prefetch_style() { // Note that this is non-virtual. return Prefetch::do_none; } + + // Bridges between _v and _nv calling sequences. + // They produce a value which accepts do_oop_nv. + // Usage: OopClosure::nv_from##nv_suffix(blk)->do_oop_nv(p) + template + static inline OopClosureType* nv_from_nv(OopClosureType* blk) { return blk; } + static inline OopClosure_nv2v* nv_from_v(OopClosure* blk); // after next class +}; + +// Fake class for giving templates access to virtual do_oop with the '_nv' suffix: +class OopClosure_nv2v : public OopClosure { + OopClosure_nv2v() { ShouldNotReachHere(); } + public: + void do_oop_nv(oop* o) { do_oop(o); } + void do_oop_nv(narrowOop* o) { do_oop(o); } + static OopClosure_nv2v* from(OopClosure* blk) { return (OopClosure_nv2v*)blk; } }; +inline OopClosure_nv2v* OopClosure::nv_from_v(OopClosure* blk) { return OopClosure_nv2v::from(blk); } // ObjectClosure is used for iterating through an object space --- old/src/share/vm/memory/oopFactory.cpp 2008-07-12 17:48:11.000000000 -0700 +++ new/src/share/vm/memory/oopFactory.cpp 2008-07-12 17:48:11.000000000 -0700 @@ -95,9 +95,9 @@ klassOop oopFactory::new_instanceKlass(int vtable_len, int itable_len, int static_field_size, - int nonstatic_oop_map_size, ReferenceType rt, TRAPS) { + int nonstatic_oop_map_count, ReferenceType rt, TRAPS) { instanceKlassKlass* ikk = instanceKlassKlass::cast(Universe::instanceKlassKlassObj()); - return ikk->allocate_instance_klass(vtable_len, itable_len, static_field_size, nonstatic_oop_map_size, rt, CHECK_NULL); + return ikk->allocate_instance_klass(vtable_len, itable_len, static_field_size, nonstatic_oop_map_count, rt, CHECK_NULL); } --- old/src/share/vm/memory/oopFactory.hpp 2008-07-12 17:48:14.000000000 -0700 +++ new/src/share/vm/memory/oopFactory.hpp 2008-07-12 17:48:13.000000000 -0700 @@ -86,7 +86,7 @@ // Instance classes static klassOop new_instanceKlass(int vtable_len, int itable_len, int static_field_size, - int nonstatic_oop_map_size, ReferenceType rt, TRAPS); + int nonstatic_oop_map_count, ReferenceType rt, TRAPS); // Methods private: --- old/src/share/vm/memory/space.cpp 2008-07-12 17:48:17.000000000 -0700 +++ new/src/share/vm/memory/space.cpp 2008-07-12 17:48:16.000000000 -0700 @@ -33,7 +33,7 @@ if (top_obj != NULL) { if (_sp->block_is_obj(top_obj)) { if (_precision == CardTableModRefBS::ObjHeadPreciseArray) { - if (oop(top_obj)->is_objArray() || oop(top_obj)->is_typeArray()) { + if (oop(top_obj)->is_array()) { // %%% should generalize to has_length_field // An arrayOop is starting on the dirty card - since we do exact // store checks for objArrays we are done. } else { --- old/src/share/vm/memory/specialized_oop_closures.cpp 2008-07-12 17:48:19.000000000 -0700 +++ new/src/share/vm/memory/specialized_oop_closures.cpp 2008-07-12 17:48:19.000000000 -0700 @@ -49,12 +49,15 @@ } void SpecializationStats::print() { + int k; const char* header_format = " %20s %10s %11s %10s"; const char* line_format = " %20s %10d %11d %9.2f%%"; - int all_numCallsTotal = - _numCallsTotal[ik] + _numCallsTotal[irk] + _numCallsTotal[oa]; - int all_numCalls_nv = - _numCalls_nv[ik] + _numCalls_nv[irk] + _numCalls_nv[oa]; + int all_numCallsTotal = 0; + int all_numCalls_nv = 0; + for (k = 0; k < NUM_Kinds; k++) { + all_numCallsTotal += _numCallsTotal[k]; + all_numCalls_nv += _numCalls_nv[k]; + } gclog_or_tty->print_cr("\nOf %d oop_oop_iterate calls %d (%6.3f%%) are in (ik, irk, oa).", _numCallsAll, all_numCallsTotal, 100.0 * (float)all_numCallsTotal / (float)_numCallsAll); @@ -91,16 +94,18 @@ "----------", "-----------", "----------"); - int all_numDoOopCallsTotal = - _numDoOopCallsTotal[ik] + _numDoOopCallsTotal[irk] + _numDoOopCallsTotal[oa]; - int all_numDoOopCalls_nv = - _numDoOopCalls_nv[ik] + _numDoOopCalls_nv[irk] + _numDoOopCalls_nv[oa]; + int all_numDoOopCallsTotal = 0; + int all_numDoOopCalls_nv = 0; + for (k = 0; k < NUM_Kinds; k++) { + all_numDoOopCallsTotal += _numDoOopCallsTotal[k]; + all_numDoOopCalls_nv += _numDoOopCalls_nv[k]; + } gclog_or_tty->print_cr(line_format, "all", all_numDoOopCallsTotal, all_numDoOopCalls_nv, 100.0 * (float)all_numDoOopCalls_nv / (float)all_numDoOopCallsTotal); - const char* kind_names[] = { "ik", "irk", "oa" }; - for (int k = ik; k < NUM_Kinds; k++) { + const char* kind_names[] = { SpecializationStats_KIND_NAMES }; + for (k = ik; k < NUM_Kinds; k++) { gclog_or_tty->print_cr(line_format, kind_names[k], _numDoOopCallsTotal[k], _numDoOopCalls_nv[k], (_numDoOopCallsTotal[k] > 0 ? --- old/src/share/vm/memory/specialized_oop_closures.hpp 2008-07-12 17:48:22.000000000 -0700 +++ new/src/share/vm/memory/specialized_oop_closures.hpp 2008-07-12 17:48:22.000000000 -0700 @@ -171,6 +171,7 @@ class SpecializationStats { public: enum Kind { + prim, // allprim (array or instance) ik, // instanceKlass irk, // instanceRefKlass oa, // objArrayKlass @@ -178,6 +179,8 @@ }; #if ENABLE_SPECIALIZATION_STATS + #define SpecializationStats_KIND_NAMES "prim", "ik", "irk", "oa" + private: static int _numCallsAll; --- old/src/share/vm/oops/arrayKlass.cpp 2008-07-12 17:48:25.000000000 -0700 +++ new/src/share/vm/oops/arrayKlass.cpp 2008-07-12 17:48:24.000000000 -0700 @@ -83,7 +83,7 @@ assert(!k()->is_parsable(), "not expecting parsability yet."); k->set_super(Universe::is_bootstrapping() ? (klassOop)NULL : SystemDictionary::object_klass()); - k->set_layout_helper(Klass::_lh_neutral_value); + k->set_layout_helper(LayoutHelper()); k->set_dimension(1); k->set_higher_dimension(NULL); k->set_lower_dimension(NULL); --- old/src/share/vm/oops/arrayKlass.hpp 2008-07-12 17:48:28.000000000 -0700 +++ new/src/share/vm/oops/arrayKlass.hpp 2008-07-12 17:48:28.000000000 -0700 @@ -36,7 +36,8 @@ public: // Testing operation - bool oop_is_array() const { return true; } + bool oop_is_array_slow() const { return true; } + bool oop_has_length_field_slow() const { return true; } // Instance variables int dimension() const { return _dimension; } @@ -55,10 +56,10 @@ void set_alloc_size(juint n) { _alloc_size = n; } // offset of first element, including any padding for the sake of alignment - int array_header_in_bytes() const { return layout_helper_header_size(layout_helper()); } - int log2_element_size() const { return layout_helper_log2_element_size(layout_helper()); } + int array_header_in_bytes() const { return layout_helper().header_size_in_bytes(); } + int log2_element_size() const { return layout_helper().log2_element_size(); } // type of elements (T_OBJECT for both oop arrays and array-arrays) - BasicType element_type() const { return layout_helper_element_type(layout_helper()); } + BasicType element_type() const { return layout_helper().element_type(); } oop component_mirror() const { return _component_mirror; } void set_component_mirror(oop m) { oop_store((oop*) &_component_mirror, m); } @@ -81,7 +82,7 @@ // Casting from klassOop static arrayKlass* cast(klassOop k) { Klass* kp = k->klass_part(); - assert(kp->null_vtbl() || kp->oop_is_array(), "cast to arrayKlass"); + assert(kp->null_vtbl() || kp->oop_is_array_slow(), "cast to arrayKlass"); return (arrayKlass*) kp; } --- old/src/share/vm/oops/arrayOop.hpp 2008-07-12 17:48:32.000000000 -0700 +++ new/src/share/vm/oops/arrayOop.hpp 2008-07-12 17:48:31.000000000 -0700 @@ -86,6 +86,12 @@ *(int*)(((intptr_t)this) + length_offset_in_bytes()) = length; } + // These accessors work on any object with a length field: + static jint length_of(oop obj) { + assert(obj->has_length_field(), "must have length field"); + return *(int*)(((intptr_t)obj) + length_offset_in_bytes()); + } + // Should only be called with constants as argument // (will not constant fold otherwise) // Returns the header size in words aligned to the requirements of the --- old/src/share/vm/oops/instanceKlass.cpp 2008-07-12 17:48:34.000000000 -0700 +++ new/src/share/vm/oops/instanceKlass.cpp 2008-07-12 17:48:34.000000000 -0700 @@ -524,7 +524,8 @@ instanceOop instanceKlass::allocate_instance(TRAPS) { bool has_finalizer_flag = has_finalizer(); // Query before possible GC - int size = size_helper(); // Query before forming handle. + int size = new_instance_size(); // Query before forming handle. + // (Using new_instance_size instead of fixed_size to support mixed arrays.) KlassHandle h_k(THREAD, as_klassOop()); @@ -543,7 +544,7 @@ // instances so simply disallow finalizable perm objects. This can // be relaxed if a need for it is found. assert(!has_finalizer(), "perm objects not allowed to have finalizers"); - int size = size_helper(); // Query before forming handle. + int size = new_instance_size(); // Query before forming handle. KlassHandle h_k(THREAD, as_klassOop()); instanceOop i = (instanceOop) CollectedHeap::permanent_obj_allocate(h_k, size, CHECK_NULL); @@ -1254,299 +1255,165 @@ } #endif //PRODUCT - +// Oop assertions. #ifdef ASSERT -template void assert_is_in(T *p) { +template inline T* assert_is_in(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop o = oopDesc::decode_heap_oop_not_null(heap_oop); assert(Universe::heap()->is_in(o), "should be in heap"); } + return p; } -template void assert_is_in_closed_subset(T *p) { +template inline T* assert_is_in_closed_subset(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop o = oopDesc::decode_heap_oop_not_null(heap_oop); assert(Universe::heap()->is_in_closed_subset(o), "should be in closed"); } + return p; } -template void assert_is_in_reserved(T *p) { +template inline T* assert_is_in_reserved(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop o = oopDesc::decode_heap_oop_not_null(heap_oop); assert(Universe::heap()->is_in_reserved(o), "should be in reserved"); } + return p; } -template void assert_nothing(T *p) {} #else -template void assert_is_in(T *p) {} -template void assert_is_in_closed_subset(T *p) {} -template void assert_is_in_reserved(T *p) {} -template void assert_nothing(T *p) {} +template inline T* assert_is_in(T* p) { return p; } +template inline T* assert_is_in_closed_subset(T* p) { return p; } +template inline T* assert_is_in_reserved(T* p) { return p; } #endif // ASSERT -// -// Macros that iterate over areas of oops which are specialized on type of -// oop pointer either narrow or wide, depending on UseCompressedOops -// -// Parameters are: -// T - type of oop to point to (either oop or narrowOop) -// start_p - starting pointer for region to iterate over -// count - number of oops or narrowOops to iterate over -// do_oop - action to perform on each oop (it's arbitrary C code which -// makes it more efficient to put in a macro rather than making -// it a template function) -// assert_fn - assert function which is template function because performance -// doesn't matter when enabled. -#define InstanceKlass_SPECIALIZED_OOP_ITERATE( \ - T, start_p, count, do_oop, \ - assert_fn) \ -{ \ - T* p = (T*)(start_p); \ - T* const end = p + (count); \ - while (p < end) { \ - (assert_fn)(p); \ - do_oop; \ - ++p; \ - } \ -} - -#define InstanceKlass_SPECIALIZED_OOP_REVERSE_ITERATE( \ - T, start_p, count, do_oop, \ - assert_fn) \ -{ \ - T* const start = (T*)(start_p); \ - T* p = start + (count); \ - while (start < p) { \ - --p; \ - (assert_fn)(p); \ - do_oop; \ - } \ -} - -#define InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE( \ - T, start_p, count, low, high, \ - do_oop, assert_fn) \ -{ \ - T* const l = (T*)(low); \ - T* const h = (T*)(high); \ - assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && \ - mask_bits((intptr_t)h, sizeof(T)-1) == 0, \ - "bounded region must be properly aligned"); \ - T* p = (T*)(start_p); \ - T* end = p + (count); \ - if (p < l) p = l; \ - if (end > h) end = h; \ - while (p < end) { \ - (assert_fn)(p); \ - do_oop; \ - ++p; \ - } \ -} - - -// The following macros call specialized macros, passing either oop or -// narrowOop as the specialization type. These test the UseCompressedOops -// flag. -#define InstanceKlass_OOP_ITERATE(start_p, count, \ - do_oop, assert_fn) \ -{ \ - if (UseCompressedOops) { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(narrowOop, \ - start_p, count, \ - do_oop, assert_fn) \ - } else { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(oop, \ - start_p, count, \ - do_oop, assert_fn) \ - } \ -} - -#define InstanceKlass_BOUNDED_OOP_ITERATE(start_p, count, low, high, \ - do_oop, assert_fn) \ -{ \ - if (UseCompressedOops) { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(narrowOop, \ - start_p, count, \ - low, high, \ - do_oop, assert_fn) \ - } else { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(oop, \ - start_p, count, \ - low, high, \ - do_oop, assert_fn) \ - } \ -} - -#define InstanceKlass_OOP_MAP_ITERATE(obj, do_oop, assert_fn) \ -{ \ - /* Compute oopmap block range. The common case \ - is nonstatic_oop_map_size == 1. */ \ - OopMapBlock* map = start_of_nonstatic_oop_maps(); \ - OopMapBlock* const end_map = map + nonstatic_oop_map_size(); \ - if (UseCompressedOops) { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(narrowOop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - ++map; \ - } \ - } else { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(oop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - ++map; \ - } \ - } \ -} - -#define InstanceKlass_OOP_MAP_REVERSE_ITERATE(obj, do_oop, assert_fn) \ -{ \ - OopMapBlock* const start_map = start_of_nonstatic_oop_maps(); \ - OopMapBlock* map = start_map + nonstatic_oop_map_size(); \ - if (UseCompressedOops) { \ - while (start_map < map) { \ - --map; \ - InstanceKlass_SPECIALIZED_OOP_REVERSE_ITERATE(narrowOop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - } \ - } else { \ - while (start_map < map) { \ - --map; \ - InstanceKlass_SPECIALIZED_OOP_REVERSE_ITERATE(oop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - } \ - } \ -} - -#define InstanceKlass_BOUNDED_OOP_MAP_ITERATE(obj, low, high, do_oop, \ - assert_fn) \ -{ \ - /* Compute oopmap block range. The common case is \ - nonstatic_oop_map_size == 1, so we accept the \ - usually non-existent extra overhead of examining \ - all the maps. */ \ - OopMapBlock* map = start_of_nonstatic_oop_maps(); \ - OopMapBlock* const end_map = map + nonstatic_oop_map_size(); \ - if (UseCompressedOops) { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(narrowOop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - low, high, \ - do_oop, assert_fn) \ - ++map; \ - } \ - } else { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(oop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - low, high, \ - do_oop, assert_fn) \ - ++map; \ - } \ - } \ -} - void instanceKlass::follow_static_fields() { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - MarkSweep::mark_and_push(p), \ - assert_is_in_closed_subset) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + MarkSweep::mark_and_push(assert_is_in_closed_subset(addr(p))); + } + }); } #ifndef SERIALGC void instanceKlass::follow_static_fields(ParCompactionManager* cm) { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - PSParallelCompact::mark_and_push(cm, p), \ - assert_is_in) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + PSParallelCompact::mark_and_push(cm, assert_is_in(addr(p))); + } + }); } #endif // SERIALGC void instanceKlass::adjust_static_fields() { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - MarkSweep::adjust_pointer(p), \ - assert_nothing) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + MarkSweep::adjust_pointer(addr(p)); + } + }); } #ifndef SERIALGC void instanceKlass::update_static_fields() { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); } void instanceKlass::update_static_fields(HeapWord* beg_addr, HeapWord* end_addr) { - InstanceKlass_BOUNDED_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - beg_addr, end_addr, \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing ) + MemRegion mr(beg_addr, end_addr); + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = oop_chunk::bounded(mr, static_oop_fields(pmax), pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); } #endif // SERIALGC void instanceKlass::oop_follow_contents(oop obj) { - assert(obj != NULL, "can't follow the content of NULL object"); obj->follow_header(); - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - MarkSweep::mark_and_push(p), \ - assert_is_in_closed_subset) + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + MarkSweep::mark_and_push(assert_is_in_closed_subset(addr(p))); + } + }); } #ifndef SERIALGC void instanceKlass::oop_follow_contents(ParCompactionManager* cm, oop obj) { - assert(obj != NULL, "can't follow the content of NULL object"); obj->follow_header(cm); - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - PSParallelCompact::mark_and_push(cm, p), \ - assert_is_in) + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + PSParallelCompact::mark_and_push(cm, assert_is_in(addr(p))); + } + }); } #endif // SERIALGC // closure's do_header() method dicates whether the given closure should be // applied to the klass ptr in the object header. +// Templated iterator loops. + +template +inline int instanceKlass::oop_oop_iterate_tp(oop obj, + OopClosureType* closure) { + SpecializationStats::record_iterate_call_nv(SpecializationStats::ik); + /* header */ + if (closure->do_header()) { + obj->oop_iterate_header_nv(closure); + } + if (has_nonstatic_oop_maps()) { + for (oop_map::iterator pmax, p = + nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + SpecializationStats::record_do_oop_call_nv(SpecializationStats::ik); + closure->do_oop_nv(assert_is_in_closed_subset(addr(p))); + } + } + return cheap_fixed_size(); +} + +template +inline int instanceKlass::oop_oop_iterate_tp_m(oop obj, + OopClosureType* closure, + MemRegion mr) { + SpecializationStats::record_iterate_call_nv(SpecializationStats::ik); + if (closure->do_header()) { + obj->oop_iterate_header_nv(closure, mr); + } + if (has_nonstatic_oop_maps()) { + for (oop_map::bounded_iterator pmax, p = + bounded_nonstatic_oop_fields(obj, mr, pmax); p < pmax; p++) { + SpecializationStats::record_do_oop_call_nv(SpecializationStats::ik); + closure->do_oop_nv(assert_is_in_closed_subset(addr(p))); + } + } + return cheap_fixed_size(); +} + #define InstanceKlass_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ \ int instanceKlass::oop_oop_iterate##nv_suffix(oop obj, \ OopClosureType* closure) {\ - SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik);\ - /* header */ \ - if (closure->do_header()) { \ - obj->oop_iterate_header(closure); \ - } \ - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - SpecializationStats:: \ - record_do_oop_call##nv_suffix(SpecializationStats::ik); \ - (closure)->do_oop##nv_suffix(p), \ - assert_is_in_closed_subset) \ - return size_helper(); \ + WITH_BOTH_OOP_KINDS(T, { \ + return oop_oop_iterate_tp(obj, \ + OopClosure::nv_from##nv_suffix(closure)); \ + }); \ } #define InstanceKlass_OOP_OOP_ITERATE_DEFN_m(OopClosureType, nv_suffix) \ \ int instanceKlass::oop_oop_iterate##nv_suffix##_m(oop obj, \ OopClosureType* closure, \ - MemRegion mr) { \ - SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik);\ - if (closure->do_header()) { \ - obj->oop_iterate_header(closure, mr); \ - } \ - InstanceKlass_BOUNDED_OOP_MAP_ITERATE( \ - obj, mr.start(), mr.end(), \ - (closure)->do_oop##nv_suffix(p), \ - assert_is_in_closed_subset) \ - return size_helper(); \ + MemRegion mr) { \ + WITH_BOTH_OOP_KINDS(T, { \ + return oop_oop_iterate_tp_m(obj, \ + OopClosure::nv_from##nv_suffix(closure), mr); \ + }); \ } ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN) @@ -1554,95 +1421,112 @@ ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN_m) ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DEFN_m) + void instanceKlass::iterate_static_fields(OopClosure* closure) { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - closure->do_oop(p), \ - assert_is_in_reserved) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + closure->do_oop(assert_is_in_reserved(addr(p))); + } + }); } void instanceKlass::iterate_static_fields(OopClosure* closure, MemRegion mr) { - InstanceKlass_BOUNDED_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - mr.start(), mr.end(), \ - (closure)->do_oop_v(p), \ - assert_is_in_closed_subset) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = oop_chunk::bounded(mr, static_oop_fields(pmax), pmax); p < pmax; p++) { + closure->do_oop(assert_is_in_closed_subset(addr(p))); + } + }); } int instanceKlass::oop_adjust_pointers(oop obj) { - int size = size_helper(); - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - MarkSweep::adjust_pointer(p), \ - assert_is_in) - obj->adjust_header(); + int size = cheap_fixed_size(); + if (!has_nonstatic_oop_maps()) return size; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + MarkSweep::adjust_pointer(assert_is_in(addr(p))); + } + }); return size; } #ifndef SERIALGC void instanceKlass::oop_copy_contents(PSPromotionManager* pm, oop obj) { assert(!pm->depth_first(), "invariant"); - InstanceKlass_OOP_MAP_REVERSE_ITERATE( \ - obj, \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_breadth(p); \ - }, \ - assert_nothing ) + if (!has_nonstatic_oop_maps()) return; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmin, p = iterator_reverse(nonstatic_oop_fields(obj, pmin), pmin); p >= pmin; p--) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_breadth(addr(p)); + } + } + }); } void instanceKlass::oop_push_contents(PSPromotionManager* pm, oop obj) { assert(pm->depth_first(), "invariant"); - InstanceKlass_OOP_MAP_REVERSE_ITERATE( \ - obj, \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_depth(p); \ - }, \ - assert_nothing ) + if (!has_nonstatic_oop_maps()) return; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmin, p = iterator_reverse(nonstatic_oop_fields(obj, pmin), pmin); p >= pmin; p--) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_depth(addr(p)); + } + } + }); } int instanceKlass::oop_update_pointers(ParCompactionManager* cm, oop obj) { - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing) - return size_helper(); + int size = cheap_fixed_size(); + if (!has_nonstatic_oop_maps()) return size; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); + return size; } int instanceKlass::oop_update_pointers(ParCompactionManager* cm, oop obj, HeapWord* beg_addr, HeapWord* end_addr) { - InstanceKlass_BOUNDED_OOP_MAP_ITERATE( \ - obj, beg_addr, end_addr, \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing) - return size_helper(); + int size = cheap_fixed_size(); + if (!has_nonstatic_oop_maps()) return size; + MemRegion mr(beg_addr, end_addr); + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::bounded_iterator pmax, p = + bounded_nonstatic_oop_fields(obj, mr, pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); + return size; } void instanceKlass::copy_static_fields(PSPromotionManager* pm) { assert(!pm->depth_first(), "invariant"); - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_breadth(p); \ - }, \ - assert_nothing ) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_breadth(addr(p)); + } + } + }); } void instanceKlass::push_static_fields(PSPromotionManager* pm) { - assert(pm->depth_first(), "invariant"); - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_depth(p); \ - }, \ - assert_nothing ) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_depth(addr(p)); + } + } + }); } void instanceKlass::copy_static_fields(ParCompactionManager* cm) { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - PSParallelCompact::adjust_pointer(p), \ - assert_is_in) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(assert_is_in(addr(p))); + } + }); } #endif // SERIALGC @@ -2040,7 +1924,7 @@ class VerifyFieldClosure: public OopClosure { protected: - template void do_oop_work(T* p) { + template inline void do_oop_work(T* p) { guarantee(Universe::heap()->is_in_closed_subset(p), "should be in heap"); oop obj = oopDesc::load_decode_heap_oop(p); if (!obj->is_oop_or_null()) { @@ -2054,10 +1938,106 @@ virtual void do_oop(narrowOop* p) { VerifyFieldClosure::do_oop_work(p); } }; +#ifndef PRODUCT +template +void instanceKlass::oop_verify_oop_map_iterators(oop obj) { + ResourceMark rm; + const int max_locs = 100; + T* locs[max_locs]; + int num_locs = 0, total_locs = 0; + + // critical points for testing bounded iteration + // each point has 3 sides; each region has 3 ends; obj + 4 oop blocks, plus 2 +/- 0x10 + const int max_crits = 3 * 3 * (1+4) + 2; + T* obj_start = obj->obj_field_addr(0); + T* crits[max_crits]; + int num_crits = 0; + crits[num_crits++] = obj_start - 0x10; + #define ADD_CRIT(p) { if ((T*)(p) > crits[num_crits-1]) crits[num_crits++] = (T*)(p); } + #define ADD_CRIT_3(p) { ADD_CRIT((T*)p - 1); ADD_CRIT((T*)p); ADD_CRIT((T*)p + 1); } + ADD_CRIT_3(obj_start); + + // iterate the normal way, and record everything + OopMapBlock* mmin = start_of_nonstatic_oop_maps(); + OopMapBlock* mmax = mmin + nonstatic_oop_map_count(); + for (OopMapBlock* m = mmin; m < mmax; m++) { + T* pmin = obj->obj_field_addr(m->offset()); + T* pmax = obj->obj_field_addr(m->offset() + m->size()); + assert(pmin < pmax, "non-empty oop map block"); + if (m < mmin+2 || m >= mmax-2) { + T* pmid = pmin + (pmax - pmin) / 2; + ADD_CRIT_3(pmin); ADD_CRIT_3(pmid); ADD_CRIT_3(pmax); + } + for (T* p = pmin; p < pmax; p++) { + assert(p != NULL, ""); + total_locs += 1; + assert(num_locs == 0 || locs[num_locs-1] < p, "monotonic"); + if (num_locs < max_locs) + locs[num_locs++] = p; + } + } + + // Fill remainder of locs array with nulls for an error check. + { int n = num_locs; while (n < max_locs) locs[n++] = NULL; } + + T* obj_end = (T*)( (HeapWord*)obj + obj->size() ); + ADD_CRIT_3(obj_end); + ADD_CRIT(obj_end + 0x10); + assert(num_crits <= max_crits, "oob"); + #undef ADD_CRIT + #undef ADD_CRIT_0 + #undef ADD_CRIT_3 + + // use a forward iterator + oop_map::iterator p, pmin, pmax; + oop_map::bounded_iterator bp, bpmax; + int n = 0; + for (p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + T* loc1 = addr(p); + T* loc0 = (n < max_locs ? locs[n] : loc1); n++; + assert(loc0 == loc1, "same result"); + } + assert(n == total_locs, "same count"); + n = total_locs; + for (p = reverse_nonstatic_oop_fields(obj, pmin); p >= pmin; p--) { + T* loc1 = addr(p); + T* loc0 = (--n < max_locs ? locs[n] : loc1); + assert(loc0 == loc1, "same result"); + } + assert(n == 0, "same count"); + for (int i = 0; i < num_crits; i++) { + for (int j = i; j < num_crits; j++) { + MemRegion mr(crits[i], crits[j]); + n = 0; + while (n < num_locs && !mr.contains(locs[n])) ++n; + for (bp = bounded_nonstatic_oop_fields(obj, mr, bpmax); bp < bpmax; bp++) { + T* loc1 = addr(bp); + assert(mr.contains(loc1), "in region"); + T* loc0 = (n < max_locs ? locs[n] : loc1); n++; + assert(loc0 == loc1, "same result"); + } + while (n < num_locs) { + assert(!mr.contains(locs[n]), "outside region"); + n++; + } + } + } +} + +static int verify_iterator_count = 10000; +#endif //PRODUCT + void instanceKlass::oop_verify_on(oop obj, outputStream* st) { Klass::oop_verify_on(obj, st); VerifyFieldClosure blk; oop_oop_iterate(obj, &blk); + +#ifndef PRODUCT + if (has_nonstatic_oop_maps() && verify_iterator_count > 0) { + --verify_iterator_count; + WITH_BOTH_OOP_KINDS(T, oop_verify_oop_map_iterators(obj)); + } +#endif //PRODUCT } #ifndef PRODUCT @@ -2083,14 +2063,14 @@ first_time = false; const int extra = java_lang_Class::number_of_fake_oop_fields; guarantee(ik->nonstatic_field_size() == extra, "just checking"); - guarantee(ik->nonstatic_oop_map_size() == 1, "just checking"); - guarantee(ik->size_helper() == align_object_size(instanceOopDesc::header_size() + extra), "just checking"); + guarantee(ik->nonstatic_oop_map_count() == 1, "just checking"); + guarantee(ik->fixed_size() == align_object_size(instanceOopDesc::header_size() + extra), "just checking"); // Check that the map is (2,extra) int offset = java_lang_Class::klass_offset; OopMapBlock* map = ik->start_of_nonstatic_oop_maps(); - guarantee(map->offset() == offset && map->length() == extra, "just checking"); + guarantee(map->offset() == offset && map->size() == extra * heapOopSize, "just checking"); } } --- old/src/share/vm/oops/instanceKlass.hpp 2008-07-12 17:48:37.000000000 -0700 +++ new/src/share/vm/oops/instanceKlass.hpp 2008-07-12 17:48:36.000000000 -0700 @@ -53,7 +53,7 @@ // [static field size ] // [nonstatic field size ] // [static oop fields size ] -// [nonstatic oop maps size ] +// [nonstatic oop maps offset ] // [has finalize method ] // [deoptimization mark bit ] // [initialization state ] @@ -63,7 +63,7 @@ // [EMBEDDED Java vtable ] size in words = vtable_len // [EMBEDDED static oop fields ] size in words = static_oop_fields_size // [ static non-oop fields ] size in words = static_field_size - static_oop_fields_size -// [EMBEDDED nonstatic oop-map blocks] size in words = nonstatic_oop_map_size +// [EMBEDDED nonstatic oop-map blocks] size in words = oop_map_block_size // // The embedded nonstatic oop-map blocks are short pairs (offset, length) indicating // where oops are located in instances of this klass. @@ -180,13 +180,14 @@ // End of the oop block. // + int _object_size; // object size in words // number of words used by non-static fields in this klass (including // inherited fields but after header_size()). If fields are compressed into - // header, this can be zero so it's not the same as number of static fields. + // header, this can be slightly less than the combined size of all nonstatic fields. int _nonstatic_field_size; int _static_field_size; // number words used by static fields (oop and non-oop) in this klass int _static_oop_field_size;// number of static oop fields in this klass - int _nonstatic_oop_map_size;// number of nonstatic oop-map blocks allocated at end of this klass + int _map_offset_in_bytes; // offset of nonstatic oop-map blocks allocated at end of this klass bool _is_marked_dependent; // used for marking during flushing and deoptimization bool _rewritten; // methods rewritten. bool _has_nonstatic_fields; // for sizing with UseCompressedOops @@ -205,6 +206,7 @@ nmethod* _osr_nmethods_head; // Head of list of on-stack replacement nmethods for this class BreakpointInfo* _breakpoints; // bpt lists, managed by methodOop int _nof_implementors; // No of implementors of this interface (zero if not an interface) + juint _alloc_size; // allocation profiling support // Array of interesting part(s) of the previous version(s) of this // instanceKlass. See PreviousVersionWalker below. GrowableArray* _previous_versions; @@ -228,6 +230,9 @@ bool has_nonstatic_fields() const { return _has_nonstatic_fields; } void set_has_nonstatic_fields(bool b) { _has_nonstatic_fields = b; } + int object_size() const { return _object_size; } + void set_object_size(int size) { _object_size = size; } + // field sizes int nonstatic_field_size() const { return _nonstatic_field_size; } void set_nonstatic_field_size(int size) { _nonstatic_field_size = size; } @@ -395,8 +400,8 @@ void set_source_debug_extension(symbolOop n){ oop_store_without_check((oop*) &_source_debug_extension, (oop) n); } // nonstatic oop-map blocks - int nonstatic_oop_map_size() const { return _nonstatic_oop_map_size; } - void set_nonstatic_oop_map_size(int size) { _nonstatic_oop_map_size = size; } + int map_offset_in_bytes() const { return _map_offset_in_bytes; } + void set_map_offset_in_bytes(int n) { _map_offset_in_bytes = n; } // RedefineClasses() support for previous versions: void add_previous_version(instanceKlassHandle ikh, BitMap *emcp_methods, @@ -544,9 +549,10 @@ bool compute_is_subtype_of(klassOop k); bool can_be_primary_super_slow() const; klassOop java_super() const { return super(); } - int oop_size(oop obj) const { return size_helper(); } + int oop_size(oop obj) const { return fixed_size(); } int klass_oop_size() const { return object_size(); } bool oop_is_instance_slow() const { return true; } + bool oop_is_allprim_slow() const { return !has_nonstatic_oop_maps(); } // Iterators void do_local_static_fields(FieldClosure* cl); @@ -567,7 +573,10 @@ // Sizing (in words) static int header_size() { return align_object_offset(oopDesc::header_size() + sizeof(instanceKlass)/HeapWordSize); } - int object_size() const { return object_size(align_object_offset(vtable_length()) + align_object_offset(itable_length()) + static_field_size() + nonstatic_oop_map_size()); } + void init_object_size() { _object_size = object_size(align_object_offset(vtable_length()) + + align_object_offset(itable_length()) + + static_field_size() + + oop_map_block_size()); } static int vtable_start_offset() { return header_size(); } static int vtable_length_offset() { return oopDesc::header_size() + offset_of(instanceKlass, _vtable_len) / HeapWordSize; } static int object_size(int extra) { return align_object_size(header_size() + extra); } @@ -588,18 +597,44 @@ return (intptr_t)start_of_static_fields() - (intptr_t)as_klassOop(); } + HeapWord* end_of_static_fields() const { + return start_of_static_fields() + static_field_size(); + } + + protected: + // store size value in header of oop map block: + inline void set_nonstatic_oop_map_count(int size); // see below + + // return size in words: + static inline int oop_map_block_size(int num_blocks); // see below + inline int oop_map_block_size() const { + return oop_map_block_size(nonstatic_oop_map_count()); + } + + public: + bool has_nonstatic_oop_maps() const { + return map_offset_in_bytes() != 0; + } + OopMapBlock* start_of_nonstatic_oop_maps() const { - return (OopMapBlock*) (start_of_static_fields() + static_field_size()); +#ifdef ASSERT + if (!has_nonstatic_oop_maps()) return (OopMapBlock*) badAddress; + // else any garbage value will do... +#endif + return (OopMapBlock*) ((intptr_t) as_klassOop() + map_offset_in_bytes()); } + inline int nonstatic_oop_map_count() const; // see below + // Allocation profiling support - juint alloc_size() const { return _alloc_count * size_helper(); } - void set_alloc_size(juint n) {} + juint alloc_size() const { return _alloc_size; } + void set_alloc_size(juint n) { _alloc_size = n; } - // Use this to return the size of an instance in heap words: - int size_helper() const { - return layout_helper_to_size_helper(layout_helper()); - } + // use this one only on non-variable instances: + int fixed_size() const { return layout_helper().fixed_size_in_bytes() >> LogHeapWordSize; } + + // the more general version allows for mixed arrays (default length is zero): + int new_instance_size() const { return layout_helper().new_instance_size_in_bytes() >> LogHeapWordSize; } // This bit is initialized in classFileParser.cpp. // It is false under any of the following conditions: @@ -608,7 +643,7 @@ // - the class size is larger than FastAllocateSizeLimit // - the class is java/lang/Class, which cannot be allocated directly bool can_be_fastpath_allocated() const { - return !layout_helper_needs_slow_path(layout_helper()); + return !layout_helper().needs_slow_path(); } // Java vtable/itable @@ -648,26 +683,285 @@ char* signature_name() const; // Iterators - int oop_oop_iterate(oop obj, OopClosure* blk) { - return oop_oop_iterate_v(obj, blk); - } - - int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr) { - return oop_oop_iterate_v_m(obj, blk, mr); - } + int oop_oop_iterate(oop obj, OopClosure* blk); + int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr); + // Populate the vtable with customized versions of the iterators. #define InstanceKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \ int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, \ - MemRegion mr); + MemRegion mr); - ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DECL) - ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DECL) + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DECL) + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DECL) void iterate_static_fields(OopClosure* closure); void iterate_static_fields(OopClosure* closure, MemRegion mr); -private: + private: + // file-local common implementation for the iterators + template + inline int oop_oop_iterate_tp(oop obj, OopClosureType* blk); + template + inline int oop_oop_iterate_tp_m(oop obj, OopClosureType* blk, MemRegion mr); + + // This is a return value which is valid for normal instances: + int cheap_fixed_size() const { return layout_helper().fixed_size_in_bytes(true) >> LogHeapWordSize; } + + protected: + // Common code for iterating over oops. + template + typename oop_chunk::iterator static_oop_fields(typename oop_chunk::iterator& pmax) { + return oop_chunk::forward((T*)start_of_static_fields(), static_oop_field_size(), pmax); + } + + // Iterator algorithm: Mixing in upper and lower bounds. + // Assumes an iterator of the above form. + template + class iterator_bound_mixin : public I { + typedef typename I::addr_t addr_t; + addr_t _bound; + + static bool bounded_nonempty_forward(iterator_bound_mixin& beg, + iterator_bound_mixin& end) { + I& ibeg = beg; I& iend = end; + while (ibeg < iend) { + I::enforce_bounds(beg, end, beg._bound, end._bound); + if (I::is_primed(beg, end)) + return true; + // else try another go of an outer loop + } + return false; + } + + public: + iterator_bound_mixin() { } + iterator_bound_mixin(I p, addr_t bound) + : iterator(p), _bound(bound) { } + bool operator<(iterator_bound_mixin& end) { + return I::is_primed(*this, end) || bounded_nonempty_forward(*this, end); + } + bool operator>=(iterator_bound_mixin& beg) { + ShouldNotReachHere(); return false; + } + static iterator_bound_mixin make(MemRegion mr, + I ubeg, I uend, + iterator_bound_mixin& end) { + end = iterator_bound_mixin(uend, (addr_t) mr.end()); + return iterator_bound_mixin(ubeg, (addr_t) mr.start()); + } + }; + + template + class oop_map : AllStatic { + typedef OopMapBlock Map; // shorthand + public: + class iterator VALUE_OBJ_CLASS_SPEC { + friend class oop_map; + address _base; // loop invariant base for each oop map block + Map* _map; // outer loop: pointer to oop map blocks + T* _p; // inner loop: pointer to current oop + + // Iterators occur in pairs. The pair is a loop cursor and a limit. + + // The loop, over an iterator pair {beg,end}, works like this: + // int map_count = ...; + // address beg_base = ...; + // address end_base = beg_base; + // Map* beg_map = ...; + // Map* end_map = beg_map + map_count; + // for (; beg_map < end_map; beg_map++) { + // T* beg_p = (T*)( beg_base + beg_map->offset() ); + // T* end_p = (T*)( beg_base + beg_map->offset() + beg_map->size() ); + // for (; beg_p < end_p; beg_p++) { + // ... + // } + // } + + // The limit is upper exclusive for forward loops and lower inclusive + // for reverse loops. So both forward and reverse loops generate + // the interval [beg,end). Since the reverse loop pattern pre-decrements + // the loop cursor, a reverse loop body executes the same interval, + // but in the form [beg,end-1]. + + // The inner loop generates the addresses from the current oop map block, + // in [beg.p,end.p). (Or in reverse, [beg.p,end.p-1].) + // The initial and final states of an iterator pair specify + // an empty iteration space as [q,q) or [q,q-1], for some arbitrary 'q'. + + // An iterator pair can be in one of three states: + // - unprimed: they have identical arbitrary p values + // p=[q,q), map=[maps,maps+map_count) + // - working: p values are in the current oop map block + // p=[m.start,m.limit), map=[m1,m2) + // - exhausted: both index pairs are empty ("upside down") + // p=[q,q), map=[m,m) + + // It is important *not* to prime iterators eagerly (on construction), + // because we do not know which of the pair will serve + // as a loop variable ('p') or a loop limit ('pmin' or 'pmax'). + + // For degenerate loop iterations, the unprimed and exhausted states are + // identical. For other iterations, the first '<' or '>=' operation + // promotes the iterator pair from the unprimed to the working state. + + // Until that moment, there is no sure way to tell which iterator + // is serving as the loop cursor and which is the loop limit. + // The case 'p < pmax' is a forward loop, and 'p >= pmin' is in reverse. + + // In both cases, the left-hand operand of the comparison is the + // loop cursor. When the inner loop runs out of values, the + // left-hand operand's outer loop state is bumped, while the + // right-hand operand contains a loop-invariant limit. + // When the inner loop is reprimed from a new iteration of + // the outer loop, the two iterators are reprimed as a pair. + + public: + iterator() { } + iterator(T* obj, Map* maps) { + reset_map(obj, maps); + } + + protected: + void reset_map(T* base, Map* maps) { + _base = (address) base; + _map = maps; + _p = unprimed(base); + } + + static void reprime_chunk(iterator &beg, iterator& end, T* p, T* q) { + beg._p = p; end._p = q; + } + + static bool is_primed(iterator& beg, iterator& end, int pd = 0) { + return beg._p < end._p + pd; + } + + static void unprime(iterator& beg, iterator& end, void* q) { + reprime_chunk(beg, end, unprimed(q), unprimed(q)); + } + + // Goofy function to create unprimed states. + // The result should not be null, and predecrementing it should make sense. + template + static Q* unprimed(void* q) { return (Q*)q; } + + public: + // We bump the inner loop blindly. It's simpler that way. + // The ignored operand requests the postfix version of the operator. + void operator++(int ignore) { _p++; } + void operator--(int ignore) { _p--; } + + bool operator<(iterator& end) { return map_nonempty_forward(*this, end); } + bool operator>=(iterator& beg) { return map_nonempty_reverse(beg, *this); } + + // get the current address: + typedef T* addr_t; // needed by Klass::addr, etc. + addr_t addr() { return _p; } + + protected: + // To keep generated code simple, we manage the transitions + // between inner and outer loops in operator< and operator>=. + // These operators are not just queries, but also make + // sure the inner loop always has elements to process. + // This design relies, delicately, on the fact that + // oop maps never contain degenerate (zero-sized) entries. + // The 'is_primed' assertions check this. + + enum { PD = 1 }; // marks instances of predecrementing + + static void assert_pair_invariants(iterator& beg, iterator& end, int pd = 0) { + //assert(beg._base == end._base, "matching endpoints"); + assert(beg._p <= end._p+pd, "correct order"); + assert(beg._map <= end._map, "correct order"); + } + + // Forward loop: Always take new values from the lower endpoint. + static bool map_nonempty_forward(iterator& beg, iterator& end) { + assert_pair_invariants(beg, end); + // invariant: beg.map not not yet begun to be used + Map* m; + if (beg._p < end._p) { + // inner loop not yet exhausted + return true; + } else if ((m = beg._map) < end._map) { + // refresh inner loop indexes from outer loop + reprime_chunk(beg, end, + m->start(beg._base), + m->limit(beg._base)); + beg._map = m + 1; // advance outer loop + assert(is_primed(beg, end), "outer loop produced at least one element"); + return true; + } else { + return false; + } + } + + // Reverse loop: Always take new values from the upper endpoint. + static bool map_nonempty_reverse(iterator& beg, iterator& end) { + assert_pair_invariants(beg, end, PD); + // invariant: end.map-1 not not yet begun to be used + Map* m; + if (end._p >= beg._p) { + // inner loop not yet exhausted + return true; + } else if ((m = end._map) > beg._map) { + m -= PD; + // refresh inner loop indexes from outer loop + reprime_chunk(beg, end, + m->start(beg._base), + m->limit(beg._base) - PD); + end._map = m; // advance outer loop; PD already applied + assert(is_primed(beg, end, PD), "outer loop produced at least one element"); + return true; + } else { + return false; + } + } + + // If lower bound intersects [beg.p,end.p), push end.p downwards. + // If upper bound intersects [beg.p,end.p), push beg.p upwards. + static void enforce_bounds(iterator& beg, iterator& end, T* lo, T* hi) { + if (end._p > hi) end._p = (hi > beg._p) ? hi : beg._p; + if (beg._p < lo) beg._p = (lo < end._p) ? lo : end._p; + // the ?: conditions help us respect the invariant that (beg.p <= end.p) + } + }; + + // for (iterator pmax, p = forward(base, maps, count, pmax); p < pmax; p++) + static iterator forward(T* base, OopMapBlock* maps, size_t count, iterator& end) { + end .reset_map(base, maps + count); + return iterator(base, maps); + } + + typedef iterator_bound_mixin bounded_iterator; + }; + + template + typename oop_map::iterator nonstatic_oop_fields(oop obj, + typename oop_map::iterator& pmax) { + assert(obj != NULL, "can't follow the content of NULL object"); + return oop_map::forward(obj->obj_field_addr(0), + start_of_nonstatic_oop_maps(), + nonstatic_oop_map_count(), + pmax); + } + + template + typename oop_map::iterator reverse_nonstatic_oop_fields(oop obj, + typename oop_map::iterator& pmin) { + return Klass::iterator_reverse(nonstatic_oop_fields(obj, pmin), pmin); + } + + template + typename oop_map::bounded_iterator + bounded_nonstatic_oop_fields(oop obj, MemRegion mr, + typename oop_map::bounded_iterator& pmax) { + typename oop_map::iterator uend, ubeg = nonstatic_oop_fields(obj, uend); + return typename oop_map::bounded_iterator::make(mr, ubeg, uend, pmax); + } + + private: // initialization state #ifdef ASSERT void set_init_state(ClassState state); @@ -763,6 +1057,7 @@ void oop_verify_on(oop obj, outputStream* st); #ifndef PRODUCT + template void oop_verify_oop_map_iterators(oop obj); static void verify_class_klass_nonstatic_oop_maps(klassOop k) PRODUCT_RETURN; #endif }; @@ -800,17 +1095,53 @@ class OopMapBlock VALUE_OBJ_CLASS_SPEC { private: - jushort _offset; // Offset of first oop in oop-map block - jushort _length; // Length of oop-map block + jushort _offset; // Offset of first oop in oop-map block, in bytes + jushort _size; // Length of oop-map block, in bytes public: // Accessors - jushort offset() const { return _offset; } - void set_offset(jushort offset) { _offset = offset; } + int offset() const { return _offset; } + void set_offset(int offset) { _offset = offset; assert(_offset == offset, "no trunc"); } - jushort length() const { return _length; } - void set_length(jushort length) { _length = length; } + template T* start(address oop_base) { + T* addr = (T*)(oop_base + offset()); + assert(addr == ((oopDesc*)oop_base)->obj_field_addr(offset()), "consistent offset calculation"); + return addr; + } + template T* limit(address oop_base) { + return start(oop_base + size()); + } + + int size() const { return _size; } + void set_size(int size) { _size = size; assert(_size == size, "no trunc"); } + + int limit() const { return _offset + _size; } }; +// These require a complete declaration of OopMapBlock: +inline int instanceKlass::nonstatic_oop_map_count() const { + // First block, if any, contains the count of remaining blocks. + if (!has_nonstatic_oop_maps()) return 0; + return start_of_nonstatic_oop_maps()[-1].size(); +} +inline void instanceKlass::set_nonstatic_oop_map_count(int count) { + if (count == 0) { + set_map_offset_in_bytes(0); + } else { + OopMapBlock* block = 1 + (OopMapBlock*) end_of_static_fields(); + int off = (int)( (intptr_t) block - (intptr_t) as_klassOop() ); + set_map_offset_in_bytes(off); + block[-1].set_size(count); + } + assert(nonstatic_oop_map_count() == count, "it worked"); +} +inline int instanceKlass::oop_map_block_size(int count) { + if (count == 0) + return 0; // no blocks at all + // otherwise, allocate a hidden leading block to carry the count: + count += 1; + return align_size_up(count * sizeof(OopMapBlock), BytesPerWord) >> LogBytesPerWord; +} + /* JNIid class for jfieldIDs only */ class JNIid: public CHeapObj { friend class VMStructs; --- old/src/share/vm/oops/instanceKlassKlass.cpp 2008-07-12 17:48:40.000000000 -0700 +++ new/src/share/vm/oops/instanceKlassKlass.cpp 2008-07-12 17:48:39.000000000 -0700 @@ -381,9 +381,13 @@ #endif // SERIALGC klassOop instanceKlassKlass::allocate_instance_klass(int vtable_len, int itable_len, int static_field_size, - int nonstatic_oop_map_size, ReferenceType rt, TRAPS) { - - int size = instanceKlass::object_size(align_object_offset(vtable_len) + align_object_offset(itable_len) + static_field_size + nonstatic_oop_map_size); + int oop_map_count, ReferenceType rt, TRAPS) { + // oop map blocks + int oop_map_block_size = instanceKlass::oop_map_block_size(oop_map_count); + int size = instanceKlass::object_size(align_object_offset(vtable_len) + + align_object_offset(itable_len) + + static_field_size + + oop_map_block_size); // Allocation KlassHandle h_this_klass(THREAD, as_klassOop()); @@ -402,13 +406,15 @@ instanceKlass* ik = (instanceKlass*) k()->klass_part(); assert(!k()->is_parsable(), "not expecting parsability yet."); - // The sizes of these these three variables are used for determining the - // size of the instanceKlassOop. It is critical that these are set to the right - // sizes before the first GC, i.e., when we allocate the mirror. + // The value of this variable is used for determining the + // size of the instanceKlassOop. It is critical that it is set to the right + // value before the first GC, i.e., when we allocate the mirror. + ik->set_object_size(size); + + // These variables determine the variable internal layout of the object: ik->set_vtable_length(vtable_len); ik->set_itable_length(itable_len); ik->set_static_field_size(static_field_size); - ik->set_nonstatic_oop_map_size(nonstatic_oop_map_size); assert(k()->size() == size, "wrong size for object"); ik->set_array_klasses(NULL); @@ -450,10 +456,13 @@ assert(k()->is_parsable(), "should be parsable here."); // initialize the non-header words to zero - intptr_t* p = (intptr_t*)k(); - for (int index = instanceKlass::header_size(); index < size; index++) { - p[index] = NULL_WORD; - } + HeapWord* beg = (HeapWord*)k(); + HeapWord* end = beg + size; + beg += instanceKlass::header_size(); + Copy::zero_to_words(beg, end - beg); + + // Initialize the oop map mini-header after the bulk zeroing: + ik->set_nonstatic_oop_map_count(oop_map_count); // To get verify to work - must be set to partial loaded before first GC point. k()->set_partially_loaded(); @@ -480,7 +489,7 @@ instanceKlass* ik = instanceKlass::cast(klassOop(obj)); klassKlass::oop_print_on(obj, st); - st->print(" - instance size: %d", ik->size_helper()); st->cr(); + st->print(" - instance size: %d", ik->new_instance_size()); st->cr(); st->print(" - klass size: %d", ik->object_size()); st->cr(); st->print(" - access: "); ik->access_flags().print_on(st); st->cr(); st->print(" - state: "); st->print_cr(state_names[ik->_init_state]); @@ -537,8 +546,6 @@ ik->source_debug_extension()->print_value_on(st); st->cr(); } - - st->print_cr(" - previous version: "); { ResourceMark rm; // PreviousVersionInfo objects returned via PreviousVersionWalker @@ -546,18 +553,22 @@ // GrowableArray _after_ the PreviousVersionWalker destructor // has destroyed the handles. { + bool pvcount = 0; PreviousVersionWalker pvw(ik); for (PreviousVersionInfo * pv_info = pvw.next_previous_version(); pv_info != NULL; pv_info = pvw.next_previous_version()) { + if (++pvcount == 1) + st->print_cr(" - previous version: "); pv_info->prev_constant_pool_handle()()->print_value_on(st); } - st->cr(); + if (pvcount != 0) st->cr(); } // pvw is cleaned up } // rm is cleaned up if (ik->generic_signature() != NULL) { st->print(" - generic signature: "); ik->generic_signature()->print_value_on(st); + st->cr(); } st->print(" - inner classes: "); ik->inner_classes()->print_value_on(st); st->cr(); st->print(" - java mirror: "); ik->java_mirror()->print_value_on(st); st->cr(); @@ -579,9 +590,9 @@ st->print(" - non-static oop maps: "); OopMapBlock* map = ik->start_of_nonstatic_oop_maps(); - OopMapBlock* end_map = map + ik->nonstatic_oop_map_size(); + OopMapBlock* end_map = map + ik->nonstatic_oop_map_count(); while (map < end_map) { - st->print("%d-%d ", map->offset(), map->offset() + oopSize*(map->length() - 1)); + st->print("%d-%d ", map->offset(), map->offset() + map->size()); map++; } st->cr(); @@ -804,7 +815,9 @@ instanceKlass* ik = instanceKlass::cast(klassOop(obj)); // Set the layout helper to a place-holder value, until fuller initialization. // (This allows asserts in oop_is_instance to succeed.) - ik->set_layout_helper(Klass::instance_layout_helper(0, true)); + LayoutHelper lh = LayoutHelper::for_instance(oopDesc::header_size()); + lh.set_low_bit(LayoutHelper::_slow_path_low_bit); + ik->set_layout_helper(lh); assert(ik->oop_is_instance(), "object must be instanceKlass"); assert(ik->transitive_interfaces() == NULL, "just checking"); ik->set_transitive_interfaces((objArrayOop) obj); // Temporarily set transitive_interfaces to point to self --- old/src/share/vm/oops/instanceKlassKlass.hpp 2008-07-12 17:48:42.000000000 -0700 +++ new/src/share/vm/oops/instanceKlassKlass.hpp 2008-07-12 17:48:42.000000000 -0700 @@ -39,7 +39,7 @@ klassOop allocate_instance_klass(int vtable_len, int itable_len, int static_field_size, - int nonstatic_oop_map_size, + int nonstatic_oop_map_count, ReferenceType rt, TRAPS); --- old/src/share/vm/oops/instanceRefKlass.cpp 2008-07-12 17:48:45.000000000 -0700 +++ new/src/share/vm/oops/instanceRefKlass.cpp 2008-07-12 17:48:45.000000000 -0700 @@ -164,7 +164,7 @@ } int instanceRefKlass::oop_adjust_pointers(oop obj) { - int size = size_helper(); + int size = fixed_size(); instanceKlass::oop_adjust_pointers(obj); if (UseCompressedOops) { @@ -325,7 +325,7 @@ } else { specialized_oop_update_pointers(this, cm, obj); } - return size_helper(); + return fixed_size(); } @@ -352,7 +352,7 @@ } else { specialized_oop_update_pointers(cm, obj, beg_addr, end_addr); } - return size_helper(); + return fixed_size(); } #endif // SERIALGC @@ -369,7 +369,7 @@ assert(k == SystemDictionary::reference_klass() && first_time, "Invalid update of maps"); debug_only(first_time = false); - assert(ik->nonstatic_oop_map_size() == 1, "just checking"); + assert(ik->nonstatic_oop_map_count() == 1, "just checking"); OopMapBlock* map = ik->start_of_nonstatic_oop_maps(); @@ -381,14 +381,14 @@ if (UseSharedSpaces) { assert(map->offset() == java_lang_ref_Reference::queue_offset && - map->length() == 1, "just checking"); + map->size() == heapOopSize, "just checking"); } else { - assert(map->offset() == offset && map->length() == length, + assert(map->offset() == offset && map->size() == length * heapOopSize, "just checking"); // Update map to (3,1) - point to offset of 3 (words) with 1 map entry. map->set_offset(java_lang_ref_Reference::queue_offset); - map->set_length(1); + map->set_size(heapOopSize); } } --- old/src/share/vm/oops/instanceRefKlass.hpp 2008-07-12 17:48:48.000000000 -0700 +++ new/src/share/vm/oops/instanceRefKlass.hpp 2008-07-12 17:48:47.000000000 -0700 @@ -60,13 +60,6 @@ // Parallel Scavenge and Parallel Old PARALLEL_GC_DECLS - int oop_oop_iterate(oop obj, OopClosure* blk) { - return oop_oop_iterate_v(obj, blk); - } - int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr) { - return oop_oop_iterate_v_m(obj, blk, mr); - } - #define InstanceRefKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \ int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, MemRegion mr); --- old/src/share/vm/oops/klass.cpp 2008-07-12 17:48:50.000000000 -0700 +++ new/src/share/vm/oops/klass.cpp 2008-07-12 17:48:50.000000000 -0700 @@ -135,7 +135,7 @@ kl->set_java_mirror(NULL); kl->set_modifier_flags(0); - kl->set_layout_helper(Klass::_lh_neutral_value); + kl->set_layout_helper(LayoutHelper()); kl->set_name(NULL); AccessFlags af; af.set_flags(0); @@ -178,26 +178,6 @@ return k->klass_part(); } -jint Klass::array_layout_helper(BasicType etype) { - assert(etype >= T_BOOLEAN && etype <= T_OBJECT, "valid etype"); - // Note that T_ARRAY is not allowed here. - int hsize = arrayOopDesc::base_offset_in_bytes(etype); - int esize = type2aelembytes(etype); - bool isobj = (etype == T_OBJECT); - int tag = isobj ? _lh_array_tag_obj_value : _lh_array_tag_type_value; - int lh = array_layout_helper(tag, hsize, etype, exact_log2(esize)); - - assert(lh < (int)_lh_neutral_value, "must look like an array layout"); - assert(layout_helper_is_javaArray(lh), "correct kind"); - assert(layout_helper_is_objArray(lh) == isobj, "correct kind"); - assert(layout_helper_is_typeArray(lh) == !isobj, "correct kind"); - assert(layout_helper_header_size(lh) == hsize, "correct decode"); - assert(layout_helper_element_type(lh) == etype, "correct decode"); - assert(1 << layout_helper_log2_element_size(lh) == esize, "correct decode"); - - return lh; -} - bool Klass::can_be_primary_super_slow() const { if (super() == NULL) return true; --- old/src/share/vm/oops/klass.hpp 2008-07-12 17:48:54.000000000 -0700 +++ new/src/share/vm/oops/klass.hpp 2008-07-12 17:48:53.000000000 -0700 @@ -158,33 +158,9 @@ // for better cache behavior (may not make much of a difference but sure won't hurt) enum { _primary_super_limit = 8 }; - // The "layout helper" is a combined descriptor of object layout. - // For klasses which are neither instance nor array, the value is zero. - // - // For instances, layout helper is a positive number, the instance size. - // This size is already passed through align_object_size and scaled to bytes. - // The low order bit is set if instances of this class cannot be - // allocated using the fastpath. - // - // For arrays, layout helper is a negative number, containing four - // distinct bytes, as follows: - // MSB:[tag, hsz, ebt, log2(esz)]:LSB - // where: - // tag is 0x80 if the elements are oops, 0xC0 if non-oops - // hsz is array header size in bytes (i.e., offset of first element) - // ebt is the BasicType of the elements - // esz is the element size in bytes - // This packed word is arranged so as to be quickly unpacked by the - // various fast paths that use the various subfields. - // - // The esz bits can be used directly by a SLL instruction, without masking. - // - // Note that the array-kind tag looks like 0x00 for instance klasses, - // since their length in bytes is always less than 24Mb. - // - // Final note: This comes first, immediately after Klass_vtbl, + // This comes first, immediately after Klass_vtbl, // because it is frequently queried. - jint _layout_helper; + LayoutHelper _layout_helper; // The fields _super_check_offset, _secondary_super_cache, _secondary_supers // and _primary_supers all help make fast subtype checks. See big discussion @@ -319,11 +295,11 @@ jint modifier_flags() const { return _modifier_flags; } void set_modifier_flags(jint flags) { _modifier_flags = flags; } - // size helper - int layout_helper() const { return _layout_helper; } - void set_layout_helper(int lh) { _layout_helper = lh; } + // layout helper + const LayoutHelper layout_helper() const { return _layout_helper; } + void set_layout_helper(LayoutHelper lh) { _layout_helper = lh; } - // Note: for instances layout_helper() may include padding. + // Note: the size reported by layout_helper may include padding. // Use instanceKlass::contains_field_offset to classify field offsets. // sub/superklass links @@ -365,80 +341,6 @@ static int layout_helper_offset_in_bytes() { return offset_of(Klass, _layout_helper); } static int access_flags_offset_in_bytes() { return offset_of(Klass, _access_flags); } - // Unpacking layout_helper: - enum { - _lh_neutral_value = 0, // neutral non-array non-instance value - _lh_instance_slow_path_bit = 0x01, - _lh_log2_element_size_shift = BitsPerByte*0, - _lh_log2_element_size_mask = BitsPerLong-1, - _lh_element_type_shift = BitsPerByte*1, - _lh_element_type_mask = right_n_bits(BitsPerByte), // shifted mask - _lh_header_size_shift = BitsPerByte*2, - _lh_header_size_mask = right_n_bits(BitsPerByte), // shifted mask - _lh_array_tag_bits = 2, - _lh_array_tag_shift = BitsPerInt - _lh_array_tag_bits, - _lh_array_tag_type_value = ~0x00, // 0xC0000000 >> 30 - _lh_array_tag_obj_value = ~0x01 // 0x80000000 >> 30 - }; - - static int layout_helper_size_in_bytes(jint lh) { - assert(lh > (jint)_lh_neutral_value, "must be instance"); - return (int) lh & ~_lh_instance_slow_path_bit; - } - static bool layout_helper_needs_slow_path(jint lh) { - assert(lh > (jint)_lh_neutral_value, "must be instance"); - return (lh & _lh_instance_slow_path_bit) != 0; - } - static bool layout_helper_is_instance(jint lh) { - return (jint)lh > (jint)_lh_neutral_value; - } - static bool layout_helper_is_javaArray(jint lh) { - return (jint)lh < (jint)_lh_neutral_value; - } - static bool layout_helper_is_typeArray(jint lh) { - // _lh_array_tag_type_value == (lh >> _lh_array_tag_shift); - return (juint)lh >= (juint)(_lh_array_tag_type_value << _lh_array_tag_shift); - } - static bool layout_helper_is_objArray(jint lh) { - // _lh_array_tag_obj_value == (lh >> _lh_array_tag_shift); - return (jint)lh < (jint)(_lh_array_tag_type_value << _lh_array_tag_shift); - } - static int layout_helper_header_size(jint lh) { - assert(lh < (jint)_lh_neutral_value, "must be array"); - int hsize = (lh >> _lh_header_size_shift) & _lh_header_size_mask; - assert(hsize > 0 && hsize < (int)sizeof(oopDesc)*3, "sanity"); - return hsize; - } - static BasicType layout_helper_element_type(jint lh) { - assert(lh < (jint)_lh_neutral_value, "must be array"); - int btvalue = (lh >> _lh_element_type_shift) & _lh_element_type_mask; - assert(btvalue >= T_BOOLEAN && btvalue <= T_OBJECT, "sanity"); - return (BasicType) btvalue; - } - static int layout_helper_log2_element_size(jint lh) { - assert(lh < (jint)_lh_neutral_value, "must be array"); - int l2esz = (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask; - assert(l2esz <= LogBitsPerLong, "sanity"); - return l2esz; - } - static jint array_layout_helper(jint tag, int hsize, BasicType etype, int log2_esize) { - return (tag << _lh_array_tag_shift) - | (hsize << _lh_header_size_shift) - | ((int)etype << _lh_element_type_shift) - | (log2_esize << _lh_log2_element_size_shift); - } - static jint instance_layout_helper(jint size, bool slow_path_flag) { - return (size << LogHeapWordSize) - | (slow_path_flag ? _lh_instance_slow_path_bit : 0); - } - static int layout_helper_to_size_helper(jint lh) { - assert(lh > (jint)_lh_neutral_value, "must be instance"); - // Note that the following expression discards _lh_instance_slow_path_bit. - return lh >> LogHeapWordSize; - } - // Out-of-line version computes everything based on the etype: - static jint array_layout_helper(BasicType etype); - // What is the maximum number of primary superclasses any klass can have? #ifdef PRODUCT static juint primary_super_limit() { return _primary_super_limit; } @@ -557,9 +459,11 @@ public: // type testing operations + virtual bool oop_is_allprim_slow() const { return false; } virtual bool oop_is_instance_slow() const { return false; } virtual bool oop_is_instanceRef() const { return false; } - virtual bool oop_is_array() const { return false; } + virtual bool oop_is_array_slow() const { return false; } + virtual bool oop_has_length_field_slow() const { return false; } virtual bool oop_is_objArray_slow() const { return false; } virtual bool oop_is_symbol() const { return false; } virtual bool oop_is_klass() const { return false; } @@ -576,9 +480,8 @@ virtual bool oop_is_compiledICHolder() const { return false; } virtual bool oop_is_instanceKlass() const { return false; } - bool oop_is_javaArray_slow() const { - return oop_is_objArray_slow() || oop_is_typeArray_slow(); - } + // for old code (all arrays are java arrays now): + bool oop_is_javaArray() const { return oop_is_array(); } // Fast non-virtual versions, used by oop.inline.hpp and elsewhere: #ifndef ASSERT @@ -592,16 +495,22 @@ public: #endif inline bool oop_is_instance() const { return assert_same_query( - layout_helper_is_instance(layout_helper()), + layout_helper().is_instance(), oop_is_instance_slow()); } - inline bool oop_is_javaArray() const { return assert_same_query( - layout_helper_is_javaArray(layout_helper()), - oop_is_javaArray_slow()); } + inline bool oop_has_length_field() const { return assert_same_query( + layout_helper().has_length_field(), + oop_has_length_field_slow()); } + inline bool oop_is_allprim() const { return assert_same_query( + layout_helper().is_allprim(), + oop_is_allprim_slow()); } + inline bool oop_is_array() const { return assert_same_query( + layout_helper().is_array(), + oop_is_array_slow()); } inline bool oop_is_objArray() const { return assert_same_query( - layout_helper_is_objArray(layout_helper()), + layout_helper().is_objArray(), oop_is_objArray_slow()); } inline bool oop_is_typeArray() const { return assert_same_query( - layout_helper_is_typeArray(layout_helper()), + layout_helper().is_typeArray(), oop_is_typeArray_slow()); } #undef assert_same_query @@ -688,20 +597,73 @@ } \ } - // iterators + protected: + // Basic iterator for working on embedded oop fields. + // The style is similar to standard C++ iterators. + // (This first instance is overkill, since the iteration + // is over simple C pointer pairs. It conforms to a pattern + // which we reuse uniformly for more complex iterations.) + template + class oop_chunk : AllStatic { + public: + typedef T* iterator; + typedef T* bounded_iterator; + + // for (iterator pmax, p = forward(base, count, pmax); p < pmax; p++) + static iterator forward(T* base, size_t count, iterator& end_out) { + end_out = base + count; return base; + } + // for (iterator pmax, p = region(mr, pmax); p < pmax; p++) + static iterator forward(MemRegion mr, iterator& end) { + return forward((T*)mr.start(), + pointer_delta((T*)mr.end(), (T*)mr.start()), + end); + } + static bounded_iterator bounded(MemRegion mr, iterator beg, + bounded_iterator& end) { + end = MIN2(end, (T*) mr.end()); + return MAX2(beg, (T*) mr.start()); + } + }; + // Function for forcing an iterator to yield an oop*: + static oop* addr(oop* p) { return p; } + static narrowOop* addr(narrowOop* p) { return p; } + template static + typename I::addr_t addr(I p) { return p.addr(); } + // (We do not use the '*' operator as with the C++ STL, because we don't + // want to indirect the oop*. We want the pointer, not the point-ee.) + + // Iterator reversal is generic: + // for (iterator pmin, p = iterator_reverse(forward(base, count, pmin), pmin); p >= pmin; p--) + template static + I iterator_reverse(I ret1, I& ret2) { + // Swap bounds, and pre-decrement the end pointer. (Do not double-reverse.) + I end = ret2; ret2 = ret1; end--; return end; + } + + // Query iteration range. + template static + MemRegion iterator_region(I beg, I end) { + typename I::addr_t lo = (beg < end ? addr(beg) : NULL); + typename I::addr_t hi = (end >= beg ? addr(end) : NULL); + return MemRegion((HeapWord*)lo, + (HeapWord*)hi); + } + + // Usage: + // for (oop_chunk::iterator pmax, p = oop_fields(oop, off, cnt, pmax); p < pmax; p++) { + // (do something with T* addr(p)) + // } + + public: + // internal iterators virtual int oop_oop_iterate(oop obj, OopClosure* blk) = 0; - virtual int oop_oop_iterate_v(oop obj, OopClosure* blk) { - return oop_oop_iterate(obj, blk); - } // Iterates "blk" over all the oops in "obj" (of type "this") within "mr". // (I don't see why the _m should be required, but without it the Solaris // C++ gives warning messages about overridings of the "oop_oop_iterate" // defined above "hiding" this virtual function. (DLD, 6/20/00)) */ virtual int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr) = 0; - virtual int oop_oop_iterate_v_m(oop obj, OopClosure* blk, MemRegion mr) { - return oop_oop_iterate_m(obj, blk, mr); - } // Versions of the above iterators specialized to particular subtypes // of OopClosure, to avoid closure virtual calls. @@ -724,6 +686,16 @@ SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(Klass_OOP_OOP_ITERATE_DECL) SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_3(Klass_OOP_OOP_ITERATE_DECL) + // oop_oop_iterate_v is an alias for oop_oop_iterate_v. + // The _v variants are just macro-friendly aliases for the general virtuals. + // (The macros have to paste something instead of _nv, so we paste _v. + // It is illegal for a C macro argument to be an empty token sequence.) + // (We don't generate these above because they are non-virtual.) +#define oop_oop_iterate_v oop_oop_iterate +#define oop_oop_iterate_v_m oop_oop_iterate_m + // Yes, these are global token replacements! + + public: virtual void array_klasses_do(void f(klassOop k)) {} virtual void with_array_klasses_do(void f(klassOop k)); --- old/src/share/vm/oops/klassKlass.cpp 2008-07-12 17:48:57.000000000 -0700 +++ new/src/share/vm/oops/klassKlass.cpp 2008-07-12 17:48:57.000000000 -0700 @@ -199,7 +199,9 @@ // Printing void klassKlass::oop_print_on(oop obj, outputStream* st) { + Klass* k = Klass::cast(klassOop(obj)); Klass::oop_print_on(obj, st); + st->print(" - layout: "); k->layout_helper().print_on(st); st->cr(); } --- old/src/share/vm/oops/objArrayKlass.cpp 2008-07-12 17:49:00.000000000 -0700 +++ new/src/share/vm/oops/objArrayKlass.cpp 2008-07-12 17:48:59.000000000 -0700 @@ -80,9 +80,28 @@ return h_array(); } +klassOop objArrayKlass::store_check_klass(oop s, oop d) { + if (s == d) { + // since source and destination are equal we do not need conversion checks. + return NULL; + } else { + // We have to make sure all elements conform to the destination array + klassOop bound = objArrayKlass::cast(d->klass())->element_klass(); + klassOop stype = objArrayKlass::cast(s->klass())->element_klass(); + if (stype == bound || Klass::cast(stype)->is_subtype_of(bound)) { + // elements are guaranteed to be subtypes, so no check necessary + return NULL; + } else { + // slow case: need individual subtype checks + return bound; + } + } +} + // Either oop or narrowOop depending on UseCompressedOops. -template void objArrayKlass::do_copy(arrayOop s, T* src, - arrayOop d, T* dst, int length, TRAPS) { +// If d is NULL, store checks are suppressed (i.e., mixed array). +template void objArrayKlass::do_copy(T* src, T* dst, size_t length, + klassOop bound, TRAPS) { const size_t word_len = objArrayOopDesc::array_size(length); @@ -91,39 +110,30 @@ BarrierSet* bs = Universe::heap()->barrier_set(); assert(bs->has_write_ref_array_opt(), "Barrier set must have ref array opt"); - if (s == d) { - // since source and destination are equal we do not need conversion checks. + if (bound == NULL || bound == SystemDictionary::object_klass()) { assert(length > 0, "sanity check"); Copy::conjoint_oops_atomic(src, dst, length); } else { - // We have to make sure all elements conform to the destination array - klassOop bound = objArrayKlass::cast(d->klass())->element_klass(); - klassOop stype = objArrayKlass::cast(s->klass())->element_klass(); - if (stype == bound || Klass::cast(stype)->is_subtype_of(bound)) { - // elements are guaranteed to be subtypes, so no check necessary - Copy::conjoint_oops_atomic(src, dst, length); - } else { - // slow case: need individual subtype checks - // note: don't use obj_at_put below because it includes a redundant store check - T* from = src; - T* end = from + length; - for (T* p = dst; from < end; from++, p++) { - // XXX this is going to be slow. - T element = *from; - if (oopDesc::is_null(element) || - Klass::cast(oopDesc::decode_heap_oop_not_null(element)->klass())->is_subtype_of(bound)) { - *p = *from; - } else { - // We must do a barrier to cover the partial copy. - const size_t pd = pointer_delta(p, dst, (size_t)heapOopSize); - // pointer delta is scaled to number of elements (length field in - // objArrayOop) which we assume is 32 bit. - assert(pd == (size_t)(int)pd, "length field overflow"); - const size_t done_word_len = objArrayOopDesc::array_size((int)pd); - bs->write_ref_array(MemRegion((HeapWord*)dst, done_word_len)); - THROW(vmSymbols::java_lang_ArrayStoreException()); - return; - } + // slow case: need individual subtype checks + // note: don't use obj_at_put below because it includes a redundant store check + T* from = src; + T* end = from + length; + for (T* p = dst; from < end; from++, p++) { + // XXX this is going to be slow. + T element = *from; + if (oopDesc::is_null(element) || + Klass::cast(oopDesc::decode_heap_oop_not_null(element)->klass())->is_subtype_of(bound)) { + *p = *from; + } else { + // We must do a barrier to cover the partial copy. + const size_t pd = pointer_delta(p, dst, (size_t)heapOopSize); + // pointer delta is scaled to number of elements (length field in + // objArrayOop) which we assume is 32 bit. + assert(pd == (size_t)(int)pd, "length field overflow"); + const size_t done_word_len = objArrayOopDesc::array_size((int)pd); + bs->write_ref_array(MemRegion((HeapWord*)dst, done_word_len)); + THROW(vmSymbols::java_lang_ArrayStoreException()); + return; } } } @@ -143,8 +153,8 @@ THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); } // Check if the ranges are valid - if ( (((unsigned int) length + (unsigned int) src_pos) > (unsigned int) s->length()) - || (((unsigned int) length + (unsigned int) dst_pos) > (unsigned int) d->length()) ) { + if ( (((unsigned int) length + (unsigned int) src_pos) > (unsigned int) arrayOopDesc::length_of(s)) + || (((unsigned int) length + (unsigned int) dst_pos) > (unsigned int) arrayOopDesc::length_of(d)) ) { THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); } @@ -155,15 +165,15 @@ if (length==0) { return; } - if (UseCompressedOops) { - narrowOop* const src = objArrayOop(s)->obj_at_addr(src_pos); - narrowOop* const dst = objArrayOop(d)->obj_at_addr(dst_pos); - do_copy(s, src, d, dst, length, CHECK); - } else { - oop* const src = objArrayOop(s)->obj_at_addr(src_pos); - oop* const dst = objArrayOop(d)->obj_at_addr(dst_pos); - do_copy (s, src, d, dst, length, CHECK); - } + + assert(d->blueprint()->layout_helper().element_type() == T_OBJECT, "simple refs here"); + + klassOop store_check = store_check_klass(s, d); + WITH_BOTH_OOP_KINDS(T, { + T* const src = objArrayOop(s)->obj_at_addr(src_pos); + T* const dst = objArrayOop(d)->obj_at_addr(dst_pos); + do_copy(src, dst, length, store_check, THREAD); + }); } @@ -263,56 +273,16 @@ Klass::cast(bottom_klass())->initialize(THREAD); // dispatches to either instanceKlass or typeArrayKlass } -#define ObjArrayKlass_SPECIALIZED_OOP_ITERATE(T, a, p, do_oop) \ -{ \ - T* p = (T*)(a)->base(); \ - T* const end = p + (a)->length(); \ - while (p < end) { \ - do_oop; \ - p++; \ - } \ -} - -#define ObjArrayKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(T, a, p, low, high, do_oop) \ -{ \ - T* const l = (T*)(low); \ - T* const h = (T*)(high); \ - T* p = (T*)(a)->base(); \ - T* end = p + (a)->length(); \ - if (p < l) p = l; \ - if (end > h) end = h; \ - while (p < end) { \ - do_oop; \ - ++p; \ - } \ -} - -#define ObjArrayKlass_OOP_ITERATE(a, p, do_oop) \ - if (UseCompressedOops) { \ - ObjArrayKlass_SPECIALIZED_OOP_ITERATE(narrowOop, \ - a, p, do_oop) \ - } else { \ - ObjArrayKlass_SPECIALIZED_OOP_ITERATE(oop, \ - a, p, do_oop) \ - } - -#define ObjArrayKlass_BOUNDED_OOP_ITERATE(a, p, low, high, do_oop) \ - if (UseCompressedOops) { \ - ObjArrayKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(narrowOop, \ - a, p, low, high, do_oop) \ - } else { \ - ObjArrayKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(oop, \ - a, p, low, high, do_oop) \ - } - void objArrayKlass::oop_follow_contents(oop obj) { assert (obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); a->follow_header(); - ObjArrayKlass_OOP_ITERATE( \ - a, p, \ - /* we call mark_and_follow here to avoid excessive marking stack usage */ \ - MarkSweep::mark_and_follow(p)) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = array_element_oops(a, pmax); p < pmax; p++) { + /* we call mark_and_follow here to avoid excessive marking stack usage */ + MarkSweep::mark_and_follow(addr(p)); + } + }); } #ifndef SERIALGC @@ -321,28 +291,65 @@ assert (obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); a->follow_header(cm); - ObjArrayKlass_OOP_ITERATE( \ - a, p, \ - /* we call mark_and_follow here to avoid excessive marking stack usage */ \ - PSParallelCompact::mark_and_follow(cm, p)) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = array_element_oops(a, pmax); p < pmax; p++) { + /* we call mark_and_follow here to avoid excessive marking stack usage */ + PSParallelCompact::mark_and_follow(cm, addr(p)); + } + }); } #endif // SERIALGC +template +inline int objArrayKlass::oop_oop_iterate_tp(oop obj, + OopClosureType* closure) { + SpecializationStats::record_iterate_call_nv(SpecializationStats::oa); + assert (obj->is_array(), "obj must be array"); + objArrayOop a = objArrayOop(obj); + /* Get size before changing pointers. */ + /* Don't call size() or oop_size() since that is a virtual call. */ + int size = a->object_size(); + if (closure->do_header()) { + a->oop_iterate_header_nv(closure); + } + for (oop_chunk::iterator pmax, p = array_element_oops(a, pmax); + p < pmax; p++) { + SpecializationStats::record_do_oop_call_nv(SpecializationStats::oa); + (closure)->do_oop_nv(addr(p)); + } + return size; +} + +template +inline int objArrayKlass::oop_oop_iterate_tp_m(oop obj, + OopClosureType* closure, + MemRegion mr) { + SpecializationStats::record_iterate_call_nv(SpecializationStats::oa); + assert(obj->is_array(), "obj must be array"); + objArrayOop a = objArrayOop(obj); + /* Get size before changing pointers. */ + /* Don't call size() or oop_size() since that is a virtual call */ + int size = a->object_size(); + if (closure->do_header()) { + a->oop_iterate_header_nv(closure, mr); + } + for (oop_chunk::iterator pmax, p = bounded_array_element_oops(a, mr, + pmax); + p < pmax; p++) { + SpecializationStats::record_do_oop_call_nv(SpecializationStats::oa); + (closure)->do_oop_nv(addr(p)); + } + return size; +} + #define ObjArrayKlass_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ \ int objArrayKlass::oop_oop_iterate##nv_suffix(oop obj, \ OopClosureType* closure) { \ - SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::oa); \ - assert (obj->is_array(), "obj must be array"); \ - objArrayOop a = objArrayOop(obj); \ - /* Get size before changing pointers. */ \ - /* Don't call size() or oop_size() since that is a virtual call. */ \ - int size = a->object_size(); \ - if (closure->do_header()) { \ - a->oop_iterate_header(closure); \ - } \ - ObjArrayKlass_OOP_ITERATE(a, p, (closure)->do_oop##nv_suffix(p)) \ - return size; \ + WITH_BOTH_OOP_KINDS(T, { \ + return oop_oop_iterate_tp(obj, \ + OopClosure::nv_from##nv_suffix(closure)); \ + }); \ } #define ObjArrayKlass_OOP_OOP_ITERATE_DEFN_m(OopClosureType, nv_suffix) \ @@ -350,62 +357,16 @@ int objArrayKlass::oop_oop_iterate##nv_suffix##_m(oop obj, \ OopClosureType* closure, \ MemRegion mr) { \ - SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::oa); \ - assert(obj->is_array(), "obj must be array"); \ - objArrayOop a = objArrayOop(obj); \ - /* Get size before changing pointers. */ \ - /* Don't call size() or oop_size() since that is a virtual call */ \ - int size = a->object_size(); \ - if (closure->do_header()) { \ - a->oop_iterate_header(closure, mr); \ - } \ - ObjArrayKlass_BOUNDED_OOP_ITERATE( \ - a, p, mr.start(), mr.end(), (closure)->do_oop##nv_suffix(p)) \ - return size; \ -} - -// Like oop_oop_iterate but only iterates over a specified range and only used -// for objArrayOops. -#define ObjArrayKlass_OOP_OOP_ITERATE_DEFN_r(OopClosureType, nv_suffix) \ - \ -int objArrayKlass::oop_oop_iterate_range##nv_suffix(oop obj, \ - OopClosureType* closure, \ - int start, int end) { \ - SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::oa); \ - assert(obj->is_array(), "obj must be array"); \ - objArrayOop a = objArrayOop(obj); \ - /* Get size before changing pointers. */ \ - /* Don't call size() or oop_size() since that is a virtual call */ \ - int size = a->object_size(); \ - if (UseCompressedOops) { \ - HeapWord* low = start == 0 ? (HeapWord*)a : (HeapWord*)a->obj_at_addr(start);\ - /* this might be wierd if end needs to be aligned on HeapWord boundary */ \ - HeapWord* high = (HeapWord*)((narrowOop*)a->base() + end); \ - MemRegion mr(low, high); \ - if (closure->do_header()) { \ - a->oop_iterate_header(closure, mr); \ - } \ - ObjArrayKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(narrowOop, \ - a, p, low, high, (closure)->do_oop##nv_suffix(p)) \ - } else { \ - HeapWord* low = start == 0 ? (HeapWord*)a : (HeapWord*)a->obj_at_addr(start); \ - HeapWord* high = (HeapWord*)((oop*)a->base() + end); \ - MemRegion mr(low, high); \ - if (closure->do_header()) { \ - a->oop_iterate_header(closure, mr); \ - } \ - ObjArrayKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(oop, \ - a, p, low, high, (closure)->do_oop##nv_suffix(p)) \ - } \ - return size; \ + WITH_BOTH_OOP_KINDS(T, { \ + return oop_oop_iterate_tp_m(obj, \ + OopClosure::nv_from##nv_suffix(closure), mr); \ + }); \ } ALL_OOP_OOP_ITERATE_CLOSURES_1(ObjArrayKlass_OOP_OOP_ITERATE_DEFN) ALL_OOP_OOP_ITERATE_CLOSURES_3(ObjArrayKlass_OOP_OOP_ITERATE_DEFN) ALL_OOP_OOP_ITERATE_CLOSURES_1(ObjArrayKlass_OOP_OOP_ITERATE_DEFN_m) ALL_OOP_OOP_ITERATE_CLOSURES_3(ObjArrayKlass_OOP_OOP_ITERATE_DEFN_m) -ALL_OOP_OOP_ITERATE_CLOSURES_1(ObjArrayKlass_OOP_OOP_ITERATE_DEFN_r) -ALL_OOP_OOP_ITERATE_CLOSURES_3(ObjArrayKlass_OOP_OOP_ITERATE_DEFN_r) int objArrayKlass::oop_adjust_pointers(oop obj) { assert(obj->is_objArray(), "obj must be obj array"); @@ -414,7 +375,11 @@ // Don't call size() or oop_size() since that is a virtual call. int size = a->object_size(); a->adjust_header(); - ObjArrayKlass_OOP_ITERATE(a, p, MarkSweep::adjust_pointer(p)) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = array_element_oops(a, pmax); p < pmax; p++) { + MarkSweep::adjust_pointer(addr(p)); + } + }); return size; } @@ -422,27 +387,37 @@ void objArrayKlass::oop_copy_contents(PSPromotionManager* pm, oop obj) { assert(!pm->depth_first(), "invariant"); assert(obj->is_objArray(), "obj must be obj array"); - ObjArrayKlass_OOP_ITERATE( \ - objArrayOop(obj), p, \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_breadth(p); \ - }) + objArrayOop a = objArrayOop(obj); + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = array_element_oops(a, pmax); p < pmax; p++) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_breadth(addr(p)); + } + } + }); } void objArrayKlass::oop_push_contents(PSPromotionManager* pm, oop obj) { assert(pm->depth_first(), "invariant"); assert(obj->is_objArray(), "obj must be obj array"); - ObjArrayKlass_OOP_ITERATE( \ - objArrayOop(obj), p, \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_depth(p); \ - }) + objArrayOop a = objArrayOop(obj); + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = array_element_oops(a, pmax); p < pmax; p++) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_depth(addr(p)); + } + } + }); } int objArrayKlass::oop_update_pointers(ParCompactionManager* cm, oop obj) { assert (obj->is_objArray(), "obj must be obj array"); objArrayOop a = objArrayOop(obj); - ObjArrayKlass_OOP_ITERATE(a, p, PSParallelCompact::adjust_pointer(p)) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = array_element_oops(a, pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); return a->object_size(); } @@ -450,9 +425,13 @@ HeapWord* beg_addr, HeapWord* end_addr) { assert (obj->is_objArray(), "obj must be obj array"); objArrayOop a = objArrayOop(obj); - ObjArrayKlass_BOUNDED_OOP_ITERATE( \ - a, p, beg_addr, end_addr, \ - PSParallelCompact::adjust_pointer(p)) + MemRegion mr(beg_addr, end_addr); + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = + bounded_array_element_oops(a, mr, pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); return a->object_size(); } #endif // SERIALGC --- old/src/share/vm/oops/objArrayKlass.hpp 2008-07-12 17:49:02.000000000 -0700 +++ new/src/share/vm/oops/objArrayKlass.hpp 2008-07-12 17:49:02.000000000 -0700 @@ -63,11 +63,11 @@ // Compute class loader oop class_loader() const { return Klass::cast(bottom_klass())->class_loader(); } - private: // Either oop or narrowOop depending on UseCompressedOops. - // must be called from within objArrayKlass.cpp - template void do_copy(arrayOop s, T* src, arrayOop d, - T* dst, int length, TRAPS); + template + static void do_copy(T* src, T* dst, size_t length, + klassOop bound, TRAPS); + klassOop store_check_klass(oop s, oop d); protected: // Returns the objArrayKlass for n'th dimension. virtual klassOop array_klass_impl(bool or_null, int n, TRAPS); @@ -97,22 +97,51 @@ PARALLEL_GC_DECLS // Iterators - int oop_oop_iterate(oop obj, OopClosure* blk) { - return oop_oop_iterate_v(obj, blk); - } - int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr) { - return oop_oop_iterate_v_m(obj, blk, mr); - } #define ObjArrayKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \ int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, \ - MemRegion mr); \ - int oop_oop_iterate_range##nv_suffix(oop obj, OopClosureType* blk, \ - int start, int end); + MemRegion mr); ALL_OOP_OOP_ITERATE_CLOSURES_1(ObjArrayKlass_OOP_OOP_ITERATE_DECL) ALL_OOP_OOP_ITERATE_CLOSURES_3(ObjArrayKlass_OOP_OOP_ITERATE_DECL) + template + void oop_oop_iterate_range(oop obj, OopClosureType* blk, int start, int end) { + assert(obj->is_objArray(), ""); + oop_oop_iterate_nv_m(obj, blk, objArrayOop(obj)->element_range(start, end)); + } + + private: + // file-local common implementation for the iterators + template + inline int oop_oop_iterate_tp(oop obj, OopClosureType* blk); + template + inline int oop_oop_iterate_tp_m(oop obj, OopClosureType* blk, MemRegion mr); + + protected: + // Common code for iterating over oops. + + template + typename oop_chunk::iterator array_element_oops(objArrayOop obj, + typename oop_chunk::iterator& pmax) { + return oop_chunk::forward((T*)obj->base(), obj->length(), pmax); + } + + template + typename oop_chunk::iterator reverse_array_element_oops(objArrayOop obj, + typename oop_chunk::iterator& pmin) { + return Klass::iterator_reverse(array_element_oops(obj, pmin), pmin); + } + + template + typename oop_chunk::bounded_iterator + bounded_array_element_oops(objArrayOop obj, MemRegion mr, + typename oop_chunk::bounded_iterator& pmax) { + return typename oop_chunk::bounded(mr, array_element_oops(obj, pmax), pmax); + } + + public: + // JVM support jint compute_modifier_flags(TRAPS) const; --- old/src/share/vm/oops/objArrayKlassKlass.cpp 2008-07-12 17:49:04.000000000 -0700 +++ new/src/share/vm/oops/objArrayKlassKlass.cpp 2008-07-12 17:49:04.000000000 -0700 @@ -151,8 +151,8 @@ assert(bk != NULL && (Klass::cast(bk)->oop_is_instance() || Klass::cast(bk)->oop_is_typeArray()), "invalid bottom klass"); oak->set_bottom_klass(bk); - oak->set_layout_helper(array_layout_helper(T_OBJECT)); - assert(oak->oop_is_javaArray(), "sanity"); + oak->set_layout_helper(LayoutHelper::for_array(T_OBJECT)); + assert(oak->oop_is_array(), "sanity"); assert(oak->oop_is_objArray(), "sanity"); // Call complete_create_array_klass after all instance variables has been initialized. --- old/src/share/vm/oops/objArrayOop.cpp 2008-07-12 17:49:08.000000000 -0700 +++ new/src/share/vm/oops/objArrayOop.cpp 2008-07-12 17:49:07.000000000 -0700 @@ -25,12 +25,4 @@ # include "incls/_precompiled.incl" # include "incls/_objArrayOop.cpp.incl" -#define ObjArrayOop_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ - \ -int objArrayOopDesc::oop_iterate_range(OopClosureType* blk, int start, int end) { \ - SpecializationStats::record_call(); \ - return ((objArrayKlass*)blueprint())->oop_oop_iterate_range##nv_suffix(this, blk, start, end); \ -} - -ALL_OOP_OOP_ITERATE_CLOSURES_1(ObjArrayOop_OOP_ITERATE_DEFN) -ALL_OOP_OOP_ITERATE_CLOSURES_3(ObjArrayOop_OOP_ITERATE_DEFN) +// blank file --- old/src/share/vm/oops/objArrayOop.hpp 2008-07-12 17:49:11.000000000 -0700 +++ new/src/share/vm/oops/objArrayOop.hpp 2008-07-12 17:49:10.000000000 -0700 @@ -69,24 +69,40 @@ // Give size of objArrayOop in HeapWords minus the header static int array_size(int length) { + int fast_result = ((((size_t)length << LogBytesPerHeapOop) + + HeapWordSize - 1) + >> LogHeapWordSize); +#ifdef ASSERT + int check_result; // Without UseCompressedOops, this is simply: // oop->length() * HeapWordsPerOop; // With narrowOops, HeapWordsPerOop is 1/2 or equal 0 as an integer. // The oop elements are aligned up to wordSize const int HeapWordsPerOop = heapOopSize/HeapWordSize; if (HeapWordsPerOop > 0) { - return length * HeapWordsPerOop; + check_result = length * HeapWordsPerOop; } else { const int OopsPerHeapWord = HeapWordSize/heapOopSize; int word_len = align_size_up(length, OopsPerHeapWord)/OopsPerHeapWord; - return word_len; + check_result = word_len; } + assert(fast_result == check_result, "correct optimization"); +#endif + return fast_result; } - // special iterators for index ranges, returns size of object -#define ObjArrayOop_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ - int oop_iterate_range(OopClosureType* blk, int start, int end); + enum { before_first_element = -1 }; + MemRegion element_range(jint start, jint end) { + assert(start >= before_first_element, "valid index or special sentinel value"); + assert(end >= start && end >= 0, "valid range"); + size_t base_off = base_offset_in_bytes(T_OBJECT); + int shift = LogBytesPerHeapOop; + size_t lo_off = (start < 0 ? 0 : base_off + (start << shift)); + size_t hi_off = ( base_off + (end << shift)); + return MemRegion(field_base(lo_off), field_base(hi_off)); + } - ALL_OOP_OOP_ITERATE_CLOSURES_1(ObjArrayOop_OOP_ITERATE_DECL) - ALL_OOP_OOP_ITERATE_CLOSURES_3(ObjArrayOop_OOP_ITERATE_DECL) + // special iterators for index ranges + template + inline void oop_iterate_range(OopClosureType* blk, int start, int end); }; --- old/src/share/vm/oops/oop.hpp 2008-07-12 17:49:14.000000000 -0700 +++ new/src/share/vm/oops/oop.hpp 2008-07-12 17:49:13.000000000 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 1997-2007 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 1997-2008 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -107,6 +107,8 @@ bool is_instance() const; bool is_instanceRef() const; bool is_array() const; + bool has_length_field() const; + bool is_allprim() const; // can be conservatively false bool is_objArray() const; bool is_symbol() const; bool is_klass() const; @@ -122,21 +124,20 @@ private: // field addresses in oop - void* field_base(int offset) const; - - jbyte* byte_field_addr(int offset) const; - jchar* char_field_addr(int offset) const; - jboolean* bool_field_addr(int offset) const; - jint* int_field_addr(int offset) const; - jshort* short_field_addr(int offset) const; - jlong* long_field_addr(int offset) const; - jfloat* float_field_addr(int offset) const; - jdouble* double_field_addr(int offset) const; - address* address_field_addr(int offset) const; + jbyte* byte_field_addr(size_t offset) const; + jchar* char_field_addr(size_t offset) const; + jboolean* bool_field_addr(size_t offset) const; + jint* int_field_addr(size_t offset) const; + jshort* short_field_addr(size_t offset) const; + jlong* long_field_addr(size_t offset) const; + jfloat* float_field_addr(size_t offset) const; + jdouble* double_field_addr(size_t offset) const; + address* address_field_addr(size_t offset) const; public: - // Need this as public for garbage collection. - template T* obj_field_addr(int offset) const; + // Need these as public for garbage collection. + void* field_base(size_t offset) const; + template T* obj_field_addr(size_t offset) const; // Oop encoding heap max static const uint64_t OopEncodingHeapMax = @@ -193,63 +194,63 @@ oop compare_value); // Access to fields in a instanceOop through these methods. - oop obj_field(int offset) const; - void obj_field_put(int offset, oop value); - void obj_field_raw_put(int offset, oop value); + oop obj_field(size_t offset) const; + void obj_field_put(size_t offset, oop value); + void obj_field_raw_put(size_t offset, oop value); - jbyte byte_field(int offset) const; - void byte_field_put(int offset, jbyte contents); + jbyte byte_field(size_t offset) const; + void byte_field_put(size_t offset, jbyte contents); - jchar char_field(int offset) const; - void char_field_put(int offset, jchar contents); + jchar char_field(size_t offset) const; + void char_field_put(size_t offset, jchar contents); - jboolean bool_field(int offset) const; - void bool_field_put(int offset, jboolean contents); + jboolean bool_field(size_t offset) const; + void bool_field_put(size_t offset, jboolean contents); - jint int_field(int offset) const; - void int_field_put(int offset, jint contents); + jint int_field(size_t offset) const; + void int_field_put(size_t offset, jint contents); - jshort short_field(int offset) const; - void short_field_put(int offset, jshort contents); + jshort short_field(size_t offset) const; + void short_field_put(size_t offset, jshort contents); - jlong long_field(int offset) const; - void long_field_put(int offset, jlong contents); + jlong long_field(size_t offset) const; + void long_field_put(size_t offset, jlong contents); - jfloat float_field(int offset) const; - void float_field_put(int offset, jfloat contents); + jfloat float_field(size_t offset) const; + void float_field_put(size_t offset, jfloat contents); - jdouble double_field(int offset) const; - void double_field_put(int offset, jdouble contents); + jdouble double_field(size_t offset) const; + void double_field_put(size_t offset, jdouble contents); - address address_field(int offset) const; - void address_field_put(int offset, address contents); + address address_field(size_t offset) const; + void address_field_put(size_t offset, address contents); - oop obj_field_acquire(int offset) const; - void release_obj_field_put(int offset, oop value); + oop obj_field_acquire(size_t offset) const; + void release_obj_field_put(size_t offset, oop value); - jbyte byte_field_acquire(int offset) const; - void release_byte_field_put(int offset, jbyte contents); + jbyte byte_field_acquire(size_t offset) const; + void release_byte_field_put(size_t offset, jbyte contents); - jchar char_field_acquire(int offset) const; - void release_char_field_put(int offset, jchar contents); + jchar char_field_acquire(size_t offset) const; + void release_char_field_put(size_t offset, jchar contents); - jboolean bool_field_acquire(int offset) const; - void release_bool_field_put(int offset, jboolean contents); + jboolean bool_field_acquire(size_t offset) const; + void release_bool_field_put(size_t offset, jboolean contents); - jint int_field_acquire(int offset) const; - void release_int_field_put(int offset, jint contents); + jint int_field_acquire(size_t offset) const; + void release_int_field_put(size_t offset, jint contents); - jshort short_field_acquire(int offset) const; - void release_short_field_put(int offset, jshort contents); + jshort short_field_acquire(size_t offset) const; + void release_short_field_put(size_t offset, jshort contents); - jlong long_field_acquire(int offset) const; - void release_long_field_put(int offset, jlong contents); + jlong long_field_acquire(size_t offset) const; + void release_long_field_put(size_t offset, jlong contents); - jfloat float_field_acquire(int offset) const; - void release_float_field_put(int offset, jfloat contents); + jfloat float_field_acquire(size_t offset) const; + void release_float_field_put(size_t offset, jfloat contents); - jdouble double_field_acquire(int offset) const; - void release_double_field_put(int offset, jdouble contents); + jdouble double_field_acquire(size_t offset) const; + void release_double_field_put(size_t offset, jdouble contents); // printing functions for VM debugging void print_on(outputStream* st) const; // First level print @@ -358,15 +359,39 @@ static void set_bs(BarrierSet* bs) { _bs = bs; } // iterators, returns size of object -#define OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ - int oop_iterate(OopClosureType* blk); \ - int oop_iterate(OopClosureType* blk, MemRegion mr); // Only in mr. + + // these macro-defined guys are all overloaded together: +#define OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ + int oop_iterate(OopClosureType* blk) \ + { return oop_iterate##nv_suffix(blk); } \ + int oop_iterate(OopClosureType* blk, MemRegion mr) /* Only in mr. */ \ + { return oop_iterate##nv_suffix(blk, mr); } \ + void oop_iterate_header(OopClosureType* blk) \ + { oop_iterate_header##nv_suffix(blk); } \ + void oop_iterate_header(OopClosureType* blk, MemRegion mr) \ + { oop_iterate_header##nv_suffix(blk, mr); } \ + /*end*/ ALL_OOP_OOP_ITERATE_CLOSURES_1(OOP_ITERATE_DECL) ALL_OOP_OOP_ITERATE_CLOSURES_3(OOP_ITERATE_DECL) - void oop_iterate_header(OopClosure* blk); - void oop_iterate_header(OopClosure* blk, MemRegion mr); + // non-virtual versions are templatized for easy debugging: + template + inline int oop_iterate_nv(OopClosureType* blk); + template + inline int oop_iterate_nv(OopClosureType* blk, MemRegion mr); + + // virtual (slow) versions, renamed with _v to make the macros work better + inline int oop_iterate_v(OopClosure* blk); + inline int oop_iterate_v(OopClosure* blk, MemRegion mr); + + // header-only iterators: + template + inline void oop_iterate_header_nv(OopClosureType* blk); + template + inline void oop_iterate_header_nv(OopClosureType* blk, MemRegion mr); + inline void oop_iterate_header_v(OopClosure* blk); + inline void oop_iterate_header_v(OopClosure* blk, MemRegion mr); // identity hash; returns the identity hash key (computes it if necessary) // NOTE with the introduction of UseBiasedLocking that identity_hash() might reach a --- old/src/share/vm/oops/oop.inline.hpp 2008-07-12 17:49:17.000000000 -0700 +++ new/src/share/vm/oops/oop.inline.hpp 2008-07-12 17:49:17.000000000 -0700 @@ -91,9 +91,10 @@ inline bool oopDesc::is_instance() const { return blueprint()->oop_is_instance(); } inline bool oopDesc::is_instanceRef() const { return blueprint()->oop_is_instanceRef(); } inline bool oopDesc::is_array() const { return blueprint()->oop_is_array(); } +inline bool oopDesc::has_length_field() const { return blueprint()->oop_has_length_field(); } +inline bool oopDesc::is_allprim() const { return blueprint()->oop_is_allprim(); } inline bool oopDesc::is_objArray() const { return blueprint()->oop_is_objArray(); } inline bool oopDesc::is_typeArray() const { return blueprint()->oop_is_typeArray(); } -inline bool oopDesc::is_javaArray() const { return blueprint()->oop_is_javaArray(); } inline bool oopDesc::is_symbol() const { return blueprint()->oop_is_symbol(); } inline bool oopDesc::is_klass() const { return blueprint()->oop_is_klass(); } inline bool oopDesc::is_thread() const { return blueprint()->oop_is_thread(); } @@ -104,18 +105,21 @@ inline bool oopDesc::is_constantPoolCache() const { return blueprint()->oop_is_constantPoolCache(); } inline bool oopDesc::is_compiledICHolder() const { return blueprint()->oop_is_compiledICHolder(); } -inline void* oopDesc::field_base(int offset) const { return (void*)&((char*)this)[offset]; } +// for old code (all arrays are java arrays now): +inline bool oopDesc::is_javaArray() const { return is_array(); } -template inline T* oopDesc::obj_field_addr(int offset) const { return (T*)field_base(offset); } -inline jbyte* oopDesc::byte_field_addr(int offset) const { return (jbyte*) field_base(offset); } -inline jchar* oopDesc::char_field_addr(int offset) const { return (jchar*) field_base(offset); } -inline jboolean* oopDesc::bool_field_addr(int offset) const { return (jboolean*)field_base(offset); } -inline jint* oopDesc::int_field_addr(int offset) const { return (jint*) field_base(offset); } -inline jshort* oopDesc::short_field_addr(int offset) const { return (jshort*) field_base(offset); } -inline jlong* oopDesc::long_field_addr(int offset) const { return (jlong*) field_base(offset); } -inline jfloat* oopDesc::float_field_addr(int offset) const { return (jfloat*) field_base(offset); } -inline jdouble* oopDesc::double_field_addr(int offset) const { return (jdouble*) field_base(offset); } -inline address* oopDesc::address_field_addr(int offset) const { return (address*) field_base(offset); } +inline void* oopDesc::field_base(size_t offset) const { return (void*)&((char*)this)[offset]; } + +template inline T* oopDesc::obj_field_addr(size_t offset) const { return (T*)field_base(offset); } +inline jbyte* oopDesc::byte_field_addr(size_t offset) const { return (jbyte*) field_base(offset); } +inline jchar* oopDesc::char_field_addr(size_t offset) const { return (jchar*) field_base(offset); } +inline jboolean* oopDesc::bool_field_addr(size_t offset) const { return (jboolean*)field_base(offset); } +inline jint* oopDesc::int_field_addr(size_t offset) const { return (jint*) field_base(offset); } +inline jshort* oopDesc::short_field_addr(size_t offset) const { return (jshort*) field_base(offset); } +inline jlong* oopDesc::long_field_addr(size_t offset) const { return (jlong*) field_base(offset); } +inline jfloat* oopDesc::float_field_addr(size_t offset) const { return (jfloat*) field_base(offset); } +inline jdouble* oopDesc::double_field_addr(size_t offset) const { return (jdouble*) field_base(offset); } +inline address* oopDesc::address_field_addr(size_t offset) const { return (address*) field_base(offset); } // Functions for getting and setting oops within instance objects. @@ -253,88 +257,88 @@ // In order to put or get a field out of an instance, must first check // if the field has been compressed and uncompress it. -inline oop oopDesc::obj_field(int offset) const { +inline oop oopDesc::obj_field(size_t offset) const { return UseCompressedOops ? load_decode_heap_oop(obj_field_addr(offset)) : load_decode_heap_oop(obj_field_addr(offset)); } -inline void oopDesc::obj_field_put(int offset, oop value) { +inline void oopDesc::obj_field_put(size_t offset, oop value) { UseCompressedOops ? oop_store(obj_field_addr(offset), value) : oop_store(obj_field_addr(offset), value); } -inline void oopDesc::obj_field_raw_put(int offset, oop value) { +inline void oopDesc::obj_field_raw_put(size_t offset, oop value) { UseCompressedOops ? encode_store_heap_oop(obj_field_addr(offset), value) : encode_store_heap_oop(obj_field_addr(offset), value); } -inline jbyte oopDesc::byte_field(int offset) const { return (jbyte) *byte_field_addr(offset); } -inline void oopDesc::byte_field_put(int offset, jbyte contents) { *byte_field_addr(offset) = (jint) contents; } +inline jbyte oopDesc::byte_field(size_t offset) const { return (jbyte) *byte_field_addr(offset); } +inline void oopDesc::byte_field_put(size_t offset, jbyte contents) { *byte_field_addr(offset) = (jint) contents; } -inline jboolean oopDesc::bool_field(int offset) const { return (jboolean) *bool_field_addr(offset); } -inline void oopDesc::bool_field_put(int offset, jboolean contents) { *bool_field_addr(offset) = (jint) contents; } +inline jboolean oopDesc::bool_field(size_t offset) const { return (jboolean) *bool_field_addr(offset); } +inline void oopDesc::bool_field_put(size_t offset, jboolean contents) { *bool_field_addr(offset) = (jint) contents; } -inline jchar oopDesc::char_field(int offset) const { return (jchar) *char_field_addr(offset); } -inline void oopDesc::char_field_put(int offset, jchar contents) { *char_field_addr(offset) = (jint) contents; } +inline jchar oopDesc::char_field(size_t offset) const { return (jchar) *char_field_addr(offset); } +inline void oopDesc::char_field_put(size_t offset, jchar contents) { *char_field_addr(offset) = (jint) contents; } -inline jint oopDesc::int_field(int offset) const { return *int_field_addr(offset); } -inline void oopDesc::int_field_put(int offset, jint contents) { *int_field_addr(offset) = contents; } +inline jint oopDesc::int_field(size_t offset) const { return *int_field_addr(offset); } +inline void oopDesc::int_field_put(size_t offset, jint contents) { *int_field_addr(offset) = contents; } -inline jshort oopDesc::short_field(int offset) const { return (jshort) *short_field_addr(offset); } -inline void oopDesc::short_field_put(int offset, jshort contents) { *short_field_addr(offset) = (jint) contents;} +inline jshort oopDesc::short_field(size_t offset) const { return (jshort) *short_field_addr(offset); } +inline void oopDesc::short_field_put(size_t offset, jshort contents) { *short_field_addr(offset) = (jint) contents;} -inline jlong oopDesc::long_field(int offset) const { return *long_field_addr(offset); } -inline void oopDesc::long_field_put(int offset, jlong contents) { *long_field_addr(offset) = contents; } +inline jlong oopDesc::long_field(size_t offset) const { return *long_field_addr(offset); } +inline void oopDesc::long_field_put(size_t offset, jlong contents) { *long_field_addr(offset) = contents; } -inline jfloat oopDesc::float_field(int offset) const { return *float_field_addr(offset); } -inline void oopDesc::float_field_put(int offset, jfloat contents) { *float_field_addr(offset) = contents; } +inline jfloat oopDesc::float_field(size_t offset) const { return *float_field_addr(offset); } +inline void oopDesc::float_field_put(size_t offset, jfloat contents) { *float_field_addr(offset) = contents; } -inline jdouble oopDesc::double_field(int offset) const { return *double_field_addr(offset); } -inline void oopDesc::double_field_put(int offset, jdouble contents) { *double_field_addr(offset) = contents; } +inline jdouble oopDesc::double_field(size_t offset) const { return *double_field_addr(offset); } +inline void oopDesc::double_field_put(size_t offset, jdouble contents) { *double_field_addr(offset) = contents; } -inline address oopDesc::address_field(int offset) const { return *address_field_addr(offset); } -inline void oopDesc::address_field_put(int offset, address contents) { *address_field_addr(offset) = contents; } +inline address oopDesc::address_field(size_t offset) const { return *address_field_addr(offset); } +inline void oopDesc::address_field_put(size_t offset, address contents) { *address_field_addr(offset) = contents; } -inline oop oopDesc::obj_field_acquire(int offset) const { +inline oop oopDesc::obj_field_acquire(size_t offset) const { return UseCompressedOops ? decode_heap_oop((narrowOop) OrderAccess::load_acquire(obj_field_addr(offset))) : decode_heap_oop((oop) OrderAccess::load_ptr_acquire(obj_field_addr(offset))); } -inline void oopDesc::release_obj_field_put(int offset, oop value) { +inline void oopDesc::release_obj_field_put(size_t offset, oop value) { UseCompressedOops ? oop_store((volatile narrowOop*)obj_field_addr(offset), value) : oop_store((volatile oop*) obj_field_addr(offset), value); } -inline jbyte oopDesc::byte_field_acquire(int offset) const { return OrderAccess::load_acquire(byte_field_addr(offset)); } -inline void oopDesc::release_byte_field_put(int offset, jbyte contents) { OrderAccess::release_store(byte_field_addr(offset), contents); } +inline jbyte oopDesc::byte_field_acquire(size_t offset) const { return OrderAccess::load_acquire(byte_field_addr(offset)); } +inline void oopDesc::release_byte_field_put(size_t offset, jbyte contents) { OrderAccess::release_store(byte_field_addr(offset), contents); } -inline jboolean oopDesc::bool_field_acquire(int offset) const { return OrderAccess::load_acquire(bool_field_addr(offset)); } -inline void oopDesc::release_bool_field_put(int offset, jboolean contents) { OrderAccess::release_store(bool_field_addr(offset), contents); } +inline jboolean oopDesc::bool_field_acquire(size_t offset) const { return OrderAccess::load_acquire(bool_field_addr(offset)); } +inline void oopDesc::release_bool_field_put(size_t offset, jboolean contents) { OrderAccess::release_store(bool_field_addr(offset), contents); } -inline jchar oopDesc::char_field_acquire(int offset) const { return OrderAccess::load_acquire(char_field_addr(offset)); } -inline void oopDesc::release_char_field_put(int offset, jchar contents) { OrderAccess::release_store(char_field_addr(offset), contents); } +inline jchar oopDesc::char_field_acquire(size_t offset) const { return OrderAccess::load_acquire(char_field_addr(offset)); } +inline void oopDesc::release_char_field_put(size_t offset, jchar contents) { OrderAccess::release_store(char_field_addr(offset), contents); } -inline jint oopDesc::int_field_acquire(int offset) const { return OrderAccess::load_acquire(int_field_addr(offset)); } -inline void oopDesc::release_int_field_put(int offset, jint contents) { OrderAccess::release_store(int_field_addr(offset), contents); } +inline jint oopDesc::int_field_acquire(size_t offset) const { return OrderAccess::load_acquire(int_field_addr(offset)); } +inline void oopDesc::release_int_field_put(size_t offset, jint contents) { OrderAccess::release_store(int_field_addr(offset), contents); } -inline jshort oopDesc::short_field_acquire(int offset) const { return (jshort)OrderAccess::load_acquire(short_field_addr(offset)); } -inline void oopDesc::release_short_field_put(int offset, jshort contents) { OrderAccess::release_store(short_field_addr(offset), contents); } +inline jshort oopDesc::short_field_acquire(size_t offset) const { return (jshort)OrderAccess::load_acquire(short_field_addr(offset)); } +inline void oopDesc::release_short_field_put(size_t offset, jshort contents) { OrderAccess::release_store(short_field_addr(offset), contents); } -inline jlong oopDesc::long_field_acquire(int offset) const { return OrderAccess::load_acquire(long_field_addr(offset)); } -inline void oopDesc::release_long_field_put(int offset, jlong contents) { OrderAccess::release_store(long_field_addr(offset), contents); } +inline jlong oopDesc::long_field_acquire(size_t offset) const { return OrderAccess::load_acquire(long_field_addr(offset)); } +inline void oopDesc::release_long_field_put(size_t offset, jlong contents) { OrderAccess::release_store(long_field_addr(offset), contents); } -inline jfloat oopDesc::float_field_acquire(int offset) const { return OrderAccess::load_acquire(float_field_addr(offset)); } -inline void oopDesc::release_float_field_put(int offset, jfloat contents) { OrderAccess::release_store(float_field_addr(offset), contents); } +inline jfloat oopDesc::float_field_acquire(size_t offset) const { return OrderAccess::load_acquire(float_field_addr(offset)); } +inline void oopDesc::release_float_field_put(size_t offset, jfloat contents) { OrderAccess::release_store(float_field_addr(offset), contents); } -inline jdouble oopDesc::double_field_acquire(int offset) const { return OrderAccess::load_acquire(double_field_addr(offset)); } -inline void oopDesc::release_double_field_put(int offset, jdouble contents) { OrderAccess::release_store(double_field_addr(offset), contents); } +inline jdouble oopDesc::double_field_acquire(size_t offset) const { return OrderAccess::load_acquire(double_field_addr(offset)); } +inline void oopDesc::release_double_field_put(size_t offset, jdouble contents) { OrderAccess::release_store(double_field_addr(offset), contents); } inline int oopDesc::size_given_klass(Klass* klass) { - int lh = klass->layout_helper(); - int s = lh >> LogHeapWordSize; // deliver size scaled by wordSize + LayoutHelper lh = klass->layout_helper(); + int s = lh.as_int() >> LogHeapWordSize; // deliver size scaled by wordSize // lh is now a value computed at class initialization that may hint // at the size. For instances, this is positive and equal to the @@ -347,38 +351,15 @@ // alive or dead. So the speed here is equal in importance to the // speed of allocation. - if (lh <= Klass::_lh_neutral_value) { + if (!lh.is_fixed_instance()) { // The most common case is instances; fall through if so. - if (lh < Klass::_lh_neutral_value) { + if (lh.has_length_field()) { // Second most common case is arrays. We have to fetch the // length of the array, shift (multiply) it appropriately, // up to wordSize, add the header, and align to object size. - size_t size_in_bytes; -#ifdef _M_IA64 - // The Windows Itanium Aug 2002 SDK hoists this load above - // the check for s < 0. An oop at the end of the heap will - // cause an access violation if this load is performed on a non - // array oop. Making the reference volatile prohibits this. - // (%%% please explain by what magic the length is actually fetched!) - volatile int *array_length; - array_length = (volatile int *)( (intptr_t)this + - arrayOopDesc::length_offset_in_bytes() ); - assert(array_length > 0, "Integer arithmetic problem somewhere"); - // Put into size_t to avoid overflow. - size_in_bytes = (size_t) array_length; - size_in_bytes = size_in_bytes << Klass::layout_helper_log2_element_size(lh); -#else - size_t array_length = (size_t) ((arrayOop)this)->length(); - size_in_bytes = array_length << Klass::layout_helper_log2_element_size(lh); -#endif - size_in_bytes += Klass::layout_helper_header_size(lh); - - // This code could be simplified, but by keeping array_header_in_bytes - // in units of bytes and doing it this way we can round up just once, - // skipping the intermediate round to HeapWordSize. Cast the result - // of round_to to size_t to guarantee unsigned division == right shift. - s = (int)((size_t)round_to(size_in_bytes, MinObjAlignmentInBytes) / - HeapWordSize); + jint array_length = arrayOopDesc::length_of(this); + + s = lh.new_array_size_in_bytes(array_length) >> LogHeapWordSize; // UseParNewGC can change the length field of an "old copy" of an object // array in the young gen so it indicates the stealable portion of @@ -637,54 +618,119 @@ } } -inline void oopDesc::oop_iterate_header(OopClosure* blk) { +inline int oopDesc::adjust_pointers() { + debug_only(int check_size = size()); + int s = blueprint()->oop_adjust_pointers(this); + assert(s == check_size, "should be the same"); + return s; +} + +inline void oopDesc::adjust_header() { if (UseCompressedOops) { - blk->do_oop(compressed_klass_addr()); + MarkSweep::adjust_pointer(compressed_klass_addr()); } else { - blk->do_oop(klass_addr()); + MarkSweep::adjust_pointer(klass_addr()); } } -inline void oopDesc::oop_iterate_header(OopClosure* blk, MemRegion mr) { - if (UseCompressedOops) { - if (mr.contains(compressed_klass_addr())) { - blk->do_oop(compressed_klass_addr()); - } +// four oop iterator versions: {_v, _nv} x {, _m} +// the _nv versions are templatized to customize the code + +template +inline int oopDesc::oop_iterate_nv(OopClosureType* blk) { + SpecializationStats::record_call(); + Klass* k = blueprint(); + if (!k->oop_is_allprim()) { + return k->oop_oop_iterate_nv(this, blk); } else { - if (mr.contains(klass_addr())) blk->do_oop(klass_addr()); + int s = size_given_klass(k); + SpecializationStats::record_do_oop_call_nv(SpecializationStats::prim); + oop_iterate_header_nv(blk); + return s; } } -inline int oopDesc::adjust_pointers() { - debug_only(int check_size = size()); - int s = blueprint()->oop_adjust_pointers(this); - assert(s == check_size, "should be the same"); - return s; +template +inline int oopDesc::oop_iterate_nv(OopClosureType* blk, MemRegion mr) { + SpecializationStats::record_call(); + Klass* k = blueprint(); + if (!k->oop_is_allprim()) { + return k->oop_oop_iterate_nv_m(this, blk, mr); + } else { + int s = size_given_klass(k); + SpecializationStats::record_do_oop_call_nv(SpecializationStats::prim); + oop_iterate_header_nv(blk, mr); + return s; + } } -inline void oopDesc::adjust_header() { +inline int oopDesc::oop_iterate_v(OopClosure* blk) { + SpecializationStats::record_call(); + Klass* k = blueprint(); + if (!k->oop_is_allprim()) { + return k->oop_oop_iterate(this, blk); + } else { + int s = size_given_klass(k); + SpecializationStats::record_do_oop_call_v(SpecializationStats::prim); + oop_iterate_header_v(blk); + return s; + } +} + +inline int oopDesc::oop_iterate_v(OopClosure* blk, MemRegion mr) { + SpecializationStats::record_call(); + Klass* k = blueprint(); + if (!k->oop_is_allprim()) { + return k->oop_oop_iterate_m(this, blk, mr); + } else { + int s = size_given_klass(k); + SpecializationStats::record_do_oop_call_v(SpecializationStats::prim); + oop_iterate_header_v(blk, mr); + return s; + } +} + +// four header-only versions: {_v, _nv} x {, _m} + +template +inline void oopDesc::oop_iterate_header_nv(OopClosureType* blk) { + if (!blk->do_header()) return; if (UseCompressedOops) { - MarkSweep::adjust_pointer(compressed_klass_addr()); + blk->do_oop_nv(compressed_klass_addr()); } else { - MarkSweep::adjust_pointer(klass_addr()); + blk->do_oop_nv(klass_addr()); } } -#define OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ - \ -inline int oopDesc::oop_iterate(OopClosureType* blk) { \ - SpecializationStats::record_call(); \ - return blueprint()->oop_oop_iterate##nv_suffix(this, blk); \ -} \ - \ -inline int oopDesc::oop_iterate(OopClosureType* blk, MemRegion mr) { \ - SpecializationStats::record_call(); \ - return blueprint()->oop_oop_iterate##nv_suffix##_m(this, blk, mr); \ +template +inline void oopDesc::oop_iterate_header_nv(OopClosureType* blk, MemRegion mr) { + if (!blk->do_header()) return; + if (UseCompressedOops) { + if (mr.contains(compressed_klass_addr())) { + blk->do_oop_nv(compressed_klass_addr()); + } + } else { + if (mr.contains(klass_addr())) blk->do_oop_nv(klass_addr()); + } } -ALL_OOP_OOP_ITERATE_CLOSURES_1(OOP_ITERATE_DEFN) -ALL_OOP_OOP_ITERATE_CLOSURES_3(OOP_ITERATE_DEFN) +inline void oopDesc::oop_iterate_header_v(OopClosure* blk) { + if (UseCompressedOops) { + blk->do_oop(compressed_klass_addr()); + } else { + blk->do_oop(klass_addr()); + } +} +inline void oopDesc::oop_iterate_header_v(OopClosure* blk, MemRegion mr) { + if (UseCompressedOops) { + if (mr.contains(compressed_klass_addr())) { + blk->do_oop(compressed_klass_addr()); + } + } else { + if (mr.contains(klass_addr())) blk->do_oop(klass_addr()); + } +} inline bool oopDesc::is_shared() const { return CompactingPermGenGen::is_shared(this); --- old/src/share/vm/oops/oopsHierarchy.hpp 2008-07-12 17:49:20.000000000 -0700 +++ new/src/share/vm/oops/oopsHierarchy.hpp 2008-07-12 17:49:20.000000000 -0700 @@ -182,3 +182,24 @@ class constantPoolCacheKlass; class symbolKlass; class compiledICHolderKlass; + +// Many snippets of code have to be repeated twice for UseCompressedOops. +// This macro can help manage the duplication. +#define WITH_BOTH_OOP_KINDS(T, code) { \ + if (UseCompressedOops) { \ + typedef narrowOop T; { code; } \ + } else { \ + typedef oop T; { code; } \ + } } + +// Example usage: +// WITH_BOTH_OOP_KINDS(T, { +// printf("addr=%p\n", x->obj_field_addr(0)); +// }); + +// To debug with this, you can comment out the first and last +// lines, and insert a temporary definition for T. + +// { typedef oop T; //WITH_BOTH_OOP_KINDS(T, { +// printf("addr=%p\n", x->obj_field_addr(0)); +// } //}); --- old/src/share/vm/oops/typeArrayKlass.cpp 2008-07-12 17:49:23.000000000 -0700 +++ new/src/share/vm/oops/typeArrayKlass.cpp 2008-07-12 17:49:23.000000000 -0700 @@ -49,9 +49,9 @@ arrayKlassHandle k = base_create_array_klass(o.vtbl_value(), header_size(), klassklass, CHECK_NULL); typeArrayKlass* ak = typeArrayKlass::cast(k()); ak->set_name(sym()); - ak->set_layout_helper(array_layout_helper(type)); + ak->set_layout_helper(LayoutHelper::for_array(type)); assert(scale == (1 << ak->log2_element_size()), "scale must check out"); - assert(ak->oop_is_javaArray(), "sanity"); + assert(ak->oop_is_array(), "sanity"); assert(ak->oop_is_typeArray(), "sanity"); ak->set_max_length(arrayOopDesc::max_array_length(type)); assert(k()->size() > header_size(), "bad size"); @@ -117,20 +117,33 @@ THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); } // Check if the ranges are valid - if ( (((unsigned int) length + (unsigned int) src_pos) > (unsigned int) s->length()) - || (((unsigned int) length + (unsigned int) dst_pos) > (unsigned int) d->length()) ) { + if ( (((unsigned int) length + (unsigned int) src_pos) > (unsigned int) arrayOopDesc::length_of(s)) + || (((unsigned int) length + (unsigned int) dst_pos) > (unsigned int) arrayOopDesc::length_of(d)) ) { THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); } - // This is an attempt to make the copy_array fast. - // NB: memmove takes care of overlapping memory segments. - // Potential problem: memmove is not guaranteed to be word atomic - // Revisit in Merlin - int l2es = log2_element_size(); - int ihs = array_header_in_bytes() / wordSize; - char* src = (char*) ((oop*)s + ihs) + (src_pos << l2es); - char* dst = (char*) ((oop*)d + ihs) + (dst_pos << l2es); - memmove(dst, src, length << l2es); + if (length == 0) { + return; + } + + LayoutHelper lh = layout_helper(); + assert(!lh.has_oops_in_tail(), ""); + size_t src_off = lh.element_offset(src_pos, true); + size_t dst_off = lh.element_offset(dst_pos, true); + size_t cpy_len = lh.scale_index(length, true); + if (d->klass() != this->as_klassOop()) { + // Destination might have a different header size. + LayoutHelper d_lh = Klass::cast(d->klass())->layout_helper(); + assert(d_lh.element_size() == lh.element_size(), ""); + assert(d_lh.element_type() == lh.element_type(), ""); + assert(!d_lh.has_oops_in_tail(), ""); + dst_off = d_lh.element_offset(dst_pos, true); + } + + // Note: memmove is not guaranteed to be word atomic. + Copy::conjoint_memory_atomic((address)s + src_off, + (address)d + dst_off, + cpy_len); } --- old/src/share/vm/oops/typeArrayKlass.hpp 2008-07-12 17:49:26.000000000 -0700 +++ new/src/share/vm/oops/typeArrayKlass.hpp 2008-07-12 17:49:25.000000000 -0700 @@ -35,6 +35,7 @@ void set_max_length(jint m) { _max_length = m; } // testers + bool oop_is_allprim_slow() const { return true; } bool oop_is_typeArray_slow() const { return true; } // klass allocation --- old/src/share/vm/oops/typeArrayOop.hpp 2008-07-12 17:49:29.000000000 -0700 +++ new/src/share/vm/oops/typeArrayOop.hpp 2008-07-12 17:49:28.000000000 -0700 @@ -36,6 +36,7 @@ jshort* short_base() const { return (jshort*) base(T_SHORT); } jfloat* float_base() const { return (jfloat*) base(T_FLOAT); } jdouble* double_base() const { return (jdouble*) base(T_DOUBLE); } + HeapWord* obj_base() const { return (HeapWord*) arrayOopDesc::base(T_OBJECT); } friend class typeArrayKlass; @@ -120,24 +121,18 @@ // Returns the number of words necessary to hold an array of "len" // elements each of the given "byte_size". private: - static int object_size(int lh, int length) { - int instance_header_size = Klass::layout_helper_header_size(lh); - int element_shift = Klass::layout_helper_log2_element_size(lh); - DEBUG_ONLY(BasicType etype = Klass::layout_helper_element_type(lh)); + static int object_size(LayoutHelper lh, jint length) { + DEBUG_ONLY(BasicType etype = lh.element_type()); assert(length <= arrayOopDesc::max_array_length(etype), "no overflow"); - - julong size_in_bytes = length; - size_in_bytes <<= element_shift; - size_in_bytes += instance_header_size; - julong size_in_words = ((size_in_bytes + (HeapWordSize-1)) >> LogHeapWordSize); + const bool log2_only = true; + julong size_in_bytes = lh.new_array_size_in_bytes(length, log2_only); + julong size_in_words = size_in_bytes >> LogHeapWordSize; assert(size_in_words <= (julong)max_jint, "no overflow"); - - return align_object_size((intptr_t)size_in_words); + return size_in_words; } public: int object_size() { - typeArrayKlass* tk = typeArrayKlass::cast(klass()); - return object_size(tk->layout_helper(), length()); + return object_size(blueprint()->layout_helper(), length()); } }; --- old/src/share/vm/opto/compile.cpp 2008-07-12 17:49:31.000000000 -0700 +++ new/src/share/vm/opto/compile.cpp 2008-07-12 17:49:31.000000000 -0700 @@ -1077,7 +1077,7 @@ // First handle header references such as a LoadKlassNode, even if the // object's klass is unloaded at compile time (4965979). tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, NULL, offset, to->instance_id()); - } else if (offset < 0 || offset >= k->size_helper() * wordSize) { + } else if (offset < 0 || offset >= k->layout_helper().new_instance_size_in_bytes()) { to = NULL; tj = TypeOopPtr::BOTTOM; offset = tj->offset(); --- old/src/share/vm/opto/graphKit.cpp 2008-07-12 17:49:34.000000000 -0700 +++ new/src/share/vm/opto/graphKit.cpp 2008-07-12 17:49:33.000000000 -0700 @@ -2738,20 +2738,19 @@ // layout helper value, and return the node which represents it. // This two-faced routine is useful because allocation sites // almost always feature constant types. -Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) { +Node* GraphKit::get_layout_helper(Node* klass_node, LayoutHelper& constant_value) { const TypeKlassPtr* inst_klass = _gvn.type(klass_node)->isa_klassptr(); if (!StressReflectiveCode && inst_klass != NULL) { ciKlass* klass = inst_klass->klass(); bool xklass = inst_klass->klass_is_exact(); if (xklass || klass->is_array_klass()) { - jint lhelper = klass->layout_helper(); - if (lhelper != Klass::_lh_neutral_value) { - constant_value = lhelper; + constant_value = klass->layout_helper(); + if (!constant_value.is_neutral()) { return (Node*) NULL; } } } - constant_value = Klass::_lh_neutral_value; // put in a known value + assert(constant_value.is_neutral(), "known value"); Node* lhp = basic_plus_adr(klass_node, klass_node, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc)); return make_load(NULL, lhp, TypeInt::INT, T_INT); } @@ -2870,7 +2869,7 @@ // The size is always an integral number of doublewords, represented // as a positive bytewise size stored in the klass's layout_helper. // The layout_helper also encodes (in a low bit) the need for a slow path. - jint layout_con = Klass::_lh_neutral_value; + LayoutHelper layout_con; Node* layout_val = get_layout_helper(klass_node, layout_con); int layout_is_con = (layout_val == NULL); @@ -2881,14 +2880,14 @@ Node* initial_slow_test = NULL; if (layout_is_con) { assert(!StressReflectiveCode, "stress mode does not use these paths"); - bool must_go_slow = Klass::layout_helper_needs_slow_path(layout_con); + bool must_go_slow = layout_con.needs_slow_path(); initial_slow_test = must_go_slow? intcon(1): extra_slow_test; } else { // reflective case // This reflective path is used by Unsafe.allocateInstance. // (It may be stress-tested by specifying StressReflectiveCode.) // Basically, we want to get into the VM is there's an illegal argument. - Node* bit = intcon(Klass::_lh_instance_slow_path_bit); + Node* bit = intcon(LayoutHelper::_slow_path_low_bit); initial_slow_test = _gvn.transform( new (C, 3) AndINode(layout_val, bit) ); if (extra_slow_test != intcon(0)) { initial_slow_test = _gvn.transform( new (C, 3) OrINode(initial_slow_test, extra_slow_test) ); @@ -2900,15 +2899,20 @@ // The size value must be valid even if the slow path is taken. Node* size = NULL; if (layout_is_con) { - size = MakeConX(Klass::layout_helper_size_in_bytes(layout_con)); + size = MakeConX(layout_con.new_instance_size_in_bytes()); } else { // reflective case // This reflective path is used by clone and Unsafe.allocateInstance. size = ConvI2X(layout_val); - // Clear the low bits to extract layout_helper_size_in_bytes: - assert((int)Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit"); - Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong)); - size = _gvn.transform( new (C, 3) AndXNode(size, mask) ); + if (!MixedArrays) { + // Clear the low bits to extract fixed_size_in_bytes: + assert((int)LayoutHelper::_slow_path_low_bit < BytesPerLong, "clear bit"); + Node* mask = MakeConX(~ (intptr_t)LayoutHelper::_size_low_mask); + size = _gvn.transform( new (C, 3) AndXNode(size, mask) ); + } else { + // NYI: Need open code for new_instance_size_in_bytes. + ShouldNotReachHere(); + } } if (return_size_val != NULL) { (*return_size_val) = size; @@ -2952,7 +2956,7 @@ Node* length, // number of array elements bool raw_mem_only, // affect only raw memory Node* *return_size_val) { - jint layout_con = Klass::_lh_neutral_value; + LayoutHelper layout_con; Node* layout_val = get_layout_helper(klass_node, layout_con); int layout_is_con = (layout_val == NULL); @@ -2961,8 +2965,8 @@ // This is a reflective array creation site. // Optimistically assume that it is a subtype of Object[], // so that we can fold up all the address arithmetic. - layout_con = Klass::array_layout_helper(T_OBJECT); - Node* cmp_lh = _gvn.transform( new(C, 3) CmpINode(layout_val, intcon(layout_con)) ); + layout_con = LayoutHelper::for_array(T_OBJECT); + Node* cmp_lh = _gvn.transform( new(C, 3) CmpINode(layout_val, intcon(layout_con.as_int())) ); Node* bol_lh = _gvn.transform( new(C, 2) BoolNode(cmp_lh, BoolTest::eq) ); { BuildCutout unless(this, bol_lh, PROB_MAX); uncommon_trap(Deoptimization::Reason_class_check, @@ -2982,8 +2986,18 @@ if (layout_is_con) { assert(!StressReflectiveCode, "stress mode does not use these paths"); // Increase the size limit if we have exact knowledge of array type. - int log2_esize = Klass::layout_helper_log2_element_size(layout_con); + int log2_esize = layout_con.log2_element_size(); fast_size_limit <<= (LogBytesPerLong - log2_esize); + } else { + if (MixedArrays) { + // Mixed arrays can have large elements. Be careful not to overflow the address word. + int log2_max_elt_size = (LayoutHelper::_element_sizem_bits + + LayoutHelper::_element_scale_mask); + assert(log2_max_elt_size <= 16, "not monstrous"); // actually, it's 12 + intptr_t length_limit = (uintptr_t)-1 >> (log2_max_elt_size + 2); + if ((uintptr_t) fast_size_limit > length_limit) + fast_size_limit = (int) length_limit; + } } Node* initial_slow_cmp = _gvn.transform( new (C, 3) CmpUNode( length, intcon( fast_size_limit ) ) ); @@ -3003,19 +3017,21 @@ int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE); // (T_BYTE has the weakest alignment and size restrictions...) if (layout_is_con) { - int hsize = Klass::layout_helper_header_size(layout_con); - int eshift = Klass::layout_helper_log2_element_size(layout_con); - BasicType etype = Klass::layout_helper_element_type(layout_con); - if ((round_mask & ~right_n_bits(eshift)) == 0) + int hsize = layout_con.header_size_in_bytes(); + int esize = layout_con.element_size(); + BasicType etype = layout_con.element_type(); + int eshift = 0; + int emask = lowest_bit_set(esize) - 1; // low zero bits + if ((round_mask & ~emask) == 0) round_mask = 0; // strength-reduce it if it goes away completely - assert((hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded"); + assert((hsize & emask) == 0, "hsize is pre-rounded"); assert(header_size_min <= hsize, "generic minimum is smallest"); header_size_min = hsize; header_size = intcon(hsize + round_mask); } else { - Node* hss = intcon(Klass::_lh_header_size_shift); - Node* hsm = intcon(Klass::_lh_header_size_mask); - Node* hsize = _gvn.transform( new(C, 3) URShiftINode(layout_val, hss) ); + assert(LayoutHelper::_header_size_shift == 0, ""); + Node* hsm = intcon(LayoutHelper::_header_size_mask); + Node* hsize = layout_val; hsize = _gvn.transform( new(C, 3) AndINode(hsize, hsm) ); Node* mask = intcon(round_mask); header_size = _gvn.transform( new(C, 3) AddINode(hsize, mask) ); @@ -3023,14 +3039,14 @@ Node* elem_shift = NULL; if (layout_is_con) { - int eshift = Klass::layout_helper_log2_element_size(layout_con); + int eshift = layout_con.log2_element_size(); if (eshift != 0) elem_shift = intcon(eshift); } else { - // There is no need to mask or shift this value. - // The semantics of LShiftINode include an implicit mask to 0x1F. - assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place"); - elem_shift = layout_val; + // There is no need to mask this value. + // The semantics of LShiftXNode include an implicit mask to 0x1F/0x3F. + Node* ess = intcon(LayoutHelper::_element_size_shift); + elem_shift = _gvn.transform( new(C, 3) URShiftINode(layout_val, ess) ); } // Transition to native address size for all offset calculations: --- old/src/share/vm/opto/graphKit.hpp 2008-07-12 17:49:37.000000000 -0700 +++ new/src/share/vm/opto/graphKit.hpp 2008-07-12 17:49:36.000000000 -0700 @@ -660,7 +660,7 @@ Node* set_output_for_allocation(AllocateNode* alloc, const TypeOopPtr* oop_type, bool raw_mem_only); - Node* get_layout_helper(Node* klass_node, jint& constant_value); + Node* get_layout_helper(Node* klass_node, LayoutHelper& constant_value); Node* new_instance(Node* klass_node, Node* slow_test = NULL, bool raw_mem_only = false, --- old/src/share/vm/opto/library_call.cpp 2008-07-12 17:49:40.000000000 -0700 +++ new/src/share/vm/opto/library_call.cpp 2008-07-12 17:49:39.000000000 -0700 @@ -122,7 +122,16 @@ return generate_array_guard_common(kls, region, true, true); } Node* generate_array_guard_common(Node* kls, RegionNode* region, - bool obj_array, bool not_array); + bool obj_array, bool not_array) { + LayoutHelper layout_con; + Node* layout_val = get_layout_helper(kls, layout_con); + return generate_array_guard_common(kls, region, + obj_array, not_array, + layout_con, layout_val); + } + Node* generate_array_guard_common(Node* kls, RegionNode* region, + bool obj_array, bool not_array, + LayoutHelper layout_con, Node* layout_val); Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region); CallJavaNode* generate_method_call(vmIntrinsics::ID method_id, bool is_virtual = false, bool is_static = false); @@ -2788,7 +2797,8 @@ //---------------------generate_array_guard_common------------------------ Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, - bool obj_array, bool not_array) { + bool obj_array, bool not_array, + LayoutHelper layout_con, Node* layout_val) { // If obj_array/non_array==false/false: // Branch around if the given klass is in fact an array (either obj or prim). // If obj_array/non_array==false/true: @@ -2799,12 +2809,10 @@ // Branch around if the kls is an oop array (Object[] or subtype) // // Like generate_guard, adds a new path onto the region. - jint layout_con = 0; - Node* layout_val = get_layout_helper(kls, layout_con); if (layout_val == NULL) { bool query = (obj_array - ? Klass::layout_helper_is_objArray(layout_con) - : Klass::layout_helper_is_javaArray(layout_con)); + ? layout_con.is_objArray() + : layout_con.is_array()); if (query == not_array) { return NULL; // never a branch } else { // always a branch @@ -2816,12 +2824,23 @@ } } // Now test the correct condition. - jint nval = (obj_array - ? ((jint)Klass::_lh_array_tag_type_value - << Klass::_lh_array_tag_shift) - : Klass::_lh_neutral_value); - Node* cmp = _gvn.transform( new(C, 3) CmpINode(layout_val, intcon(nval)) ); - BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array + BoolTest::mask btest; + Node* fval; + int nval; + if (obj_array) { + // as in LayoutHelper::is_objArray, extract the flags and compare. + LayoutHelper lh = LayoutHelper::for_array(T_OBJECT); + Node* con = intcon(LayoutHelper::_flags_shift); + fval = _gvn.transform( new(C, 3) URShiftINode(layout_val, con) ); + btest = BoolTest::eq; + nval = ((juint)lh.as_int() >> LayoutHelper::_flags_shift); + } else { + Node* con = intcon(LayoutHelper::_array_flag << LayoutHelper::_flags_shift); + fval = _gvn.transform( new(C, 3) AndINode(layout_val, con) ); + btest = BoolTest::ne; + nval = 0; + } + Node* cmp = _gvn.transform( new(C, 3) CmpINode(fval, intcon(nval)) ); // invert the test if we are looking for a non-array if (not_array) btest = BoolTest(btest).negate(); Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, btest) ); --- old/src/share/vm/opto/macro.cpp 2008-07-12 17:49:43.000000000 -0700 +++ new/src/share/vm/opto/macro.cpp 2008-07-12 17:49:42.000000000 -0700 @@ -1298,7 +1298,7 @@ header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE); ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); if (k->is_array_klass()) // we know the exact header size in most cases: - header_size = Klass::layout_helper_header_size(k->layout_helper()); + header_size = k->layout_helper().header_size_in_bytes(); } // Clear the object body, if necessary. --- old/src/share/vm/opto/memnode.cpp 2008-07-12 17:49:47.000000000 -0700 +++ new/src/share/vm/opto/memnode.cpp 2008-07-12 17:49:46.000000000 -0700 @@ -1333,7 +1333,7 @@ if (tkls->offset() == Klass::layout_helper_offset_in_bytes() + (int)sizeof(oopDesc)) { // The field is Klass::_layout_helper. Return its constant value if known. assert(this->Opcode() == Op_LoadI, "must load an int from _layout_helper"); - return TypeInt::make(klass->layout_helper()); + return TypeInt::make(klass->layout_helper().as_int()); } // No match. @@ -1495,7 +1495,7 @@ // Note: When interfaces are reliable, we can narrow the interface // test to (klass != Serializable && klass != Cloneable). assert(Opcode() == Op_LoadI, "must load an int from _layout_helper"); - jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false); + jint min_size = LayoutHelper::for_instance(oopDesc::header_size()).as_int(); // The key property of this type is that it folds up tests // for array-ness, since it proves that the layout_helper is positive. // Thus, a generic value like the basic object layout helper works fine. @@ -3228,7 +3228,8 @@ assert(allocation() != NULL, ""); Node* klass_node = allocation()->in(AllocateNode::KlassNode); ciKlass* k = phase->type(klass_node)->is_klassptr()->klass(); - if (zeroes_done == k->layout_helper()) + if (k->layout_helper().is_fixed_instance() && + zeroes_done == k->layout_helper().fixed_size_in_bytes()) zeroes_done = size_limit; } if (zeroes_done < size_limit) { --- old/src/share/vm/opto/runtime.cpp 2008-07-12 17:49:51.000000000 -0700 +++ new/src/share/vm/opto/runtime.cpp 2008-07-12 17:49:50.000000000 -0700 @@ -163,8 +163,7 @@ assert(check_compiled_frame(thread), "incorrect caller"); // These checks are cheap to make and support reflective allocation. - int lh = Klass::cast(klass)->layout_helper(); - if (Klass::layout_helper_needs_slow_path(lh) + if (Klass::cast(klass)->layout_helper().needs_slow_path() || !instanceKlass::cast(klass)->is_initialized()) { KlassHandle kh(THREAD, klass); kh->check_valid_for_instantiation(false, THREAD); --- old/src/share/vm/runtime/globals.hpp 2008-07-12 17:49:55.000000000 -0700 +++ new/src/share/vm/runtime/globals.hpp 2008-07-12 17:49:54.000000000 -0700 @@ -3050,6 +3050,9 @@ develop(intx, FastSuperclassLimit, 8, \ "Depth of hardwired instanceof accelerator array") \ \ + product(bool, MixedArrays, false, \ + "Support hybrid instance/array data types") \ + \ /* Properties for Java libraries */ \ \ product(intx, MaxDirectMemorySize, -1, \ --- old/src/share/vm/runtime/handles.hpp 2008-07-12 17:49:58.000000000 -0700 +++ new/src/share/vm/runtime/handles.hpp 2008-07-12 17:49:58.000000000 -0700 @@ -215,7 +215,7 @@ DEF_KLASS_HANDLE(arrayKlassKlass , oop_is_arrayKlass ) DEF_KLASS_HANDLE(objArrayKlassKlass , oop_is_objArrayKlass ) DEF_KLASS_HANDLE(typeArrayKlassKlass , oop_is_typeArrayKlass) -DEF_KLASS_HANDLE(arrayKlass , oop_is_array ) +DEF_KLASS_HANDLE(arrayKlass , oop_is_array_slow ) DEF_KLASS_HANDLE(typeArrayKlass , oop_is_typeArray_slow) DEF_KLASS_HANDLE(objArrayKlass , oop_is_objArray_slow ) DEF_KLASS_HANDLE(symbolKlass , oop_is_symbol ) --- old/src/share/vm/runtime/vmStructs.cpp 2008-07-12 17:50:01.000000000 -0700 +++ new/src/share/vm/runtime/vmStructs.cpp 2008-07-12 17:50:01.000000000 -0700 @@ -89,6 +89,7 @@ nonstatic_field(constantPoolOopDesc, _length, int) \ nonstatic_field(constantPoolCacheOopDesc, _length, int) \ nonstatic_field(constantPoolCacheOopDesc, _constant_pool, constantPoolOop) \ + nonstatic_field(instanceKlass, _alloc_size, juint) \ nonstatic_field(instanceKlass, _array_klasses, klassOop) \ nonstatic_field(instanceKlass, _methods, objArrayOop) \ nonstatic_field(instanceKlass, _method_ordering, typeArrayOop) \ @@ -104,10 +105,11 @@ nonstatic_field(instanceKlass, _source_file_name, symbolOop) \ nonstatic_field(instanceKlass, _source_debug_extension, symbolOop) \ nonstatic_field(instanceKlass, _inner_classes, typeArrayOop) \ + nonstatic_field(instanceKlass, _object_size, int) \ nonstatic_field(instanceKlass, _nonstatic_field_size, int) \ nonstatic_field(instanceKlass, _static_field_size, int) \ nonstatic_field(instanceKlass, _static_oop_field_size, int) \ - nonstatic_field(instanceKlass, _nonstatic_oop_map_size, int) \ + nonstatic_field(instanceKlass, _map_offset_in_bytes, int) \ nonstatic_field(instanceKlass, _is_marked_dependent, bool) \ nonstatic_field(instanceKlass, _minor_version, u2) \ nonstatic_field(instanceKlass, _major_version, u2) \ @@ -136,7 +138,7 @@ nonstatic_field(Klass, _java_mirror, oop) \ nonstatic_field(Klass, _modifier_flags, jint) \ nonstatic_field(Klass, _super, klassOop) \ - nonstatic_field(Klass, _layout_helper, jint) \ + nonstatic_field(Klass, _layout_helper, LayoutHelper) \ nonstatic_field(Klass, _name, symbolOop) \ nonstatic_field(Klass, _access_flags, AccessFlags) \ nonstatic_field(Klass, _subklass, klassOop) \ @@ -1261,6 +1263,7 @@ declare_toplevel_type(JNIid) \ declare_toplevel_type(JNIid*) \ declare_toplevel_type(jmethodID*) \ + declare_integer_type(LayoutHelper) /* FIXME: wrong type (not integer) */\ declare_toplevel_type(Mutex*) \ declare_toplevel_type(nmethod*) \ declare_toplevel_type(ObjectMonitor*) \ @@ -1463,13 +1466,28 @@ /******************************/ \ \ declare_constant(Klass::_primary_super_limit) \ - declare_constant(Klass::_lh_instance_slow_path_bit) \ - declare_constant(Klass::_lh_log2_element_size_shift) \ - declare_constant(Klass::_lh_element_type_shift) \ - declare_constant(Klass::_lh_header_size_shift) \ - declare_constant(Klass::_lh_array_tag_shift) \ - declare_constant(Klass::_lh_array_tag_type_value) \ - declare_constant(Klass::_lh_array_tag_obj_value) \ + \ + /*******************************/ \ + /* LayoutHelper enum constants */ \ + /*******************************/ \ + \ + declare_constant(LayoutHelper::_flags_bits) \ + declare_constant(LayoutHelper::_flags_shift) \ + declare_constant(LayoutHelper::_flags_low_bits) \ + declare_constant(LayoutHelper::_variable_flag) \ + declare_constant(LayoutHelper::_array_flag) \ + declare_constant(LayoutHelper::_flags_ebt_mask) \ + declare_constant(LayoutHelper::_element_size_bits) \ + declare_constant(LayoutHelper::_element_size_shift) \ + declare_constant(LayoutHelper::_element_scale_bits) \ + declare_constant(LayoutHelper::_element_sizem_bits) \ + declare_constant(LayoutHelper::_element_sizem_mask_ip) \ + declare_constant(LayoutHelper::_size_low_bits) \ + declare_constant(LayoutHelper::_slow_path_low_bit) \ + declare_constant(LayoutHelper::_header_size_bits) \ + declare_constant(LayoutHelper::_header_size_shift) \ + declare_constant(LayoutHelper::_fixed_size_bits) \ + declare_constant(LayoutHelper::_fixed_size_shift) \ \ /********************************/ \ /* constMethodOopDesc anon-enum */ \ --- old/src/share/vm/utilities/globalDefinitions.hpp 2008-07-12 17:50:04.000000000 -0700 +++ new/src/share/vm/utilities/globalDefinitions.hpp 2008-07-12 17:50:04.000000000 -0700 @@ -902,6 +902,18 @@ return log2_intptr(x); } +//* smallest i (a power of 2) such that (x & i) != 0 +// A zero value of 'x' will return '0'. +inline uintptr_t lowest_bit_set(uintptr_t x) { + return ((x ^ (x - 1)) >> 1) + 1; +} + +//* largest i (a power of 2) such that (x & i) != 0 +// A zero value of 'x' will return '0'. +inline uintptr_t highest_bit_set(uintptr_t x) { + return (x == 0) ? 0 : (uintptr_t)1 << log2_intptr(x); +} + // returns integer round-up to the nearest multiple of s (s must be a power of two) inline intptr_t round_to(intptr_t x, uintx s) { --- /dev/null 2008-07-12 17:50:07.000000000 -0700 +++ new/src/share/vm/oops/layoutHelper.cpp 2008-07-12 17:50:06.000000000 -0700 @@ -0,0 +1,209 @@ +/* + * Copyright 2008 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + */ + +# include "incls/_precompiled.incl" +# include "incls/_layoutHelper.cpp.incl" + + +int LayoutHelper::new_instance_size_in_bytes() const { + if (!MixedArrays) { + // The variable case will not appear. + return fixed_size_in_bytes(); + } + assert(as_int() != (jint)_neutral_value, "must be instance or array"); + jint varmask = ((jint)as_int() >> (BitsPerInt-1)); // replicate sign bit + varmask <<= (BitsPerInt - _header_size_bits); // 0xFFFF0000 or zero + jint size = as_int(); + size += _header_size_odd_mask; // align size, in case variable + size &= ~_size_low_mask; // clear alignment & flag bits + size &= ~varmask; // strip high 16 bits + assert(size == (is_fixed_instance() + ? fixed_size_in_bytes() + : align_size_up_( + header_size_in_bytes(), MinObjAlignmentInBytes )), + "size calcs agree"); + return size; +} + + +LayoutHelper LayoutHelper::for_array(BasicType etype) { + assert(etype >= T_BOOLEAN && etype <= T_OBJECT, "valid etype"); + // Note that T_ARRAY is not allowed here. + int hsize = arrayOopDesc::base_offset_in_bytes(etype); + int esize = type2aelembytes(etype); + int flags = _variable_flag | _array_flag; + assert((hsize & _size_low_mask) == 0, "aligned to int"); + assert(hsize == round_to(hsize, esize), "already rounded"); + int hfield = hsize; + bool isobj = (etype == T_OBJECT); + if (!isobj) { + flags |= _element_allprim_flag; + hfield |= _obj_allprim_low_bit; + } + flags |= _header_allprim_flag; // true of all arrays + LayoutHelper lh = for_variable_object(flags, hfield, etype, exact_log2(esize)); + + assert(lh.has_length_field(), "must look like an array layout"); + assert(lh.is_array(), "correct kind"); + assert(lh.is_objArray() == isobj, "correct kind"); + assert(lh.is_typeArray() == !isobj, "correct kind"); + assert(!lh.is_instance(), "correct kind"); + assert(lh.header_size_in_bytes() == hsize, "correct decode"); + assert(lh.element_type() == etype, "correct decode"); + assert(1 << lh.log2_element_size() == esize, "correct decode"); + + // Might as well put this here: + assert(sizeof(LayoutHelper) == sizeof(jint), "no header or padding"); + + return lh; +} + +LayoutHelper LayoutHelper::for_mixed_array(jint header_size_in_bytes, + Handle element_type_mirror, + TRAPS) { + assert(MixedArrays, "else do not use"); + int esize = 0, ealign = 0; + BasicType etype = T_ILLEGAL; + instanceKlassHandle tklass; + if (java_lang_Class::is_primitive(element_type_mirror())) { + etype = java_lang_Class::primitive_type(element_type_mirror()); + if (etype == T_VOID) { + // this only happens for an abstract class + esize = ealign = 1; + } else { + assert(etype >= T_BOOLEAN && etype < T_OBJECT, "valid etype"); + esize = ealign = type2aelembytes(etype); + } + } else { + klassOop eklass = java_lang_Class::as_klassOop(element_type_mirror()); + if (eklass == SystemDictionary::object_klass()) { + esize = ealign = type2aelembytes(T_OBJECT); + } else if (Klass::cast(eklass)->oop_is_instance()) { + // mixed array of tuples (inline structs of given concrete type) + tklass = instanceKlassHandle(THREAD, eklass); + esize = tklass->nonstatic_field_size() * wordSize; + ealign = BytesPerLong; + // TO DO: if fields are all subwords, shrink esize and ealign + if (!tklass->can_be_fastpath_allocated()) { + esize = ealign = 0; + } + if (esize > 0) { + int esize_field = compute_esize_field(esize); + if (esize_field > _element_size_mask) { + // overflow; round up + int upsize = decode_esize_field(esize_field); + if (upsize < esize) { + esize = ealign = 0; // fall through to error + } else { + esize_field &= _element_size_mask; + assert(esize_field == compute_esize_field(upsize), "stable"); + esize = upsize; + } + } + } + } + } + if (esize == 0 || ealign == 0) { + ResourceMark rm(THREAD); + THROW_MSG_(vmSymbols::java_lang_InternalError(), "bad @MixedArray.Of annotation", 0); + } + int hsize = round_to(header_size_in_bytes, ealign); + int flags = _variable_flag; // but not _array_flag + if (etype < T_OBJECT) { + flags |= _element_allprim_flag; + } + int esize_field = compute_esize_field(esize); + assert(esize_field <= _element_size_mask, "no overflow this time"); + LayoutHelper lh = for_variable_object(flags, hsize, etype, esize_field); + + assert(lh.has_length_field(), "must look like an array layout"); + assert(!lh.is_array(), "correct kind"); + assert(!lh.is_objArray(), "correct kind"); + assert(!lh.is_typeArray(), "correct kind"); + assert(lh.is_instance(), "correct kind"); + assert(lh.is_mixedArray(), "correct kind"); + assert(lh.header_size_in_bytes() == hsize, "correct decode"); + assert(lh.element_type() == etype, "correct decode"); + assert(lh.element_size() == esize, "correct decode"); + + return lh; +} + +// Pack the given size (which can be any number) into the 8-bit element size field. +// If there is a problem representing this value in 8 bits, +// set the next-higher bit of the result. +// If it needs a little rounding up, do so. +int LayoutHelper::compute_esize_field(uintptr_t raw_size_in_bytes) { + int overflow_flag = (1 << _element_size_bits); + if (raw_size_in_bytes == 0) return overflow_flag; + int low_zeroes = exact_log2(lowest_bit_set(raw_size_in_bytes)); + if (low_zeroes > _element_scale_mask) + low_zeroes = _element_scale_mask; + uintptr_t sizem = ((raw_size_in_bytes >> low_zeroes) - 1); + if (sizem <= right_n_bits(_element_sizem_bits)) { + assert(((sizem+1) << low_zeroes) == raw_size_in_bytes, ""); + return (sizem << _element_scale_bits) | low_zeroes; + } + // Overflow. Round up low bits of sizem, if possible. + while (low_zeroes < _element_scale_mask) { + sizem = (sizem + 1) >> 1; + low_zeroes += 1; + if (sizem <= right_n_bits(_element_sizem_bits)) { + // Return the next higher size, with a flag bit. + assert(((sizem+1) << low_zeroes) > raw_size_in_bytes, ""); + return ((sizem << _element_scale_bits) | low_zeroes | overflow_flag); + } + } + // Not possible to round up. (Size is more than (31+1)**7 == 2**12.) + assert(raw_size_in_bytes > ((uintptr_t)(1<<_element_sizem_bits) << _element_scale_mask), ""); + return overflow_flag; +} + +#ifndef PRODUCT + +void LayoutHelper::print_on(outputStream* st) const { + if (is_fixed_instance()) { + st->print("fixed=%d", fixed_size_in_bytes()); + } else if (has_length_field()) { + st->print("variable=%d[%d]", header_size_in_bytes(), element_size()); + } else { + st->print("none"); + } + // print various flags + int flags = this->flags(); + if (flags & _array_flag) st->print(",array"); + if (flags & _element_allprim_flag) st->print(",[allprim]"); + if (flags & _header_allprim_flag) st->print(",allprim[]"); + if (flags & _flags_ebt_mask) { + BasicType ebt = (BasicType)(flags & _flags_ebt_mask); + const char* ebt_name = type2name(ebt); + if (ebt_name != NULL) + st->print(",[bt=%s]", ebt_name); + else st->print(",[bt=%d]", (int) ebt); + } + if (is_allprim()) st->print(",allprim"); + if (needs_slow_path()) st->print(",slow_path"); +} + +#endif --- /dev/null 2008-07-12 17:50:09.000000000 -0700 +++ new/src/share/vm/oops/layoutHelper.hpp 2008-07-12 17:50:08.000000000 -0700 @@ -0,0 +1,445 @@ +/* + * Copyright 2008 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + */ + +// The "layout helper" is a combined 32-bit descriptor of object layout. +// For klasses which are neither instance nor array, the value is zero. +// +// For fixed sized instances, layout helper is a positive number, the size. +// The three low order bits may be used as flags, and must be masked. +// +// For variable sized objects, layout helper is a negative number, +// containing header and element size, and various flag bits. +// +// >0: fixed: [_:8, fixed:24] +// where fixed = (f_size_8:21, _, allprim, slow) +// <0: variable: [flags:8, e_size:8, header:16] +// where flags = (sign=1, array, e_allprim, h_allprim, ebt:4) +// & e_size = (e_sizem:5, e_scale:3) +// & header = (h_size_4:14, allprim, slow) +// =0: other: [_:32] +// +// f_size_8: Object size after align_object_size and scaled to jlong. +// h_size_4: Header size in ints (i.e., offset of first element). +// allprim: All words in the object (incl. tail) are known to be non-oops. +// slow: Object allocation must take a slow path through the JVM. +// flags: Flag bits which are either all zero or describe a variable obj. +// sign: Always set, for variable-sized objects. +// array: Object is a true array (not a variable sized instance). +// e_allprim: All words in the object tail are known to be non-oops. +// h_allprim: Non-klass words in the object header are known to be non-oops. +// ebt: BasicType of array elements. +// e_size: Virtual field, equal to ((1+e_sizem)<> _flags_shift; } + int esize() const { return (as_int() >> _element_size_shift) & _element_size_mask; } + + /// Classification: + bool is_neutral() const { + // a neutral LH is used only for non-Java objects + return as_int() == _neutral_value; + } + bool is_fixed_instance() const { + // use clever range comparison instead of bitfield extraction + return (jint)as_int() > (jint)_neutral_value; + } + bool has_length_field() const { + // use clever range comparison instead of bitfield extraction + return (jint)as_int() < (jint)_neutral_value; + } + // this query can return a conservative 'false': + bool is_allprim() const { + const jint testbit = _obj_allprim_low_bit; + assert((_fixed_size_mask & testbit) == 0, "compatible with fixed layout"); + // simple in-place test of single bit + return (as_int() & testbit) != 0; + } + bool needs_slow_path() const { + const jint testbit = _slow_path_low_bit; + assert((_fixed_size_mask & testbit) == 0, "compatible with fixed layout"); + // simple in-place test of single bit + return (as_int() & testbit) != 0; + } + bool is_instance() const { + const jint testbit = (_array_flag << _flags_shift); + assert((_fixed_size_mask & testbit) == 0, "compatible with fixed layout"); + // two-step check is required because of mixed arrays + return as_int() != 0 && (as_int() & testbit) == 0; + } + bool is_array() const { + const jint testbit = (_array_flag << _flags_shift); + assert((_fixed_size_mask & testbit) == 0, "compatible with fixed layout"); + // simple in-place test of single bit + return (as_int() & testbit) != 0; + } + bool is_typeArray() const { + const jint testbits = ((_variable_flag | + _array_flag | + _element_allprim_flag | + _header_allprim_flag ) + << _flags_shift); + // use clever range comparison instead of bitfield extraction + assert(testbits == (-1 << (BitsPerInt - 4)), "high unsigned value"); + return (juint)as_int() >= (juint)testbits; + } + bool is_objArray() const { + const juint testbits = (_variable_flag | + _array_flag | + _header_allprim_flag | + T_OBJECT); + // cannot do a clever j[u]int range comparison; just extract and compare + return flags() == testbits; + } + bool is_mixedArray() const { + const jint testbits = ((_variable_flag | + _array_flag) + << _flags_shift); + assert(testbits == (-1 << (BitsPerInt - 2)), "high unsigned value"); + assert(((testbits - 1) & (_array_flag << _flags_shift)) == 0, ""); + // signed comparison requires high bit pattern of '10x*' + return (jint)as_int() < (jint)testbits; + } + bool has_oops_in_tail() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + const jint testbit = (_element_allprim_flag << _flags_shift); + // simple in-place test of single bit + return (as_int() & testbit) == 0; + } + bool has_non_oops_in_tail() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + // in-place test of element basic type + return element_type() != T_OBJECT; + } + bool has_oops_in_header() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + const jint testbit = (_header_allprim_flag << _flags_shift); + // simple in-place test of single bit + return (as_int() & testbit) == 0; + } + + /// Size queries: + + int fixed_size_in_bytes(bool allow_variable = false) const { + if (allow_variable) { + // allow a garbage return value in the non-fixed case + } else { + assert(as_int() > (jint)_neutral_value, "must be instance"); + assert((as_int() & _header_size_odd_mask) == 0, "size not odd"); + } + return (int) as_int() & ~_size_low_mask; + } + int header_size_in_bytes() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + // mask out both high and low bits: + return (int) as_int() & _header_size_mask; + } + + // The size of an object created without a length, or with zero length. + int new_instance_size_in_bytes() const; + + // The size of an object created with the given length. + // Rounding to MinObjAlignment is performed. + // If log2_only, the element size must be a power of two (and small, <= 2^7). + size_t new_array_size_in_bytes(jint length, bool log2_only = false) const { + return new_array_size_in_bytes_common(length, log2_only); + } + // Result is a julong to avoid overflows. + julong new_array_size_in_bytes_no_overflow(jint length, bool log2_only = false) const { + if (sizeof(julong) == sizeof(size_t)) + // Overflow is impossible on LP64; use size_t always. + return new_array_size_in_bytes(length, log2_only); + else + return new_array_size_in_bytes_common(length, log2_only); + } + + private: + template + T new_array_size_in_bytes_common(jint length, bool log2_only = false) const { + assert(as_int() < (jint)_neutral_value, "must be variable instance or array"); + T size_in_bytes = scale_index_common(length, log2_only); + size_in_bytes += header_size_in_bytes(); + size_in_bytes = align_size_up_(size_in_bytes, MinObjAlignmentInBytes); + return size_in_bytes; + } + + public: + /// Element queries. + + BasicType element_type() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + int btvalue = (as_int() >> _flags_shift) & _flags_ebt_mask; + assert(btvalue >= T_BOOLEAN && btvalue <= T_ARRAY, "sanity"); + return (BasicType) btvalue; + } + int element_size_has_log2() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + int esize = (as_int() >> _element_size_shift) & _element_size_mask; + // true if the sizem bits are all zero: + return esize <= _element_scale_mask; + } + int log2_element_size() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + // for this accessor, the sizem bits must all be zero: + assert(element_size_has_log2(), "caller resp."); + int esize = (as_int() >> _element_size_shift) & _element_size_mask; + return esize; + } + int element_size() const { + assert(as_int() < (jint)_neutral_value, "must be array"); + int esize = (as_int() >> _element_size_shift) & _element_size_mask; + return decode_esize_field(esize); + } + // Scale a given index by the element size. + // Result must not overflow native address size. + // If log2_only, multiplies are excluded; only shifts are allowed. + size_t scale_index(jint index, bool log2_only = false) const { + return scale_index_common(index, log2_only); + } + // Result is 64 bits to avoid any 32-bit overflow. + julong scale_index_no_overflow(jint index, bool log2_only = false) const { + if (sizeof(julong) == sizeof(size_t)) + // Overflow is impossible on LP64; use size_t always. + return scale_index(index, log2_only); + else + return scale_index_common(index, log2_only); + } + // Combine the header size with the scaled index. + size_t element_offset(jint index, bool log2_only = false) const { + return header_size_in_bytes() + scale_index(index, log2_only); + } + julong element_offset_no_overflow(jint index, bool log2_only = false) const { + return header_size_in_bytes() + scale_index_no_overflow(index, log2_only); + } + // Compute an index given an offset. Works like a div/mod. + // Returns zero if the offset is before the first element. + // Returns max_jint (with an accurate remainder) if the offset is beyond + // any indexable array element. + // Except for those two limiting cases, the remainder will always + // be in the range zero to element_size()-1. + jint index_from_element_offset(size_t offset, + size_t *remainder = NULL, // 2nd ret val + bool log2_only = false) const { + size_t hsize = header_size_in_bytes(); + if (offset <= hsize) { + // Just return zero. Remainder might be negative. + if (remainder != NULL) + (*remainder) = offset - hsize; + return 0; + } else { + return unscale_offset_common(offset - hsize, remainder, log2_only); + } + } + + // Working with the 8-bit esize field. + static int decode_esize_field(int esize) { + assert((esize & ~_element_size_mask) == 0, "no bad bits"); + int scale = (esize & _element_scale_mask); + int sizem = (esize >> _element_scale_bits); + return (sizem + 1) << scale; + } + + static int element_size_limit() { + return decode_esize_field(_element_size_mask); + } + + // Compute an esize field for a given raw element size. + // If the requested raw size cannot be represented, return > 255. + // If the result is exactly 256, the element size is too large. + // Otherwise, the low-order bits will be the best approximation. + static int compute_esize_field(uintptr_t raw_size_in_bytes); + + private: + template + T scale_index_common(jint index, bool log2_only = false) const { + int esize = (as_int() >> _element_size_shift); + int scale = esize; + // Note: Semantics of C operator<< always include a mask. + //scale &= _element_size_mask; + T size_in_bytes = ((T)index << scale); + // The following check (for non-power-of-2 element size) is + // only relevant to tuple arrays and mixed arrays: + int sizem_ip = (esize & _element_sizem_mask_ip); + if (log2_only) { + assert(sizem_ip == 0, "no multiplies allowed"); + } else if (sizem_ip != 0) { + scale -= sizem_ip; // scale might have been polluted by sizem bits + int sizem = (sizem_ip >> _element_scale_bits); + size_in_bytes = ((T)index * (1+sizem)) << scale; + } + assert((julong)size_in_bytes == (julong)index * element_size(), + "same result as if using direct multiplication"); + return size_in_bytes; + } + + jint unscale_offset_common(size_t offset, size_t *remainder, + bool log2_only = false) const { + int esize = (as_int() >> _element_size_shift); + int scale = esize; + size_t index = (offset >> scale); + size_t rem = (offset & (size_t)right_n_bits(scale)); + // The following check (for non-power-of-2 element size) is + // only relevant to tuple arrays and mixed arrays: + int sizem_ip = (esize & _element_sizem_mask_ip); + if (log2_only) { + assert(sizem_ip == 0, "no multiplies allowed"); + if (index != (size_t)(jint)index) { + index = max_jint; + rem = offset - (index << scale); + } + } else if (sizem_ip != 0) { + size_t es = element_size(); + index = offset / es; + if (index != (size_t)(jint)index) { + index = max_jint; + } + if (remainder != NULL) + rem = offset - (index * es); + } + if (remainder != NULL) + (*remainder) = rem; + return index; + } + + /// Type-specific constructors: + + public: + static LayoutHelper for_variable_object(int v_flags, + int h_size, + BasicType e_type, + int e_size) { + assert((v_flags & _variable_flag) != 0, "must be marked"); + assert((v_flags & _flags_ebt_mask) == 0, "disjoint bits"); + assert(((int)e_type & ~_flags_ebt_mask) == 0, "disjoint bits"); + v_flags |= (e_type & _flags_ebt_mask); + return LayoutHelper((v_flags << _flags_shift) | + (e_size << _element_size_shift) | + (h_size << _header_size_shift) ); + } + static LayoutHelper for_instance(jint size_in_words) { + jint lh = (size_in_words << LogHeapWordSize); + assert((lh & ~_fixed_size_mask) == 0 && lh > _neutral_value, "size must fit"); + assert(LayoutHelper(lh).is_instance(), "correct kind"); + return LayoutHelper(lh); + } + + // Out-of-line version computes everything based on the etype: + static LayoutHelper for_array(BasicType etype); + + static LayoutHelper for_mixed_array(jint header_size_in_bytes, + Handle element_type_mirror, + TRAPS); + + // Mutators, for editing new values. (Note that Klass::lh is const.) + void set_flag(int flag) { + assert((flag & ~(_flags_mask << _flags_shift)) == 0, "preshifted, no bad bits"); + _int_value |= flag; + } + void clear_flag(int flag) { + assert((flag & ~(_flags_mask << _flags_shift)) == 0, "preshifted, no bad bits"); + _int_value &= ~flag; + } + void set_low_bit(int bit) { + assert((bit & ~_size_low_mask) == 0, "no bad bits"); + _int_value |= bit; + } + void clear_low_bit(int bit) { + assert((bit & ~_size_low_mask) == 0, "no bad bits"); + _int_value &= ~bit; + } + + // Printing/debugging + void print_on(outputStream* st) const PRODUCT_RETURN; +}; --- /dev/null 2008-07-12 17:50:11.000000000 -0700 +++ new/src/share/vm/oops/mixedArrayKlass.cpp 2008-07-12 17:50:11.000000000 -0700 @@ -0,0 +1,348 @@ +/* + * Copyright 2008 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + */ + +# include "incls/_precompiled.incl" +# include "incls/_mixedArrayKlass.cpp.incl" + +int mixedArrayKlass::oop_size(oop obj) const { + assert(obj->is_mixedArray(), "must be object array"); + return mixedArrayOop(obj)->object_size(); +} + +mixedArrayOop mixedArrayKlass::allocate_array(int length, TRAPS) { + if (length >= 0) { + if (length <= max_array_length()) { + int size = new_array_size(length); + KlassHandle h_k(THREAD, as_klassOop()); + mixedArrayOop a = (mixedArrayOop)CollectedHeap::array_allocate(h_k, size, length, CHECK_NULL); + assert(a->is_parsable(), "Can't publish unless parsable"); + return a; + } else { + THROW_OOP_0(Universe::out_of_memory_error_array_size()); + } + } else { + THROW_0(vmSymbols::java_lang_NegativeArraySizeException()); + } +} + +void mixedArrayKlass::copy_array(oop s, int src_pos, oop d, + int dst_pos, int length, + bool allow_mixed, TRAPS) { + assert(s->is_mixedArray(), "must be mixed array"); + + bool d_is_objArray = d->is_objArray(); + + if (!allow_mixed || !is_conformable_to(d->blueprint())) { + THROW(vmSymbols::java_lang_ArrayStoreException()); + } + + // Check is all offsets and lengths are non negative + if (src_pos < 0 || dst_pos < 0 || length < 0) { + THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); + } + // Check if the ranges are valid + if ( (((unsigned int) length + (unsigned int) src_pos) > (unsigned int) arrayOopDesc::length_of(s)) + || (((unsigned int) length + (unsigned int) dst_pos) > (unsigned int) arrayOopDesc::length_of(d)) ) { + THROW(vmSymbols::java_lang_ArrayIndexOutOfBoundsException()); + } + + if (length == 0) { + return; + } + + LayoutHelper lh = layout_helper(); + size_t src_off = lh.element_offset(src_pos); + size_t dst_off = lh.element_offset(dst_pos); + size_t elt_siz = lh.element_size(); + size_t cpy_len = lh.scale_index(length, true); + if (d->klass() != this->as_klassOop()) { + // Destination might have a different header size. + LayoutHelper d_lh = Klass::cast(d->klass())->layout_helper(); + assert(d_lh.element_size() == lh.element_size(), "same element layout"); + assert(d_lh.element_type() == lh.element_type(), "same element layout"); + assert(!d_lh.has_oops_in_tail(), ""); + dst_off = d_lh.element_offset(dst_pos, true); + } + address src = (address)s + src_off; + address dst = (address)d + dst_off; + + if (!lh.has_oops_in_tail()) { + // Simple bytewise copy. + // Note: memmove is not guaranteed to be word atomic. + Copy::conjoint_memory_atomic(src, dst, cpy_len); + } else if (!lh.has_non_oops_in_tail()) { + // We nay need to perform store checks here. + klassOop store_check = NULL; + if (d_is_objArray) { + // Mixed arrays can contain any object, so d needs a store check. + store_check = objArrayKlass::cast(d->klass())->element_klass(); + assert(!lh.has_non_oops_in_tail(), ""); + assert(elt_siz == heapOopSize, ""); + } else { + // %%% TO DO: If T_OBJECT mixed arrays get narrowed element types, + // put in a store check here. + // Note: Maybe lh.has_non_oops_in_tail() is true. + } + WITH_BOTH_OOP_KINDS(T, { + assert(sizeof(T) == lh.element_size(), "correct lh for oops"); + size_t oop_len = cpy_len / sizeof(T); + assert(elt_siz % sizeof(T) == 0, ""); + objArrayKlass::do_copy((T*)src, (T*)dst, + oop_len, store_check, THREAD); + }); + } else { + assert(!d_is_objArray, ""); + assert((src_off|dst_off|elt_siz) % heapOopSize == 0, "aligned oop-wise copy"); + + // For performance reasons, we assume we are using a card marking write + // barrier. The assert will fail if this is not the case. + BarrierSet* bs = Universe::heap()->barrier_set(); + assert(bs->has_write_ref_array_opt(), "Barrier set must have ref array opt"); + + // Copy the whole mess of oops and non-oops. + Copy::conjoint_memory_atomic(src, dst, cpy_len); + + size_t aligned_word_len = cpy_len; + // include partial word before dst: + aligned_word_len += (dst_off % HeapWordSize); + // include partial word after dst+len: + aligned_word_len = align_size_up_(aligned_word_len, HeapWordSize); + // scale down to heap words: + aligned_word_len >>= LogHeapWordSize; + + bs->write_ref_array(MemRegion((HeapWord*)dst, aligned_word_len)); + } +} + +void mixedArrayKlass::oop_follow_contents(oop obj) { + instanceKlass::oop_follow_contents(obj); + WITH_BOTH_OOP_KINDS(T, { + for (mixed_tail::iterator pmax, p = tail_oop_fields(obj, pmax); p < pmax; p++) { + MarkSweep::mark_and_push(assert_is_in_closed_subset(addr(p))); + } + }); +} + +#if 0 //@@@ +#ifndef SERIALGC +void mixedArrayKlass::oop_follow_contents(ParCompactionManager* cm, + oop obj) { + assert(obj != NULL, "can't follow the content of NULL object"); + obj->follow_header(cm); + PSParallelCompact::KeepAliveClosure closure(cm); + oop_oop_tail_iterate_nv + (obj, + start_of_nonstatic_oop_maps(), + nonstatic_oop_map_count(), + &closure, + AssertIsIn::instance.or_null()); +} +#endif // SERIALGC + +// closure's do_header() method dicates whether the given closure should be +// applied to the klass ptr in the object header. + +#define InstanceKlass_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ + \ +int mixedArrayKlass::oop_oop_iterate##nv_suffix(oop obj, \ + OopClosureType* closure) { \ + SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik); \ + /* header */ \ + if (closure->do_header()) { \ + obj->oop_iterate_header(closure); \ + } \ + oop_oop_tail_iterate##nv_suffix \ + (obj, \ + start_of_nonstatic_oop_maps(), \ + nonstatic_oop_map_count(), \ + closure, \ + AssertIsInClosedSubset::instance.or_null()); \ + return new_instance_size(); \ +} + +#define InstanceKlass_OOP_OOP_ITERATE_DEFN_m(OopClosureType, nv_suffix) \ + \ +int mixedArrayKlass::oop_oop_iterate##nv_suffix##_m(oop obj, \ + OopClosureType* closure, \ + MemRegion mr) { \ + SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik);\ + if (closure->do_header()) { \ + obj->oop_iterate_header(closure, mr); \ + } \ + oop_oop_bounded_tail_iterate##nv_suffix \ + (obj, \ + start_of_nonstatic_oop_maps(), \ + nonstatic_oop_map_count(), \ + closure, &mr, \ + AssertIsInClosedSubset::instance.or_null()); \ + return new_instance_size(); \ +} + +ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN) +ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DEFN) +ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN_m) +ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DEFN_m) + +void mixedArrayKlass::iterate_static_fields(OopClosure* closure) { + oop_oop_chunk_iterate_v(start_of_static_fields(), + static_oop_field_size(), + closure, + AssertIsInReserved::instance.or_null()); +} + +void mixedArrayKlass::iterate_static_fields(OopClosure* closure, + MemRegion mr) { + oop_oop_bounded_chunk_iterate_v(start_of_static_fields(), + static_oop_field_size(), + closure, &mr, + AssertIsInClosedSubset::instance.or_null()); +} + +int mixedArrayKlass::oop_adjust_pointers(oop obj) { + int size = new_instance_size(); + oop_oop_tail_iterate_nv + (obj, + start_of_nonstatic_oop_maps(), + nonstatic_oop_map_count(), + &MarkSweep::adjust_pointer_closure, + AssertIsIn::instance.or_null()); + return size; +} + +#ifndef SERIALGC +void mixedArrayKlass::oop_copy_contents(PSPromotionManager* pm, oop obj) { + assert(!pm->depth_first(), "invariant"); + PSPromotionManager::ForwardClosure closure(pm, /*breadth=*/ true); + oop_oop_reverse_tail_iterate_nv + (obj, + start_of_nonstatic_oop_maps(), + nonstatic_oop_map_count(), + &closure); +} + +void mixedArrayKlass::oop_push_contents(PSPromotionManager* pm, oop obj) { + assert(pm->depth_first(), "invariant"); + PSPromotionManager::ForwardClosure closure(pm, /*breadth=*/ false); + oop_oop_reverse_tail_iterate_nv + (obj, + start_of_nonstatic_oop_maps(), + nonstatic_oop_map_count(), + &closure); +} + +int mixedArrayKlass::oop_update_pointers(ParCompactionManager* cm, oop obj) { + oop_oop_tail_iterate_nv + (obj, + start_of_nonstatic_oop_maps(), + nonstatic_oop_map_count(), + PSParallelCompact::adjust_pointer_closure()); + return new_instance_size(); +} + +int mixedArrayKlass::oop_update_pointers(ParCompactionManager* cm, oop obj, + HeapWord* beg_addr, HeapWord* end_addr) { + MemRegion mr(beg_addr, end_addr); + oop_oop_bounded_tail_iterate_nv + (obj, + start_of_nonstatic_oop_maps(), + nonstatic_oop_map_count(), + PSParallelCompact::adjust_pointer_closure(), + &mr); + return new_instance_size(); +} + +void mixedArrayKlass::copy_static_fields(PSPromotionManager* pm) { + assert(!pm->depth_first(), "invariant"); + PSPromotionManager::ForwardClosure closure(pm, /*breadth=*/ true); + oop_oop_chunk_iterate_nv + (start_of_static_fields(), + static_field_size(), + &closure); +} + +void mixedArrayKlass::push_static_fields(PSPromotionManager* pm) { + assert(pm->depth_first(), "invariant"); + PSPromotionManager::ForwardClosure closure(pm, /*breadth=*/ false); + oop_oop_chunk_iterate_nv + (start_of_static_fields(), + static_field_size(), + &closure); +} + +void mixedArrayKlass::copy_static_fields(ParCompactionManager* cm) { + oop_oop_chunk_iterate_nv + (start_of_static_fields(), + static_field_size(), + PSParallelCompact::adjust_pointer_closure(), + AssertIsIn::instance.or_null()); +} +#endif // SERIALGC + +// ----------------------------------------------------------------------------------------------------- +#ifndef PRODUCT + +// Printing + +static void print_tuple_at(address elem, mixedArrayKlass* self, outputStream* st) { + // FIXME: Factor relevant field-printing code from instanceKlass::oop_print_on. + st->print("(%d bytes)@%p", (int)self->layout_helper().element_size(), elem); +} + +void mixedArrayKlass::oop_print_on(oop obj, outputStream* st) { + instanceKlass::oop_print_on(obj, st); + assert(obj->is_mixedArray(), "must be mixedArray"); + mixedArrayOop ma = mixedArrayOop(obj); + st->print_cr(" - length: %d", ma->length()); + BasicType etype = layout_helper().element_type(); + int print_len = MIN2((intx) ma->length(), MaxElementPrintSize); + for (int index = 0; index < print_len; index++) { + address elem = ma->element_at_addr(index); + st->print(" - %3d: ", index); + switch (etype) { + case T_BOOLEAN: st->print("%s", *(jboolean*)elem == 0 ? "false" : "true"); break; + case T_CHAR: st->print("%x %c", *(jchar*)elem, isprint(*(jchar*)elem) ? *(jchar*)elem : ' '); break; + case T_FLOAT: st->print("%g", *(jfloat*)elem); break; + case T_DOUBLE: st->print("%g", *(jdouble*)elem); break; + case T_BYTE: st->print("%x %c", *(jbyte*)elem, isprint(*(jbyte*)elem) ? *(jbyte*)elem : ' '); break; + case T_SHORT: st->print("0x%x\t %d", *(jushort*)elem, *(jushort*)elem); break; + case T_INT: st->print("0x%x %d", *(jint*)elem, *(jint*)elem); break; + case T_LONG: st->print("0x%x 0x%x", high(*(jlong*)elem), low(*(jlong*)elem)); break; + case T_OBJECT: ma->obj_at(index)->print_value_on(st); break; + default: print_tuple_at(elem, this, st); break; + } + st->cr(); + } + int remaining = ma->length() - print_len; + if (remaining > 0) { + tty->print_cr(" - <%d more elements, increase MaxElementPrintSize to print>", remaining); + } +} + +void mixedArrayKlass::oop_print_value_on(oop obj, outputStream* st) { + instanceKlass::oop_print_value_on(obj, st); + st->print(" [%d]", arrayOopDesc::length_of(obj)); +} + +#endif // ndef PRODUCT +#endif //@@@ --- /dev/null 2008-07-12 17:50:13.000000000 -0700 +++ new/src/share/vm/oops/mixedArrayKlass.hpp 2008-07-12 17:50:13.000000000 -0700 @@ -0,0 +1,332 @@ +/* + * Copyright 2008 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + */ + +// mixedArrayKlass is the klass for mixedArrays + +class mixedArrayKlass : public instanceKlass { + friend class VMStructs; + + public: + // Dispatched operations + bool oop_is_variable_slow() const { return true; } + bool oop_is_mixedArray_slow() const { return true; } + int oop_size(oop obj) const; + int klass_oop_size() const { return object_size(); } + + // Allocation + DEFINE_ALLOCATE_PERMANENT(mixedArrayKlass); + mixedArrayOop allocate_array(jint length, TRAPS); + + // Casting from klassOop + static mixedArrayKlass* cast(klassOop k) { + assert(k->klass_part()->oop_is_mixedArray(), "cast to mixedArrayKlass"); + return (mixedArrayKlass*) k->klass_part(); + } + + // Sizing + int element_size() { return layout_helper().element_size(); } + int header_size_in_bytes() { return layout_helper().header_size_in_bytes(); } + size_t new_array_size_in_bytes(int l) { return layout_helper().new_array_size_in_bytes(l); } + + int new_array_size(int l) { + size_t sib = new_array_size_in_bytes(l); + size_t siw = sib >> LogHeapWordSize; + assert((size_t)(int)siw == siw, "no overflow"); + return (int)siw; + } + + jint max_array_length() { + LayoutHelper lh = layout_helper(); + // Currently, object_size is restricted to max_jint. + julong max_body_size_in_bytes = (julong)max_jint * HeapWordSize; + // Trim the roast, just the way arrayOopDesc::max_array_length does it: + max_body_size_in_bytes -= 2 * HeapWordSize; + // Cut off the header: + max_body_size_in_bytes -= lh.header_size_in_bytes(); + julong len; + if (lh.element_size_has_log2()) + len = max_body_size_in_bytes >> lh.log2_element_size(); + else len = max_body_size_in_bytes / lh.element_size(); + len >>= LogHeapWordSize; // convert to words now + return (len > max_jint) ? max_jint : (jint)len; + } + + // Copying + void copy_array(oop s, int src_pos, oop d, int dst_pos, int length, bool allow_mixed, TRAPS); + bool is_conformable_to(Klass* other) { + // Extended array copy works between *identical* mixed array types, + // or else between the mixed array and a java array of the same basic type. + if (this == other) { + return true; + } else if (other->oop_is_javaArray()) { + BasicType other_bt = other->layout_helper().element_type(); + assert(other_bt != 0, "NYI: tuple arrays"); + return (other_bt == layout_helper().element_type()); + } else { + return false; + } + } + + // Garbage collection + void oop_follow_contents(oop obj); + int oop_adjust_pointers(oop obj); + + // Parallel Scavenge and Parallel Old + PARALLEL_GC_DECLS + + // Iterators + + // Populate the vtable with customized versions of the iterators. +#define MixedArrayKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ + int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \ + int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, \ + MemRegion mr); + // %%% TO DO: virtualize, update ParScanThreadState::scan_partial_array + // int oop_oop_iterate_range##nv_suffix(oop obj, OopClosureType* blk, + // int start, int end); + + ALL_OOP_OOP_ITERATE_CLOSURES_1(MixedArrayKlass_OOP_OOP_ITERATE_DECL) + ALL_OOP_OOP_ITERATE_CLOSURES_3(MixedArrayKlass_OOP_OOP_ITERATE_DECL) + + protected: + // Common code for iterating over oops. + template + class mixed_tail : AllStatic { + typedef OopMapBlock Map; // shorthand + public: + class iterator : public oop_map::iterator { + friend class mixed_tail; + int _stride; // loop invariant element size + Map* _elt_map; // loop invariant element map + address _elt_base; // address of next element to tackle + + // See comments on the base class (instanceKlass::oop_map::iterator) + // to find out about the design of this iterator. + + // A pair of these iterators implements this triply nested loop: + // int obj_length = ...; + // int elt_map_count = ...; + // Map* beg_elt_map = ...; + // Map* end_elt_map = beg_elt_map + elt_map_count; + // int beg_stride = ...; + // int end_stride = beg_stride; + // address beg_elt_base = ...; + // address end_elt_base = beg_elt_base + (beg_stride * obj_length); + // for (; beg_elt_base < end_elt_base; beg_elt_base += beg_elt_stride) { + // Map* beg_map = beg_elt_map; + // Map* end_map = end_elt_map; + // for (; beg_map < end_map; beg_map++) { + // T* beg_p = (T*)( beg_elt_base + beg_map->offset() ); + // T* end_p = (T*)( beg_elt_base + beg_map->offset() + beg_map->size() ); + // for (; beg_p < end_p; beg_p++) { + // ... + // } + // } + // } + + public: + iterator() { } + iterator(T* base, int stride, Map* elt_maps) { + reset_tail(base, stride, elt_maps); + } + + protected: + void reset_tail(T* base, int stride, Map* elt_maps) { + _stride = stride; + _elt_map = elt_maps; + // Both inner loops have to start out unprimed: + reset_map((address) base, unprimed(elt_maps)); + } + + static void reprime_elt_forward(iterator &beg, iterator& end, address b) { + beg.reset_map((T*)b, beg._elt_map); + // load the first map chunk: + bool z = map_nonempty_forward(beg, end); + assert(z, "progress"); + } + static void reprime_elt_reverse(iterator &beg, iterator& end, address b) { + end.reset_map((T*)b, end._elt_map); + // load the first map chunk: + bool z = map_nonempty_backward(beg, end); + assert(z, "progress"); + } + + public: + // inner loop operators are inherited unchanged from base class: + // ++, --, addr + + bool operator<(iterator& end) { return tail_nonempty_forward(*this, end); } + bool operator>=(iterator& beg) { return tail_nonempty_reverse(beg, *this); } + + protected: + static void assert_pair_invariants(iterator& beg, iterator& end, int PD = 0) { + //oop_map::iterator::assert_pair_invariants(beg, end, PD); + assert(beg._stride == end._stride, "matching endpoints"); + assert(beg._elt_map <= end._elt_map, "correct order"); + assert(beg._elt_base <= end._elt_base, "correct order"); + } + + static bool tail_nonempty_forward(iterator& beg, iterator& end) { + assert_pair_invariants(beg, end); + address b; + // invariant: beg.base not not yet begun to be used + if (map_nonempty_forward(beg, end)) { + // inner loops not yet exhausted + return true; + } else if ((b = beg._elt_base) < end._elt_base) { + // refresh inner loop indexes from outer loop + reprime_elt_forward(beg, end, b); + assert(is_primed(beg, end), "outer loop produced at least one element"); + beg._elt_base = b + beg._stride; + assert(beg._elt_base <= end._elt_base, "no partial elements"); + return true; + } else { + return false; + } + } + + static bool tail_nonempty_reverse(iterator& beg, iterator& end) { + assert_pair_invariants(beg, end, PD); + address b; + // invariant: end.base-stride not not yet begun to be used + if (p >= beg.p) { + // inner loops not yet exhausted + return true; + } else if ((b = end._elt_base) > beg.p._elt_base) { + b -= end._stride; // predecrement + end._elt_base = b; + assert(end._elt_base >= beg._elt_base, "no partial elements"); + // refresh inner loop indexes from outer loop + reprime_elt_reverse(beg, end, p); + assert(is_primed(beg, end), "outer loop produced at least one element"); + return true; + } else { + return false; + } + } + + }; + + // for (iterator pmax, p = forward(base, stride, limit, maps, count, pmax); p < pmax; p++) + static iterator forward(T* base, int stride, T* limit, + OopMapBlock* maps, size_t map_count, + iterator& end) { + end = iterator(limit, stride, maps); + return iterator(base, stride, maps + map_count); + } + + static iterator single_chunk(T* base, T* limit, iterator& end) { + OopMapBlock* no_map = NULL; + iterator beg; + beg = end = iterator(base, 1, no_map); + iterator::reprime_chunk(beg, end, base, limit); + assert(iterator::is_primed(beg, end) && addr(beg) == base && addr(end) == limit, ""); + return beg; + } + + typedef iterator_bound_mixin bounded_iterator; + }; + + template + typename mixed_tail::iterator tail_oop_fields(oop obj, + typename mixed_tail::iterator& pmax) { + return tail_oop_fields(obj, 0, arrayOopDesc::array_length(obj), pmax); + } + + template + typename mixed_tail::iterator tail_oop_fields(oop obj, jint from, jint to, + typename mixed_tail::iterator& pmax) { + assert(obj != NULL, "can't follow the content of NULL object"); + LayoutHelper lh = layout_helper(); + if (!lh.has_oops_in_tail()) { + // return an empty iterator pair + T* no_base = obj->obj_field_addr(0); + return single_chunk(no_base, no_base, pmax); + } + T* base = obj->obj_field_addr(lh.element_offset(from)); + T* limit = obj->obj_field_addr(lh.element_offset(to)); + if (!lh.has_non_oops_in_tail()) { + // collapse all elements into a single run + return single_chunk(base, limit, pmax); + } + // Else do it the hard way, with a triply nested loop. + return mixed_tail::forward(base, lh.element_size(), limit, + start_of_element_oop_maps(), + element_oop_map_size(), + pmax); + } + + template + typename mixed_tail::iterator + reverse_tail_oop_fields(oop obj, typename mixed_tail::iterator& pmin) { + return Klass::iterator_reverse(tail_oop_fields(obj, pmin), pmin); + } + + template + typename mixed_tail::bounded_iterator + bounded_tail_oop_fields(oop obj, MemRegion mr, + typename mixed_tail::bounded_iterator& pmax) { + jint from = 0, to = arrayOopDesc::array_length(obj); + restrict_indexes_to_region(mr, obj, from, to); + typename mixed_tail::iterator uend, ubeg + = tail_oop_fields(obj, from, to, uend); + return mixed_tail::bounded_iterator::make(mr, ubeg, uend, pmax); + } + + template + void restrict_indexes_to_region(MemRegion mr, oop obj, jint& from, jint& to) { + LayoutHelper lh = layout_helper(); + address mlo = (address) mr.start(); + address mhi = (address) mr.end(); + address olo = (address) obj->obj_field_addr(lh.header_size_in_bytes()); + address ohi = (address) obj->obj_field_addr(lh.new_array_size_in_bytes(to)); + if (mlo > olo) { + if (mlo >= ohi) { + ohi = olo; + to = 0; + } else { + from = lh.index_from_element_offset(mlo); + } + } + if (mhi < ohi) { + if (mhi <= olo) { + ohi = olo; + to = 0; + } else { + size_t rem; + int mid = lh.index_from_element_offset(mhi, &rem); + if (rem != 0 && mid < to) mid += 1; + to = mid; + } + } + } + + +#ifndef PRODUCT + public: + // Printing + void oop_print_on (oop obj, outputStream* st); + void oop_print_value_on(oop obj, outputStream* st); +#endif +}; --- /dev/null 2008-07-12 17:50:16.000000000 -0700 +++ new/src/share/vm/oops/objArrayOop.inline.hpp 2008-07-12 17:50:15.000000000 -0700 @@ -0,0 +1,29 @@ +/* + * Copyright 2008 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + */ + +template +inline void objArrayOopDesc::oop_iterate_range(OopClosureType* blk, int start, int end) { + SpecializationStats::record_call(); + ((objArrayKlass*)blueprint())->oop_oop_iterate_range(this, blk, start, end); +}