--- old/agent/src/share/classes/sun/jvm/hotspot/oops/Array.java 2008-07-12 17:46:55.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/Array.java 2008-07-12 17:46:55.000000000 -0700 @@ -99,7 +99,7 @@ // We have to fetch the length of the array, shift (multiply) it // appropriately, up to wordSize, add the header, and align to // object size. - long s = getLength() << klass.getLog2ElementSize(); + long s = getLength() * klass.getElementSize(); s += klass.getArrayHeaderInBytes(); s = Oop.alignObjectSize(s); return s; --- old/agent/src/share/classes/sun/jvm/hotspot/oops/ArrayKlass.java 2008-07-12 17:46:58.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/ArrayKlass.java 2008-07-12 17:46:57.000000000 -0700 @@ -114,15 +114,7 @@ } public long getArrayHeaderInBytes() { - return Bits.maskBits(getLayoutHelper() >> LH_HEADER_SIZE_SHIFT, 0xFF); - } - - public int getLog2ElementSize() { - return Bits.maskBits(getLayoutHelper() >> LH_LOG2_ELEMENT_SIZE_SHIFT, 0xFF); - } - - public int getElementType() { - return Bits.maskBits(getLayoutHelper() >> LH_ELEMENT_TYPE_SHIFT, 0xFF); + return getHeaderSizeInBytes(); } boolean computeSubtypeOf(Klass k) { --- old/agent/src/share/classes/sun/jvm/hotspot/oops/InstanceKlass.java 2008-07-12 17:47:00.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/InstanceKlass.java 2008-07-12 17:47:00.000000000 -0700 @@ -88,7 +88,8 @@ nonstaticFieldSize = new CIntField(type.getCIntegerField("_nonstatic_field_size"), Oop.getHeaderSize()); staticFieldSize = new CIntField(type.getCIntegerField("_static_field_size"), Oop.getHeaderSize()); staticOopFieldSize = new CIntField(type.getCIntegerField("_static_oop_field_size"), Oop.getHeaderSize()); - nonstaticOopMapSize = new CIntField(type.getCIntegerField("_nonstatic_oop_map_size"), Oop.getHeaderSize()); + mapOffsetInBytes = new CIntField(type.getCIntegerField("_map_offset_in_bytes"), Oop.getHeaderSize()); + objectSize = new CIntField(type.getCIntegerField("_object_size"), Oop.getHeaderSize()); isMarkedDependent = new CIntField(type.getCIntegerField("_is_marked_dependent"), Oop.getHeaderSize()); initState = new CIntField(type.getCIntegerField("_init_state"), Oop.getHeaderSize()); vtableLen = new CIntField(type.getCIntegerField("_vtable_len"), Oop.getHeaderSize()); @@ -141,7 +142,8 @@ private static CIntField nonstaticFieldSize; private static CIntField staticFieldSize; private static CIntField staticOopFieldSize; - private static CIntField nonstaticOopMapSize; + private static CIntField mapOffsetInBytes; + private static CIntField objectSize; private static CIntField isMarkedDependent; private static CIntField initState; private static CIntField vtableLen; @@ -263,7 +265,8 @@ public long getNonstaticFieldSize() { return nonstaticFieldSize.getValue(this); } public long getStaticFieldSize() { return staticFieldSize.getValue(this); } public long getStaticOopFieldSize() { return staticOopFieldSize.getValue(this); } - public long getNonstaticOopMapSize() { return nonstaticOopMapSize.getValue(this); } + public long getMapOffsetInBytes() { return mapOffsetInBytes.getValue(this); } + public long objectSize() { return objectSize.getValue(this); } public boolean getIsMarkedDependent() { return isMarkedDependent.getValue(this) != 0; } public long getVtableLen() { return vtableLen.getValue(this); } public long getItableLen() { return itableLen.getValue(this); } @@ -275,8 +278,9 @@ public long getSizeHelper() { int lh = getLayoutHelper(); if (Assert.ASSERTS_ENABLED) { - Assert.that(lh > 0, "layout helper initialized for instance class"); + Assert.that(isFixedInstance(), "layout helper initialized for instance class"); } + lh = Bits.maskBits(lh, ~Bits.rightNBits(LH_SIZE_LOW_BITS)); return lh / VM.getVM().getAddressSize(); } @@ -455,7 +459,8 @@ visitor.doCInt(nonstaticFieldSize, true); visitor.doCInt(staticFieldSize, true); visitor.doCInt(staticOopFieldSize, true); - visitor.doCInt(nonstaticOopMapSize, true); + visitor.doCInt(mapOffsetInBytes, true); + visitor.doCInt(objectSize, true); visitor.doCInt(isMarkedDependent, true); visitor.doCInt(initState, true); visitor.doCInt(vtableLen, true); @@ -690,13 +695,6 @@ } - public long getObjectSize() { - long bodySize = alignObjectOffset(getVtableLen() * getHeap().getOopSize()) - + alignObjectOffset(getItableLen() * getHeap().getOopSize()) - + (getStaticFieldSize() + getNonstaticOopMapSize()) * getHeap().getOopSize(); - return alignObjectSize(headerSize + bodySize); - } - public Klass arrayKlassImpl(boolean orNull, int n) { // FIXME: in reflective system this would need to change to // actually allocate --- old/agent/src/share/classes/sun/jvm/hotspot/oops/Klass.java 2008-07-12 17:47:03.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/oops/Klass.java 2008-07-12 17:47:02.000000000 -0700 @@ -29,6 +29,7 @@ import sun.jvm.hotspot.debugger.*; import sun.jvm.hotspot.runtime.*; import sun.jvm.hotspot.types.*; +import sun.jvm.hotspot.utilities.*; public class Klass extends Oop implements ClassConstants { static { @@ -40,13 +41,24 @@ } // anon-enum constants for _layout_helper. - public static int LH_INSTANCE_SLOW_PATH_BIT; - public static int LH_LOG2_ELEMENT_SIZE_SHIFT; - public static int LH_ELEMENT_TYPE_SHIFT; + public static int LH_FLAGS_BITS; + public static int LH_FLAGS_SHIFT; + public static int LH_FLAGS_LOW_BITS; + public static int LH_VARIABLE_FLAG; + public static int LH_ARRAY_FLAG; + public static int LH_FLAGS_EBT_MASK; + public static int LH_ELEMENT_SIZE_BITS; + public static int LH_ELEMENT_SIZE_SHIFT; + public static int LH_ELEMENT_SCALE_BITS; + public static int LH_ELEMENT_SIZEM_BITS; + public static int LH_ELEMENT_SIZEM_MASK_IP; + public static int LH_SIZE_LOW_BITS; + public static int LH_SLOW_PATH_LOW_BIT; + public static int LH_HEADER_SIZE_BITS; public static int LH_HEADER_SIZE_SHIFT; - public static int LH_ARRAY_TAG_SHIFT; - public static int LH_ARRAY_TAG_TYPE_VALUE; - public static int LH_ARRAY_TAG_OBJ_VALUE; + public static int LH_FIXED_SIZE_BITS; + public static int LH_FIXED_SIZE_SHIFT; + private static synchronized void initialize(TypeDataBase db) throws WrongTypeException { Type type = db.lookupType("Klass"); @@ -59,13 +71,23 @@ nextSibling = new OopField(type.getOopField("_next_sibling"), Oop.getHeaderSize()); allocCount = new CIntField(type.getCIntegerField("_alloc_count"), Oop.getHeaderSize()); - LH_INSTANCE_SLOW_PATH_BIT = db.lookupIntConstant("Klass::_lh_instance_slow_path_bit").intValue(); - LH_LOG2_ELEMENT_SIZE_SHIFT = db.lookupIntConstant("Klass::_lh_log2_element_size_shift").intValue(); - LH_ELEMENT_TYPE_SHIFT = db.lookupIntConstant("Klass::_lh_element_type_shift").intValue(); - LH_HEADER_SIZE_SHIFT = db.lookupIntConstant("Klass::_lh_header_size_shift").intValue(); - LH_ARRAY_TAG_SHIFT = db.lookupIntConstant("Klass::_lh_array_tag_shift").intValue(); - LH_ARRAY_TAG_TYPE_VALUE = db.lookupIntConstant("Klass::_lh_array_tag_type_value").intValue(); - LH_ARRAY_TAG_OBJ_VALUE = db.lookupIntConstant("Klass::_lh_array_tag_obj_value").intValue(); + LH_FLAGS_BITS = db.lookupIntConstant("LayoutHelper::_flags_bits").intValue(); + LH_FLAGS_SHIFT = db.lookupIntConstant("LayoutHelper::_flags_shift").intValue(); + LH_FLAGS_LOW_BITS = db.lookupIntConstant("LayoutHelper::_flags_low_bits").intValue(); + LH_VARIABLE_FLAG = db.lookupIntConstant("LayoutHelper::_variable_flag").intValue(); + LH_ARRAY_FLAG = db.lookupIntConstant("LayoutHelper::_array_flag").intValue(); + LH_FLAGS_EBT_MASK = db.lookupIntConstant("LayoutHelper::_flags_ebt_mask").intValue(); + LH_ELEMENT_SIZE_BITS = db.lookupIntConstant("LayoutHelper::_element_size_bits").intValue(); + LH_ELEMENT_SIZE_SHIFT = db.lookupIntConstant("LayoutHelper::_element_size_shift").intValue(); + LH_ELEMENT_SCALE_BITS = db.lookupIntConstant("LayoutHelper::_element_scale_bits").intValue(); + LH_ELEMENT_SIZEM_BITS = db.lookupIntConstant("LayoutHelper::_element_sizem_bits").intValue(); + LH_ELEMENT_SIZEM_MASK_IP = db.lookupIntConstant("LayoutHelper::_element_sizem_mask_ip").intValue(); + LH_SIZE_LOW_BITS = db.lookupIntConstant("LayoutHelper::_fixed_size_low_bits").intValue(); + LH_SLOW_PATH_LOW_BIT = db.lookupIntConstant("LayoutHelper::_slow_path_low_bit").intValue(); + LH_HEADER_SIZE_BITS = db.lookupIntConstant("LayoutHelper::_header_size_bits").intValue(); + LH_HEADER_SIZE_SHIFT = db.lookupIntConstant("LayoutHelper::_header_size_shift").intValue(); + LH_FIXED_SIZE_BITS = db.lookupIntConstant("LayoutHelper::_fixed_size_bits").intValue(); + LH_FIXED_SIZE_SHIFT = db.lookupIntConstant("LayoutHelper::_fixed_size_shift").intValue(); } Klass(OopHandle handle, ObjectHeap heap) { @@ -102,6 +124,38 @@ public Klass getNextSiblingKlass() { return (Klass) nextSibling.getValue(this); } public long getAllocCount() { return allocCount.getValue(this); } + public boolean isVariableObject() { return getLayoutHelper() < 0; } + public boolean isFixedInstance() { return getLayoutHelper() > 0; } + + public long getHeaderSizeInBytes() { + if (Assert.ASSERTS_ENABLED) { + Assert.that(isVariableObject(), "layout helper initialized for variable class"); + } + return Bits.maskBits(getLayoutHelper() >> LH_HEADER_SIZE_SHIFT, + (Bits.rightNBits(LH_HEADER_SIZE_BITS) + - Bits.rightNBits(LH_SIZE_LOW_BITS))); + } + + public int getElementSize() { + if (Assert.ASSERTS_ENABLED) { + Assert.that(isVariableObject(), "layout helper initialized for variable class"); + } + int lh = getLayoutHelper(); + int scale = Bits.maskBits(lh >> LH_ELEMENT_SIZE_SHIFT, + Bits.rightNBits(LH_ELEMENT_SCALE_BITS)); + int sizem = Bits.maskBits(lh >> (LH_ELEMENT_SIZE_SHIFT + LH_ELEMENT_SCALE_BITS), + Bits.rightNBits(LH_ELEMENT_SIZEM_BITS)); + return (sizem + 1) << scale; + } + + public int getElementType() { + if (Assert.ASSERTS_ENABLED) { + Assert.that(isVariableObject(), "layout helper initialized for variable class"); + } + return Bits.maskBits(getLayoutHelper() >> LH_FLAGS_SHIFT, + Bits.rightNBits(LH_FLAGS_EBT_MASK)); + } + // computed access flags - takes care of inner classes etc. // This is closer to actual source level than getAccessFlags() etc. public long computeModifierFlags() { --- old/agent/src/share/classes/sun/jvm/hotspot/utilities/HeapGXLWriter.java 2008-07-12 17:47:05.000000000 -0700 +++ new/agent/src/share/classes/sun/jvm/hotspot/utilities/HeapGXLWriter.java 2008-07-12 17:47:04.000000000 -0700 @@ -176,6 +176,7 @@ if (isInstanceKlass) { // write object-size as an attribute long sizeInBytes = reflectedType.getLayoutHelper(); + sizeInBytes &= ~Bits.rightNBits(Klass.LH_SIZE_LOW_BITS); writeAttribute("object-size", "int", Long.toString(sizeInBytes)); // write static fields of this class. --- old/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp 2008-07-12 17:47:07.000000000 -0700 +++ new/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp 2008-07-12 17:47:07.000000000 -0700 @@ -343,6 +343,94 @@ return oop_maps; } +inline Address layout_helper_addr(Register klass) { + return Address(klass, 0, (klassOopDesc::header_size() * HeapWordSize + + Klass::layout_helper_offset_in_bytes())); +} + +static void compute_instance_size(Register obj_size, Register klass, + Label& have_obj_size, Label& size_is_variable, + Register t1, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(obj_size, klass, t1); + + if (hot_part) { + __ lduw(layout_helper_addr(klass), obj_size); + __ andcc(obj_size, ~LayoutHelper::_size_low_mask, obj_size); + if (mixed_arrays) { + __ jcc(Assembler::negative, size_is_variable); + __ bind(have_obj_size); + } + } else { + if (MixedArrays) { + // Side path for fixing up the initial size of a variable object. + // Must round from int to object alignment, and clear high bits. + __ bind(size_is_variable); + __ add(obj_size, LayoutHelper::_header_size_odd_mask, obj_size); + __ set((LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask), t1); + __ and3(obj_size, t1, obj_size); + __ jmp(have_obj_size); + } + } +} + +static void compute_array_size(Register arr_size, Register klass, Register length, + Label& have_scaled_length, Label& need_multiply, + Register t1, Register t2, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(arr_size, klass, length, t1, t2); + + if (hot_part) { + // get the allocation size: round_up(hdr + length << (lh>>16 & 0x1F)) + __ lduw(layout_helper_addr(klass), t1); + // int scale = (lh >> LayoutHelper::_element_size_shift); + __ mov(t1, t2); // spill layout helper + __ srl(t1, LayoutHelper::_element_size_shift); // lh>>16, no mask needed + // size_t arr_size = (array_length << scale); + __ sllx(length, t1, arr_size); + if (MixedArrays) { + // int sizem_ip = (scale & (LayoutHelper::_element_sizem_mask_ip)); + __ andcc(t1, LayoutHelper::_element_sizem_mask_ip, t1); + __ jcc(Assembler::notZero, need_multiply); + __ bind(have_scaled_length); + } + // arr_size += Klass::layout_helper_header_size_in_bytes(lh); + assert(LayoutHelper::_header_size_shift == 0, ""); + // sll(t2, LayoutHelper::_header_size_shift, t2); + __ set((LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask), t1); + __ and(t2, t1, t2); // t2 = lh & _header_size_mask + __ add(arr_size, t2, arr_size); + __ add(arr_size, MinObjAlignmentInBytesMask, arr_size); // align up + __ and3(arr_size, ~MinObjAlignmentInBytesMask, arr_size); + } else { + if (MixedArrays) { + // Side path for scaling by a non-power-of-two array element size. + __ bind(need_multiply); + // int sizem = (sizem_ip >> LayoutHelper::_element_scale_bits); + __ srl(t1, LayoutHelper::_element_scale_bits, t1); + // arr_size = (array_length * (1+sizem)) << scale; + __ add(t1, 1, t1); + // previous value of arr_size is junk; start from length again + assert(wordSize == jintSize, "else use mulx"); + __ smul(length, t1, arr_size); + __ srl(t2, LayoutHelper::_element_size_shift, t1); // lh>>16, mask needed + __ and3(t1, LayoutHelper::_element_scale_mask, t1); + __ sllx(arr_size, t1, arr_size); + __ jmp(have_scaled_length); + } + } +} + +static void initialize_array_body(Register obj, Register arr_size, Register t1, Register t2, StubAssembler* sasm) { + assert_different_registers(obj, arr_size, t1, t2); + + int min_header_size = arrayOopDesc::header_size(T_BYTE); + __ sub(arr_size, min_header_size, arr_size); // body length + __ add(obj, min_header_size, t1); // body start + __ initialize_body(t1, arr_size, 0, t2); +} + + OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { OopMapSet* oop_maps = NULL; @@ -413,15 +501,11 @@ #ifdef ASSERT // assert object can be fast path allocated { - Label ok, not_ok; + Label ok; __ ld(G5_klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc), G1_obj_size); - __ cmp(G1_obj_size, 0); // make sure it's an instance (LH > 0) - __ br(Assembler::lessEqual, false, Assembler::pn, not_ok); - __ delayed()->nop(); - __ btst(Klass::_lh_instance_slow_path_bit, G1_obj_size); + __ btst(LayoutHelper::_slow_path_low_bit, G1_obj_size); __ br(Assembler::zero, false, Assembler::pn, ok); __ delayed()->nop(); - __ bind(not_ok); __ stop("assert(can be fast path allocated)"); __ should_not_reach_here(); __ bind(ok); @@ -435,7 +519,10 @@ __ bind(retry_tlab); // get the instance size - __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); + Label have_obj_size, size_is_variable; + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size, size_is_variable, + G3_t1, sasm, true); __ tlab_allocate(O0_obj, G1_obj_size, 0, G3_t1, slow_path); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); @@ -443,9 +530,17 @@ __ ret(); __ delayed()->restore(); + // generate side path, if any + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size, size_is_variable, + G3_t1, sasm, false); + __ bind(try_eden); // get the instance size - __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); + Label have_obj_size_2, size_is_variable_2; + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size_2, size_is_variable_2, + G3_t1, sasm, true); __ eden_allocate(O0_obj, G1_obj_size, 0, G3_t1, G4_t2, slow_path); __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); @@ -453,6 +548,11 @@ __ ret(); __ delayed()->restore(); + // generate side path, if any + compute_instance_size(G1_obj_size, G5_klass, + have_obj_size_2, size_is_variable_2, + G3_t1, sasm, false); + __ bind(slow_path); // pop this frame so generate_stub_call can push it's own @@ -479,14 +579,6 @@ Register G4_length = G4; // Incoming Register O0_obj = O0; // Outgoing - Address klass_lh(G5_klass, 0, ((klassOopDesc::header_size() * HeapWordSize) - + Klass::layout_helper_offset_in_bytes())); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask == 0xFF, "bytewise"); - // Use this offset to pick out an individual byte of the layout_helper: - const int klass_lh_header_size_offset = ((BytesPerInt - 1) // 3 - 2 selects byte {0,1,0,0} - - Klass::_lh_header_size_shift / BitsPerByte); - if (id == new_type_array_id) { __ set_info("new_type_array", dont_gc_arguments); } else { @@ -498,11 +590,15 @@ { Label ok; Register G3_t1 = G3; - __ ld(klass_lh, G3_t1); - __ sra(G3_t1, Klass::_lh_array_tag_shift, G3_t1); - int tag = ((id == new_type_array_id) - ? Klass::_lh_array_tag_type_value - : Klass::_lh_array_tag_obj_value); + __ lduw(layout_helper_addr(G5_klass), G3_t1); + int flag_shift = LayoutHelper::_flags_shift; + flag_shift += LayoutHelper::_flags_low_bits; // shift out BasicType also + int tag = LayoutHelper::for_array((id == new_type_array_id) ? T_BYTE : T_OBJECT).as_int(); + tag >>= flag_shift; + __ sra(G3_t1, flag_shift, G3_t1); + assert(id == new_object_array_id || + tag == (LayoutHelper::for_array(T_DOUBLE).as_int() >> flag_shift), + "same tag for all type arrays"); __ cmp(G3_t1, tag); __ brx(Assembler::equal, false, Assembler::pt, ok); __ delayed()->nop(); @@ -533,46 +629,42 @@ __ bind(retry_tlab); // get the allocation size: (length << (layout_helper & 0x1F)) + header_size - __ ld(klass_lh, G3_t1); - __ sll(G4_length, G3_t1, G1_arr_size); - __ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1); - __ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1); - __ add(G1_arr_size, G3_t1, G1_arr_size); - __ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size); // align up - __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); - + Label have_scaled_length, need_multiply; + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length, need_multiply, + G3_t1, O1_t2, sasm, true); __ tlab_allocate(O0_obj, G1_arr_size, 0, G3_t1, slow_path); // preserves G1_arr_size __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); - __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); - __ sub(G1_arr_size, G3_t1, O1_t2); // body length - __ add(O0_obj, G3_t1, G3_t1); // body start - __ initialize_body(G3_t1, O1_t2); + initialize_array_body(O0_obj, G1_arr_size, G3_t1, O1_t2, sasm); __ verify_oop(O0_obj); __ retl(); __ delayed()->nop(); + // generate side path, if any + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length, need_multiply, + G3_t1, O1_t2, sasm, false); + __ bind(try_eden); + Label have_scaled_length_2, need_multiply_2; // get the allocation size: (length << (layout_helper & 0x1F)) + header_size - __ ld(klass_lh, G3_t1); - __ sll(G4_length, G3_t1, G1_arr_size); - __ srl(G3_t1, Klass::_lh_header_size_shift, G3_t1); - __ and3(G3_t1, Klass::_lh_header_size_mask, G3_t1); - __ add(G1_arr_size, G3_t1, G1_arr_size); - __ add(G1_arr_size, MinObjAlignmentInBytesMask, G1_arr_size); - __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); - + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length_2, need_multiply_2, + G3_t1, O1_t2, sasm, true); __ eden_allocate(O0_obj, G1_arr_size, 0, G3_t1, O1_t2, slow_path); // preserves G1_arr_size __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); - __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); - __ sub(G1_arr_size, G3_t1, O1_t2); // body length - __ add(O0_obj, G3_t1, G3_t1); // body start - __ initialize_body(G3_t1, O1_t2); + initialize_array_body(O0_obj, G1_arr_size, G3_t1, O1_t2, sasm); __ verify_oop(O0_obj); __ retl(); __ delayed()->nop(); + // generate side path, if any + compute_array_size(G1_arr_size, G5_klass, G4_length, + have_scaled_length_2, need_multiply_2, + G3_t1, O1_t2, sasm, false); + __ bind(slow_path); } --- old/src/cpu/sparc/vm/stubGenerator_sparc.cpp 2008-07-12 17:47:10.000000000 -0700 +++ new/src/cpu/sparc/vm/stubGenerator_sparc.cpp 2008-07-12 17:47:09.000000000 -0700 @@ -2700,10 +2700,10 @@ // Load layout helper // - // |array_tag| | header_size | element_type | |log2_element_size| - // 32 30 24 16 8 2 0 + // | array_tag | element_type | log2_element_size | header_size | + // 32 28 24 16 0 // - // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 + // array_tag: typeArray = 0xF, objArray = 0xD, normal instance = 0x0 // int lh_offset = klassOopDesc::header_size() * HeapWordSize + @@ -2716,7 +2716,10 @@ __ load_klass(dst, G4_dst_klass); } // Handle objArrays completely differently... - juint objArray_lh = Klass::array_layout_helper(T_OBJECT); + // This is necessary because they have covariant element types. + // The next type check (for src.klass == dst.klass), can get a + // false negative in the case of object arrays. + jint objArray_lh = LayoutHelper::for_array(T_OBJECT).as_int(); __ set(objArray_lh, O5_temp); __ cmp(G5_lh, O5_temp); __ br(Assembler::equal, false, Assembler::pt, L_objArray); @@ -2731,25 +2734,11 @@ __ brx(Assembler::notEqual, false, Assembler::pn, L_failed); __ delayed()->nop(); - // if (!src->is_Array()) return -1; - __ cmp(G5_lh, Klass::_lh_neutral_value); // < 0 + // if (!src->has_length_field()) return -1; + __ cmp(G5_lh, LayoutHelper::_neutral_value); // < 0 __ br(Assembler::greaterEqual, false, Assembler::pn, L_failed); - // At this point, it is known to be a typeArray (array_tag 0x3). -#ifdef ASSERT - __ delayed()->nop(); - { Label L; - jint lh_prim_tag_in_place = (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift); - __ set(lh_prim_tag_in_place, O5_temp); - __ cmp(G5_lh, O5_temp); - __ br(Assembler::greaterEqual, false, Assembler::pt, L); - __ delayed()->nop(); - __ stop("must be a primitive array"); - __ bind(L); - } -#else __ delayed(); // match next insn to prev branch -#endif arraycopy_range_checks(src, src_pos, dst, dst_pos, length, O5_temp, G4_dst_klass, L_failed); @@ -2763,11 +2752,13 @@ const Register G4_offset = G4_dst_klass; // array offset const Register G3_elsize = G3_src_klass; // log2 element size - __ srl(G5_lh, Klass::_lh_header_size_shift, G4_offset); - __ and3(G4_offset, Klass::_lh_header_size_mask, G4_offset); // array_offset + assert(LayoutHelper::_header_size_shift == 0, ""); + __ set(LayoutHelper::_header_size_mask, G4_offset); + __ and3(G5_lh, G4_offset, G4_offset); // array_offset __ add(src, G4_offset, src); // src array offset __ add(dst, G4_offset, dst); // dst array offset - __ and3(G5_lh, Klass::_lh_log2_element_size_mask, G3_elsize); // log2 element size + __ srl( G5_lh, LayoutHelper::_element_size_shift, G3_elsize); + __ and3(G3_elsize, LayoutHelper::_element_size_mask, G3_elsize); // log2 element size // next registers should be set before the jump to corresponding stub const Register from = O0; // source array address @@ -2807,14 +2798,15 @@ __ br(Assembler::always,false,Assembler::pt,StubRoutines::_jlong_arraycopy); __ delayed()->signx(length, count); // length + Label L_different_objArray, L_plain_copy, L_checkcast_copy; + // objArrayKlass __ BIND(L_objArray); // live at this point: G3_src_klass, G4_dst_klass, src[_pos], dst[_pos], length - Label L_plain_copy, L_checkcast_copy; // test array classes for subtyping __ cmp(G3_src_klass, G4_dst_klass); // usual case is exact equality - __ brx(Assembler::notEqual, true, Assembler::pn, L_checkcast_copy); + __ brx(Assembler::notEqual, true, Assembler::pn, L_different_objArray); __ delayed()->lduw(G4_dst_klass, lh_offset, O5_temp); // hoisted from below // Identically typed arrays can be copied without element-wise checks. @@ -2831,7 +2823,7 @@ __ br(Assembler::always, false, Assembler::pt,StubRoutines::_oop_arraycopy); __ delayed()->signx(length, count); // length - __ BIND(L_checkcast_copy); + __ BIND(L_different_objArray); // live at this point: G3_src_klass, G4_dst_klass { // Before looking at dst.length, make sure dst is also an objArray. --- old/src/cpu/sparc/vm/templateTable_sparc.cpp 2008-07-12 17:47:12.000000000 -0700 +++ new/src/cpu/sparc/vm/templateTable_sparc.cpp 2008-07-12 17:47:12.000000000 -0700 @@ -25,6 +25,8 @@ #include "incls/_precompiled.incl" #include "incls/_templateTable_sparc.cpp.incl" +#error "NYI: mixa.patch not yet implemented for this platform" + #ifndef CC_INTERP #define __ _masm-> @@ -3092,9 +3094,9 @@ //__ ld(RinstanceKlass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc), Roffset); // make sure klass does not have has_finalizer, or is abstract, or interface or java/lang/Class - __ btst(Klass::_lh_instance_slow_path_bit, Roffset); + __ btst(LayoutHelper::_slow_path_low_bit, Roffset); __ br(Assembler::notZero, false, Assembler::pn, slow_case); - __ delayed()->nop(); + __ delayed()->and3(Roffset, ~LayoutHelper::_size_low_mask, Roffset); // allocate the instance // 1) Try to allocate in the TLAB --- old/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2008-07-12 17:47:15.000000000 -0700 +++ new/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2008-07-12 17:47:14.000000000 -0700 @@ -786,6 +786,97 @@ } +inline Address layout_helper_addr(Register klass) { + return Address(klass, (klassOopDesc::header_size() * HeapWordSize + + Klass::layout_helper_offset_in_bytes())); +} + +static void compute_instance_size(Register obj_size, Register klass, + Label& have_obj_size, Label& size_is_variable, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(obj_size, klass); + + if (hot_part) { + __ movl(obj_size, layout_helper_addr(klass)); + __ andl(obj_size, ~LayoutHelper::_size_low_mask); + if (MixedArrays) { + __ jcc(Assembler::negative, size_is_variable); + __ bind(have_obj_size); + } + } else { + if (MixedArrays) { + // Side path for fixing up the initial size of a variable object. + // Must round from int to object alignment, and clear high bits. + __ bind(size_is_variable); + __ addl(obj_size, LayoutHelper::_header_size_odd_mask); + __ andl(obj_size, (LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask)); + __ jmp(have_obj_size); + } + } +} + + +static void compute_array_size(Register arr_size, Register klass, Register length, + Label& have_scaled_length, Label& need_multiply, + Register t1, Register t2, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(arr_size, klass, length, t1, t2); + + if (hot_part) { + // get the allocation size: round_up(hdr + length << (lh>>16 & 0x1F)) + __ movl(t1, layout_helper_addr(klass)); + __ movl(arr_size, length); + // int scale = (lh >> LayoutHelper::_element_size_shift); + __ movl(t1, t2); // spill layout helper + __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, no mask needed + // size_t arr_size = (array_length << scale); + assert(t1 == rcx, "fixed register usage"); + __ shll(arr_size /* by t1=rcx, mod 32 */); + if (MixedArrays) { + // int sizem_ip = (scale & (LayoutHelper::_element_sizem_mask_ip)); + __ andl(t1, LayoutHelper::_element_sizem_mask_ip); + __ jcc(Assembler::notZero, need_multiply); + __ bind(have_scaled_length); + } + // arr_size += Klass::layout_helper_header_size_in_bytes(lh); + assert(LayoutHelper::_header_size_shift == 0, ""); + // shrl(t2, LayoutHelper::_header_size_shift); + __ andl(t2, LayoutHelper::_header_size_mask); + __ addl(arr_size, t2); + __ addl(arr_size, MinObjAlignmentInBytesMask); // align up + __ andl(arr_size, ~MinObjAlignmentInBytesMask); + } else { + if (MixedArrays) { + // Side path for scaling by a non-power-of-two array element size. + __ bind(need_multiply); + __ movl(arr_size, length); // reload; previous value was trash + // int sizem = (sizem_ip >> LayoutHelper::_element_scale_bits); + __ shrl(t1, LayoutHelper::_element_scale_bits); + // arr_size = (array_length * (1+sizem)) << scale; + __ addl(t1, 1); + assert(wordSize == jintSize, "else use imulq, shlq, etc."); + __ imull(arr_size, t1); + __ movl(t1, t2); // reload layout helper + __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, mask needed + __ andl(t1, LayoutHelper::_element_scale_mask); + assert(t1 == rcx, "fixed register usage"); + __ shll(arr_size /* by t1=rcx, mod 32 */); + __ jmp(have_scaled_length); + } + } +} + +static void initialize_array_body(Register obj, Register arr_size, Register t1, Register t2, StubAssembler* sasm) { + assert_different_registers(obj, arr_size, t1, t2); + + int min_header_size = arrayOopDesc::header_size(T_BYTE); + __ subl(arr_size, min_header_size); // body length + __ movl(t1, obj); + __ addl(t1, min_header_size); // body start + __ initialize_body(t1, arr_size, 0, t2); +} + + OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { // for better readability @@ -869,13 +960,10 @@ #ifdef ASSERT // assert object can be fast path allocated { - Label ok, not_ok; - __ movl(obj_size, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); - __ cmpl(obj_size, 0); // make sure it's an instance (LH > 0) - __ jcc(Assembler::lessEqual, not_ok); - __ testl(obj_size, Klass::_lh_instance_slow_path_bit); + Label ok; + __ movl(obj_size, layout_helper_addr(klass)); + __ testl(obj_size, LayoutHelper::_slow_path_low_bit); __ jcc(Assembler::zero, ok); - __ bind(not_ok); __ stop("assert(can be fast path allocated)"); __ should_not_reach_here(); __ bind(ok); @@ -890,7 +978,10 @@ __ bind(retry_tlab); // get the instance size - __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + Label have_obj_size, size_is_variable; + compute_instance_size(obj_size, klass, + have_obj_size, size_is_variable, + sasm, true); __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path); __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); @@ -898,9 +989,17 @@ __ popl(rdi); __ ret(0); + // generate side path, if any + compute_instance_size(obj_size, klass, + have_obj_size, size_is_variable, + sasm, false); + __ bind(try_eden); // get the instance size - __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + Label have_obj_size_2, size_is_variable_2; + compute_instance_size(obj_size, klass, + have_obj_size_2, size_is_variable_2, + sasm, true); __ eden_allocate(obj, obj_size, 0, t1, slow_path); __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); @@ -908,6 +1007,11 @@ __ popl(rdi); __ ret(0); + // generate side path, if any + compute_instance_size(obj_size, klass, + have_obj_size_2, size_is_variable_2, + sasm, false); + __ bind(slow_path); __ popl(rbx); __ popl(rdi); @@ -964,11 +1068,15 @@ { Label ok; Register t0 = obj; - __ movl(t0, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); - __ sarl(t0, Klass::_lh_array_tag_shift); - int tag = ((id == new_type_array_id) - ? Klass::_lh_array_tag_type_value - : Klass::_lh_array_tag_obj_value); + __ movl(t0, layout_helper_addr(klass)); + int flag_shift = LayoutHelper::_flags_shift; + flag_shift += LayoutHelper::_flags_low_bits; // shift out BasicType also + int tag = LayoutHelper::for_array((id == new_type_array_id) ? T_BYTE : T_OBJECT).as_int(); + tag >>= flag_shift; + __ sarl(t0, flag_shift); + assert(id == new_object_array_id || + tag == (LayoutHelper::for_array(T_DOUBLE).as_int() >> flag_shift), + "same tag for all type arrays"); __ cmpl(t0, tag); __ jcc(Assembler::equal, ok); __ stop("assert(is an array klass)"); @@ -995,55 +1103,39 @@ __ bind(retry_tlab); - // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); - __ movl(arr_size, length); - assert(t1 == rcx, "fixed register usage"); - __ shll(arr_size /* by t1=rcx, mod 32 */); - __ shrl(t1, Klass::_lh_header_size_shift); - __ andl(t1, Klass::_lh_header_size_mask); - __ addl(arr_size, t1); - __ addl(arr_size, MinObjAlignmentInBytesMask); // align up - __ andl(arr_size, ~MinObjAlignmentInBytesMask); - + Label have_scaled_length, need_multiply; + compute_array_size(arr_size, klass, length, + have_scaled_length, need_multiply, + t1, t2, sasm, true); __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path); // preserves arr_size __ initialize_header(obj, klass, length, t1, t2); - __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte))); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); - __ andl(t1, Klass::_lh_header_size_mask); - __ subl(arr_size, t1); // body length - __ addl(t1, obj); // body start - __ initialize_body(t1, arr_size, 0, t2); + initialize_array_body(obj, arr_size, t1, t2, sasm); __ verify_oop(obj); __ ret(0); - __ bind(try_eden); - // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); - __ movl(arr_size, length); - assert(t1 == rcx, "fixed register usage"); - __ shll(arr_size /* by t1=rcx, mod 32 */); - __ shrl(t1, Klass::_lh_header_size_shift); - __ andl(t1, Klass::_lh_header_size_mask); - __ addl(arr_size, t1); - __ addl(arr_size, MinObjAlignmentInBytesMask); // align up - __ andl(arr_size, ~MinObjAlignmentInBytesMask); + // generate side path, if any + compute_array_size(arr_size, klass, length, + have_scaled_length, need_multiply, + t1, t2, sasm, false); + __ bind(try_eden); + Label have_scaled_length_2, need_multiply_2; + compute_array_size(arr_size, klass, length, + have_scaled_length_2, need_multiply_2, + t1, t2, sasm, true); __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size __ initialize_header(obj, klass, length, t1, t2); - __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte))); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); - __ andl(t1, Klass::_lh_header_size_mask); - __ subl(arr_size, t1); // body length - __ addl(t1, obj); // body start - __ initialize_body(t1, arr_size, 0, t2); + initialize_array_body(obj, arr_size, t1, t2, sasm); __ verify_oop(obj); __ ret(0); + // generate side path, if any + compute_array_size(arr_size, klass, length, + have_scaled_length_2, need_multiply_2, + t1, t2, sasm, false); + __ bind(slow_path); } --- old/src/cpu/x86/vm/stubGenerator_x86_32.cpp 2008-07-12 17:47:18.000000000 -0700 +++ new/src/cpu/x86/vm/stubGenerator_x86_32.cpp 2008-07-12 17:47:17.000000000 -0700 @@ -1464,6 +1464,7 @@ // to a long, int, short, or byte copy loop. // address generate_unsafe_copy(const char *name, + address* entry, address byte_copy_entry, address short_copy_entry, address int_copy_entry, @@ -1494,6 +1495,9 @@ // bump this on entry, not on exit: inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); + *entry = __ pc(); // Entry point from generic arraycopy stub. + BLOCK_COMMENT("Entry:"); + const Register bits = rsi; __ movl(bits, from); __ orl(bits, to); @@ -1575,6 +1579,7 @@ address entry_jint_arraycopy, address entry_oop_arraycopy, address entry_jlong_arraycopy, + address entry_unsafe_arraycopy, address entry_checkcast_arraycopy) { Label L_failed, L_failed_0, L_objArray; @@ -1676,10 +1681,10 @@ // Load layout helper (32-bits) // - // |array_tag| | header_size | element_type | |log2_element_size| - // 32 30 24 16 8 2 0 + // | array_tag | element_type | log2_element_size | header_size | + // 32 28 24 16 0 // - // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 + // array_tag: typeArray = 0xF, objArray = 0xD, normal instance = 0x0 // int lh_offset = klassOopDesc::header_size() * HeapWordSize + @@ -1687,32 +1692,25 @@ Address src_klass_lh_addr(rcx_src_klass, lh_offset); // Handle objArrays completely differently... - jint objArray_lh = Klass::array_layout_helper(T_OBJECT); + // This is necessary because they have covariant element types. + // The next type check (for src.klass == dst.klass), can get a + // false negative in the case of object arrays. + jint objArray_lh = LayoutHelper::for_array(T_OBJECT).as_int(); __ cmpl(src_klass_lh_addr, objArray_lh); __ jcc(Assembler::equal, L_objArray); // if (src->klass() != dst->klass()) return -1; __ cmpl(rcx_src_klass, dst_klass_addr); - __ jccb(Assembler::notEqual, L_failed_0); + __ jcc(Assembler::notEqual, L_failed_0); const Register rcx_lh = rcx; // layout helper assert(rcx_lh == rcx_src_klass, "known alias"); __ movl(rcx_lh, src_klass_lh_addr); - // if (!src->is_Array()) return -1; - __ cmpl(rcx_lh, Klass::_lh_neutral_value); + // if (!src->has_length_field()) return -1; + __ cmpl(rcx_lh, LayoutHelper::_neutral_value); __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp - // At this point, it is known to be a typeArray (array_tag 0x3). -#ifdef ASSERT - { Label L; - __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); - __ jcc(Assembler::greaterEqual, L); // signed cmp - __ stop("must be a primitive array"); - __ bind(L); - } -#endif - assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh); arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed); @@ -1727,11 +1725,12 @@ const Register rdi_elsize = rdi; // log2 element size __ movl(rsi_offset, rcx_lh); - __ shrl(rsi_offset, Klass::_lh_header_size_shift); - __ andl(rsi_offset, Klass::_lh_header_size_mask); // array_offset + assert(LayoutHelper::_header_size_shift == 0, ""); + __ andl(rsi_offset, LayoutHelper::_header_size_mask); // array_offset __ addl(src_array, rsi_offset); // src array offset __ addl(dst_array, rsi_offset); // dst array offset - __ andl(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize + __ shrl(rcx_lh, LayoutHelper::_element_size_shift); + __ andl(rcx_lh, LayoutHelper::_element_size_mask); // log2 elsize // next registers should be set before the jump to corresponding stub const Register from = src; // source array address @@ -1747,12 +1746,11 @@ __ shll(rsi); // src_pos << rcx (log2 elsize) assert(src_array == from, ""); __ addl(from, rsi); // from = src_array + SRC_POS << log2 elsize - __ movl(rdi, DST_POS); // dst_pos - __ shll(rdi); // dst_pos << rcx (log2 elsize) + __ movl(rsi, DST_POS); // dst_pos (reuse rsi for a temp) + __ shll(rsi); // dst_pos << rcx (log2 elsize) assert(dst_array == to, ""); - __ addl(to, rdi); // to = dst_array + DST_POS << log2 elsize + __ addl(to, rsi); // to = dst_array + DST_POS << log2 elsize __ movl(FROM, from); // src_addr - __ movl(rdi_elsize, rcx_lh); // log2 elsize __ movl(count, LENGTH); // elements count BLOCK_COMMENT("choose copy loop based on element size"); @@ -1779,14 +1777,15 @@ __ leave(); // required for proper stackwalking of RuntimeStub frame __ ret(0); + Label L_different_objArray, L_plain_copy, L_checkcast_copy; + // objArrayKlass __ BIND(L_objArray); // live at this point: rcx_src_klass, src[_pos], dst[_pos] - Label L_plain_copy, L_checkcast_copy; // test array classes for subtyping __ cmpl(rcx_src_klass, dst_klass_addr); // usual case is exact equality - __ jccb(Assembler::notEqual, L_checkcast_copy); + __ jccb(Assembler::notEqual, L_different_objArray); // Identically typed arrays can be copied without element-wise checks. assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass); @@ -1805,7 +1804,7 @@ __ movl(COUNT, count); // count __ jump(RuntimeAddress(entry_oop_arraycopy)); - __ BIND(L_checkcast_copy); + __ BIND(L_different_objArray); // live at this point: rcx_src_klass, dst[_pos], src[_pos] { // Handy offsets: @@ -1823,7 +1822,7 @@ // Before looking at dst.length, make sure dst is also an objArray. __ movl(rsi_dst_klass, dst_klass_addr); __ cmpl(dst_klass_lh_addr, objArray_lh); - __ jccb(Assembler::notEqual, L_failed); + __ jcc(Assembler::notEqual, L_failed); // It is safe to examine both src.length and dst.length. __ movl(src_pos, SRC_POS); // reload rsi @@ -1897,6 +1896,7 @@ address entry_jint_arraycopy; address entry_oop_arraycopy; address entry_jlong_arraycopy; + address entry_unsafe_arraycopy; address entry_checkcast_arraycopy; StubRoutines::_arrayof_jbyte_disjoint_arraycopy = @@ -1963,6 +1963,7 @@ StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", + &entry_unsafe_arraycopy, entry_jbyte_arraycopy, entry_jshort_arraycopy, entry_jint_arraycopy, @@ -1975,6 +1976,7 @@ entry_jint_arraycopy, entry_oop_arraycopy, entry_jlong_arraycopy, + entry_unsafe_arraycopy, entry_checkcast_arraycopy); } --- old/src/cpu/x86/vm/stubGenerator_x86_64.cpp 2008-07-12 17:47:21.000000000 -0700 +++ new/src/cpu/x86/vm/stubGenerator_x86_64.cpp 2008-07-12 17:47:20.000000000 -0700 @@ -1044,6 +1044,7 @@ static address oop_copy_entry; static address checkcast_copy_entry; + static address unsafe_copy_entry; // // Verify that a register contains clean 32-bits positive value @@ -2328,6 +2329,9 @@ __ enter(); // required for proper stackwalking of RuntimeStub frame + checkcast_copy_entry = __ pc(); + BLOCK_COMMENT("Entry:"); + // bump this on entry, not on exit: inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); @@ -2515,10 +2519,10 @@ // Load layout helper (32-bits) // - // |array_tag| | header_size | element_type | |log2_element_size| - // 32 30 24 16 8 2 0 + // | array_tag | element_type | log2_element_size | header_size | + // 32 28 24 16 0 // - // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 + // array_tag: typeArray = 0xF, objArray = 0xD, normal instance = 0x0 // int lh_offset = klassOopDesc::header_size() * HeapWordSize + @@ -2529,7 +2533,10 @@ __ movl(rax_lh, Address(r10_src_klass, lh_offset)); // Handle objArrays completely differently... - jint objArray_lh = Klass::array_layout_helper(T_OBJECT); + // This is necessary because they have covariant element types. + // The next type check (for src.klass == dst.klass), can get a + // false negative in the case of object arrays. + jint objArray_lh = LayoutHelper::for_array(T_OBJECT).as_int(); __ cmpl(rax_lh, objArray_lh); __ jcc(Assembler::equal, L_objArray); @@ -2538,20 +2545,10 @@ __ cmpq(r10_src_klass, r9_dst_klass); __ jcc(Assembler::notEqual, L_failed); - // if (!src->is_Array()) return -1; - __ cmpl(rax_lh, Klass::_lh_neutral_value); + // if (!src->has_length_field()) return -1; + __ cmpl(rax_lh, LayoutHelper::_neutral_value); __ jcc(Assembler::greaterEqual, L_failed); - // At this point, it is known to be a typeArray (array_tag 0x3). -#ifdef ASSERT - { Label L; - __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); - __ jcc(Assembler::greaterEqual, L); - __ stop("must be a primitive array"); - __ bind(L); - } -#endif - arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, r10, L_failed); @@ -2565,12 +2562,13 @@ const Register rax_elsize = rax_lh; // element size __ movl(r10_offset, rax_lh); - __ shrl(r10_offset, Klass::_lh_header_size_shift); - __ andq(r10_offset, Klass::_lh_header_size_mask); // array_offset + assert(LayoutHelper::_header_size_shift == 0, ""); + __ andl(r10_offset, LayoutHelper::_header_size_mask); // array_offset __ addq(src, r10_offset); // src array offset __ addq(dst, r10_offset); // dst array offset BLOCK_COMMENT("choose copy loop based on element size"); - __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize + __ shrl(rax_lh, LayoutHelper::_element_size_shift); + __ andl(rax_lh, LayoutHelper::_element_size_mask); // rax_lh -> rax_elsize // next registers should be set before the jump to corresponding stub const Register from = c_rarg0; // source array address @@ -2618,15 +2616,16 @@ __ movslq(count, r11_length); // length __ jump(RuntimeAddress(long_copy_entry)); + Label L_different_objArray, L_plain_copy, L_checkcast_copy; + // objArrayKlass __ BIND(L_objArray); // live at this point: r10_src_klass, src[_pos], dst[_pos] - Label L_plain_copy, L_checkcast_copy; // test array classes for subtyping __ load_klass(r9_dst_klass, dst); __ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality - __ jcc(Assembler::notEqual, L_checkcast_copy); + __ jcc(Assembler::notEqual, L_different_objArray); // Identically typed arrays can be copied without element-wise checks. arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, @@ -2640,7 +2639,7 @@ __ BIND(L_plain_copy); __ jump(RuntimeAddress(oop_copy_entry)); - __ BIND(L_checkcast_copy); + __ BIND(L_different_objArray); // live at this point: r10_src_klass, !r11_length { // assert(r11_length == C_RARG4); // will reload from here --- old/src/cpu/x86/vm/templateTable_x86_32.cpp 2008-07-12 17:47:23.000000000 -0700 +++ new/src/cpu/x86/vm/templateTable_x86_32.cpp 2008-07-12 17:47:23.000000000 -0700 @@ -3040,8 +3040,9 @@ // get instance_size in instanceKlass (scaled to a count of bytes) __ movl(rdx, Address(rcx, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); // test to see if it has a finalizer or is malformed in some way - __ testl(rdx, Klass::_lh_instance_slow_path_bit); + __ testl(rdx, LayoutHelper::_slow_path_low_bit); __ jcc(Assembler::notZero, slow_case); + __ andl(rdx, ~LayoutHelper::_size_low_mask); // // Allocate the instance --- old/src/cpu/x86/vm/templateTable_x86_64.cpp 2008-07-12 17:47:26.000000000 -0700 +++ new/src/cpu/x86/vm/templateTable_x86_64.cpp 2008-07-12 17:47:25.000000000 -0700 @@ -3071,8 +3071,9 @@ Address(rsi, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); // test to see if it has a finalizer or is malformed in some way - __ testl(rdx, Klass::_lh_instance_slow_path_bit); + __ testl(rdx, LayoutHelper::_slow_path_low_bit); __ jcc(Assembler::notZero, slow_case); + __ andl(rdx, ~LayoutHelper::_size_low_mask); // Allocate the instance // 1) Try to allocate in the TLAB --- old/src/share/vm/c1/c1_LIRGenerator.cpp 2008-07-12 17:47:28.000000000 -0700 +++ new/src/share/vm/c1/c1_LIRGenerator.cpp 2008-07-12 17:47:28.000000000 -0700 @@ -652,7 +652,7 @@ jobject2reg_with_patching(klass_reg, klass, info); // If klass is not loaded we do not know if the klass has finalizers: if (UseFastNewInstance && klass->is_loaded() - && !Klass::layout_helper_needs_slow_path(klass->layout_helper())) { + && !klass->layout_helper().needs_slow_path()) { Runtime1::StubID stub_id = klass->is_initialized() ? Runtime1::fast_new_instance_id : Runtime1::fast_new_instance_init_check_id; @@ -660,8 +660,9 @@ assert(klass->is_loaded(), "must be loaded"); // allocate space for instance - assert(klass->size_helper() >= 0, "illegal instance size"); - const int instance_size = align_object_size(klass->size_helper()); + const int instance_size = klass->new_instance_size(); + assert(instance_size == align_object_size(instance_size), "pre-aligned"); + assert(instance_size > 0, "illegal instance size"); __ allocate_object(dst, scratch1, scratch2, scratch3, scratch4, oopDesc::header_size(), instance_size, klass_reg, !klass->is_initialized(), slow_path); } else { --- old/src/share/vm/ci/ciArray.hpp 2008-07-12 17:47:31.000000000 -0700 +++ new/src/share/vm/ci/ciArray.hpp 2008-07-12 17:47:31.000000000 -0700 @@ -47,6 +47,7 @@ int length() { return _length; } // What kind of ciObject is this? - bool is_array() { return true; } + bool is_java_array() { return true; } bool is_java_object() { return true; } + bool has_length_field() { return true; } }; --- old/src/share/vm/ci/ciInstance.cpp 2008-07-12 17:47:33.000000000 -0700 +++ new/src/share/vm/ci/ciInstance.cpp 2008-07-12 17:47:33.000000000 -0700 @@ -31,6 +31,12 @@ // machine. // ------------------------------------------------------------------ +// ciInstance::has_length_field +bool ciInstance::has_length_field() { + return klass()->layout_helper().has_length_field(); +} + +// ------------------------------------------------------------------ // ciObject::java_mirror_type ciType* ciInstance::java_mirror_type() { VM_ENTRY_MARK; --- old/src/share/vm/ci/ciInstance.hpp 2008-07-12 17:47:36.000000000 -0700 +++ new/src/share/vm/ci/ciInstance.hpp 2008-07-12 17:47:35.000000000 -0700 @@ -53,6 +53,8 @@ bool is_instance() { return true; } bool is_java_object() { return true; } + bool has_length_field(); // klass()->layout_helper().has_length_field() + // Constant value of a field. ciConstant field_value(ciField* field); --- old/src/share/vm/ci/ciInstanceKlass.cpp 2008-07-12 17:47:38.000000000 -0700 +++ new/src/share/vm/ci/ciInstanceKlass.cpp 2008-07-12 17:47:38.000000000 -0700 @@ -195,12 +195,13 @@ // ciInstanceKlass* ciInstanceKlass::get_canonical_holder(int offset) { #ifdef ASSERT - if (!(offset >= 0 && offset < layout_helper())) { + int inst_size = layout_helper().new_instance_size_in_bytes(); + if (!(offset >= 0 && offset < inst_size)) { tty->print("*** get_canonical_holder(%d) on ", offset); this->print(); tty->print_cr(" ***"); }; - assert(offset >= 0 && offset < layout_helper(), "offset must be tame"); + assert(offset >= 0 && offset < inst_size, "offset must be tame"); #endif if (offset < instanceOopDesc::base_offset_in_bytes()) { @@ -244,12 +245,12 @@ ciKlass::print_impl(st); GUARDED_VM_ENTRY(st->print(" loader=0x%x", (address)loader());) if (is_loaded()) { - st->print(" loaded=true initialized=%s finalized=%s subklass=%s size=%d flags=", + st->print(" loaded=true initialized=%s finalized=%s subklass=%s ", bool_to_str(is_initialized()), bool_to_str(has_finalizer()), - bool_to_str(has_subklass()), - layout_helper()); - + bool_to_str(has_subklass())); + layout_helper().print_on(st); + st->print(" flags="); _flags.print_klass_flags(); if (_super) { --- old/src/share/vm/ci/ciInstanceKlass.hpp 2008-07-12 17:47:41.000000000 -0700 +++ new/src/share/vm/ci/ciInstanceKlass.hpp 2008-07-12 17:47:40.000000000 -0700 @@ -47,7 +47,7 @@ ciFlags _flags; jint _nonstatic_field_size; - jint _nonstatic_oop_map_size; + jint _nonstatic_oop_map_count; // Lazy fields get filled in only upon request. ciInstanceKlass* _super; @@ -126,9 +126,8 @@ } return _has_subklass; } - jint size_helper() { - return (Klass::layout_helper_size_in_bytes(layout_helper()) - >> LogHeapWordSize); + jint new_instance_size() { + return layout_helper().new_instance_size_in_bytes() >> LogHeapWordSize; } jint nonstatic_field_size() { assert(is_loaded(), "must be loaded"); @@ -136,9 +135,9 @@ jint has_nonstatic_fields() { assert(is_loaded(), "must be loaded"); return _has_nonstatic_fields; } - jint nonstatic_oop_map_size() { + jint nonstatic_oop_map_count() { assert(is_loaded(), "must be loaded"); - return _nonstatic_oop_map_size; } + return _nonstatic_oop_map_count; } ciInstanceKlass* super(); jint nof_implementors() { assert(is_loaded(), "must be loaded"); --- old/src/share/vm/ci/ciKlass.cpp 2008-07-12 17:47:43.000000000 -0700 +++ new/src/share/vm/ci/ciKlass.cpp 2008-07-12 17:47:43.000000000 -0700 @@ -48,7 +48,7 @@ ciKlass::ciKlass(KlassHandle h_k, ciSymbol* name) : ciType(h_k) { assert(get_oop()->is_klass(), "wrong type"); _name = name; - _layout_helper = Klass::_lh_neutral_value; + _layout_helper = LayoutHelper::_neutral_value; } // ------------------------------------------------------------------ @@ -57,7 +57,7 @@ // Unloaded klass variant. ciKlass::ciKlass(ciSymbol* name, ciKlass* klass) : ciType(klass) { _name = name; - _layout_helper = Klass::_lh_neutral_value; + _layout_helper = LayoutHelper::_neutral_value; } // ------------------------------------------------------------------ --- old/src/share/vm/ci/ciKlass.hpp 2008-07-12 17:47:45.000000000 -0700 +++ new/src/share/vm/ci/ciKlass.hpp 2008-07-12 17:47:45.000000000 -0700 @@ -40,7 +40,7 @@ private: ciSymbol* _name; - jint _layout_helper; + LayoutHelper _layout_helper; protected: ciKlass(KlassHandle k_h, ciSymbol* name); @@ -72,7 +72,7 @@ ciSymbol* name() { return _name; } // What is its layout helper value? - jint layout_helper() { return _layout_helper; } + LayoutHelper layout_helper() { return _layout_helper; } bool is_subtype_of(ciKlass* klass); bool is_subclass_of(ciKlass* klass); --- old/src/share/vm/ci/ciObject.hpp 2008-07-12 17:47:48.000000000 -0700 +++ new/src/share/vm/ci/ciObject.hpp 2008-07-12 17:47:47.000000000 -0700 @@ -116,9 +116,11 @@ virtual bool is_instance() { return false; } virtual bool is_method() { return false; } virtual bool is_method_data() { return false; } - virtual bool is_array() { return false; } + virtual bool has_length_field() { return false; } + virtual bool is_allprim() { return false; } virtual bool is_obj_array() { return false; } virtual bool is_type_array() { return false; } + virtual bool is_java_array() { return false; } virtual bool is_symbol() { return false; } virtual bool is_type() { return false; } virtual bool is_return_address() { return false; } @@ -135,6 +137,8 @@ virtual bool is_obj_array_klass_klass() { return false; } virtual bool is_type_array_klass_klass() { return false; } + bool is_array() { return is_java_array(); } + // Is this a type or value which has no associated class? // It is true of primitive types and null objects. virtual bool is_classless() const { return false; } --- old/src/share/vm/ci/ciTypeArray.hpp 2008-07-12 17:47:51.000000000 -0700 +++ new/src/share/vm/ci/ciTypeArray.hpp 2008-07-12 17:47:50.000000000 -0700 @@ -42,6 +42,7 @@ public: // What kind of ciObject is this? + bool is_allprim() { return true; } bool is_type_array() { return true; } // Return character at index. This is only useful if the --- old/src/share/vm/ci/ciTypeArrayKlass.hpp 2008-07-12 17:47:53.000000000 -0700 +++ new/src/share/vm/ci/ciTypeArrayKlass.hpp 2008-07-12 17:47:53.000000000 -0700 @@ -44,7 +44,7 @@ public: // The type of the array elements. BasicType element_type() { - return Klass::layout_helper_element_type(layout_helper()); + return layout_helper().element_type(); } // What kind of ciObject is this? --- old/src/share/vm/classfile/classFileParser.cpp 2008-07-12 17:47:55.000000000 -0700 +++ new/src/share/vm/classfile/classFileParser.cpp 2008-07-12 17:47:55.000000000 -0700 @@ -2960,14 +2960,20 @@ } // Size of instances - int instance_size; - - instance_size = align_object_size(next_nonstatic_type_offset / wordSize); + assert(next_nonstatic_type_offset % wordSize == 0, "already aligned for instance size"); + int instance_size = align_object_size(next_nonstatic_type_offset / wordSize); assert(instance_size == align_object_size(instanceOopDesc::header_size() + nonstatic_field_size), "consistent layout helper value"); - // Size of non-static oop map blocks (in words) allocated at end of klass - int nonstatic_oop_map_size = compute_oop_map_size(super_klass, nonstatic_oop_map_count, first_nonstatic_oop_offset); + // Count of non-static oop map blocks allocated at end of klass. + int total_map_count = compute_oop_map_size(super_klass, nonstatic_oop_map_count, first_nonstatic_oop_offset); + + LayoutHelper layout_helper = LayoutHelper::for_instance(instance_size); + + if (total_map_count == 0) { + // mark this class as having no oops + layout_helper.set_low_bit(LayoutHelper::_obj_allprim_low_bit); + } // Compute reference type ReferenceType rt; @@ -2980,19 +2986,18 @@ // We can now create the basic klassOop for this klass klassOop ik = oopFactory::new_instanceKlass( vtable_size, itable_size, - static_field_size, nonstatic_oop_map_size, + static_field_size, total_map_count, rt, CHECK_(nullHandle)); instanceKlassHandle this_klass (THREAD, ik); assert(this_klass->static_field_size() == static_field_size && - this_klass->nonstatic_oop_map_size() == nonstatic_oop_map_size, "sanity check"); + this_klass->nonstatic_oop_map_count() == total_map_count, "sanity check"); // Fill in information already parsed this_klass->set_access_flags(access_flags); - jint lh = Klass::instance_layout_helper(instance_size, false); - this_klass->set_layout_helper(lh); + this_klass->set_layout_helper(layout_helper); assert(this_klass->oop_is_instance(), "layout is correct"); - assert(this_klass->size_helper() == instance_size, "correct size_helper"); + assert(this_klass->new_instance_size() == instance_size, "correct size_helper"); // Not yet: supers are done below to support the new subtype-checking fields //this_klass->set_super(super_klass()); this_klass->set_class_loader(class_loader()); @@ -3146,7 +3151,7 @@ int ClassFileParser::compute_oop_map_size(instanceKlassHandle super, int nonstatic_oop_map_count, int first_nonstatic_oop_offset) { - int map_size = super.is_null() ? 0 : super->nonstatic_oop_map_size(); + int map_size = super.is_null() ? 0 : super->nonstatic_oop_map_count(); if (nonstatic_oop_map_count > 0) { // We have oops to add to map if (map_size == 0) { @@ -3156,7 +3161,7 @@ OopMapBlock* first_map = super->start_of_nonstatic_oop_maps(); OopMapBlock* last_map = first_map + map_size - 1; - int next_offset = last_map->offset() + (last_map->length() * heapOopSize); + int next_offset = last_map->offset() + last_map->size(); if (next_offset == first_nonstatic_oop_offset) { // There is no gap bettwen superklass's last oop field and first // local oop field, merge maps. @@ -3176,10 +3181,10 @@ int nonstatic_oop_map_count, u2* nonstatic_oop_offsets, u2* nonstatic_oop_length) { OopMapBlock* this_oop_map = k->start_of_nonstatic_oop_maps(); - OopMapBlock* last_oop_map = this_oop_map + k->nonstatic_oop_map_size(); + OopMapBlock* last_oop_map = this_oop_map + k->nonstatic_oop_map_count(); instanceKlass* super = k->superklass(); if (super != NULL) { - int super_oop_map_size = super->nonstatic_oop_map_size(); + int super_oop_map_size = super->nonstatic_oop_map_count(); OopMapBlock* super_oop_map = super->start_of_nonstatic_oop_maps(); // Copy maps from superklass while (super_oop_map_size-- > 0) { @@ -3195,14 +3200,15 @@ nonstatic_oop_map_count--; nonstatic_oop_offsets++; this_oop_map--; - this_oop_map->set_length(this_oop_map->length() + *nonstatic_oop_length++); + int more_size = *nonstatic_oop_length++ << LogBytesPerHeapOop; + this_oop_map->set_size(this_oop_map->size() + more_size); this_oop_map++; } assert((this_oop_map + nonstatic_oop_map_count) == last_oop_map, "just checking"); // Add new map blocks, fill them while (nonstatic_oop_map_count-- > 0) { this_oop_map->set_offset(*nonstatic_oop_offsets++); - this_oop_map->set_length(*nonstatic_oop_length++); + this_oop_map->set_size(*nonstatic_oop_length++ << LogBytesPerHeapOop); this_oop_map++; } } @@ -3262,14 +3268,15 @@ // If it cannot be fast-path allocated, set a bit in the layout helper. // See documentation of instanceKlass::can_be_fastpath_allocated(). - assert(k->size_helper() > 0, "layout_helper is initialized"); + assert(k->layout_helper().is_instance(), "layout_helper is initialized"); if ((!RegisterFinalizersAtInit && k->has_finalizer()) || k->is_abstract() || k->is_interface() || (k->name() == vmSymbols::java_lang_Class() && k->class_loader() == NULL) - || k->size_helper() >= FastAllocateSizeLimit) { + || k->new_instance_size() >= FastAllocateSizeLimit) { // Forbid fast-path allocation. - jint lh = Klass::instance_layout_helper(k->size_helper(), true); + LayoutHelper lh = k->layout_helper(); + lh.set_low_bit(LayoutHelper::_slow_path_low_bit); k->set_layout_helper(lh); } } --- old/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep 2008-07-12 17:47:58.000000000 -0700 +++ new/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep 2008-07-12 17:47:58.000000000 -0700 @@ -108,11 +108,13 @@ compactibleFreeListSpace.cpp concurrentMarkSweepThread.hpp compactibleFreeListSpace.cpp copy.hpp compactibleFreeListSpace.cpp globals.hpp +compactibleFreeListSpace.cpp genOopClosures.inline.hpp compactibleFreeListSpace.cpp handles.inline.hpp compactibleFreeListSpace.cpp init.hpp compactibleFreeListSpace.cpp java.hpp compactibleFreeListSpace.cpp liveRange.hpp compactibleFreeListSpace.cpp oop.inline.hpp +compactibleFreeListSpace.cpp parOopClosures.inline.hpp compactibleFreeListSpace.cpp resourceArea.hpp compactibleFreeListSpace.cpp universe.inline.hpp compactibleFreeListSpace.cpp vmThread.hpp --- old/src/share/vm/gc_implementation/includeDB_gc_parNew 2008-07-12 17:48:01.000000000 -0700 +++ new/src/share/vm/gc_implementation/includeDB_gc_parNew 2008-07-12 17:48:00.000000000 -0700 @@ -79,6 +79,7 @@ parNewGeneration.hpp defNewGeneration.hpp parNewGeneration.hpp parGCAllocBuffer.hpp +parNewGeneration.hpp objArrayOop.inline.hpp parNewGeneration.hpp taskqueue.hpp parOopClosures.hpp genOopClosures.hpp --- old/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp 2008-07-12 17:48:03.000000000 -0700 +++ new/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp 2008-07-12 17:48:03.000000000 -0700 @@ -72,6 +72,7 @@ } bool ParScanThreadState::should_be_partially_scanned(oop new_obj, oop old_obj) const { + // %%% TO DO: generalize objAarry; check Klass::LayoutHelper::has_oops_in_tail return new_obj->is_objArray() && arrayOop(new_obj)->length() > ParGCArrayScanChunk && new_obj != old_obj; @@ -89,6 +90,8 @@ int start = arrayOop(old)->length(); int end = obj->length(); int remainder = end - start; + assert(objArrayOopDesc::before_first_element == -1, "else change range logic"); + assert(start >= -1, ""); assert(start <= end, "just checking"); if (remainder > 2 * ParGCArrayScanChunk) { // Test above combines last partial chunk with a full chunk @@ -106,7 +109,6 @@ } // process our set of indices (include header in first chunk) - // should make sure end is even (aligned to HeapWord in case of compressed oops) if ((HeapWord *)obj < young_old_boundary()) { // object is in to_space obj->oop_iterate_range(&_to_space_closure, start, end); @@ -125,6 +127,7 @@ note_pop(); if ((HeapWord *)obj_to_scan < young_old_boundary()) { + // %%% TO DO: generalize objAarry; check Klass::LayoutHelper::has_oops_in_tail if (obj_to_scan->is_objArray() && obj_to_scan->is_forwarded() && obj_to_scan->forwardee() != obj_to_scan) { @@ -979,11 +982,16 @@ assert(new_obj != NULL, "just checking"); if (forward_ptr == NULL) { + if (new_obj->is_allprim()) { + // Header is OK. Nothing more to scan. + assert((HeapWord*) new_obj->klass() > reserved().end_word(), "klass is in perm"); + return new_obj; + } oop obj_to_push = new_obj; if (par_scan_state->should_be_partially_scanned(obj_to_push, old)) { // Length field used as index of next element to be scanned. // Real length can be obtained from real_forwardee() - arrayOop(old)->set_length(0); + arrayOop(old)->set_length(objArrayOopDesc::before_first_element); obj_to_push = old; assert(obj_to_push->is_forwarded() && obj_to_push->forwardee() != obj_to_push, "push forwarded object"); @@ -1096,11 +1104,16 @@ } if (forward_ptr == NULL) { + if (new_obj->is_allprim()) { + // Header is OK. Nothing more to scan. + assert((HeapWord*) new_obj->klass() > reserved().end_word(), "klass is in perm"); + return new_obj; + } oop obj_to_push = new_obj; if (par_scan_state->should_be_partially_scanned(obj_to_push, old)) { // Length field used as index of next element to be scanned. // Real length can be obtained from real_forwardee() - arrayOop(old)->set_length(0); + arrayOop(old)->set_length(objArrayOopDesc::before_first_element); obj_to_push = old; assert(obj_to_push->is_forwarded() && obj_to_push->forwardee() != obj_to_push, "push forwarded object"); --- old/src/share/vm/includeDB_core 2008-07-12 17:48:06.000000000 -0700 +++ new/src/share/vm/includeDB_core 2008-07-12 17:48:05.000000000 -0700 @@ -2512,6 +2512,7 @@ klass.hpp iterator.hpp klass.hpp klassOop.hpp klass.hpp klassPS.hpp +klass.hpp layoutHelper.hpp klass.hpp memRegion.hpp klass.hpp oop.hpp klass.hpp specialized_oop_closures.hpp @@ -2568,6 +2569,14 @@ klassVtable.hpp handles.hpp klassVtable.hpp oopsHierarchy.hpp +layoutHelper.cpp arrayOop.hpp +layoutHelper.cpp instanceKlass.hpp +layoutHelper.cpp javaClasses.hpp +layoutHelper.cpp layoutHelper.hpp +layoutHelper.cpp systemDictionary.hpp + +layoutHelper.hpp top.hpp + linkResolver.cpp bytecode.hpp linkResolver.cpp collectedHeap.inline.hpp linkResolver.cpp compilationPolicy.hpp @@ -2996,11 +3005,14 @@ objArrayKlassKlass.hpp objArrayKlass.hpp objArrayOop.cpp objArrayKlass.hpp -objArrayOop.cpp objArrayOop.hpp +objArrayOop.cpp objArrayOop.inline.hpp objArrayOop.cpp oop.inline.hpp objArrayOop.hpp arrayOop.hpp +objArrayOop.inline.hpp objArrayKlass.hpp +objArrayOop.inline.hpp objArrayOop.hpp + objectMonitor.hpp os.hpp objectMonitor_.cpp dtrace.hpp @@ -3712,12 +3724,14 @@ space.cpp copy.hpp space.cpp defNewGeneration.hpp space.cpp genCollectedHeap.hpp +space.cpp genOopClosures.inline.hpp space.cpp globalDefinitions.hpp space.cpp java.hpp space.cpp liveRange.hpp space.cpp markSweep.hpp space.cpp oop.inline.hpp space.cpp oop.inline2.hpp +space.cpp parOopClosures.inline.hpp space.cpp safepoint.hpp space.cpp space.hpp space.cpp space.inline.hpp --- old/src/share/vm/memory/iterator.hpp 2008-07-12 17:48:09.000000000 -0700 +++ new/src/share/vm/memory/iterator.hpp 2008-07-12 17:48:08.000000000 -0700 @@ -25,6 +25,7 @@ // The following classes are C++ `closures` for iterating over objects, roots and spaces class ReferenceProcessor; +class OopClosure_nv2v; // OopClosure is used for iterating through roots (oop*) @@ -34,9 +35,10 @@ OopClosure(ReferenceProcessor* rp) : _ref_processor(rp) { } OopClosure() : _ref_processor(NULL) { } virtual void do_oop(oop* o) = 0; - virtual void do_oop_v(oop* o) { do_oop(o); } virtual void do_oop(narrowOop* o) = 0; - virtual void do_oop_v(narrowOop* o) { do_oop(o); } + // The _v variant is just a macro-friendly alias for the abstract virtual. + void do_oop_v(oop* o) { do_oop(o); } + void do_oop_v(narrowOop* o) { do_oop(o); } // In support of post-processing of weak links of KlassKlass objects; // see KlassKlass::oop_oop_iterate(). @@ -55,7 +57,24 @@ Prefetch::style prefetch_style() { // Note that this is non-virtual. return Prefetch::do_none; } + + // Bridges between _v and _nv calling sequences. + // They produce a value which accepts do_oop_nv. + // Usage: OopClosure::nv_from##nv_suffix(blk)->do_oop_nv(p) + template + static inline OopClosureType* nv_from_nv(OopClosureType* blk) { return blk; } + static inline OopClosure_nv2v* nv_from_v(OopClosure* blk); // after next class +}; + +// Fake class for giving templates access to virtual do_oop with the '_nv' suffix: +class OopClosure_nv2v : public OopClosure { + OopClosure_nv2v() { ShouldNotReachHere(); } + public: + void do_oop_nv(oop* o) { do_oop(o); } + void do_oop_nv(narrowOop* o) { do_oop(o); } + static OopClosure_nv2v* from(OopClosure* blk) { return (OopClosure_nv2v*)blk; } }; +inline OopClosure_nv2v* OopClosure::nv_from_v(OopClosure* blk) { return OopClosure_nv2v::from(blk); } // ObjectClosure is used for iterating through an object space --- old/src/share/vm/memory/oopFactory.cpp 2008-07-12 17:48:11.000000000 -0700 +++ new/src/share/vm/memory/oopFactory.cpp 2008-07-12 17:48:11.000000000 -0700 @@ -95,9 +95,9 @@ klassOop oopFactory::new_instanceKlass(int vtable_len, int itable_len, int static_field_size, - int nonstatic_oop_map_size, ReferenceType rt, TRAPS) { + int nonstatic_oop_map_count, ReferenceType rt, TRAPS) { instanceKlassKlass* ikk = instanceKlassKlass::cast(Universe::instanceKlassKlassObj()); - return ikk->allocate_instance_klass(vtable_len, itable_len, static_field_size, nonstatic_oop_map_size, rt, CHECK_NULL); + return ikk->allocate_instance_klass(vtable_len, itable_len, static_field_size, nonstatic_oop_map_count, rt, CHECK_NULL); } --- old/src/share/vm/memory/oopFactory.hpp 2008-07-12 17:48:14.000000000 -0700 +++ new/src/share/vm/memory/oopFactory.hpp 2008-07-12 17:48:13.000000000 -0700 @@ -86,7 +86,7 @@ // Instance classes static klassOop new_instanceKlass(int vtable_len, int itable_len, int static_field_size, - int nonstatic_oop_map_size, ReferenceType rt, TRAPS); + int nonstatic_oop_map_count, ReferenceType rt, TRAPS); // Methods private: --- old/src/share/vm/memory/space.cpp 2008-07-12 17:48:17.000000000 -0700 +++ new/src/share/vm/memory/space.cpp 2008-07-12 17:48:16.000000000 -0700 @@ -33,7 +33,7 @@ if (top_obj != NULL) { if (_sp->block_is_obj(top_obj)) { if (_precision == CardTableModRefBS::ObjHeadPreciseArray) { - if (oop(top_obj)->is_objArray() || oop(top_obj)->is_typeArray()) { + if (oop(top_obj)->is_array()) { // %%% should generalize to has_length_field // An arrayOop is starting on the dirty card - since we do exact // store checks for objArrays we are done. } else { --- old/src/share/vm/memory/specialized_oop_closures.cpp 2008-07-12 17:48:19.000000000 -0700 +++ new/src/share/vm/memory/specialized_oop_closures.cpp 2008-07-12 17:48:19.000000000 -0700 @@ -49,12 +49,15 @@ } void SpecializationStats::print() { + int k; const char* header_format = " %20s %10s %11s %10s"; const char* line_format = " %20s %10d %11d %9.2f%%"; - int all_numCallsTotal = - _numCallsTotal[ik] + _numCallsTotal[irk] + _numCallsTotal[oa]; - int all_numCalls_nv = - _numCalls_nv[ik] + _numCalls_nv[irk] + _numCalls_nv[oa]; + int all_numCallsTotal = 0; + int all_numCalls_nv = 0; + for (k = 0; k < NUM_Kinds; k++) { + all_numCallsTotal += _numCallsTotal[k]; + all_numCalls_nv += _numCalls_nv[k]; + } gclog_or_tty->print_cr("\nOf %d oop_oop_iterate calls %d (%6.3f%%) are in (ik, irk, oa).", _numCallsAll, all_numCallsTotal, 100.0 * (float)all_numCallsTotal / (float)_numCallsAll); @@ -91,16 +94,18 @@ "----------", "-----------", "----------"); - int all_numDoOopCallsTotal = - _numDoOopCallsTotal[ik] + _numDoOopCallsTotal[irk] + _numDoOopCallsTotal[oa]; - int all_numDoOopCalls_nv = - _numDoOopCalls_nv[ik] + _numDoOopCalls_nv[irk] + _numDoOopCalls_nv[oa]; + int all_numDoOopCallsTotal = 0; + int all_numDoOopCalls_nv = 0; + for (k = 0; k < NUM_Kinds; k++) { + all_numDoOopCallsTotal += _numDoOopCallsTotal[k]; + all_numDoOopCalls_nv += _numDoOopCalls_nv[k]; + } gclog_or_tty->print_cr(line_format, "all", all_numDoOopCallsTotal, all_numDoOopCalls_nv, 100.0 * (float)all_numDoOopCalls_nv / (float)all_numDoOopCallsTotal); - const char* kind_names[] = { "ik", "irk", "oa" }; - for (int k = ik; k < NUM_Kinds; k++) { + const char* kind_names[] = { SpecializationStats_KIND_NAMES }; + for (k = ik; k < NUM_Kinds; k++) { gclog_or_tty->print_cr(line_format, kind_names[k], _numDoOopCallsTotal[k], _numDoOopCalls_nv[k], (_numDoOopCallsTotal[k] > 0 ? --- old/src/share/vm/memory/specialized_oop_closures.hpp 2008-07-12 17:48:22.000000000 -0700 +++ new/src/share/vm/memory/specialized_oop_closures.hpp 2008-07-12 17:48:22.000000000 -0700 @@ -171,6 +171,7 @@ class SpecializationStats { public: enum Kind { + prim, // allprim (array or instance) ik, // instanceKlass irk, // instanceRefKlass oa, // objArrayKlass @@ -178,6 +179,8 @@ }; #if ENABLE_SPECIALIZATION_STATS + #define SpecializationStats_KIND_NAMES "prim", "ik", "irk", "oa" + private: static int _numCallsAll; --- old/src/share/vm/oops/arrayKlass.cpp 2008-07-12 17:48:25.000000000 -0700 +++ new/src/share/vm/oops/arrayKlass.cpp 2008-07-12 17:48:24.000000000 -0700 @@ -83,7 +83,7 @@ assert(!k()->is_parsable(), "not expecting parsability yet."); k->set_super(Universe::is_bootstrapping() ? (klassOop)NULL : SystemDictionary::object_klass()); - k->set_layout_helper(Klass::_lh_neutral_value); + k->set_layout_helper(LayoutHelper()); k->set_dimension(1); k->set_higher_dimension(NULL); k->set_lower_dimension(NULL); --- old/src/share/vm/oops/arrayKlass.hpp 2008-07-12 17:48:28.000000000 -0700 +++ new/src/share/vm/oops/arrayKlass.hpp 2008-07-12 17:48:28.000000000 -0700 @@ -36,7 +36,8 @@ public: // Testing operation - bool oop_is_array() const { return true; } + bool oop_is_array_slow() const { return true; } + bool oop_has_length_field_slow() const { return true; } // Instance variables int dimension() const { return _dimension; } @@ -55,10 +56,10 @@ void set_alloc_size(juint n) { _alloc_size = n; } // offset of first element, including any padding for the sake of alignment - int array_header_in_bytes() const { return layout_helper_header_size(layout_helper()); } - int log2_element_size() const { return layout_helper_log2_element_size(layout_helper()); } + int array_header_in_bytes() const { return layout_helper().header_size_in_bytes(); } + int log2_element_size() const { return layout_helper().log2_element_size(); } // type of elements (T_OBJECT for both oop arrays and array-arrays) - BasicType element_type() const { return layout_helper_element_type(layout_helper()); } + BasicType element_type() const { return layout_helper().element_type(); } oop component_mirror() const { return _component_mirror; } void set_component_mirror(oop m) { oop_store((oop*) &_component_mirror, m); } @@ -81,7 +82,7 @@ // Casting from klassOop static arrayKlass* cast(klassOop k) { Klass* kp = k->klass_part(); - assert(kp->null_vtbl() || kp->oop_is_array(), "cast to arrayKlass"); + assert(kp->null_vtbl() || kp->oop_is_array_slow(), "cast to arrayKlass"); return (arrayKlass*) kp; } --- old/src/share/vm/oops/arrayOop.hpp 2008-07-12 17:48:32.000000000 -0700 +++ new/src/share/vm/oops/arrayOop.hpp 2008-07-12 17:48:31.000000000 -0700 @@ -86,6 +86,12 @@ *(int*)(((intptr_t)this) + length_offset_in_bytes()) = length; } + // These accessors work on any object with a length field: + static jint length_of(oop obj) { + assert(obj->has_length_field(), "must have length field"); + return *(int*)(((intptr_t)obj) + length_offset_in_bytes()); + } + // Should only be called with constants as argument // (will not constant fold otherwise) // Returns the header size in words aligned to the requirements of the --- old/src/share/vm/oops/instanceKlass.cpp 2008-07-12 17:48:34.000000000 -0700 +++ new/src/share/vm/oops/instanceKlass.cpp 2008-07-12 17:48:34.000000000 -0700 @@ -524,7 +524,8 @@ instanceOop instanceKlass::allocate_instance(TRAPS) { bool has_finalizer_flag = has_finalizer(); // Query before possible GC - int size = size_helper(); // Query before forming handle. + int size = new_instance_size(); // Query before forming handle. + // (Using new_instance_size instead of fixed_size to support mixed arrays.) KlassHandle h_k(THREAD, as_klassOop()); @@ -543,7 +544,7 @@ // instances so simply disallow finalizable perm objects. This can // be relaxed if a need for it is found. assert(!has_finalizer(), "perm objects not allowed to have finalizers"); - int size = size_helper(); // Query before forming handle. + int size = new_instance_size(); // Query before forming handle. KlassHandle h_k(THREAD, as_klassOop()); instanceOop i = (instanceOop) CollectedHeap::permanent_obj_allocate(h_k, size, CHECK_NULL); @@ -1254,299 +1255,165 @@ } #endif //PRODUCT - +// Oop assertions. #ifdef ASSERT -template void assert_is_in(T *p) { +template inline T* assert_is_in(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop o = oopDesc::decode_heap_oop_not_null(heap_oop); assert(Universe::heap()->is_in(o), "should be in heap"); } + return p; } -template void assert_is_in_closed_subset(T *p) { +template inline T* assert_is_in_closed_subset(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop o = oopDesc::decode_heap_oop_not_null(heap_oop); assert(Universe::heap()->is_in_closed_subset(o), "should be in closed"); } + return p; } -template void assert_is_in_reserved(T *p) { +template inline T* assert_is_in_reserved(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop o = oopDesc::decode_heap_oop_not_null(heap_oop); assert(Universe::heap()->is_in_reserved(o), "should be in reserved"); } + return p; } -template void assert_nothing(T *p) {} #else -template void assert_is_in(T *p) {} -template void assert_is_in_closed_subset(T *p) {} -template void assert_is_in_reserved(T *p) {} -template void assert_nothing(T *p) {} +template inline T* assert_is_in(T* p) { return p; } +template inline T* assert_is_in_closed_subset(T* p) { return p; } +template inline T* assert_is_in_reserved(T* p) { return p; } #endif // ASSERT -// -// Macros that iterate over areas of oops which are specialized on type of -// oop pointer either narrow or wide, depending on UseCompressedOops -// -// Parameters are: -// T - type of oop to point to (either oop or narrowOop) -// start_p - starting pointer for region to iterate over -// count - number of oops or narrowOops to iterate over -// do_oop - action to perform on each oop (it's arbitrary C code which -// makes it more efficient to put in a macro rather than making -// it a template function) -// assert_fn - assert function which is template function because performance -// doesn't matter when enabled. -#define InstanceKlass_SPECIALIZED_OOP_ITERATE( \ - T, start_p, count, do_oop, \ - assert_fn) \ -{ \ - T* p = (T*)(start_p); \ - T* const end = p + (count); \ - while (p < end) { \ - (assert_fn)(p); \ - do_oop; \ - ++p; \ - } \ -} - -#define InstanceKlass_SPECIALIZED_OOP_REVERSE_ITERATE( \ - T, start_p, count, do_oop, \ - assert_fn) \ -{ \ - T* const start = (T*)(start_p); \ - T* p = start + (count); \ - while (start < p) { \ - --p; \ - (assert_fn)(p); \ - do_oop; \ - } \ -} - -#define InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE( \ - T, start_p, count, low, high, \ - do_oop, assert_fn) \ -{ \ - T* const l = (T*)(low); \ - T* const h = (T*)(high); \ - assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && \ - mask_bits((intptr_t)h, sizeof(T)-1) == 0, \ - "bounded region must be properly aligned"); \ - T* p = (T*)(start_p); \ - T* end = p + (count); \ - if (p < l) p = l; \ - if (end > h) end = h; \ - while (p < end) { \ - (assert_fn)(p); \ - do_oop; \ - ++p; \ - } \ -} - - -// The following macros call specialized macros, passing either oop or -// narrowOop as the specialization type. These test the UseCompressedOops -// flag. -#define InstanceKlass_OOP_ITERATE(start_p, count, \ - do_oop, assert_fn) \ -{ \ - if (UseCompressedOops) { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(narrowOop, \ - start_p, count, \ - do_oop, assert_fn) \ - } else { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(oop, \ - start_p, count, \ - do_oop, assert_fn) \ - } \ -} - -#define InstanceKlass_BOUNDED_OOP_ITERATE(start_p, count, low, high, \ - do_oop, assert_fn) \ -{ \ - if (UseCompressedOops) { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(narrowOop, \ - start_p, count, \ - low, high, \ - do_oop, assert_fn) \ - } else { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(oop, \ - start_p, count, \ - low, high, \ - do_oop, assert_fn) \ - } \ -} - -#define InstanceKlass_OOP_MAP_ITERATE(obj, do_oop, assert_fn) \ -{ \ - /* Compute oopmap block range. The common case \ - is nonstatic_oop_map_size == 1. */ \ - OopMapBlock* map = start_of_nonstatic_oop_maps(); \ - OopMapBlock* const end_map = map + nonstatic_oop_map_size(); \ - if (UseCompressedOops) { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(narrowOop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - ++map; \ - } \ - } else { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_OOP_ITERATE(oop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - ++map; \ - } \ - } \ -} - -#define InstanceKlass_OOP_MAP_REVERSE_ITERATE(obj, do_oop, assert_fn) \ -{ \ - OopMapBlock* const start_map = start_of_nonstatic_oop_maps(); \ - OopMapBlock* map = start_map + nonstatic_oop_map_size(); \ - if (UseCompressedOops) { \ - while (start_map < map) { \ - --map; \ - InstanceKlass_SPECIALIZED_OOP_REVERSE_ITERATE(narrowOop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - } \ - } else { \ - while (start_map < map) { \ - --map; \ - InstanceKlass_SPECIALIZED_OOP_REVERSE_ITERATE(oop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - do_oop, assert_fn) \ - } \ - } \ -} - -#define InstanceKlass_BOUNDED_OOP_MAP_ITERATE(obj, low, high, do_oop, \ - assert_fn) \ -{ \ - /* Compute oopmap block range. The common case is \ - nonstatic_oop_map_size == 1, so we accept the \ - usually non-existent extra overhead of examining \ - all the maps. */ \ - OopMapBlock* map = start_of_nonstatic_oop_maps(); \ - OopMapBlock* const end_map = map + nonstatic_oop_map_size(); \ - if (UseCompressedOops) { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(narrowOop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - low, high, \ - do_oop, assert_fn) \ - ++map; \ - } \ - } else { \ - while (map < end_map) { \ - InstanceKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(oop, \ - obj->obj_field_addr(map->offset()), map->length(), \ - low, high, \ - do_oop, assert_fn) \ - ++map; \ - } \ - } \ -} - void instanceKlass::follow_static_fields() { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - MarkSweep::mark_and_push(p), \ - assert_is_in_closed_subset) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + MarkSweep::mark_and_push(assert_is_in_closed_subset(addr(p))); + } + }); } #ifndef SERIALGC void instanceKlass::follow_static_fields(ParCompactionManager* cm) { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - PSParallelCompact::mark_and_push(cm, p), \ - assert_is_in) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + PSParallelCompact::mark_and_push(cm, assert_is_in(addr(p))); + } + }); } #endif // SERIALGC void instanceKlass::adjust_static_fields() { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - MarkSweep::adjust_pointer(p), \ - assert_nothing) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + MarkSweep::adjust_pointer(addr(p)); + } + }); } #ifndef SERIALGC void instanceKlass::update_static_fields() { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); } void instanceKlass::update_static_fields(HeapWord* beg_addr, HeapWord* end_addr) { - InstanceKlass_BOUNDED_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - beg_addr, end_addr, \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing ) + MemRegion mr(beg_addr, end_addr); + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = oop_chunk::bounded(mr, static_oop_fields(pmax), pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); } #endif // SERIALGC void instanceKlass::oop_follow_contents(oop obj) { - assert(obj != NULL, "can't follow the content of NULL object"); obj->follow_header(); - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - MarkSweep::mark_and_push(p), \ - assert_is_in_closed_subset) + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + MarkSweep::mark_and_push(assert_is_in_closed_subset(addr(p))); + } + }); } #ifndef SERIALGC void instanceKlass::oop_follow_contents(ParCompactionManager* cm, oop obj) { - assert(obj != NULL, "can't follow the content of NULL object"); obj->follow_header(cm); - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - PSParallelCompact::mark_and_push(cm, p), \ - assert_is_in) + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + PSParallelCompact::mark_and_push(cm, assert_is_in(addr(p))); + } + }); } #endif // SERIALGC // closure's do_header() method dicates whether the given closure should be // applied to the klass ptr in the object header. +// Templated iterator loops. + +template +inline int instanceKlass::oop_oop_iterate_tp(oop obj, + OopClosureType* closure) { + SpecializationStats::record_iterate_call_nv(SpecializationStats::ik); + /* header */ + if (closure->do_header()) { + obj->oop_iterate_header_nv(closure); + } + if (has_nonstatic_oop_maps()) { + for (oop_map::iterator pmax, p = + nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + SpecializationStats::record_do_oop_call_nv(SpecializationStats::ik); + closure->do_oop_nv(assert_is_in_closed_subset(addr(p))); + } + } + return cheap_fixed_size(); +} + +template +inline int instanceKlass::oop_oop_iterate_tp_m(oop obj, + OopClosureType* closure, + MemRegion mr) { + SpecializationStats::record_iterate_call_nv(SpecializationStats::ik); + if (closure->do_header()) { + obj->oop_iterate_header_nv(closure, mr); + } + if (has_nonstatic_oop_maps()) { + for (oop_map::bounded_iterator pmax, p = + bounded_nonstatic_oop_fields(obj, mr, pmax); p < pmax; p++) { + SpecializationStats::record_do_oop_call_nv(SpecializationStats::ik); + closure->do_oop_nv(assert_is_in_closed_subset(addr(p))); + } + } + return cheap_fixed_size(); +} + #define InstanceKlass_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ \ int instanceKlass::oop_oop_iterate##nv_suffix(oop obj, \ OopClosureType* closure) {\ - SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik);\ - /* header */ \ - if (closure->do_header()) { \ - obj->oop_iterate_header(closure); \ - } \ - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - SpecializationStats:: \ - record_do_oop_call##nv_suffix(SpecializationStats::ik); \ - (closure)->do_oop##nv_suffix(p), \ - assert_is_in_closed_subset) \ - return size_helper(); \ + WITH_BOTH_OOP_KINDS(T, { \ + return oop_oop_iterate_tp(obj, \ + OopClosure::nv_from##nv_suffix(closure)); \ + }); \ } #define InstanceKlass_OOP_OOP_ITERATE_DEFN_m(OopClosureType, nv_suffix) \ \ int instanceKlass::oop_oop_iterate##nv_suffix##_m(oop obj, \ OopClosureType* closure, \ - MemRegion mr) { \ - SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::ik);\ - if (closure->do_header()) { \ - obj->oop_iterate_header(closure, mr); \ - } \ - InstanceKlass_BOUNDED_OOP_MAP_ITERATE( \ - obj, mr.start(), mr.end(), \ - (closure)->do_oop##nv_suffix(p), \ - assert_is_in_closed_subset) \ - return size_helper(); \ + MemRegion mr) { \ + WITH_BOTH_OOP_KINDS(T, { \ + return oop_oop_iterate_tp_m(obj, \ + OopClosure::nv_from##nv_suffix(closure), mr); \ + }); \ } ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN) @@ -1554,95 +1421,112 @@ ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DEFN_m) ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DEFN_m) + void instanceKlass::iterate_static_fields(OopClosure* closure) { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - closure->do_oop(p), \ - assert_is_in_reserved) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + closure->do_oop(assert_is_in_reserved(addr(p))); + } + }); } void instanceKlass::iterate_static_fields(OopClosure* closure, MemRegion mr) { - InstanceKlass_BOUNDED_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - mr.start(), mr.end(), \ - (closure)->do_oop_v(p), \ - assert_is_in_closed_subset) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = oop_chunk::bounded(mr, static_oop_fields(pmax), pmax); p < pmax; p++) { + closure->do_oop(assert_is_in_closed_subset(addr(p))); + } + }); } int instanceKlass::oop_adjust_pointers(oop obj) { - int size = size_helper(); - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - MarkSweep::adjust_pointer(p), \ - assert_is_in) - obj->adjust_header(); + int size = cheap_fixed_size(); + if (!has_nonstatic_oop_maps()) return size; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + MarkSweep::adjust_pointer(assert_is_in(addr(p))); + } + }); return size; } #ifndef SERIALGC void instanceKlass::oop_copy_contents(PSPromotionManager* pm, oop obj) { assert(!pm->depth_first(), "invariant"); - InstanceKlass_OOP_MAP_REVERSE_ITERATE( \ - obj, \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_breadth(p); \ - }, \ - assert_nothing ) + if (!has_nonstatic_oop_maps()) return; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmin, p = iterator_reverse(nonstatic_oop_fields(obj, pmin), pmin); p >= pmin; p--) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_breadth(addr(p)); + } + } + }); } void instanceKlass::oop_push_contents(PSPromotionManager* pm, oop obj) { assert(pm->depth_first(), "invariant"); - InstanceKlass_OOP_MAP_REVERSE_ITERATE( \ - obj, \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_depth(p); \ - }, \ - assert_nothing ) + if (!has_nonstatic_oop_maps()) return; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmin, p = iterator_reverse(nonstatic_oop_fields(obj, pmin), pmin); p >= pmin; p--) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_depth(addr(p)); + } + } + }); } int instanceKlass::oop_update_pointers(ParCompactionManager* cm, oop obj) { - InstanceKlass_OOP_MAP_ITERATE( \ - obj, \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing) - return size_helper(); + int size = cheap_fixed_size(); + if (!has_nonstatic_oop_maps()) return size; + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::iterator pmax, p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); + return size; } int instanceKlass::oop_update_pointers(ParCompactionManager* cm, oop obj, HeapWord* beg_addr, HeapWord* end_addr) { - InstanceKlass_BOUNDED_OOP_MAP_ITERATE( \ - obj, beg_addr, end_addr, \ - PSParallelCompact::adjust_pointer(p), \ - assert_nothing) - return size_helper(); + int size = cheap_fixed_size(); + if (!has_nonstatic_oop_maps()) return size; + MemRegion mr(beg_addr, end_addr); + WITH_BOTH_OOP_KINDS(T, { + for (oop_map::bounded_iterator pmax, p = + bounded_nonstatic_oop_fields(obj, mr, pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(addr(p)); + } + }); + return size; } void instanceKlass::copy_static_fields(PSPromotionManager* pm) { assert(!pm->depth_first(), "invariant"); - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_breadth(p); \ - }, \ - assert_nothing ) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_breadth(addr(p)); + } + } + }); } void instanceKlass::push_static_fields(PSPromotionManager* pm) { - assert(pm->depth_first(), "invariant"); - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - if (PSScavenge::should_scavenge(p)) { \ - pm->claim_or_forward_depth(p); \ - }, \ - assert_nothing ) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + if (PSScavenge::should_scavenge(addr(p))) { + pm->claim_or_forward_depth(addr(p)); + } + } + }); } void instanceKlass::copy_static_fields(ParCompactionManager* cm) { - InstanceKlass_OOP_ITERATE( \ - start_of_static_fields(), static_oop_field_size(), \ - PSParallelCompact::adjust_pointer(p), \ - assert_is_in) + WITH_BOTH_OOP_KINDS(T, { + for (oop_chunk::iterator pmax, p = static_oop_fields(pmax); p < pmax; p++) { + PSParallelCompact::adjust_pointer(assert_is_in(addr(p))); + } + }); } #endif // SERIALGC @@ -2040,7 +1924,7 @@ class VerifyFieldClosure: public OopClosure { protected: - template void do_oop_work(T* p) { + template inline void do_oop_work(T* p) { guarantee(Universe::heap()->is_in_closed_subset(p), "should be in heap"); oop obj = oopDesc::load_decode_heap_oop(p); if (!obj->is_oop_or_null()) { @@ -2054,10 +1938,106 @@ virtual void do_oop(narrowOop* p) { VerifyFieldClosure::do_oop_work(p); } }; +#ifndef PRODUCT +template +void instanceKlass::oop_verify_oop_map_iterators(oop obj) { + ResourceMark rm; + const int max_locs = 100; + T* locs[max_locs]; + int num_locs = 0, total_locs = 0; + + // critical points for testing bounded iteration + // each point has 3 sides; each region has 3 ends; obj + 4 oop blocks, plus 2 +/- 0x10 + const int max_crits = 3 * 3 * (1+4) + 2; + T* obj_start = obj->obj_field_addr(0); + T* crits[max_crits]; + int num_crits = 0; + crits[num_crits++] = obj_start - 0x10; + #define ADD_CRIT(p) { if ((T*)(p) > crits[num_crits-1]) crits[num_crits++] = (T*)(p); } + #define ADD_CRIT_3(p) { ADD_CRIT((T*)p - 1); ADD_CRIT((T*)p); ADD_CRIT((T*)p + 1); } + ADD_CRIT_3(obj_start); + + // iterate the normal way, and record everything + OopMapBlock* mmin = start_of_nonstatic_oop_maps(); + OopMapBlock* mmax = mmin + nonstatic_oop_map_count(); + for (OopMapBlock* m = mmin; m < mmax; m++) { + T* pmin = obj->obj_field_addr(m->offset()); + T* pmax = obj->obj_field_addr(m->offset() + m->size()); + assert(pmin < pmax, "non-empty oop map block"); + if (m < mmin+2 || m >= mmax-2) { + T* pmid = pmin + (pmax - pmin) / 2; + ADD_CRIT_3(pmin); ADD_CRIT_3(pmid); ADD_CRIT_3(pmax); + } + for (T* p = pmin; p < pmax; p++) { + assert(p != NULL, ""); + total_locs += 1; + assert(num_locs == 0 || locs[num_locs-1] < p, "monotonic"); + if (num_locs < max_locs) + locs[num_locs++] = p; + } + } + + // Fill remainder of locs array with nulls for an error check. + { int n = num_locs; while (n < max_locs) locs[n++] = NULL; } + + T* obj_end = (T*)( (HeapWord*)obj + obj->size() ); + ADD_CRIT_3(obj_end); + ADD_CRIT(obj_end + 0x10); + assert(num_crits <= max_crits, "oob"); + #undef ADD_CRIT + #undef ADD_CRIT_0 + #undef ADD_CRIT_3 + + // use a forward iterator + oop_map::iterator p, pmin, pmax; + oop_map::bounded_iterator bp, bpmax; + int n = 0; + for (p = nonstatic_oop_fields(obj, pmax); p < pmax; p++) { + T* loc1 = addr(p); + T* loc0 = (n < max_locs ? locs[n] : loc1); n++; + assert(loc0 == loc1, "same result"); + } + assert(n == total_locs, "same count"); + n = total_locs; + for (p = reverse_nonstatic_oop_fields(obj, pmin); p >= pmin; p--) { + T* loc1 = addr(p); + T* loc0 = (--n < max_locs ? locs[n] : loc1); + assert(loc0 == loc1, "same result"); + } + assert(n == 0, "same count"); + for (int i = 0; i < num_crits; i++) { + for (int j = i; j < num_crits; j++) { + MemRegion mr(crits[i], crits[j]); + n = 0; + while (n < num_locs && !mr.contains(locs[n])) ++n; + for (bp = bounded_nonstatic_oop_fields(obj, mr, bpmax); bp < bpmax; bp++) { + T* loc1 = addr(bp); + assert(mr.contains(loc1), "in region"); + T* loc0 = (n < max_locs ? locs[n] : loc1); n++; + assert(loc0 == loc1, "same result"); + } + while (n < num_locs) { + assert(!mr.contains(locs[n]), "outside region"); + n++; + } + } + } +} + +static int verify_iterator_count = 10000; +#endif //PRODUCT + void instanceKlass::oop_verify_on(oop obj, outputStream* st) { Klass::oop_verify_on(obj, st); VerifyFieldClosure blk; oop_oop_iterate(obj, &blk); + +#ifndef PRODUCT + if (has_nonstatic_oop_maps() && verify_iterator_count > 0) { + --verify_iterator_count; + WITH_BOTH_OOP_KINDS(T, oop_verify_oop_map_iterators(obj)); + } +#endif //PRODUCT } #ifndef PRODUCT @@ -2083,14 +2063,14 @@ first_time = false; const int extra = java_lang_Class::number_of_fake_oop_fields; guarantee(ik->nonstatic_field_size() == extra, "just checking"); - guarantee(ik->nonstatic_oop_map_size() == 1, "just checking"); - guarantee(ik->size_helper() == align_object_size(instanceOopDesc::header_size() + extra), "just checking"); + guarantee(ik->nonstatic_oop_map_count() == 1, "just checking"); + guarantee(ik->fixed_size() == align_object_size(instanceOopDesc::header_size() + extra), "just checking"); // Check that the map is (2,extra) int offset = java_lang_Class::klass_offset; OopMapBlock* map = ik->start_of_nonstatic_oop_maps(); - guarantee(map->offset() == offset && map->length() == extra, "just checking"); + guarantee(map->offset() == offset && map->size() == extra * heapOopSize, "just checking"); } } --- old/src/share/vm/oops/instanceKlass.hpp 2008-07-12 17:48:37.000000000 -0700 +++ new/src/share/vm/oops/instanceKlass.hpp 2008-07-12 17:48:36.000000000 -0700 @@ -53,7 +53,7 @@ // [static field size ] // [nonstatic field size ] // [static oop fields size ] -// [nonstatic oop maps size ] +// [nonstatic oop maps offset ] // [has finalize method ] // [deoptimization mark bit ] // [initialization state ] @@ -63,7 +63,7 @@ // [EMBEDDED Java vtable ] size in words = vtable_len // [EMBEDDED static oop fields ] size in words = static_oop_fields_size // [ static non-oop fields ] size in words = static_field_size - static_oop_fields_size -// [EMBEDDED nonstatic oop-map blocks] size in words = nonstatic_oop_map_size +// [EMBEDDED nonstatic oop-map blocks] size in words = oop_map_block_size // // The embedded nonstatic oop-map blocks are short pairs (offset, length) indicating // where oops are located in instances of this klass. @@ -180,13 +180,14 @@ // End of the oop block. // + int _object_size; // object size in words // number of words used by non-static fields in this klass (including // inherited fields but after header_size()). If fields are compressed into - // header, this can be zero so it's not the same as number of static fields. + // header, this can be slightly less than the combined size of all nonstatic fields. int _nonstatic_field_size; int _static_field_size; // number words used by static fields (oop and non-oop) in this klass int _static_oop_field_size;// number of static oop fields in this klass - int _nonstatic_oop_map_size;// number of nonstatic oop-map blocks allocated at end of this klass + int _map_offset_in_bytes; // offset of nonstatic oop-map blocks allocated at end of this klass bool _is_marked_dependent; // used for marking during flushing and deoptimization bool _rewritten; // methods rewritten. bool _has_nonstatic_fields; // for sizing with UseCompressedOops @@ -205,6 +206,7 @@ nmethod* _osr_nmethods_head; // Head of list of on-stack replacement nmethods for this class BreakpointInfo* _breakpoints; // bpt lists, managed by methodOop int _nof_implementors; // No of implementors of this interface (zero if not an interface) + juint _alloc_size; // allocation profiling support // Array of interesting part(s) of the previous version(s) of this // instanceKlass. See PreviousVersionWalker below. GrowableArray* _previous_versions; @@ -228,6 +230,9 @@ bool has_nonstatic_fields() const { return _has_nonstatic_fields; } void set_has_nonstatic_fields(bool b) { _has_nonstatic_fields = b; } + int object_size() const { return _object_size; } + void set_object_size(int size) { _object_size = size; } + // field sizes int nonstatic_field_size() const { return _nonstatic_field_size; } void set_nonstatic_field_size(int size) { _nonstatic_field_size = size; } @@ -395,8 +400,8 @@ void set_source_debug_extension(symbolOop n){ oop_store_without_check((oop*) &_source_debug_extension, (oop) n); } // nonstatic oop-map blocks - int nonstatic_oop_map_size() const { return _nonstatic_oop_map_size; } - void set_nonstatic_oop_map_size(int size) { _nonstatic_oop_map_size = size; } + int map_offset_in_bytes() const { return _map_offset_in_bytes; } + void set_map_offset_in_bytes(int n) { _map_offset_in_bytes = n; } // RedefineClasses() support for previous versions: void add_previous_version(instanceKlassHandle ikh, BitMap *emcp_methods, @@ -544,9 +549,10 @@ bool compute_is_subtype_of(klassOop k); bool can_be_primary_super_slow() const; klassOop java_super() const { return super(); } - int oop_size(oop obj) const { return size_helper(); } + int oop_size(oop obj) const { return fixed_size(); } int klass_oop_size() const { return object_size(); } bool oop_is_instance_slow() const { return true; } + bool oop_is_allprim_slow() const { return !has_nonstatic_oop_maps(); } // Iterators void do_local_static_fields(FieldClosure* cl); @@ -567,7 +573,10 @@ // Sizing (in words) static int header_size() { return align_object_offset(oopDesc::header_size() + sizeof(instanceKlass)/HeapWordSize); } - int object_size() const { return object_size(align_object_offset(vtable_length()) + align_object_offset(itable_length()) + static_field_size() + nonstatic_oop_map_size()); } + void init_object_size() { _object_size = object_size(align_object_offset(vtable_length()) + + align_object_offset(itable_length()) + + static_field_size() + + oop_map_block_size()); } static int vtable_start_offset() { return header_size(); } static int vtable_length_offset() { return oopDesc::header_size() + offset_of(instanceKlass, _vtable_len) / HeapWordSize; } static int object_size(int extra) { return align_object_size(header_size() + extra); } @@ -588,18 +597,44 @@ return (intptr_t)start_of_static_fields() - (intptr_t)as_klassOop(); } + HeapWord* end_of_static_fields() const { + return start_of_static_fields() + static_field_size(); + } + + protected: + // store size value in header of oop map block: + inline void set_nonstatic_oop_map_count(int size); // see below + + // return size in words: + static inline int oop_map_block_size(int num_blocks); // see below + inline int oop_map_block_size() const { + return oop_map_block_size(nonstatic_oop_map_count()); + } + + public: + bool has_nonstatic_oop_maps() const { + return map_offset_in_bytes() != 0; + } + OopMapBlock* start_of_nonstatic_oop_maps() const { - return (OopMapBlock*) (start_of_static_fields() + static_field_size()); +#ifdef ASSERT + if (!has_nonstatic_oop_maps()) return (OopMapBlock*) badAddress; + // else any garbage value will do... +#endif + return (OopMapBlock*) ((intptr_t) as_klassOop() + map_offset_in_bytes()); } + inline int nonstatic_oop_map_count() const; // see below + // Allocation profiling support - juint alloc_size() const { return _alloc_count * size_helper(); } - void set_alloc_size(juint n) {} + juint alloc_size() const { return _alloc_size; } + void set_alloc_size(juint n) { _alloc_size = n; } - // Use this to return the size of an instance in heap words: - int size_helper() const { - return layout_helper_to_size_helper(layout_helper()); - } + // use this one only on non-variable instances: + int fixed_size() const { return layout_helper().fixed_size_in_bytes() >> LogHeapWordSize; } + + // the more general version allows for mixed arrays (default length is zero): + int new_instance_size() const { return layout_helper().new_instance_size_in_bytes() >> LogHeapWordSize; } // This bit is initialized in classFileParser.cpp. // It is false under any of the following conditions: @@ -608,7 +643,7 @@ // - the class size is larger than FastAllocateSizeLimit // - the class is java/lang/Class, which cannot be allocated directly bool can_be_fastpath_allocated() const { - return !layout_helper_needs_slow_path(layout_helper()); + return !layout_helper().needs_slow_path(); } // Java vtable/itable @@ -648,26 +683,285 @@ char* signature_name() const; // Iterators - int oop_oop_iterate(oop obj, OopClosure* blk) { - return oop_oop_iterate_v(obj, blk); - } - - int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr) { - return oop_oop_iterate_v_m(obj, blk, mr); - } + int oop_oop_iterate(oop obj, OopClosure* blk); + int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr); + // Populate the vtable with customized versions of the iterators. #define InstanceKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \ int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, \ - MemRegion mr); + MemRegion mr); - ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DECL) - ALL_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DECL) + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(InstanceKlass_OOP_OOP_ITERATE_DECL) + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_3(InstanceKlass_OOP_OOP_ITERATE_DECL) void iterate_static_fields(OopClosure* closure); void iterate_static_fields(OopClosure* closure, MemRegion mr); -private: + private: + // file-local common implementation for the iterators + template + inline int oop_oop_iterate_tp(oop obj, OopClosureType* blk); + template + inline int oop_oop_iterate_tp_m(oop obj, OopClosureType* blk, MemRegion mr); + + // This is a return value which is valid for normal instances: + int cheap_fixed_size() const { return layout_helper().fixed_size_in_bytes(true) >> LogHeapWordSize; } + + protected: + // Common code for iterating over oops. + template + typename oop_chunk::iterator static_oop_fields(typename oop_chunk::iterator& pmax) { + return oop_chunk::forward((T*)start_of_static_fields(), static_oop_field_size(), pmax); + } + + // Iterator algorithm: Mixing in upper and lower bounds. + // Assumes an iterator of the above form. + template + class iterator_bound_mixin : public I { + typedef typename I::addr_t addr_t; + addr_t _bound; + + static bool bounded_nonempty_forward(iterator_bound_mixin& beg, + iterator_bound_mixin& end) { + I& ibeg = beg; I& iend = end; + while (ibeg < iend) { + I::enforce_bounds(beg, end, beg._bound, end._bound); + if (I::is_primed(beg, end)) + return true; + // else try another go of an outer loop + } + return false; + } + + public: + iterator_bound_mixin() { } + iterator_bound_mixin(I p, addr_t bound) + : iterator(p), _bound(bound) { } + bool operator<(iterator_bound_mixin& end) { + return I::is_primed(*this, end) || bounded_nonempty_forward(*this, end); + } + bool operator>=(iterator_bound_mixin& beg) { + ShouldNotReachHere(); return false; + } + static iterator_bound_mixin make(MemRegion mr, + I ubeg, I uend, + iterator_bound_mixin& end) { + end = iterator_bound_mixin(uend, (addr_t) mr.end()); + return iterator_bound_mixin(ubeg, (addr_t) mr.start()); + } + }; + + template + class oop_map : AllStatic { + typedef OopMapBlock Map; // shorthand + public: + class iterator VALUE_OBJ_CLASS_SPEC { + friend class oop_map; + address _base; // loop invariant base for each oop map block + Map* _map; // outer loop: pointer to oop map blocks + T* _p; // inner loop: pointer to current oop + + // Iterators occur in pairs. The pair is a loop cursor and a limit. + + // The loop, over an iterator pair {beg,end}, works like this: + // int map_count = ...; + // address beg_base = ...; + // address end_base = beg_base; + // Map* beg_map = ...; + // Map* end_map = beg_map + map_count; + // for (; beg_map < end_map; beg_map++) { + // T* beg_p = (T*)( beg_base + beg_map->offset() ); + // T* end_p = (T*)( beg_base + beg_map->offset() + beg_map->size() ); + // for (; beg_p < end_p; beg_p++) { + // ... + // } + // } + + // The limit is upper exclusive for forward loops and lower inclusive + // for reverse loops. So both forward and reverse loops generate + // the interval [beg,end). Since the reverse loop pattern pre-decrements + // the loop cursor, a reverse loop body executes the same interval, + // but in the form [beg,end-1]. + + // The inner loop generates the addresses from the current oop map block, + // in [beg.p,end.p). (Or in reverse, [beg.p,end.p-1].) + // The initial and final states of an iterator pair specify + // an empty iteration space as [q,q) or [q,q-1], for some arbitrary 'q'. + + // An iterator pair can be in one of three states: + // - unprimed: they have identical arbitrary p values + // p=[q,q), map=[maps,maps+map_count) + // - working: p values are in the current oop map block + // p=[m.start,m.limit), map=[m1,m2) + // - exhausted: both index pairs are empty ("upside down") + // p=[q,q), map=[m,m) + + // It is important *not* to prime iterators eagerly (on construction), + // because we do not know which of the pair will serve + // as a loop variable ('p') or a loop limit ('pmin' or 'pmax'). + + // For degenerate loop iterations, the unprimed and exhausted states are + // identical. For other iterations, the first '<' or '>=' operation + // promotes the iterator pair from the unprimed to the working state. + + // Until that moment, there is no sure way to tell which iterator + // is serving as the loop cursor and which is the loop limit. + // The case 'p < pmax' is a forward loop, and 'p >= pmin' is in reverse. + + // In both cases, the left-hand operand of the comparison is the + // loop cursor. When the inner loop runs out of values, the + // left-hand operand's outer loop state is bumped, while the + // right-hand