--- old/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2008-07-12 17:47:15.000000000 -0700 +++ new/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2008-07-12 17:47:14.000000000 -0700 @@ -786,6 +786,97 @@ } +inline Address layout_helper_addr(Register klass) { + return Address(klass, (klassOopDesc::header_size() * HeapWordSize + + Klass::layout_helper_offset_in_bytes())); +} + +static void compute_instance_size(Register obj_size, Register klass, + Label& have_obj_size, Label& size_is_variable, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(obj_size, klass); + + if (hot_part) { + __ movl(obj_size, layout_helper_addr(klass)); + __ andl(obj_size, ~LayoutHelper::_size_low_mask); + if (MixedArrays) { + __ jcc(Assembler::negative, size_is_variable); + __ bind(have_obj_size); + } + } else { + if (MixedArrays) { + // Side path for fixing up the initial size of a variable object. + // Must round from int to object alignment, and clear high bits. + __ bind(size_is_variable); + __ addl(obj_size, LayoutHelper::_header_size_odd_mask); + __ andl(obj_size, (LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask)); + __ jmp(have_obj_size); + } + } +} + + +static void compute_array_size(Register arr_size, Register klass, Register length, + Label& have_scaled_length, Label& need_multiply, + Register t1, Register t2, + StubAssembler* sasm, bool hot_part) { + assert_different_registers(arr_size, klass, length, t1, t2); + + if (hot_part) { + // get the allocation size: round_up(hdr + length << (lh>>16 & 0x1F)) + __ movl(t1, layout_helper_addr(klass)); + __ movl(arr_size, length); + // int scale = (lh >> LayoutHelper::_element_size_shift); + __ movl(t1, t2); // spill layout helper + __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, no mask needed + // size_t arr_size = (array_length << scale); + assert(t1 == rcx, "fixed register usage"); + __ shll(arr_size /* by t1=rcx, mod 32 */); + if (MixedArrays) { + // int sizem_ip = (scale & (LayoutHelper::_element_sizem_mask_ip)); + __ andl(t1, LayoutHelper::_element_sizem_mask_ip); + __ jcc(Assembler::notZero, need_multiply); + __ bind(have_scaled_length); + } + // arr_size += Klass::layout_helper_header_size_in_bytes(lh); + assert(LayoutHelper::_header_size_shift == 0, ""); + // shrl(t2, LayoutHelper::_header_size_shift); + __ andl(t2, LayoutHelper::_header_size_mask); + __ addl(arr_size, t2); + __ addl(arr_size, MinObjAlignmentInBytesMask); // align up + __ andl(arr_size, ~MinObjAlignmentInBytesMask); + } else { + if (MixedArrays) { + // Side path for scaling by a non-power-of-two array element size. + __ bind(need_multiply); + __ movl(arr_size, length); // reload; previous value was trash + // int sizem = (sizem_ip >> LayoutHelper::_element_scale_bits); + __ shrl(t1, LayoutHelper::_element_scale_bits); + // arr_size = (array_length * (1+sizem)) << scale; + __ addl(t1, 1); + assert(wordSize == jintSize, "else use imulq, shlq, etc."); + __ imull(arr_size, t1); + __ movl(t1, t2); // reload layout helper + __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, mask needed + __ andl(t1, LayoutHelper::_element_scale_mask); + assert(t1 == rcx, "fixed register usage"); + __ shll(arr_size /* by t1=rcx, mod 32 */); + __ jmp(have_scaled_length); + } + } +} + +static void initialize_array_body(Register obj, Register arr_size, Register t1, Register t2, StubAssembler* sasm) { + assert_different_registers(obj, arr_size, t1, t2); + + int min_header_size = arrayOopDesc::header_size(T_BYTE); + __ subl(arr_size, min_header_size); // body length + __ movl(t1, obj); + __ addl(t1, min_header_size); // body start + __ initialize_body(t1, arr_size, 0, t2); +} + + OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { // for better readability @@ -869,13 +960,10 @@ #ifdef ASSERT // assert object can be fast path allocated { - Label ok, not_ok; - __ movl(obj_size, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); - __ cmpl(obj_size, 0); // make sure it's an instance (LH > 0) - __ jcc(Assembler::lessEqual, not_ok); - __ testl(obj_size, Klass::_lh_instance_slow_path_bit); + Label ok; + __ movl(obj_size, layout_helper_addr(klass)); + __ testl(obj_size, LayoutHelper::_slow_path_low_bit); __ jcc(Assembler::zero, ok); - __ bind(not_ok); __ stop("assert(can be fast path allocated)"); __ should_not_reach_here(); __ bind(ok); @@ -890,7 +978,10 @@ __ bind(retry_tlab); // get the instance size - __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + Label have_obj_size, size_is_variable; + compute_instance_size(obj_size, klass, + have_obj_size, size_is_variable, + sasm, true); __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path); __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); @@ -898,9 +989,17 @@ __ popl(rdi); __ ret(0); + // generate side path, if any + compute_instance_size(obj_size, klass, + have_obj_size, size_is_variable, + sasm, false); + __ bind(try_eden); // get the instance size - __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + Label have_obj_size_2, size_is_variable_2; + compute_instance_size(obj_size, klass, + have_obj_size_2, size_is_variable_2, + sasm, true); __ eden_allocate(obj, obj_size, 0, t1, slow_path); __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); @@ -908,6 +1007,11 @@ __ popl(rdi); __ ret(0); + // generate side path, if any + compute_instance_size(obj_size, klass, + have_obj_size_2, size_is_variable_2, + sasm, false); + __ bind(slow_path); __ popl(rbx); __ popl(rdi); @@ -964,11 +1068,15 @@ { Label ok; Register t0 = obj; - __ movl(t0, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc))); - __ sarl(t0, Klass::_lh_array_tag_shift); - int tag = ((id == new_type_array_id) - ? Klass::_lh_array_tag_type_value - : Klass::_lh_array_tag_obj_value); + __ movl(t0, layout_helper_addr(klass)); + int flag_shift = LayoutHelper::_flags_shift; + flag_shift += LayoutHelper::_flags_low_bits; // shift out BasicType also + int tag = LayoutHelper::for_array((id == new_type_array_id) ? T_BYTE : T_OBJECT).as_int(); + tag >>= flag_shift; + __ sarl(t0, flag_shift); + assert(id == new_object_array_id || + tag == (LayoutHelper::for_array(T_DOUBLE).as_int() >> flag_shift), + "same tag for all type arrays"); __ cmpl(t0, tag); __ jcc(Assembler::equal, ok); __ stop("assert(is an array klass)"); @@ -995,55 +1103,39 @@ __ bind(retry_tlab); - // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); - __ movl(arr_size, length); - assert(t1 == rcx, "fixed register usage"); - __ shll(arr_size /* by t1=rcx, mod 32 */); - __ shrl(t1, Klass::_lh_header_size_shift); - __ andl(t1, Klass::_lh_header_size_mask); - __ addl(arr_size, t1); - __ addl(arr_size, MinObjAlignmentInBytesMask); // align up - __ andl(arr_size, ~MinObjAlignmentInBytesMask); - + Label have_scaled_length, need_multiply; + compute_array_size(arr_size, klass, length, + have_scaled_length, need_multiply, + t1, t2, sasm, true); __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path); // preserves arr_size __ initialize_header(obj, klass, length, t1, t2); - __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte))); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); - __ andl(t1, Klass::_lh_header_size_mask); - __ subl(arr_size, t1); // body length - __ addl(t1, obj); // body start - __ initialize_body(t1, arr_size, 0, t2); + initialize_array_body(obj, arr_size, t1, t2, sasm); __ verify_oop(obj); __ ret(0); - __ bind(try_eden); - // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); - __ movl(arr_size, length); - assert(t1 == rcx, "fixed register usage"); - __ shll(arr_size /* by t1=rcx, mod 32 */); - __ shrl(t1, Klass::_lh_header_size_shift); - __ andl(t1, Klass::_lh_header_size_mask); - __ addl(arr_size, t1); - __ addl(arr_size, MinObjAlignmentInBytesMask); // align up - __ andl(arr_size, ~MinObjAlignmentInBytesMask); + // generate side path, if any + compute_array_size(arr_size, klass, length, + have_scaled_length, need_multiply, + t1, t2, sasm, false); + __ bind(try_eden); + Label have_scaled_length_2, need_multiply_2; + compute_array_size(arr_size, klass, length, + have_scaled_length_2, need_multiply_2, + t1, t2, sasm, true); __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size __ initialize_header(obj, klass, length, t1, t2); - __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte))); - assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); - assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); - __ andl(t1, Klass::_lh_header_size_mask); - __ subl(arr_size, t1); // body length - __ addl(t1, obj); // body start - __ initialize_body(t1, arr_size, 0, t2); + initialize_array_body(obj, arr_size, t1, t2, sasm); __ verify_oop(obj); __ ret(0); + // generate side path, if any + compute_array_size(arr_size, klass, length, + have_scaled_length_2, need_multiply_2, + t1, t2, sasm, false); + __ bind(slow_path); }