1027 __ movq(rsp, r12); // restore rsp
1028 __ popaq(); // pop registers (includes r12)
1029 __ ret(3 * wordSize); // pop caller saved stuff
1030
1031 return start;
1032 }
1033
1034 static address disjoint_byte_copy_entry;
1035 static address disjoint_short_copy_entry;
1036 static address disjoint_int_copy_entry;
1037 static address disjoint_long_copy_entry;
1038 static address disjoint_oop_copy_entry;
1039
1040 static address byte_copy_entry;
1041 static address short_copy_entry;
1042 static address int_copy_entry;
1043 static address long_copy_entry;
1044 static address oop_copy_entry;
1045
1046 static address checkcast_copy_entry;
1047
1048 //
1049 // Verify that a register contains clean 32-bits positive value
1050 // (high 32-bits are 0) so it could be used in 64-bits shifts.
1051 //
1052 // Input:
1053 // Rint - 32-bits value
1054 // Rtmp - scratch
1055 //
1056 void assert_clean_int(Register Rint, Register Rtmp) {
1057 #ifdef ASSERT
1058 Label L;
1059 assert_different_registers(Rtmp, Rint);
1060 __ movslq(Rtmp, Rint);
1061 __ cmpq(Rtmp, Rint);
1062 __ jcc(Assembler::equal, L);
1063 __ stop("high 32-bits of int value are not 0");
1064 __ bind(L);
1065 #endif
1066 }
2311 // to a long, int, short, or byte copy loop.
2312 //
2313 address generate_unsafe_copy(const char *name) {
2314
2315 Label L_long_aligned, L_int_aligned, L_short_aligned;
2316
2317 // Input registers (before setup_arg_regs)
2318 const Register from = c_rarg0; // source array address
2319 const Register to = c_rarg1; // destination array address
2320 const Register size = c_rarg2; // byte count (size_t)
2321
2322 // Register used as a temp
2323 const Register bits = rax; // test copy of low bits
2324
2325 __ align(CodeEntryAlignment);
2326 StubCodeMark mark(this, "StubRoutines", name);
2327 address start = __ pc();
2328
2329 __ enter(); // required for proper stackwalking of RuntimeStub frame
2330
2331 // bump this on entry, not on exit:
2332 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2333
2334 __ movq(bits, from);
2335 __ orq(bits, to);
2336 __ orq(bits, size);
2337
2338 __ testb(bits, BytesPerLong-1);
2339 __ jccb(Assembler::zero, L_long_aligned);
2340
2341 __ testb(bits, BytesPerInt-1);
2342 __ jccb(Assembler::zero, L_int_aligned);
2343
2344 __ testb(bits, BytesPerShort-1);
2345 __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2346
2347 __ BIND(L_short_aligned);
2348 __ shrq(size, LogBytesPerShort); // size => short_count
2349 __ jump(RuntimeAddress(short_copy_entry));
2350
2498
2499 __ load_klass(r10_src_klass, src);
2500 #ifdef ASSERT
2501 // assert(src->klass() != NULL);
2502 BLOCK_COMMENT("assert klasses not null");
2503 { Label L1, L2;
2504 __ testq(r10_src_klass, r10_src_klass);
2505 __ jcc(Assembler::notZero, L2); // it is broken if klass is NULL
2506 __ bind(L1);
2507 __ stop("broken null klass");
2508 __ bind(L2);
2509 __ load_klass(r9_dst_klass, dst);
2510 __ cmpq(r9_dst_klass, 0);
2511 __ jcc(Assembler::equal, L1); // this would be broken also
2512 BLOCK_COMMENT("assert done");
2513 }
2514 #endif
2515
2516 // Load layout helper (32-bits)
2517 //
2518 // |array_tag| | header_size | element_type | |log2_element_size|
2519 // 32 30 24 16 8 2 0
2520 //
2521 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2522 //
2523
2524 int lh_offset = klassOopDesc::header_size() * HeapWordSize +
2525 Klass::layout_helper_offset_in_bytes();
2526
2527 const Register rax_lh = rax; // layout helper
2528
2529 __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2530
2531 // Handle objArrays completely differently...
2532 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2533 __ cmpl(rax_lh, objArray_lh);
2534 __ jcc(Assembler::equal, L_objArray);
2535
2536 // if (src->klass() != dst->klass()) return -1;
2537 __ load_klass(r9_dst_klass, dst);
2538 __ cmpq(r10_src_klass, r9_dst_klass);
2539 __ jcc(Assembler::notEqual, L_failed);
2540
2541 // if (!src->is_Array()) return -1;
2542 __ cmpl(rax_lh, Klass::_lh_neutral_value);
2543 __ jcc(Assembler::greaterEqual, L_failed);
2544
2545 // At this point, it is known to be a typeArray (array_tag 0x3).
2546 #ifdef ASSERT
2547 { Label L;
2548 __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2549 __ jcc(Assembler::greaterEqual, L);
2550 __ stop("must be a primitive array");
2551 __ bind(L);
2552 }
2553 #endif
2554
2555 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2556 r10, L_failed);
2557
2558 // typeArrayKlass
2559 //
2560 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2561 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2562 //
2563
2564 const Register r10_offset = r10; // array offset
2565 const Register rax_elsize = rax_lh; // element size
2566
2567 __ movl(r10_offset, rax_lh);
2568 __ shrl(r10_offset, Klass::_lh_header_size_shift);
2569 __ andq(r10_offset, Klass::_lh_header_size_mask); // array_offset
2570 __ addq(src, r10_offset); // src array offset
2571 __ addq(dst, r10_offset); // dst array offset
2572 BLOCK_COMMENT("choose copy loop based on element size");
2573 __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2574
2575 // next registers should be set before the jump to corresponding stub
2576 const Register from = c_rarg0; // source array address
2577 const Register to = c_rarg1; // destination array address
2578 const Register count = c_rarg2; // elements count
2579
2580 // 'from', 'to', 'count' registers should be set in such order
2581 // since they are the same as 'src', 'src_pos', 'dst'.
2582
2583 __ BIND(L_copy_bytes);
2584 __ cmpl(rax_elsize, 0);
2585 __ jccb(Assembler::notEqual, L_copy_shorts);
2586 __ leaq(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2587 __ leaq(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2588 __ movslq(count, r11_length); // length
2589 __ jump(RuntimeAddress(byte_copy_entry));
2590
2591 __ BIND(L_copy_shorts);
2592 __ cmpl(rax_elsize, LogBytesPerShort);
2593 __ jccb(Assembler::notEqual, L_copy_ints);
2601 __ jccb(Assembler::notEqual, L_copy_longs);
2602 __ leaq(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2603 __ leaq(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2604 __ movslq(count, r11_length); // length
2605 __ jump(RuntimeAddress(int_copy_entry));
2606
2607 __ BIND(L_copy_longs);
2608 #ifdef ASSERT
2609 { Label L;
2610 __ cmpl(rax_elsize, LogBytesPerLong);
2611 __ jcc(Assembler::equal, L);
2612 __ stop("must be long copy, but elsize is wrong");
2613 __ bind(L);
2614 }
2615 #endif
2616 __ leaq(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2617 __ leaq(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2618 __ movslq(count, r11_length); // length
2619 __ jump(RuntimeAddress(long_copy_entry));
2620
2621 // objArrayKlass
2622 __ BIND(L_objArray);
2623 // live at this point: r10_src_klass, src[_pos], dst[_pos]
2624
2625 Label L_plain_copy, L_checkcast_copy;
2626 // test array classes for subtyping
2627 __ load_klass(r9_dst_klass, dst);
2628 __ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality
2629 __ jcc(Assembler::notEqual, L_checkcast_copy);
2630
2631 // Identically typed arrays can be copied without element-wise checks.
2632 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2633 r10, L_failed);
2634
2635 __ leaq(from, Address(src, src_pos, TIMES_OOP,
2636 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2637 __ leaq(to, Address(dst, dst_pos, TIMES_OOP,
2638 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2639 __ movslq(count, r11_length); // length
2640 __ BIND(L_plain_copy);
2641 __ jump(RuntimeAddress(oop_copy_entry));
2642
2643 __ BIND(L_checkcast_copy);
2644 // live at this point: r10_src_klass, !r11_length
2645 {
2646 // assert(r11_length == C_RARG4); // will reload from here
2647 Register r11_dst_klass = r11;
2648 __ load_klass(r11_dst_klass, dst);
2649
2650 // Before looking at dst.length, make sure dst is also an objArray.
2651 __ cmpl(Address(r11_dst_klass, lh_offset), objArray_lh);
2652 __ jcc(Assembler::notEqual, L_failed);
2653
2654 // It is safe to examine both src.length and dst.length.
2655 #ifndef _WIN64
2656 arraycopy_range_checks(src, src_pos, dst, dst_pos, C_RARG4,
2657 rax, L_failed);
2658 #else
2659 __ movl(r11_length, C_RARG4); // reload
2660 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2661 rax, L_failed);
2662 __ load_klass(r11_dst_klass, dst); // reload
2663 #endif
|
1027 __ movq(rsp, r12); // restore rsp
1028 __ popaq(); // pop registers (includes r12)
1029 __ ret(3 * wordSize); // pop caller saved stuff
1030
1031 return start;
1032 }
1033
1034 static address disjoint_byte_copy_entry;
1035 static address disjoint_short_copy_entry;
1036 static address disjoint_int_copy_entry;
1037 static address disjoint_long_copy_entry;
1038 static address disjoint_oop_copy_entry;
1039
1040 static address byte_copy_entry;
1041 static address short_copy_entry;
1042 static address int_copy_entry;
1043 static address long_copy_entry;
1044 static address oop_copy_entry;
1045
1046 static address checkcast_copy_entry;
1047 static address unsafe_copy_entry;
1048
1049 //
1050 // Verify that a register contains clean 32-bits positive value
1051 // (high 32-bits are 0) so it could be used in 64-bits shifts.
1052 //
1053 // Input:
1054 // Rint - 32-bits value
1055 // Rtmp - scratch
1056 //
1057 void assert_clean_int(Register Rint, Register Rtmp) {
1058 #ifdef ASSERT
1059 Label L;
1060 assert_different_registers(Rtmp, Rint);
1061 __ movslq(Rtmp, Rint);
1062 __ cmpq(Rtmp, Rint);
1063 __ jcc(Assembler::equal, L);
1064 __ stop("high 32-bits of int value are not 0");
1065 __ bind(L);
1066 #endif
1067 }
2312 // to a long, int, short, or byte copy loop.
2313 //
2314 address generate_unsafe_copy(const char *name) {
2315
2316 Label L_long_aligned, L_int_aligned, L_short_aligned;
2317
2318 // Input registers (before setup_arg_regs)
2319 const Register from = c_rarg0; // source array address
2320 const Register to = c_rarg1; // destination array address
2321 const Register size = c_rarg2; // byte count (size_t)
2322
2323 // Register used as a temp
2324 const Register bits = rax; // test copy of low bits
2325
2326 __ align(CodeEntryAlignment);
2327 StubCodeMark mark(this, "StubRoutines", name);
2328 address start = __ pc();
2329
2330 __ enter(); // required for proper stackwalking of RuntimeStub frame
2331
2332 checkcast_copy_entry = __ pc();
2333 BLOCK_COMMENT("Entry:");
2334
2335 // bump this on entry, not on exit:
2336 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2337
2338 __ movq(bits, from);
2339 __ orq(bits, to);
2340 __ orq(bits, size);
2341
2342 __ testb(bits, BytesPerLong-1);
2343 __ jccb(Assembler::zero, L_long_aligned);
2344
2345 __ testb(bits, BytesPerInt-1);
2346 __ jccb(Assembler::zero, L_int_aligned);
2347
2348 __ testb(bits, BytesPerShort-1);
2349 __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2350
2351 __ BIND(L_short_aligned);
2352 __ shrq(size, LogBytesPerShort); // size => short_count
2353 __ jump(RuntimeAddress(short_copy_entry));
2354
2502
2503 __ load_klass(r10_src_klass, src);
2504 #ifdef ASSERT
2505 // assert(src->klass() != NULL);
2506 BLOCK_COMMENT("assert klasses not null");
2507 { Label L1, L2;
2508 __ testq(r10_src_klass, r10_src_klass);
2509 __ jcc(Assembler::notZero, L2); // it is broken if klass is NULL
2510 __ bind(L1);
2511 __ stop("broken null klass");
2512 __ bind(L2);
2513 __ load_klass(r9_dst_klass, dst);
2514 __ cmpq(r9_dst_klass, 0);
2515 __ jcc(Assembler::equal, L1); // this would be broken also
2516 BLOCK_COMMENT("assert done");
2517 }
2518 #endif
2519
2520 // Load layout helper (32-bits)
2521 //
2522 // | array_tag | element_type | log2_element_size | header_size |
2523 // 32 28 24 16 0
2524 //
2525 // array_tag: typeArray = 0xF, objArray = 0xD, normal instance = 0x0
2526 //
2527
2528 int lh_offset = klassOopDesc::header_size() * HeapWordSize +
2529 Klass::layout_helper_offset_in_bytes();
2530
2531 const Register rax_lh = rax; // layout helper
2532
2533 __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2534
2535 // Handle objArrays completely differently...
2536 // This is necessary because they have covariant element types.
2537 // The next type check (for src.klass == dst.klass), can get a
2538 // false negative in the case of object arrays.
2539 jint objArray_lh = LayoutHelper::for_array(T_OBJECT).as_int();
2540 __ cmpl(rax_lh, objArray_lh);
2541 __ jcc(Assembler::equal, L_objArray);
2542
2543 // if (src->klass() != dst->klass()) return -1;
2544 __ load_klass(r9_dst_klass, dst);
2545 __ cmpq(r10_src_klass, r9_dst_klass);
2546 __ jcc(Assembler::notEqual, L_failed);
2547
2548 // if (!src->has_length_field()) return -1;
2549 __ cmpl(rax_lh, LayoutHelper::_neutral_value);
2550 __ jcc(Assembler::greaterEqual, L_failed);
2551
2552 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2553 r10, L_failed);
2554
2555 // typeArrayKlass
2556 //
2557 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2558 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2559 //
2560
2561 const Register r10_offset = r10; // array offset
2562 const Register rax_elsize = rax_lh; // element size
2563
2564 __ movl(r10_offset, rax_lh);
2565 assert(LayoutHelper::_header_size_shift == 0, "");
2566 __ andl(r10_offset, LayoutHelper::_header_size_mask); // array_offset
2567 __ addq(src, r10_offset); // src array offset
2568 __ addq(dst, r10_offset); // dst array offset
2569 BLOCK_COMMENT("choose copy loop based on element size");
2570 __ shrl(rax_lh, LayoutHelper::_element_size_shift);
2571 __ andl(rax_lh, LayoutHelper::_element_size_mask); // rax_lh -> rax_elsize
2572
2573 // next registers should be set before the jump to corresponding stub
2574 const Register from = c_rarg0; // source array address
2575 const Register to = c_rarg1; // destination array address
2576 const Register count = c_rarg2; // elements count
2577
2578 // 'from', 'to', 'count' registers should be set in such order
2579 // since they are the same as 'src', 'src_pos', 'dst'.
2580
2581 __ BIND(L_copy_bytes);
2582 __ cmpl(rax_elsize, 0);
2583 __ jccb(Assembler::notEqual, L_copy_shorts);
2584 __ leaq(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2585 __ leaq(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2586 __ movslq(count, r11_length); // length
2587 __ jump(RuntimeAddress(byte_copy_entry));
2588
2589 __ BIND(L_copy_shorts);
2590 __ cmpl(rax_elsize, LogBytesPerShort);
2591 __ jccb(Assembler::notEqual, L_copy_ints);
2599 __ jccb(Assembler::notEqual, L_copy_longs);
2600 __ leaq(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2601 __ leaq(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2602 __ movslq(count, r11_length); // length
2603 __ jump(RuntimeAddress(int_copy_entry));
2604
2605 __ BIND(L_copy_longs);
2606 #ifdef ASSERT
2607 { Label L;
2608 __ cmpl(rax_elsize, LogBytesPerLong);
2609 __ jcc(Assembler::equal, L);
2610 __ stop("must be long copy, but elsize is wrong");
2611 __ bind(L);
2612 }
2613 #endif
2614 __ leaq(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2615 __ leaq(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2616 __ movslq(count, r11_length); // length
2617 __ jump(RuntimeAddress(long_copy_entry));
2618
2619 Label L_different_objArray, L_plain_copy, L_checkcast_copy;
2620
2621 // objArrayKlass
2622 __ BIND(L_objArray);
2623 // live at this point: r10_src_klass, src[_pos], dst[_pos]
2624
2625 // test array classes for subtyping
2626 __ load_klass(r9_dst_klass, dst);
2627 __ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality
2628 __ jcc(Assembler::notEqual, L_different_objArray);
2629
2630 // Identically typed arrays can be copied without element-wise checks.
2631 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2632 r10, L_failed);
2633
2634 __ leaq(from, Address(src, src_pos, TIMES_OOP,
2635 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2636 __ leaq(to, Address(dst, dst_pos, TIMES_OOP,
2637 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2638 __ movslq(count, r11_length); // length
2639 __ BIND(L_plain_copy);
2640 __ jump(RuntimeAddress(oop_copy_entry));
2641
2642 __ BIND(L_different_objArray);
2643 // live at this point: r10_src_klass, !r11_length
2644 {
2645 // assert(r11_length == C_RARG4); // will reload from here
2646 Register r11_dst_klass = r11;
2647 __ load_klass(r11_dst_klass, dst);
2648
2649 // Before looking at dst.length, make sure dst is also an objArray.
2650 __ cmpl(Address(r11_dst_klass, lh_offset), objArray_lh);
2651 __ jcc(Assembler::notEqual, L_failed);
2652
2653 // It is safe to examine both src.length and dst.length.
2654 #ifndef _WIN64
2655 arraycopy_range_checks(src, src_pos, dst, dst_pos, C_RARG4,
2656 rax, L_failed);
2657 #else
2658 __ movl(r11_length, C_RARG4); // reload
2659 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2660 rax, L_failed);
2661 __ load_klass(r11_dst_klass, dst); // reload
2662 #endif
|