769
770 __ testl(rax, rax); // have we deoptimized?
771 __ jcc(Assembler::equal, cont); // no
772
773 // Will reexecute. Proper return address is already on the stack we just restore
774 // registers, pop all of our frame but the return address and jump to the deopt blob
775 restore_live_registers(sasm);
776 __ leave();
777 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
778
779 __ bind(cont);
780 restore_live_registers(sasm);
781 __ leave();
782 __ ret(0);
783
784 return oop_maps;
785
786 }
787
788
789 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
790
791 // for better readability
792 const bool must_gc_arguments = true;
793 const bool dont_gc_arguments = false;
794
795 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
796 bool save_fpu_registers = true;
797
798 // stub code & info for the different stubs
799 OopMapSet* oop_maps = NULL;
800 switch (id) {
801 case forward_exception_id:
802 {
803 // we're handling an exception in the context of a compiled
804 // frame. The registers have been saved in the standard
805 // places. Perform an exception lookup in the caller and
806 // dispatch to the handler if found. Otherwise unwind and
807 // dispatch to the callers exception handler.
808
852 if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
853 UseTLAB && FastTLABRefill) {
854 Label slow_path;
855 Register obj_size = rcx;
856 Register t1 = rbx;
857 Register t2 = rsi;
858 assert_different_registers(klass, obj, obj_size, t1, t2);
859
860 __ pushl(rdi);
861 __ pushl(rbx);
862
863 if (id == fast_new_instance_init_check_id) {
864 // make sure the klass is initialized
865 __ cmpl(Address(klass, instanceKlass::init_state_offset_in_bytes() + sizeof(oopDesc)), instanceKlass::fully_initialized);
866 __ jcc(Assembler::notEqual, slow_path);
867 }
868
869 #ifdef ASSERT
870 // assert object can be fast path allocated
871 {
872 Label ok, not_ok;
873 __ movl(obj_size, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc)));
874 __ cmpl(obj_size, 0); // make sure it's an instance (LH > 0)
875 __ jcc(Assembler::lessEqual, not_ok);
876 __ testl(obj_size, Klass::_lh_instance_slow_path_bit);
877 __ jcc(Assembler::zero, ok);
878 __ bind(not_ok);
879 __ stop("assert(can be fast path allocated)");
880 __ should_not_reach_here();
881 __ bind(ok);
882 }
883 #endif // ASSERT
884
885 // if we got here then the TLAB allocation failed, so try
886 // refilling the TLAB or allocating directly from eden.
887 Label retry_tlab, try_eden;
888 __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy rdx (klass)
889
890 __ bind(retry_tlab);
891
892 // get the instance size
893 __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
894 __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path);
895 __ initialize_object(obj, klass, obj_size, 0, t1, t2);
896 __ verify_oop(obj);
897 __ popl(rbx);
898 __ popl(rdi);
899 __ ret(0);
900
901 __ bind(try_eden);
902 // get the instance size
903 __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
904 __ eden_allocate(obj, obj_size, 0, t1, slow_path);
905 __ initialize_object(obj, klass, obj_size, 0, t1, t2);
906 __ verify_oop(obj);
907 __ popl(rbx);
908 __ popl(rdi);
909 __ ret(0);
910
911 __ bind(slow_path);
912 __ popl(rbx);
913 __ popl(rdi);
914 }
915
916 __ enter();
917 OopMap* map = save_live_registers(sasm, 2);
918 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
919 oop_maps = new OopMapSet();
920 oop_maps->add_gc_map(call_offset, map);
921 restore_live_registers_except_rax(sasm);
922 __ verify_oop(obj);
923 __ leave();
924 __ ret(0);
925
926 // rax,: new instance
927 }
928
929 break;
930
947 #endif // TIERED
948
949 case new_type_array_id:
950 case new_object_array_id:
951 {
952 Register length = rbx; // Incoming
953 Register klass = rdx; // Incoming
954 Register obj = rax; // Result
955
956 if (id == new_type_array_id) {
957 __ set_info("new_type_array", dont_gc_arguments);
958 } else {
959 __ set_info("new_object_array", dont_gc_arguments);
960 }
961
962 #ifdef ASSERT
963 // assert object type is really an array of the proper kind
964 {
965 Label ok;
966 Register t0 = obj;
967 __ movl(t0, Address(klass, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc)));
968 __ sarl(t0, Klass::_lh_array_tag_shift);
969 int tag = ((id == new_type_array_id)
970 ? Klass::_lh_array_tag_type_value
971 : Klass::_lh_array_tag_obj_value);
972 __ cmpl(t0, tag);
973 __ jcc(Assembler::equal, ok);
974 __ stop("assert(is an array klass)");
975 __ should_not_reach_here();
976 __ bind(ok);
977 }
978 #endif // ASSERT
979
980 if (UseTLAB && FastTLABRefill) {
981 Register arr_size = rsi;
982 Register t1 = rcx; // must be rcx for use as shift count
983 Register t2 = rdi;
984 Label slow_path;
985 assert_different_registers(length, klass, obj, arr_size, t1, t2);
986
987 // check that array length is small enough for fast path.
988 __ cmpl(length, C1_MacroAssembler::max_array_allocation_length);
989 __ jcc(Assembler::above, slow_path);
990
991 // if we got here then the TLAB allocation failed, so try
992 // refilling the TLAB or allocating directly from eden.
993 Label retry_tlab, try_eden;
994 __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves rbx, & rdx
995
996 __ bind(retry_tlab);
997
998 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
999 __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
1000 __ movl(arr_size, length);
1001 assert(t1 == rcx, "fixed register usage");
1002 __ shll(arr_size /* by t1=rcx, mod 32 */);
1003 __ shrl(t1, Klass::_lh_header_size_shift);
1004 __ andl(t1, Klass::_lh_header_size_mask);
1005 __ addl(arr_size, t1);
1006 __ addl(arr_size, MinObjAlignmentInBytesMask); // align up
1007 __ andl(arr_size, ~MinObjAlignmentInBytesMask);
1008
1009 __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path); // preserves arr_size
1010
1011 __ initialize_header(obj, klass, length, t1, t2);
1012 __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte)));
1013 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
1014 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
1015 __ andl(t1, Klass::_lh_header_size_mask);
1016 __ subl(arr_size, t1); // body length
1017 __ addl(t1, obj); // body start
1018 __ initialize_body(t1, arr_size, 0, t2);
1019 __ verify_oop(obj);
1020 __ ret(0);
1021
1022 __ bind(try_eden);
1023 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
1024 __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
1025 __ movl(arr_size, length);
1026 assert(t1 == rcx, "fixed register usage");
1027 __ shll(arr_size /* by t1=rcx, mod 32 */);
1028 __ shrl(t1, Klass::_lh_header_size_shift);
1029 __ andl(t1, Klass::_lh_header_size_mask);
1030 __ addl(arr_size, t1);
1031 __ addl(arr_size, MinObjAlignmentInBytesMask); // align up
1032 __ andl(arr_size, ~MinObjAlignmentInBytesMask);
1033
1034 __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size
1035
1036 __ initialize_header(obj, klass, length, t1, t2);
1037 __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte)));
1038 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
1039 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
1040 __ andl(t1, Klass::_lh_header_size_mask);
1041 __ subl(arr_size, t1); // body length
1042 __ addl(t1, obj); // body start
1043 __ initialize_body(t1, arr_size, 0, t2);
1044 __ verify_oop(obj);
1045 __ ret(0);
1046
1047 __ bind(slow_path);
1048 }
1049
1050 __ enter();
1051 OopMap* map = save_live_registers(sasm, 3);
1052 int call_offset;
1053 if (id == new_type_array_id) {
1054 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
1055 } else {
1056 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
1057 }
1058
1059 oop_maps = new OopMapSet();
1060 oop_maps->add_gc_map(call_offset, map);
1061 restore_live_registers_except_rax(sasm);
1062
1063 __ verify_oop(obj);
1064 __ leave();
1065 __ ret(0);
1066
|
769
770 __ testl(rax, rax); // have we deoptimized?
771 __ jcc(Assembler::equal, cont); // no
772
773 // Will reexecute. Proper return address is already on the stack we just restore
774 // registers, pop all of our frame but the return address and jump to the deopt blob
775 restore_live_registers(sasm);
776 __ leave();
777 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
778
779 __ bind(cont);
780 restore_live_registers(sasm);
781 __ leave();
782 __ ret(0);
783
784 return oop_maps;
785
786 }
787
788
789 inline Address layout_helper_addr(Register klass) {
790 return Address(klass, (klassOopDesc::header_size() * HeapWordSize
791 + Klass::layout_helper_offset_in_bytes()));
792 }
793
794 static void compute_instance_size(Register obj_size, Register klass,
795 Label& have_obj_size, Label& size_is_variable,
796 StubAssembler* sasm, bool hot_part) {
797 assert_different_registers(obj_size, klass);
798
799 if (hot_part) {
800 __ movl(obj_size, layout_helper_addr(klass));
801 __ andl(obj_size, ~LayoutHelper::_size_low_mask);
802 if (MixedArrays) {
803 __ jcc(Assembler::negative, size_is_variable);
804 __ bind(have_obj_size);
805 }
806 } else {
807 if (MixedArrays) {
808 // Side path for fixing up the initial size of a variable object.
809 // Must round from int to object alignment, and clear high bits.
810 __ bind(size_is_variable);
811 __ addl(obj_size, LayoutHelper::_header_size_odd_mask);
812 __ andl(obj_size, (LayoutHelper::_header_size_mask & ~MinObjAlignmentInBytesMask));
813 __ jmp(have_obj_size);
814 }
815 }
816 }
817
818
819 static void compute_array_size(Register arr_size, Register klass, Register length,
820 Label& have_scaled_length, Label& need_multiply,
821 Register t1, Register t2,
822 StubAssembler* sasm, bool hot_part) {
823 assert_different_registers(arr_size, klass, length, t1, t2);
824
825 if (hot_part) {
826 // get the allocation size: round_up(hdr + length << (lh>>16 & 0x1F))
827 __ movl(t1, layout_helper_addr(klass));
828 __ movl(arr_size, length);
829 // int scale = (lh >> LayoutHelper::_element_size_shift);
830 __ movl(t1, t2); // spill layout helper
831 __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, no mask needed
832 // size_t arr_size = (array_length << scale);
833 assert(t1 == rcx, "fixed register usage");
834 __ shll(arr_size /* by t1=rcx, mod 32 */);
835 if (MixedArrays) {
836 // int sizem_ip = (scale & (LayoutHelper::_element_sizem_mask_ip));
837 __ andl(t1, LayoutHelper::_element_sizem_mask_ip);
838 __ jcc(Assembler::notZero, need_multiply);
839 __ bind(have_scaled_length);
840 }
841 // arr_size += Klass::layout_helper_header_size_in_bytes(lh);
842 assert(LayoutHelper::_header_size_shift == 0, "");
843 // shrl(t2, LayoutHelper::_header_size_shift);
844 __ andl(t2, LayoutHelper::_header_size_mask);
845 __ addl(arr_size, t2);
846 __ addl(arr_size, MinObjAlignmentInBytesMask); // align up
847 __ andl(arr_size, ~MinObjAlignmentInBytesMask);
848 } else {
849 if (MixedArrays) {
850 // Side path for scaling by a non-power-of-two array element size.
851 __ bind(need_multiply);
852 __ movl(arr_size, length); // reload; previous value was trash
853 // int sizem = (sizem_ip >> LayoutHelper::_element_scale_bits);
854 __ shrl(t1, LayoutHelper::_element_scale_bits);
855 // arr_size = (array_length * (1+sizem)) << scale;
856 __ addl(t1, 1);
857 assert(wordSize == jintSize, "else use imulq, shlq, etc.");
858 __ imull(arr_size, t1);
859 __ movl(t1, t2); // reload layout helper
860 __ shrl(t1, LayoutHelper::_element_size_shift); // lh>>16, mask needed
861 __ andl(t1, LayoutHelper::_element_scale_mask);
862 assert(t1 == rcx, "fixed register usage");
863 __ shll(arr_size /* by t1=rcx, mod 32 */);
864 __ jmp(have_scaled_length);
865 }
866 }
867 }
868
869 static void initialize_array_body(Register obj, Register arr_size, Register t1, Register t2, StubAssembler* sasm) {
870 assert_different_registers(obj, arr_size, t1, t2);
871
872 int min_header_size = arrayOopDesc::header_size(T_BYTE);
873 __ subl(arr_size, min_header_size); // body length
874 __ movl(t1, obj);
875 __ addl(t1, min_header_size); // body start
876 __ initialize_body(t1, arr_size, 0, t2);
877 }
878
879
880 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
881
882 // for better readability
883 const bool must_gc_arguments = true;
884 const bool dont_gc_arguments = false;
885
886 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
887 bool save_fpu_registers = true;
888
889 // stub code & info for the different stubs
890 OopMapSet* oop_maps = NULL;
891 switch (id) {
892 case forward_exception_id:
893 {
894 // we're handling an exception in the context of a compiled
895 // frame. The registers have been saved in the standard
896 // places. Perform an exception lookup in the caller and
897 // dispatch to the handler if found. Otherwise unwind and
898 // dispatch to the callers exception handler.
899
943 if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
944 UseTLAB && FastTLABRefill) {
945 Label slow_path;
946 Register obj_size = rcx;
947 Register t1 = rbx;
948 Register t2 = rsi;
949 assert_different_registers(klass, obj, obj_size, t1, t2);
950
951 __ pushl(rdi);
952 __ pushl(rbx);
953
954 if (id == fast_new_instance_init_check_id) {
955 // make sure the klass is initialized
956 __ cmpl(Address(klass, instanceKlass::init_state_offset_in_bytes() + sizeof(oopDesc)), instanceKlass::fully_initialized);
957 __ jcc(Assembler::notEqual, slow_path);
958 }
959
960 #ifdef ASSERT
961 // assert object can be fast path allocated
962 {
963 Label ok;
964 __ movl(obj_size, layout_helper_addr(klass));
965 __ testl(obj_size, LayoutHelper::_slow_path_low_bit);
966 __ jcc(Assembler::zero, ok);
967 __ stop("assert(can be fast path allocated)");
968 __ should_not_reach_here();
969 __ bind(ok);
970 }
971 #endif // ASSERT
972
973 // if we got here then the TLAB allocation failed, so try
974 // refilling the TLAB or allocating directly from eden.
975 Label retry_tlab, try_eden;
976 __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy rdx (klass)
977
978 __ bind(retry_tlab);
979
980 // get the instance size
981 Label have_obj_size, size_is_variable;
982 compute_instance_size(obj_size, klass,
983 have_obj_size, size_is_variable,
984 sasm, true);
985 __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path);
986 __ initialize_object(obj, klass, obj_size, 0, t1, t2);
987 __ verify_oop(obj);
988 __ popl(rbx);
989 __ popl(rdi);
990 __ ret(0);
991
992 // generate side path, if any
993 compute_instance_size(obj_size, klass,
994 have_obj_size, size_is_variable,
995 sasm, false);
996
997 __ bind(try_eden);
998 // get the instance size
999 Label have_obj_size_2, size_is_variable_2;
1000 compute_instance_size(obj_size, klass,
1001 have_obj_size_2, size_is_variable_2,
1002 sasm, true);
1003 __ eden_allocate(obj, obj_size, 0, t1, slow_path);
1004 __ initialize_object(obj, klass, obj_size, 0, t1, t2);
1005 __ verify_oop(obj);
1006 __ popl(rbx);
1007 __ popl(rdi);
1008 __ ret(0);
1009
1010 // generate side path, if any
1011 compute_instance_size(obj_size, klass,
1012 have_obj_size_2, size_is_variable_2,
1013 sasm, false);
1014
1015 __ bind(slow_path);
1016 __ popl(rbx);
1017 __ popl(rdi);
1018 }
1019
1020 __ enter();
1021 OopMap* map = save_live_registers(sasm, 2);
1022 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
1023 oop_maps = new OopMapSet();
1024 oop_maps->add_gc_map(call_offset, map);
1025 restore_live_registers_except_rax(sasm);
1026 __ verify_oop(obj);
1027 __ leave();
1028 __ ret(0);
1029
1030 // rax,: new instance
1031 }
1032
1033 break;
1034
1051 #endif // TIERED
1052
1053 case new_type_array_id:
1054 case new_object_array_id:
1055 {
1056 Register length = rbx; // Incoming
1057 Register klass = rdx; // Incoming
1058 Register obj = rax; // Result
1059
1060 if (id == new_type_array_id) {
1061 __ set_info("new_type_array", dont_gc_arguments);
1062 } else {
1063 __ set_info("new_object_array", dont_gc_arguments);
1064 }
1065
1066 #ifdef ASSERT
1067 // assert object type is really an array of the proper kind
1068 {
1069 Label ok;
1070 Register t0 = obj;
1071 __ movl(t0, layout_helper_addr(klass));
1072 int flag_shift = LayoutHelper::_flags_shift;
1073 flag_shift += LayoutHelper::_flags_low_bits; // shift out BasicType also
1074 int tag = LayoutHelper::for_array((id == new_type_array_id) ? T_BYTE : T_OBJECT).as_int();
1075 tag >>= flag_shift;
1076 __ sarl(t0, flag_shift);
1077 assert(id == new_object_array_id ||
1078 tag == (LayoutHelper::for_array(T_DOUBLE).as_int() >> flag_shift),
1079 "same tag for all type arrays");
1080 __ cmpl(t0, tag);
1081 __ jcc(Assembler::equal, ok);
1082 __ stop("assert(is an array klass)");
1083 __ should_not_reach_here();
1084 __ bind(ok);
1085 }
1086 #endif // ASSERT
1087
1088 if (UseTLAB && FastTLABRefill) {
1089 Register arr_size = rsi;
1090 Register t1 = rcx; // must be rcx for use as shift count
1091 Register t2 = rdi;
1092 Label slow_path;
1093 assert_different_registers(length, klass, obj, arr_size, t1, t2);
1094
1095 // check that array length is small enough for fast path.
1096 __ cmpl(length, C1_MacroAssembler::max_array_allocation_length);
1097 __ jcc(Assembler::above, slow_path);
1098
1099 // if we got here then the TLAB allocation failed, so try
1100 // refilling the TLAB or allocating directly from eden.
1101 Label retry_tlab, try_eden;
1102 __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves rbx, & rdx
1103
1104 __ bind(retry_tlab);
1105
1106 Label have_scaled_length, need_multiply;
1107 compute_array_size(arr_size, klass, length,
1108 have_scaled_length, need_multiply,
1109 t1, t2, sasm, true);
1110 __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path); // preserves arr_size
1111
1112 __ initialize_header(obj, klass, length, t1, t2);
1113 initialize_array_body(obj, arr_size, t1, t2, sasm);
1114 __ verify_oop(obj);
1115 __ ret(0);
1116
1117 // generate side path, if any
1118 compute_array_size(arr_size, klass, length,
1119 have_scaled_length, need_multiply,
1120 t1, t2, sasm, false);
1121
1122 __ bind(try_eden);
1123 Label have_scaled_length_2, need_multiply_2;
1124 compute_array_size(arr_size, klass, length,
1125 have_scaled_length_2, need_multiply_2,
1126 t1, t2, sasm, true);
1127 __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size
1128
1129 __ initialize_header(obj, klass, length, t1, t2);
1130 initialize_array_body(obj, arr_size, t1, t2, sasm);
1131 __ verify_oop(obj);
1132 __ ret(0);
1133
1134 // generate side path, if any
1135 compute_array_size(arr_size, klass, length,
1136 have_scaled_length_2, need_multiply_2,
1137 t1, t2, sasm, false);
1138
1139 __ bind(slow_path);
1140 }
1141
1142 __ enter();
1143 OopMap* map = save_live_registers(sasm, 3);
1144 int call_offset;
1145 if (id == new_type_array_id) {
1146 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
1147 } else {
1148 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
1149 }
1150
1151 oop_maps = new OopMapSet();
1152 oop_maps->add_gc_map(call_offset, map);
1153 restore_live_registers_except_rax(sasm);
1154
1155 __ verify_oop(obj);
1156 __ leave();
1157 __ ret(0);
1158
|