1 /*
2 * Copyright 1997-2008 Sun Microsystems, Inc. All Rights Reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
20 * CA 95054 USA or visit www.sun.com if you need additional information or
21 * have any questions.
22 *
23 */
24
25 #include "incls/_precompiled.incl"
26 #include "incls/_stubGenerator_sparc.cpp.incl"
27
28 // Declaration and definition of StubGenerator (no .hpp file).
29 // For a more detailed description of the stub routine structure
30 // see the comment in stubRoutines.hpp.
31
32 #define __ _masm->
33
34 #ifdef PRODUCT
35 #define BLOCK_COMMENT(str) /* nothing */
36 #else
37 #define BLOCK_COMMENT(str) __ block_comment(str)
38 #endif
39
40 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
41
42 // Note: The register L7 is used as L7_thread_cache, and may not be used
43 // any other way within this module.
44
45
46 static const Register& Lstub_temp = L2;
47
48 // -------------------------------------------------------------------------------------------------------------------------
49 // Stub Code definitions
50
51 static address handle_unsafe_access() {
52 JavaThread* thread = JavaThread::current();
53 address pc = thread->saved_exception_pc();
54 address npc = thread->saved_exception_npc();
55 // pc is the instruction which we must emulate
56 // doing a no-op is fine: return garbage from the load
57
58 // request an async exception
59 thread->set_pending_unsafe_access_error();
60
61 // return address of next instruction to execute
62 return npc;
63 }
64
65 class StubGenerator: public StubCodeGenerator {
66 private:
67
68 #ifdef PRODUCT
69 #define inc_counter_np(a,b,c) (0)
70 #else
71 void inc_counter_np_(int& counter, Register t1, Register t2) {
72 Address counter_addr(t2, (address) &counter);
73 __ sethi(counter_addr);
74 __ ld(counter_addr, t1);
75 __ inc(t1);
76 __ st(t1, counter_addr);
77 }
78 #define inc_counter_np(counter, t1, t2) \
79 BLOCK_COMMENT("inc_counter " #counter); \
80 inc_counter_np_(counter, t1, t2);
81 #endif
82
83 //----------------------------------------------------------------------------------------------------
84 // Call stubs are used to call Java from C
85
86 address generate_call_stub(address& return_pc) {
87 StubCodeMark mark(this, "StubRoutines", "call_stub");
88 address start = __ pc();
89
90 // Incoming arguments:
91 //
92 // o0 : call wrapper address
93 // o1 : result (address)
94 // o2 : result type
95 // o3 : method
96 // o4 : (interpreter) entry point
97 // o5 : parameters (address)
98 // [sp + 0x5c]: parameter size (in words)
99 // [sp + 0x60]: thread
100 //
101 // +---------------+ <--- sp + 0
102 // | |
103 // . reg save area .
104 // | |
105 // +---------------+ <--- sp + 0x40
106 // | |
107 // . extra 7 slots .
108 // | |
109 // +---------------+ <--- sp + 0x5c
110 // | param. size |
111 // +---------------+ <--- sp + 0x60
112 // | thread |
113 // +---------------+
114 // | |
115
116 // note: if the link argument position changes, adjust
117 // the code in frame::entry_frame_call_wrapper()
118
119 const Argument link = Argument(0, false); // used only for GC
120 const Argument result = Argument(1, false);
121 const Argument result_type = Argument(2, false);
122 const Argument method = Argument(3, false);
123 const Argument entry_point = Argument(4, false);
124 const Argument parameters = Argument(5, false);
125 const Argument parameter_size = Argument(6, false);
126 const Argument thread = Argument(7, false);
127
128 // setup thread register
129 __ ld_ptr(thread.as_address(), G2_thread);
130 __ reinit_heapbase();
131
132 #ifdef ASSERT
133 // make sure we have no pending exceptions
134 { const Register t = G3_scratch;
135 Label L;
136 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), t);
137 __ br_null(t, false, Assembler::pt, L);
138 __ delayed()->nop();
139 __ stop("StubRoutines::call_stub: entered with pending exception");
140 __ bind(L);
141 }
142 #endif
143
144 // create activation frame & allocate space for parameters
145 { const Register t = G3_scratch;
146 __ ld_ptr(parameter_size.as_address(), t); // get parameter size (in words)
147 __ add(t, frame::memory_parameter_word_sp_offset, t); // add space for save area (in words)
148 __ round_to(t, WordsPerLong); // make sure it is multiple of 2 (in words)
149 __ sll(t, Interpreter::logStackElementSize(), t); // compute number of bytes
150 __ neg(t); // negate so it can be used with save
151 __ save(SP, t, SP); // setup new frame
152 }
153
154 // +---------------+ <--- sp + 0
155 // | |
156 // . reg save area .
157 // | |
158 // +---------------+ <--- sp + 0x40
159 // | |
160 // . extra 7 slots .
161 // | |
162 // +---------------+ <--- sp + 0x5c
163 // | empty slot | (only if parameter size is even)
164 // +---------------+
165 // | |
166 // . parameters .
167 // | |
168 // +---------------+ <--- fp + 0
169 // | |
170 // . reg save area .
171 // | |
172 // +---------------+ <--- fp + 0x40
173 // | |
174 // . extra 7 slots .
175 // | |
176 // +---------------+ <--- fp + 0x5c
177 // | param. size |
178 // +---------------+ <--- fp + 0x60
179 // | thread |
180 // +---------------+
181 // | |
182
183 // pass parameters if any
184 BLOCK_COMMENT("pass parameters if any");
185 { const Register src = parameters.as_in().as_register();
186 const Register dst = Lentry_args;
187 const Register tmp = G3_scratch;
188 const Register cnt = G4_scratch;
189
190 // test if any parameters & setup of Lentry_args
191 Label exit;
192 __ ld_ptr(parameter_size.as_in().as_address(), cnt); // parameter counter
193 __ add( FP, STACK_BIAS, dst );
194 __ tst(cnt);
195 __ br(Assembler::zero, false, Assembler::pn, exit);
196 __ delayed()->sub(dst, BytesPerWord, dst); // setup Lentry_args
197
198 // copy parameters if any
199 Label loop;
200 __ BIND(loop);
201 // Store tag first.
202 if (TaggedStackInterpreter) {
203 __ ld_ptr(src, 0, tmp);
204 __ add(src, BytesPerWord, src); // get next
205 __ st_ptr(tmp, dst, Interpreter::tag_offset_in_bytes());
206 }
207 // Store parameter value
208 __ ld_ptr(src, 0, tmp);
209 __ add(src, BytesPerWord, src);
210 __ st_ptr(tmp, dst, Interpreter::value_offset_in_bytes());
211 __ deccc(cnt);
212 __ br(Assembler::greater, false, Assembler::pt, loop);
213 __ delayed()->sub(dst, Interpreter::stackElementSize(), dst);
214
215 // done
216 __ BIND(exit);
217 }
218
219 // setup parameters, method & call Java function
220 #ifdef ASSERT
221 // layout_activation_impl checks it's notion of saved SP against
222 // this register, so if this changes update it as well.
223 const Register saved_SP = Lscratch;
224 __ mov(SP, saved_SP); // keep track of SP before call
225 #endif
226
227 // setup parameters
228 const Register t = G3_scratch;
229 __ ld_ptr(parameter_size.as_in().as_address(), t); // get parameter size (in words)
230 __ sll(t, Interpreter::logStackElementSize(), t); // compute number of bytes
231 __ sub(FP, t, Gargs); // setup parameter pointer
232 #ifdef _LP64
233 __ add( Gargs, STACK_BIAS, Gargs ); // Account for LP64 stack bias
234 #endif
235 __ mov(SP, O5_savedSP);
236
237
238 // do the call
239 //
240 // the following register must be setup:
241 //
242 // G2_thread
243 // G5_method
244 // Gargs
245 BLOCK_COMMENT("call Java function");
246 __ jmpl(entry_point.as_in().as_register(), G0, O7);
247 __ delayed()->mov(method.as_in().as_register(), G5_method); // setup method
248
249 BLOCK_COMMENT("call_stub_return_address:");
250 return_pc = __ pc();
251
252 // The callee, if it wasn't interpreted, can return with SP changed so
253 // we can no longer assert of change of SP.
254
255 // store result depending on type
256 // (everything that is not T_OBJECT, T_LONG, T_FLOAT, or T_DOUBLE
257 // is treated as T_INT)
258 { const Register addr = result .as_in().as_register();
259 const Register type = result_type.as_in().as_register();
260 Label is_long, is_float, is_double, is_object, exit;
261 __ cmp(type, T_OBJECT); __ br(Assembler::equal, false, Assembler::pn, is_object);
262 __ delayed()->cmp(type, T_FLOAT); __ br(Assembler::equal, false, Assembler::pn, is_float);
263 __ delayed()->cmp(type, T_DOUBLE); __ br(Assembler::equal, false, Assembler::pn, is_double);
264 __ delayed()->cmp(type, T_LONG); __ br(Assembler::equal, false, Assembler::pn, is_long);
265 __ delayed()->nop();
266
267 // store int result
268 __ st(O0, addr, G0);
269
270 __ BIND(exit);
271 __ ret();
272 __ delayed()->restore();
273
274 __ BIND(is_object);
275 __ ba(false, exit);
276 __ delayed()->st_ptr(O0, addr, G0);
277
278 __ BIND(is_float);
279 __ ba(false, exit);
280 __ delayed()->stf(FloatRegisterImpl::S, F0, addr, G0);
281
282 __ BIND(is_double);
283 __ ba(false, exit);
284 __ delayed()->stf(FloatRegisterImpl::D, F0, addr, G0);
285
286 __ BIND(is_long);
287 #ifdef _LP64
288 __ ba(false, exit);
289 __ delayed()->st_long(O0, addr, G0); // store entire long
290 #else
291 #if defined(COMPILER2)
292 // All return values are where we want them, except for Longs. C2 returns
293 // longs in G1 in the 32-bit build whereas the interpreter wants them in O0/O1.
294 // Since the interpreter will return longs in G1 and O0/O1 in the 32bit
295 // build we simply always use G1.
296 // Note: I tried to make c2 return longs in O0/O1 and G1 so we wouldn't have to
297 // do this here. Unfortunately if we did a rethrow we'd see an machepilog node
298 // first which would move g1 -> O0/O1 and destroy the exception we were throwing.
299
300 __ ba(false, exit);
301 __ delayed()->stx(G1, addr, G0); // store entire long
302 #else
303 __ st(O1, addr, BytesPerInt);
304 __ ba(false, exit);
305 __ delayed()->st(O0, addr, G0);
306 #endif /* COMPILER2 */
307 #endif /* _LP64 */
308 }
309 return start;
310 }
311
312
313 //----------------------------------------------------------------------------------------------------
314 // Return point for a Java call if there's an exception thrown in Java code.
315 // The exception is caught and transformed into a pending exception stored in
316 // JavaThread that can be tested from within the VM.
317 //
318 // Oexception: exception oop
319
320 address generate_catch_exception() {
321 StubCodeMark mark(this, "StubRoutines", "catch_exception");
322
323 address start = __ pc();
324 // verify that thread corresponds
325 __ verify_thread();
326
327 const Register& temp_reg = Gtemp;
328 Address pending_exception_addr (G2_thread, 0, in_bytes(Thread::pending_exception_offset()));
329 Address exception_file_offset_addr(G2_thread, 0, in_bytes(Thread::exception_file_offset ()));
330 Address exception_line_offset_addr(G2_thread, 0, in_bytes(Thread::exception_line_offset ()));
331
332 // set pending exception
333 __ verify_oop(Oexception);
334 __ st_ptr(Oexception, pending_exception_addr);
335 __ set((intptr_t)__FILE__, temp_reg);
336 __ st_ptr(temp_reg, exception_file_offset_addr);
337 __ set((intptr_t)__LINE__, temp_reg);
338 __ st(temp_reg, exception_line_offset_addr);
339
340 // complete return to VM
341 assert(StubRoutines::_call_stub_return_address != NULL, "must have been generated before");
342
343 Address stub_ret(temp_reg, StubRoutines::_call_stub_return_address);
344 __ jump_to(stub_ret);
345 __ delayed()->nop();
346
347 return start;
348 }
349
350
351 //----------------------------------------------------------------------------------------------------
352 // Continuation point for runtime calls returning with a pending exception
353 // The pending exception check happened in the runtime or native call stub
354 // The pending exception in Thread is converted into a Java-level exception
355 //
356 // Contract with Java-level exception handler: O0 = exception
357 // O1 = throwing pc
358
359 address generate_forward_exception() {
360 StubCodeMark mark(this, "StubRoutines", "forward_exception");
361 address start = __ pc();
362
363 // Upon entry, O7 has the return address returning into Java
364 // (interpreted or compiled) code; i.e. the return address
365 // becomes the throwing pc.
366
367 const Register& handler_reg = Gtemp;
368
369 Address exception_addr (G2_thread, 0, in_bytes(Thread::pending_exception_offset()));
370
371 #ifdef ASSERT
372 // make sure that this code is only executed if there is a pending exception
373 { Label L;
374 __ ld_ptr(exception_addr, Gtemp);
375 __ br_notnull(Gtemp, false, Assembler::pt, L);
376 __ delayed()->nop();
377 __ stop("StubRoutines::forward exception: no pending exception (1)");
378 __ bind(L);
379 }
380 #endif
381
382 // compute exception handler into handler_reg
383 __ get_thread();
384 __ ld_ptr(exception_addr, Oexception);
385 __ verify_oop(Oexception);
386 __ save_frame(0); // compensates for compiler weakness
387 __ add(O7->after_save(), frame::pc_return_offset, Lscratch); // save the issuing PC
388 BLOCK_COMMENT("call exception_handler_for_return_address");
389 __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), Lscratch);
390 __ mov(O0, handler_reg);
391 __ restore(); // compensates for compiler weakness
392
393 __ ld_ptr(exception_addr, Oexception);
394 __ add(O7, frame::pc_return_offset, Oissuing_pc); // save the issuing PC
395
396 #ifdef ASSERT
397 // make sure exception is set
398 { Label L;
399 __ br_notnull(Oexception, false, Assembler::pt, L);
400 __ delayed()->nop();
401 __ stop("StubRoutines::forward exception: no pending exception (2)");
402 __ bind(L);
403 }
404 #endif
405 // jump to exception handler
406 __ jmp(handler_reg, 0);
407 // clear pending exception
408 __ delayed()->st_ptr(G0, exception_addr);
409
410 return start;
411 }
412
413
414 //------------------------------------------------------------------------------------------------------------------------
415 // Continuation point for throwing of implicit exceptions that are not handled in
416 // the current activation. Fabricates an exception oop and initiates normal
417 // exception dispatching in this frame. Only callee-saved registers are preserved
418 // (through the normal register window / RegisterMap handling).
419 // If the compiler needs all registers to be preserved between the fault
420 // point and the exception handler then it must assume responsibility for that in
421 // AbstractCompiler::continuation_for_implicit_null_exception or
422 // continuation_for_implicit_division_by_zero_exception. All other implicit
423 // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
424 // either at call sites or otherwise assume that stack unwinding will be initiated,
425 // so caller saved registers were assumed volatile in the compiler.
426
427 // Note that we generate only this stub into a RuntimeStub, because it needs to be
428 // properly traversed and ignored during GC, so we change the meaning of the "__"
429 // macro within this method.
430 #undef __
431 #define __ masm->
432
433 address generate_throw_exception(const char* name, address runtime_entry, bool restore_saved_exception_pc) {
434 #ifdef ASSERT
435 int insts_size = VerifyThread ? 1 * K : 600;
436 #else
437 int insts_size = VerifyThread ? 1 * K : 256;
438 #endif /* ASSERT */
439 int locs_size = 32;
440
441 CodeBuffer code(name, insts_size, locs_size);
442 MacroAssembler* masm = new MacroAssembler(&code);
443
444 __ verify_thread();
445
446 // This is an inlined and slightly modified version of call_VM
447 // which has the ability to fetch the return PC out of thread-local storage
448 __ assert_not_delayed();
449
450 // Note that we always push a frame because on the SPARC
451 // architecture, for all of our implicit exception kinds at call
452 // sites, the implicit exception is taken before the callee frame
453 // is pushed.
454 __ save_frame(0);
455
456 int frame_complete = __ offset();
457
458 if (restore_saved_exception_pc) {
459 Address saved_exception_pc(G2_thread, 0, in_bytes(JavaThread::saved_exception_pc_offset()));
460 __ ld_ptr(saved_exception_pc, I7);
461 __ sub(I7, frame::pc_return_offset, I7);
462 }
463
464 // Note that we always have a runtime stub frame on the top of stack by this point
465 Register last_java_sp = SP;
466 // 64-bit last_java_sp is biased!
467 __ set_last_Java_frame(last_java_sp, G0);
468 if (VerifyThread) __ mov(G2_thread, O0); // about to be smashed; pass early
469 __ save_thread(noreg);
470 // do the call
471 BLOCK_COMMENT("call runtime_entry");
472 __ call(runtime_entry, relocInfo::runtime_call_type);
473 if (!VerifyThread)
474 __ delayed()->mov(G2_thread, O0); // pass thread as first argument
475 else
476 __ delayed()->nop(); // (thread already passed)
477 __ restore_thread(noreg);
478 __ reset_last_Java_frame();
479
480 // check for pending exceptions. use Gtemp as scratch register.
481 #ifdef ASSERT
482 Label L;
483
484 Address exception_addr(G2_thread, 0, in_bytes(Thread::pending_exception_offset()));
485 Register scratch_reg = Gtemp;
486 __ ld_ptr(exception_addr, scratch_reg);
487 __ br_notnull(scratch_reg, false, Assembler::pt, L);
488 __ delayed()->nop();
489 __ should_not_reach_here();
490 __ bind(L);
491 #endif // ASSERT
492 BLOCK_COMMENT("call forward_exception_entry");
493 __ call(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
494 // we use O7 linkage so that forward_exception_entry has the issuing PC
495 __ delayed()->restore();
496
497 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, masm->total_frame_size_in_bytes(0), NULL, false);
498 return stub->entry_point();
499 }
500
501 #undef __
502 #define __ _masm->
503
504
505 // Generate a routine that sets all the registers so we
506 // can tell if the stop routine prints them correctly.
507 address generate_test_stop() {
508 StubCodeMark mark(this, "StubRoutines", "test_stop");
509 address start = __ pc();
510
511 int i;
512
513 __ save_frame(0);
514
515 static jfloat zero = 0.0, one = 1.0;
516
517 // put addr in L0, then load through L0 to F0
518 __ set((intptr_t)&zero, L0); __ ldf( FloatRegisterImpl::S, L0, 0, F0);
519 __ set((intptr_t)&one, L0); __ ldf( FloatRegisterImpl::S, L0, 0, F1); // 1.0 to F1
520
521 // use add to put 2..18 in F2..F18
522 for ( i = 2; i <= 18; ++i ) {
523 __ fadd( FloatRegisterImpl::S, F1, as_FloatRegister(i-1), as_FloatRegister(i));
524 }
525
526 // Now put double 2 in F16, double 18 in F18
527 __ ftof( FloatRegisterImpl::S, FloatRegisterImpl::D, F2, F16 );
528 __ ftof( FloatRegisterImpl::S, FloatRegisterImpl::D, F18, F18 );
529
530 // use add to put 20..32 in F20..F32
531 for (i = 20; i < 32; i += 2) {
532 __ fadd( FloatRegisterImpl::D, F16, as_FloatRegister(i-2), as_FloatRegister(i));
533 }
534
535 // put 0..7 in i's, 8..15 in l's, 16..23 in o's, 24..31 in g's
536 for ( i = 0; i < 8; ++i ) {
537 if (i < 6) {
538 __ set( i, as_iRegister(i));
539 __ set(16 + i, as_oRegister(i));
540 __ set(24 + i, as_gRegister(i));
541 }
542 __ set( 8 + i, as_lRegister(i));
543 }
544
545 __ stop("testing stop");
546
547
548 __ ret();
549 __ delayed()->restore();
550
551 return start;
552 }
553
554
555 address generate_stop_subroutine() {
556 StubCodeMark mark(this, "StubRoutines", "stop_subroutine");
557 address start = __ pc();
558
559 __ stop_subroutine();
560
561 return start;
562 }
563
564 address generate_flush_callers_register_windows() {
565 StubCodeMark mark(this, "StubRoutines", "flush_callers_register_windows");
566 address start = __ pc();
567
568 __ flush_windows();
569 __ retl(false);
570 __ delayed()->add( FP, STACK_BIAS, O0 );
571 // The returned value must be a stack pointer whose register save area
572 // is flushed, and will stay flushed while the caller executes.
573
574 return start;
575 }
576
577 // Helper functions for v8 atomic operations.
578 //
579 void get_v8_oop_lock_ptr(Register lock_ptr_reg, Register mark_oop_reg, Register scratch_reg) {
580 if (mark_oop_reg == noreg) {
581 address lock_ptr = (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr();
582 __ set((intptr_t)lock_ptr, lock_ptr_reg);
583 } else {
584 assert(scratch_reg != noreg, "just checking");
585 address lock_ptr = (address)StubRoutines::Sparc::_v8_oop_lock_cache;
586 __ set((intptr_t)lock_ptr, lock_ptr_reg);
587 __ and3(mark_oop_reg, StubRoutines::Sparc::v8_oop_lock_mask_in_place, scratch_reg);
588 __ add(lock_ptr_reg, scratch_reg, lock_ptr_reg);
589 }
590 }
591
592 void generate_v8_lock_prologue(Register lock_reg, Register lock_ptr_reg, Register yield_reg, Label& retry, Label& dontyield, Register mark_oop_reg = noreg, Register scratch_reg = noreg) {
593
594 get_v8_oop_lock_ptr(lock_ptr_reg, mark_oop_reg, scratch_reg);
595 __ set(StubRoutines::Sparc::locked, lock_reg);
596 // Initialize yield counter
597 __ mov(G0,yield_reg);
598
599 __ BIND(retry);
600 __ cmp(yield_reg, V8AtomicOperationUnderLockSpinCount);
601 __ br(Assembler::less, false, Assembler::pt, dontyield);
602 __ delayed()->nop();
603
604 // This code can only be called from inside the VM, this
605 // stub is only invoked from Atomic::add(). We do not
606 // want to use call_VM, because _last_java_sp and such
607 // must already be set.
608 //
609 // Save the regs and make space for a C call
610 __ save(SP, -96, SP);
611 __ save_all_globals_into_locals();
612 BLOCK_COMMENT("call os::naked_sleep");
613 __ call(CAST_FROM_FN_PTR(address, os::naked_sleep));
614 __ delayed()->nop();
615 __ restore_globals_from_locals();
616 __ restore();
617 // reset the counter
618 __ mov(G0,yield_reg);
619
620 __ BIND(dontyield);
621
622 // try to get lock
623 __ swap(lock_ptr_reg, 0, lock_reg);
624
625 // did we get the lock?
626 __ cmp(lock_reg, StubRoutines::Sparc::unlocked);
627 __ br(Assembler::notEqual, true, Assembler::pn, retry);
628 __ delayed()->add(yield_reg,1,yield_reg);
629
630 // yes, got lock. do the operation here.
631 }
632
633 void generate_v8_lock_epilogue(Register lock_reg, Register lock_ptr_reg, Register yield_reg, Label& retry, Label& dontyield, Register mark_oop_reg = noreg, Register scratch_reg = noreg) {
634 __ st(lock_reg, lock_ptr_reg, 0); // unlock
635 }
636
637 // Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest).
638 //
639 // Arguments :
640 //
641 // exchange_value: O0
642 // dest: O1
643 //
644 // Results:
645 //
646 // O0: the value previously stored in dest
647 //
648 address generate_atomic_xchg() {
649 StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
650 address start = __ pc();
651
652 if (UseCASForSwap) {
653 // Use CAS instead of swap, just in case the MP hardware
654 // prefers to work with just one kind of synch. instruction.
655 Label retry;
656 __ BIND(retry);
657 __ mov(O0, O3); // scratch copy of exchange value
658 __ ld(O1, 0, O2); // observe the previous value
659 // try to replace O2 with O3
660 __ cas_under_lock(O1, O2, O3,
661 (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr(),false);
662 __ cmp(O2, O3);
663 __ br(Assembler::notEqual, false, Assembler::pn, retry);
664 __ delayed()->nop();
665
666 __ retl(false);
667 __ delayed()->mov(O2, O0); // report previous value to caller
668
669 } else {
670 if (VM_Version::v9_instructions_work()) {
671 __ retl(false);
672 __ delayed()->swap(O1, 0, O0);
673 } else {
674 const Register& lock_reg = O2;
675 const Register& lock_ptr_reg = O3;
676 const Register& yield_reg = O4;
677
678 Label retry;
679 Label dontyield;
680
681 generate_v8_lock_prologue(lock_reg, lock_ptr_reg, yield_reg, retry, dontyield);
682 // got the lock, do the swap
683 __ swap(O1, 0, O0);
684
685 generate_v8_lock_epilogue(lock_reg, lock_ptr_reg, yield_reg, retry, dontyield);
686 __ retl(false);
687 __ delayed()->nop();
688 }
689 }
690
691 return start;
692 }
693
694
695 // Support for jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value)
696 //
697 // Arguments :
698 //
699 // exchange_value: O0
700 // dest: O1
701 // compare_value: O2
702 //
703 // Results:
704 //
705 // O0: the value previously stored in dest
706 //
707 // Overwrites (v8): O3,O4,O5
708 //
709 address generate_atomic_cmpxchg() {
710 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
711 address start = __ pc();
712
713 // cmpxchg(dest, compare_value, exchange_value)
714 __ cas_under_lock(O1, O2, O0,
715 (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr(),false);
716 __ retl(false);
717 __ delayed()->nop();
718
719 return start;
720 }
721
722 // Support for jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value)
723 //
724 // Arguments :
725 //
726 // exchange_value: O1:O0
727 // dest: O2
728 // compare_value: O4:O3
729 //
730 // Results:
731 //
732 // O1:O0: the value previously stored in dest
733 //
734 // This only works on V9, on V8 we don't generate any
735 // code and just return NULL.
736 //
737 // Overwrites: G1,G2,G3
738 //
739 address generate_atomic_cmpxchg_long() {
740 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
741 address start = __ pc();
742
743 if (!VM_Version::supports_cx8())
744 return NULL;;
745 __ sllx(O0, 32, O0);
746 __ srl(O1, 0, O1);
747 __ or3(O0,O1,O0); // O0 holds 64-bit value from compare_value
748 __ sllx(O3, 32, O3);
749 __ srl(O4, 0, O4);
750 __ or3(O3,O4,O3); // O3 holds 64-bit value from exchange_value
751 __ casx(O2, O3, O0);
752 __ srl(O0, 0, O1); // unpacked return value in O1:O0
753 __ retl(false);
754 __ delayed()->srlx(O0, 32, O0);
755
756 return start;
757 }
758
759
760 // Support for jint Atomic::add(jint add_value, volatile jint* dest).
761 //
762 // Arguments :
763 //
764 // add_value: O0 (e.g., +1 or -1)
765 // dest: O1
766 //
767 // Results:
768 //
769 // O0: the new value stored in dest
770 //
771 // Overwrites (v9): O3
772 // Overwrites (v8): O3,O4,O5
773 //
774 address generate_atomic_add() {
775 StubCodeMark mark(this, "StubRoutines", "atomic_add");
776 address start = __ pc();
777 __ BIND(_atomic_add_stub);
778
779 if (VM_Version::v9_instructions_work()) {
780 Label(retry);
781 __ BIND(retry);
782
783 __ lduw(O1, 0, O2);
784 __ add(O0, O2, O3);
785 __ cas(O1, O2, O3);
786 __ cmp( O2, O3);
787 __ br(Assembler::notEqual, false, Assembler::pn, retry);
788 __ delayed()->nop();
789 __ retl(false);
790 __ delayed()->add(O0, O2, O0); // note that cas made O2==O3
791 } else {
792 const Register& lock_reg = O2;
793 const Register& lock_ptr_reg = O3;
794 const Register& value_reg = O4;
795 const Register& yield_reg = O5;
796
797 Label(retry);
798 Label(dontyield);
799
800 generate_v8_lock_prologue(lock_reg, lock_ptr_reg, yield_reg, retry, dontyield);
801 // got lock, do the increment
802 __ ld(O1, 0, value_reg);
803 __ add(O0, value_reg, value_reg);
804 __ st(value_reg, O1, 0);
805
806 // %%% only for RMO and PSO
807 __ membar(Assembler::StoreStore);
808
809 generate_v8_lock_epilogue(lock_reg, lock_ptr_reg, yield_reg, retry, dontyield);
810
811 __ retl(false);
812 __ delayed()->mov(value_reg, O0);
813 }
814
815 return start;
816 }
817 Label _atomic_add_stub; // called from other stubs
818
819
820 // Support for void OrderAccess::fence().
821 //
822 address generate_fence() {
823 StubCodeMark mark(this, "StubRoutines", "fence");
824 address start = __ pc();
825
826 __ membar(Assembler::Membar_mask_bits(Assembler::LoadLoad | Assembler::LoadStore |
827 Assembler::StoreLoad | Assembler::StoreStore));
828 __ retl(false);
829 __ delayed()->nop();
830
831 return start;
832 }
833
834
835 //------------------------------------------------------------------------------------------------------------------------
836 // The following routine generates a subroutine to throw an asynchronous
837 // UnknownError when an unsafe access gets a fault that could not be
838 // reasonably prevented by the programmer. (Example: SIGBUS/OBJERR.)
839 //
840 // Arguments :
841 //
842 // trapping PC: O7
843 //
844 // Results:
845 // posts an asynchronous exception, skips the trapping instruction
846 //
847
848 address generate_handler_for_unsafe_access() {
849 StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
850 address start = __ pc();
851
852 const int preserve_register_words = (64 * 2);
853 Address preserve_addr(FP, 0, (-preserve_register_words * wordSize) + STACK_BIAS);
854
855 Register Lthread = L7_thread_cache;
856 int i;
857
858 __ save_frame(0);
859 __ mov(G1, L1);
860 __ mov(G2, L2);
861 __ mov(G3, L3);
862 __ mov(G4, L4);
863 __ mov(G5, L5);
864 for (i = 0; i < (VM_Version::v9_instructions_work() ? 64 : 32); i += 2) {
865 __ stf(FloatRegisterImpl::D, as_FloatRegister(i), preserve_addr, i * wordSize);
866 }
867
868 address entry_point = CAST_FROM_FN_PTR(address, handle_unsafe_access);
869 BLOCK_COMMENT("call handle_unsafe_access");
870 __ call(entry_point, relocInfo::runtime_call_type);
871 __ delayed()->nop();
872
873 __ mov(L1, G1);
874 __ mov(L2, G2);
875 __ mov(L3, G3);
876 __ mov(L4, G4);
877 __ mov(L5, G5);
878 for (i = 0; i < (VM_Version::v9_instructions_work() ? 64 : 32); i += 2) {
879 __ ldf(FloatRegisterImpl::D, preserve_addr, as_FloatRegister(i), i * wordSize);
880 }
881
882 __ verify_thread();
883
884 __ jmp(O0, 0);
885 __ delayed()->restore();
886
887 return start;
888 }
889
890
891 // Support for uint StubRoutine::Sparc::partial_subtype_check( Klass sub, Klass super );
892 // Arguments :
893 //
894 // ret : O0, returned
895 // icc/xcc: set as O0 (depending on wordSize)
896 // sub : O1, argument, not changed
897 // super: O2, argument, not changed
898 // raddr: O7, blown by call
899 address generate_partial_subtype_check() {
900 __ align(CodeEntryAlignment);
901 StubCodeMark mark(this, "StubRoutines", "partial_subtype_check");
902 address start = __ pc();
903 Label loop, miss;
904
905 // Compare super with sub directly, since super is not in its own SSA.
906 // The compiler used to emit this test, but we fold it in here,
907 // to increase overall code density, with no real loss of speed.
908 { Label L;
909 __ cmp(O1, O2);
910 __ brx(Assembler::notEqual, false, Assembler::pt, L);
911 __ delayed()->nop();
912 __ retl();
913 __ delayed()->addcc(G0,0,O0); // set Z flags, zero result
914 __ bind(L);
915 }
916
917 #if defined(COMPILER2) && !defined(_LP64)
918 // Do not use a 'save' because it blows the 64-bit O registers.
919 __ add(SP,-4*wordSize,SP); // Make space for 4 temps (stack must be 2 words aligned)
920 __ st_ptr(L0,SP,(frame::register_save_words+0)*wordSize);
921 __ st_ptr(L1,SP,(frame::register_save_words+1)*wordSize);
922 __ st_ptr(L2,SP,(frame::register_save_words+2)*wordSize);
923 __ st_ptr(L3,SP,(frame::register_save_words+3)*wordSize);
924 Register Rret = O0;
925 Register Rsub = O1;
926 Register Rsuper = O2;
927 #else
928 __ save_frame(0);
929 Register Rret = I0;
930 Register Rsub = I1;
931 Register Rsuper = I2;
932 #endif
933
934 Register L0_ary_len = L0;
935 Register L1_ary_ptr = L1;
936 Register L2_super = L2;
937 Register L3_index = L3;
938
939 #ifdef _LP64
940 Register L4_ooptmp = L4;
941
942 if (UseCompressedOops) {
943 // this must be under UseCompressedOops check, as we rely upon fact
944 // that L4 not clobbered in C2 on 32-bit platforms, where we do explicit save
945 // on stack, see several lines above
946 __ encode_heap_oop(Rsuper, L4_ooptmp);
947 }
948 #endif
949
950 inc_counter_np(SharedRuntime::_partial_subtype_ctr, L0, L1);
951
952 __ ld_ptr( Rsub, sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes(), L3 );
953 __ lduw(L3,arrayOopDesc::length_offset_in_bytes(),L0_ary_len);
954 __ add(L3,arrayOopDesc::base_offset_in_bytes(T_OBJECT),L1_ary_ptr);
955 __ clr(L3_index); // zero index
956 // Load a little early; will load 1 off the end of the array.
957 // Ok for now; revisit if we have other uses of this routine.
958 if (UseCompressedOops) {
959 __ lduw(L1_ary_ptr,0,L2_super);// Will load a little early
960 } else {
961 __ ld_ptr(L1_ary_ptr,0,L2_super);// Will load a little early
962 }
963
964 assert(heapOopSize != 0, "heapOopSize should be initialized");
965 // The scan loop
966 __ BIND(loop);
967 __ add(L1_ary_ptr, heapOopSize, L1_ary_ptr); // Bump by OOP size
968 __ cmp(L3_index,L0_ary_len);
969 __ br(Assembler::equal,false,Assembler::pn,miss);
970 __ delayed()->inc(L3_index); // Bump index
971
972 if (UseCompressedOops) {
973 #ifdef _LP64
974 __ subcc(L2_super,L4_ooptmp,Rret); // Check for match; zero in Rret for a hit
975 __ br( Assembler::notEqual, false, Assembler::pt, loop );
976 __ delayed()->lduw(L1_ary_ptr,0,L2_super);// Will load a little early
977 #else
978 ShouldNotReachHere();
979 #endif
980 } else {
981 __ subcc(L2_super,Rsuper,Rret); // Check for match; zero in Rret for a hit
982 __ brx( Assembler::notEqual, false, Assembler::pt, loop );
983 __ delayed()->ld_ptr(L1_ary_ptr,0,L2_super);// Will load a little early
984 }
985
986 // Got a hit; report success; set cache. Cache load doesn't
987 // happen here; for speed it is directly emitted by the compiler.
988 __ st_ptr( Rsuper, Rsub, sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() );
989
990 #if defined(COMPILER2) && !defined(_LP64)
991 __ ld_ptr(SP,(frame::register_save_words+0)*wordSize,L0);
992 __ ld_ptr(SP,(frame::register_save_words+1)*wordSize,L1);
993 __ ld_ptr(SP,(frame::register_save_words+2)*wordSize,L2);
994 __ ld_ptr(SP,(frame::register_save_words+3)*wordSize,L3);
995 __ retl(); // Result in Rret is zero; flags set to Z
996 __ delayed()->add(SP,4*wordSize,SP);
997 #else
998 __ ret(); // Result in Rret is zero; flags set to Z
999 __ delayed()->restore();
1000 #endif
1001
1002 // Hit or miss falls through here
1003 __ BIND(miss);
1004 __ addcc(G0,1,Rret); // set NZ flags, NZ result
1005
1006 #if defined(COMPILER2) && !defined(_LP64)
1007 __ ld_ptr(SP,(frame::register_save_words+0)*wordSize,L0);
1008 __ ld_ptr(SP,(frame::register_save_words+1)*wordSize,L1);
1009 __ ld_ptr(SP,(frame::register_save_words+2)*wordSize,L2);
1010 __ ld_ptr(SP,(frame::register_save_words+3)*wordSize,L3);
1011 __ retl(); // Result in Rret is != 0; flags set to NZ
1012 __ delayed()->add(SP,4*wordSize,SP);
1013 #else
1014 __ ret(); // Result in Rret is != 0; flags set to NZ
1015 __ delayed()->restore();
1016 #endif
1017
1018 return start;
1019 }
1020
1021
1022 // Called from MacroAssembler::verify_oop
1023 //
1024 address generate_verify_oop_subroutine() {
1025 StubCodeMark mark(this, "StubRoutines", "verify_oop_stub");
1026
1027 address start = __ pc();
1028
1029 __ verify_oop_subroutine();
1030
1031 return start;
1032 }
1033
1034 static address disjoint_byte_copy_entry;
1035 static address disjoint_short_copy_entry;
1036 static address disjoint_int_copy_entry;
1037 static address disjoint_long_copy_entry;
1038 static address disjoint_oop_copy_entry;
1039
1040 static address byte_copy_entry;
1041 static address short_copy_entry;
1042 static address int_copy_entry;
1043 static address long_copy_entry;
1044 static address oop_copy_entry;
1045
1046 static address checkcast_copy_entry;
1047
1048 //
1049 // Verify that a register contains clean 32-bits positive value
1050 // (high 32-bits are 0) so it could be used in 64-bits shifts (sllx, srax).
1051 //
1052 // Input:
1053 // Rint - 32-bits value
1054 // Rtmp - scratch
1055 //
1056 void assert_clean_int(Register Rint, Register Rtmp) {
1057 #if defined(ASSERT) && defined(_LP64)
1058 __ signx(Rint, Rtmp);
1059 __ cmp(Rint, Rtmp);
1060 __ breakpoint_trap(Assembler::notEqual, Assembler::xcc);
1061 #endif
1062 }
1063
1064 //
1065 // Generate overlap test for array copy stubs
1066 //
1067 // Input:
1068 // O0 - array1
1069 // O1 - array2
1070 // O2 - element count
1071 //
1072 // Kills temps: O3, O4
1073 //
1074 void array_overlap_test(address no_overlap_target, int log2_elem_size) {
1075 assert(no_overlap_target != NULL, "must be generated");
1076 array_overlap_test(no_overlap_target, NULL, log2_elem_size);
1077 }
1078 void array_overlap_test(Label& L_no_overlap, int log2_elem_size) {
1079 array_overlap_test(NULL, &L_no_overlap, log2_elem_size);
1080 }
1081 void array_overlap_test(address no_overlap_target, Label* NOLp, int log2_elem_size) {
1082 const Register from = O0;
1083 const Register to = O1;
1084 const Register count = O2;
1085 const Register to_from = O3; // to - from
1086 const Register byte_count = O4; // count << log2_elem_size
1087
1088 __ subcc(to, from, to_from);
1089 __ sll_ptr(count, log2_elem_size, byte_count);
1090 if (NOLp == NULL)
1091 __ brx(Assembler::lessEqualUnsigned, false, Assembler::pt, no_overlap_target);
1092 else
1093 __ brx(Assembler::lessEqualUnsigned, false, Assembler::pt, (*NOLp));
1094 __ delayed()->cmp(to_from, byte_count);
1095 if (NOLp == NULL)
1096 __ brx(Assembler::greaterEqual, false, Assembler::pt, no_overlap_target);
1097 else
1098 __ brx(Assembler::greaterEqual, false, Assembler::pt, (*NOLp));
1099 __ delayed()->nop();
1100 }
1101
1102 //
1103 // Generate pre-write barrier for array.
1104 //
1105 // Input:
1106 // addr - register containing starting address
1107 // count - register containing element count
1108 // tmp - scratch register
1109 //
1110 // The input registers are overwritten.
1111 //
1112 void gen_write_ref_array_pre_barrier(Register addr, Register count) {
1113 BarrierSet* bs = Universe::heap()->barrier_set();
1114 if (bs->has_write_ref_pre_barrier()) {
1115 assert(bs->has_write_ref_array_pre_opt(),
1116 "Else unsupported barrier set.");
1117
1118 __ save_frame(0);
1119 // Save the necessary global regs... will be used after.
1120 if (addr->is_global()) {
1121 __ mov(addr, L0);
1122 }
1123 if (count->is_global()) {
1124 __ mov(count, L1);
1125 }
1126 __ mov(addr->after_save(), O0);
1127 // Get the count into O1
1128 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre));
1129 __ delayed()->mov(count->after_save(), O1);
1130 if (addr->is_global()) {
1131 __ mov(L0, addr);
1132 }
1133 if (count->is_global()) {
1134 __ mov(L1, count);
1135 }
1136 __ restore();
1137 }
1138 }
1139 //
1140 // Generate post-write barrier for array.
1141 //
1142 // Input:
1143 // addr - register containing starting address
1144 // count - register containing element count
1145 // tmp - scratch register
1146 //
1147 // The input registers are overwritten.
1148 //
1149 void gen_write_ref_array_post_barrier(Register addr, Register count,
1150 Register tmp) {
1151 BarrierSet* bs = Universe::heap()->barrier_set();
1152
1153 switch (bs->kind()) {
1154 case BarrierSet::G1SATBCT:
1155 case BarrierSet::G1SATBCTLogging:
1156 {
1157 // Get some new fresh output registers.
1158 __ save_frame(0);
1159 __ mov(addr->after_save(), O0);
1160 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post));
1161 __ delayed()->mov(count->after_save(), O1);
1162 __ restore();
1163 }
1164 break;
1165 case BarrierSet::CardTableModRef:
1166 case BarrierSet::CardTableExtension:
1167 {
1168 CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1169 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1170 assert_different_registers(addr, count, tmp);
1171
1172 Label L_loop;
1173
1174 __ sll_ptr(count, LogBytesPerHeapOop, count);
1175 __ sub(count, BytesPerHeapOop, count);
1176 __ add(count, addr, count);
1177 // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
1178 __ srl_ptr(addr, CardTableModRefBS::card_shift, addr);
1179 __ srl_ptr(count, CardTableModRefBS::card_shift, count);
1180 __ sub(count, addr, count);
1181 Address rs(tmp, (address)ct->byte_map_base);
1182 __ load_address(rs);
1183 __ BIND(L_loop);
1184 __ stb(G0, rs.base(), addr);
1185 __ subcc(count, 1, count);
1186 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1187 __ delayed()->add(addr, 1, addr);
1188
1189 }
1190 break;
1191 case BarrierSet::ModRef:
1192 break;
1193 default :
1194 ShouldNotReachHere();
1195
1196 }
1197 }
1198
1199
1200 // Copy big chunks forward with shift
1201 //
1202 // Inputs:
1203 // from - source arrays
1204 // to - destination array aligned to 8-bytes
1205 // count - elements count to copy >= the count equivalent to 16 bytes
1206 // count_dec - elements count's decrement equivalent to 16 bytes
1207 // L_copy_bytes - copy exit label
1208 //
1209 void copy_16_bytes_forward_with_shift(Register from, Register to,
1210 Register count, int count_dec, Label& L_copy_bytes) {
1211 Label L_loop, L_aligned_copy, L_copy_last_bytes;
1212
1213 // if both arrays have the same alignment mod 8, do 8 bytes aligned copy
1214 __ andcc(from, 7, G1); // misaligned bytes
1215 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1216 __ delayed()->nop();
1217
1218 const Register left_shift = G1; // left shift bit counter
1219 const Register right_shift = G5; // right shift bit counter
1220
1221 __ sll(G1, LogBitsPerByte, left_shift);
1222 __ mov(64, right_shift);
1223 __ sub(right_shift, left_shift, right_shift);
1224
1225 //
1226 // Load 2 aligned 8-bytes chunks and use one from previous iteration
1227 // to form 2 aligned 8-bytes chunks to store.
1228 //
1229 __ deccc(count, count_dec); // Pre-decrement 'count'
1230 __ andn(from, 7, from); // Align address
1231 __ ldx(from, 0, O3);
1232 __ inc(from, 8);
1233 __ align(16);
1234 __ BIND(L_loop);
1235 __ ldx(from, 0, O4);
1236 __ deccc(count, count_dec); // Can we do next iteration after this one?
1237 __ ldx(from, 8, G4);
1238 __ inc(to, 16);
1239 __ inc(from, 16);
1240 __ sllx(O3, left_shift, O3);
1241 __ srlx(O4, right_shift, G3);
1242 __ bset(G3, O3);
1243 __ stx(O3, to, -16);
1244 __ sllx(O4, left_shift, O4);
1245 __ srlx(G4, right_shift, G3);
1246 __ bset(G3, O4);
1247 __ stx(O4, to, -8);
1248 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1249 __ delayed()->mov(G4, O3);
1250
1251 __ inccc(count, count_dec>>1 ); // + 8 bytes
1252 __ brx(Assembler::negative, true, Assembler::pn, L_copy_last_bytes);
1253 __ delayed()->inc(count, count_dec>>1); // restore 'count'
1254
1255 // copy 8 bytes, part of them already loaded in O3
1256 __ ldx(from, 0, O4);
1257 __ inc(to, 8);
1258 __ inc(from, 8);
1259 __ sllx(O3, left_shift, O3);
1260 __ srlx(O4, right_shift, G3);
1261 __ bset(O3, G3);
1262 __ stx(G3, to, -8);
1263
1264 __ BIND(L_copy_last_bytes);
1265 __ srl(right_shift, LogBitsPerByte, right_shift); // misaligned bytes
1266 __ br(Assembler::always, false, Assembler::pt, L_copy_bytes);
1267 __ delayed()->sub(from, right_shift, from); // restore address
1268
1269 __ BIND(L_aligned_copy);
1270 }
1271
1272 // Copy big chunks backward with shift
1273 //
1274 // Inputs:
1275 // end_from - source arrays end address
1276 // end_to - destination array end address aligned to 8-bytes
1277 // count - elements count to copy >= the count equivalent to 16 bytes
1278 // count_dec - elements count's decrement equivalent to 16 bytes
1279 // L_aligned_copy - aligned copy exit label
1280 // L_copy_bytes - copy exit label
1281 //
1282 void copy_16_bytes_backward_with_shift(Register end_from, Register end_to,
1283 Register count, int count_dec,
1284 Label& L_aligned_copy, Label& L_copy_bytes) {
1285 Label L_loop, L_copy_last_bytes;
1286
1287 // if both arrays have the same alignment mod 8, do 8 bytes aligned copy
1288 __ andcc(end_from, 7, G1); // misaligned bytes
1289 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1290 __ delayed()->deccc(count, count_dec); // Pre-decrement 'count'
1291
1292 const Register left_shift = G1; // left shift bit counter
1293 const Register right_shift = G5; // right shift bit counter
1294
1295 __ sll(G1, LogBitsPerByte, left_shift);
1296 __ mov(64, right_shift);
1297 __ sub(right_shift, left_shift, right_shift);
1298
1299 //
1300 // Load 2 aligned 8-bytes chunks and use one from previous iteration
1301 // to form 2 aligned 8-bytes chunks to store.
1302 //
1303 __ andn(end_from, 7, end_from); // Align address
1304 __ ldx(end_from, 0, O3);
1305 __ align(16);
1306 __ BIND(L_loop);
1307 __ ldx(end_from, -8, O4);
1308 __ deccc(count, count_dec); // Can we do next iteration after this one?
1309 __ ldx(end_from, -16, G4);
1310 __ dec(end_to, 16);
1311 __ dec(end_from, 16);
1312 __ srlx(O3, right_shift, O3);
1313 __ sllx(O4, left_shift, G3);
1314 __ bset(G3, O3);
1315 __ stx(O3, end_to, 8);
1316 __ srlx(O4, right_shift, O4);
1317 __ sllx(G4, left_shift, G3);
1318 __ bset(G3, O4);
1319 __ stx(O4, end_to, 0);
1320 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1321 __ delayed()->mov(G4, O3);
1322
1323 __ inccc(count, count_dec>>1 ); // + 8 bytes
1324 __ brx(Assembler::negative, true, Assembler::pn, L_copy_last_bytes);
1325 __ delayed()->inc(count, count_dec>>1); // restore 'count'
1326
1327 // copy 8 bytes, part of them already loaded in O3
1328 __ ldx(end_from, -8, O4);
1329 __ dec(end_to, 8);
1330 __ dec(end_from, 8);
1331 __ srlx(O3, right_shift, O3);
1332 __ sllx(O4, left_shift, G3);
1333 __ bset(O3, G3);
1334 __ stx(G3, end_to, 0);
1335
1336 __ BIND(L_copy_last_bytes);
1337 __ srl(left_shift, LogBitsPerByte, left_shift); // misaligned bytes
1338 __ br(Assembler::always, false, Assembler::pt, L_copy_bytes);
1339 __ delayed()->add(end_from, left_shift, end_from); // restore address
1340 }
1341
1342 //
1343 // Generate stub for disjoint byte copy. If "aligned" is true, the
1344 // "from" and "to" addresses are assumed to be heapword aligned.
1345 //
1346 // Arguments for generated stub:
1347 // from: O0
1348 // to: O1
1349 // count: O2 treated as signed
1350 //
1351 address generate_disjoint_byte_copy(bool aligned, const char * name) {
1352 __ align(CodeEntryAlignment);
1353 StubCodeMark mark(this, "StubRoutines", name);
1354 address start = __ pc();
1355
1356 Label L_skip_alignment, L_align;
1357 Label L_copy_byte, L_copy_byte_loop, L_exit;
1358
1359 const Register from = O0; // source array address
1360 const Register to = O1; // destination array address
1361 const Register count = O2; // elements count
1362 const Register offset = O5; // offset from start of arrays
1363 // O3, O4, G3, G4 are used as temp registers
1364
1365 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1366
1367 if (!aligned) disjoint_byte_copy_entry = __ pc();
1368 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1369 if (!aligned) BLOCK_COMMENT("Entry:");
1370
1371 // for short arrays, just do single element copy
1372 __ cmp(count, 23); // 16 + 7
1373 __ brx(Assembler::less, false, Assembler::pn, L_copy_byte);
1374 __ delayed()->mov(G0, offset);
1375
1376 if (aligned) {
1377 // 'aligned' == true when it is known statically during compilation
1378 // of this arraycopy call site that both 'from' and 'to' addresses
1379 // are HeapWordSize aligned (see LibraryCallKit::basictype2arraycopy()).
1380 //
1381 // Aligned arrays have 4 bytes alignment in 32-bits VM
1382 // and 8 bytes - in 64-bits VM. So we do it only for 32-bits VM
1383 //
1384 #ifndef _LP64
1385 // copy a 4-bytes word if necessary to align 'to' to 8 bytes
1386 __ andcc(to, 7, G0);
1387 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment);
1388 __ delayed()->ld(from, 0, O3);
1389 __ inc(from, 4);
1390 __ inc(to, 4);
1391 __ dec(count, 4);
1392 __ st(O3, to, -4);
1393 __ BIND(L_skip_alignment);
1394 #endif
1395 } else {
1396 // copy bytes to align 'to' on 8 byte boundary
1397 __ andcc(to, 7, G1); // misaligned bytes
1398 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1399 __ delayed()->neg(G1);
1400 __ inc(G1, 8); // bytes need to copy to next 8-bytes alignment
1401 __ sub(count, G1, count);
1402 __ BIND(L_align);
1403 __ ldub(from, 0, O3);
1404 __ deccc(G1);
1405 __ inc(from);
1406 __ stb(O3, to, 0);
1407 __ br(Assembler::notZero, false, Assembler::pt, L_align);
1408 __ delayed()->inc(to);
1409 __ BIND(L_skip_alignment);
1410 }
1411 #ifdef _LP64
1412 if (!aligned)
1413 #endif
1414 {
1415 // Copy with shift 16 bytes per iteration if arrays do not have
1416 // the same alignment mod 8, otherwise fall through to the next
1417 // code for aligned copy.
1418 // The compare above (count >= 23) guarantes 'count' >= 16 bytes.
1419 // Also jump over aligned copy after the copy with shift completed.
1420
1421 copy_16_bytes_forward_with_shift(from, to, count, 16, L_copy_byte);
1422 }
1423
1424 // Both array are 8 bytes aligned, copy 16 bytes at a time
1425 __ and3(count, 7, G4); // Save count
1426 __ srl(count, 3, count);
1427 generate_disjoint_long_copy_core(aligned);
1428 __ mov(G4, count); // Restore count
1429
1430 // copy tailing bytes
1431 __ BIND(L_copy_byte);
1432 __ br_zero(Assembler::zero, false, Assembler::pt, count, L_exit);
1433 __ delayed()->nop();
1434 __ align(16);
1435 __ BIND(L_copy_byte_loop);
1436 __ ldub(from, offset, O3);
1437 __ deccc(count);
1438 __ stb(O3, to, offset);
1439 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_byte_loop);
1440 __ delayed()->inc(offset);
1441
1442 __ BIND(L_exit);
1443 // O3, O4 are used as temp registers
1444 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr, O3, O4);
1445 __ retl();
1446 __ delayed()->mov(G0, O0); // return 0
1447 return start;
1448 }
1449
1450 //
1451 // Generate stub for conjoint byte copy. If "aligned" is true, the
1452 // "from" and "to" addresses are assumed to be heapword aligned.
1453 //
1454 // Arguments for generated stub:
1455 // from: O0
1456 // to: O1
1457 // count: O2 treated as signed
1458 //
1459 address generate_conjoint_byte_copy(bool aligned, const char * name) {
1460 // Do reverse copy.
1461
1462 __ align(CodeEntryAlignment);
1463 StubCodeMark mark(this, "StubRoutines", name);
1464 address start = __ pc();
1465 address nooverlap_target = aligned ?
1466 StubRoutines::arrayof_jbyte_disjoint_arraycopy() :
1467 disjoint_byte_copy_entry;
1468
1469 Label L_skip_alignment, L_align, L_aligned_copy;
1470 Label L_copy_byte, L_copy_byte_loop, L_exit;
1471
1472 const Register from = O0; // source array address
1473 const Register to = O1; // destination array address
1474 const Register count = O2; // elements count
1475 const Register end_from = from; // source array end address
1476 const Register end_to = to; // destination array end address
1477
1478 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1479
1480 if (!aligned) byte_copy_entry = __ pc();
1481 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1482 if (!aligned) BLOCK_COMMENT("Entry:");
1483
1484 array_overlap_test(nooverlap_target, 0);
1485
1486 __ add(to, count, end_to); // offset after last copied element
1487
1488 // for short arrays, just do single element copy
1489 __ cmp(count, 23); // 16 + 7
1490 __ brx(Assembler::less, false, Assembler::pn, L_copy_byte);
1491 __ delayed()->add(from, count, end_from);
1492
1493 {
1494 // Align end of arrays since they could be not aligned even
1495 // when arrays itself are aligned.
1496
1497 // copy bytes to align 'end_to' on 8 byte boundary
1498 __ andcc(end_to, 7, G1); // misaligned bytes
1499 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1500 __ delayed()->nop();
1501 __ sub(count, G1, count);
1502 __ BIND(L_align);
1503 __ dec(end_from);
1504 __ dec(end_to);
1505 __ ldub(end_from, 0, O3);
1506 __ deccc(G1);
1507 __ brx(Assembler::notZero, false, Assembler::pt, L_align);
1508 __ delayed()->stb(O3, end_to, 0);
1509 __ BIND(L_skip_alignment);
1510 }
1511 #ifdef _LP64
1512 if (aligned) {
1513 // Both arrays are aligned to 8-bytes in 64-bits VM.
1514 // The 'count' is decremented in copy_16_bytes_backward_with_shift()
1515 // in unaligned case.
1516 __ dec(count, 16);
1517 } else
1518 #endif
1519 {
1520 // Copy with shift 16 bytes per iteration if arrays do not have
1521 // the same alignment mod 8, otherwise jump to the next
1522 // code for aligned copy (and substracting 16 from 'count' before jump).
1523 // The compare above (count >= 11) guarantes 'count' >= 16 bytes.
1524 // Also jump over aligned copy after the copy with shift completed.
1525
1526 copy_16_bytes_backward_with_shift(end_from, end_to, count, 16,
1527 L_aligned_copy, L_copy_byte);
1528 }
1529 // copy 4 elements (16 bytes) at a time
1530 __ align(16);
1531 __ BIND(L_aligned_copy);
1532 __ dec(end_from, 16);
1533 __ ldx(end_from, 8, O3);
1534 __ ldx(end_from, 0, O4);
1535 __ dec(end_to, 16);
1536 __ deccc(count, 16);
1537 __ stx(O3, end_to, 8);
1538 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
1539 __ delayed()->stx(O4, end_to, 0);
1540 __ inc(count, 16);
1541
1542 // copy 1 element (2 bytes) at a time
1543 __ BIND(L_copy_byte);
1544 __ br_zero(Assembler::zero, false, Assembler::pt, count, L_exit);
1545 __ delayed()->nop();
1546 __ align(16);
1547 __ BIND(L_copy_byte_loop);
1548 __ dec(end_from);
1549 __ dec(end_to);
1550 __ ldub(end_from, 0, O4);
1551 __ deccc(count);
1552 __ brx(Assembler::greater, false, Assembler::pt, L_copy_byte_loop);
1553 __ delayed()->stb(O4, end_to, 0);
1554
1555 __ BIND(L_exit);
1556 // O3, O4 are used as temp registers
1557 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr, O3, O4);
1558 __ retl();
1559 __ delayed()->mov(G0, O0); // return 0
1560 return start;
1561 }
1562
1563 //
1564 // Generate stub for disjoint short copy. If "aligned" is true, the
1565 // "from" and "to" addresses are assumed to be heapword aligned.
1566 //
1567 // Arguments for generated stub:
1568 // from: O0
1569 // to: O1
1570 // count: O2 treated as signed
1571 //
1572 address generate_disjoint_short_copy(bool aligned, const char * name) {
1573 __ align(CodeEntryAlignment);
1574 StubCodeMark mark(this, "StubRoutines", name);
1575 address start = __ pc();
1576
1577 Label L_skip_alignment, L_skip_alignment2;
1578 Label L_copy_2_bytes, L_copy_2_bytes_loop, L_exit;
1579
1580 const Register from = O0; // source array address
1581 const Register to = O1; // destination array address
1582 const Register count = O2; // elements count
1583 const Register offset = O5; // offset from start of arrays
1584 // O3, O4, G3, G4 are used as temp registers
1585
1586 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1587
1588 if (!aligned) disjoint_short_copy_entry = __ pc();
1589 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1590 if (!aligned) BLOCK_COMMENT("Entry:");
1591
1592 // for short arrays, just do single element copy
1593 __ cmp(count, 11); // 8 + 3 (22 bytes)
1594 __ brx(Assembler::less, false, Assembler::pn, L_copy_2_bytes);
1595 __ delayed()->mov(G0, offset);
1596
1597 if (aligned) {
1598 // 'aligned' == true when it is known statically during compilation
1599 // of this arraycopy call site that both 'from' and 'to' addresses
1600 // are HeapWordSize aligned (see LibraryCallKit::basictype2arraycopy()).
1601 //
1602 // Aligned arrays have 4 bytes alignment in 32-bits VM
1603 // and 8 bytes - in 64-bits VM.
1604 //
1605 #ifndef _LP64
1606 // copy a 2-elements word if necessary to align 'to' to 8 bytes
1607 __ andcc(to, 7, G0);
1608 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1609 __ delayed()->ld(from, 0, O3);
1610 __ inc(from, 4);
1611 __ inc(to, 4);
1612 __ dec(count, 2);
1613 __ st(O3, to, -4);
1614 __ BIND(L_skip_alignment);
1615 #endif
1616 } else {
1617 // copy 1 element if necessary to align 'to' on an 4 bytes
1618 __ andcc(to, 3, G0);
1619 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1620 __ delayed()->lduh(from, 0, O3);
1621 __ inc(from, 2);
1622 __ inc(to, 2);
1623 __ dec(count);
1624 __ sth(O3, to, -2);
1625 __ BIND(L_skip_alignment);
1626
1627 // copy 2 elements to align 'to' on an 8 byte boundary
1628 __ andcc(to, 7, G0);
1629 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment2);
1630 __ delayed()->lduh(from, 0, O3);
1631 __ dec(count, 2);
1632 __ lduh(from, 2, O4);
1633 __ inc(from, 4);
1634 __ inc(to, 4);
1635 __ sth(O3, to, -4);
1636 __ sth(O4, to, -2);
1637 __ BIND(L_skip_alignment2);
1638 }
1639 #ifdef _LP64
1640 if (!aligned)
1641 #endif
1642 {
1643 // Copy with shift 16 bytes per iteration if arrays do not have
1644 // the same alignment mod 8, otherwise fall through to the next
1645 // code for aligned copy.
1646 // The compare above (count >= 11) guarantes 'count' >= 16 bytes.
1647 // Also jump over aligned copy after the copy with shift completed.
1648
1649 copy_16_bytes_forward_with_shift(from, to, count, 8, L_copy_2_bytes);
1650 }
1651
1652 // Both array are 8 bytes aligned, copy 16 bytes at a time
1653 __ and3(count, 3, G4); // Save
1654 __ srl(count, 2, count);
1655 generate_disjoint_long_copy_core(aligned);
1656 __ mov(G4, count); // restore
1657
1658 // copy 1 element at a time
1659 __ BIND(L_copy_2_bytes);
1660 __ br_zero(Assembler::zero, false, Assembler::pt, count, L_exit);
1661 __ delayed()->nop();
1662 __ align(16);
1663 __ BIND(L_copy_2_bytes_loop);
1664 __ lduh(from, offset, O3);
1665 __ deccc(count);
1666 __ sth(O3, to, offset);
1667 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_2_bytes_loop);
1668 __ delayed()->inc(offset, 2);
1669
1670 __ BIND(L_exit);
1671 // O3, O4 are used as temp registers
1672 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr, O3, O4);
1673 __ retl();
1674 __ delayed()->mov(G0, O0); // return 0
1675 return start;
1676 }
1677
1678 //
1679 // Generate stub for conjoint short copy. If "aligned" is true, the
1680 // "from" and "to" addresses are assumed to be heapword aligned.
1681 //
1682 // Arguments for generated stub:
1683 // from: O0
1684 // to: O1
1685 // count: O2 treated as signed
1686 //
1687 address generate_conjoint_short_copy(bool aligned, const char * name) {
1688 // Do reverse copy.
1689
1690 __ align(CodeEntryAlignment);
1691 StubCodeMark mark(this, "StubRoutines", name);
1692 address start = __ pc();
1693 address nooverlap_target = aligned ?
1694 StubRoutines::arrayof_jshort_disjoint_arraycopy() :
1695 disjoint_short_copy_entry;
1696
1697 Label L_skip_alignment, L_skip_alignment2, L_aligned_copy;
1698 Label L_copy_2_bytes, L_copy_2_bytes_loop, L_exit;
1699
1700 const Register from = O0; // source array address
1701 const Register to = O1; // destination array address
1702 const Register count = O2; // elements count
1703 const Register end_from = from; // source array end address
1704 const Register end_to = to; // destination array end address
1705
1706 const Register byte_count = O3; // bytes count to copy
1707
1708 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1709
1710 if (!aligned) short_copy_entry = __ pc();
1711 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1712 if (!aligned) BLOCK_COMMENT("Entry:");
1713
1714 array_overlap_test(nooverlap_target, 1);
1715
1716 __ sllx(count, LogBytesPerShort, byte_count);
1717 __ add(to, byte_count, end_to); // offset after last copied element
1718
1719 // for short arrays, just do single element copy
1720 __ cmp(count, 11); // 8 + 3 (22 bytes)
1721 __ brx(Assembler::less, false, Assembler::pn, L_copy_2_bytes);
1722 __ delayed()->add(from, byte_count, end_from);
1723
1724 {
1725 // Align end of arrays since they could be not aligned even
1726 // when arrays itself are aligned.
1727
1728 // copy 1 element if necessary to align 'end_to' on an 4 bytes
1729 __ andcc(end_to, 3, G0);
1730 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1731 __ delayed()->lduh(end_from, -2, O3);
1732 __ dec(end_from, 2);
1733 __ dec(end_to, 2);
1734 __ dec(count);
1735 __ sth(O3, end_to, 0);
1736 __ BIND(L_skip_alignment);
1737
1738 // copy 2 elements to align 'end_to' on an 8 byte boundary
1739 __ andcc(end_to, 7, G0);
1740 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment2);
1741 __ delayed()->lduh(end_from, -2, O3);
1742 __ dec(count, 2);
1743 __ lduh(end_from, -4, O4);
1744 __ dec(end_from, 4);
1745 __ dec(end_to, 4);
1746 __ sth(O3, end_to, 2);
1747 __ sth(O4, end_to, 0);
1748 __ BIND(L_skip_alignment2);
1749 }
1750 #ifdef _LP64
1751 if (aligned) {
1752 // Both arrays are aligned to 8-bytes in 64-bits VM.
1753 // The 'count' is decremented in copy_16_bytes_backward_with_shift()
1754 // in unaligned case.
1755 __ dec(count, 8);
1756 } else
1757 #endif
1758 {
1759 // Copy with shift 16 bytes per iteration if arrays do not have
1760 // the same alignment mod 8, otherwise jump to the next
1761 // code for aligned copy (and substracting 8 from 'count' before jump).
1762 // The compare above (count >= 11) guarantes 'count' >= 16 bytes.
1763 // Also jump over aligned copy after the copy with shift completed.
1764
1765 copy_16_bytes_backward_with_shift(end_from, end_to, count, 8,
1766 L_aligned_copy, L_copy_2_bytes);
1767 }
1768 // copy 4 elements (16 bytes) at a time
1769 __ align(16);
1770 __ BIND(L_aligned_copy);
1771 __ dec(end_from, 16);
1772 __ ldx(end_from, 8, O3);
1773 __ ldx(end_from, 0, O4);
1774 __ dec(end_to, 16);
1775 __ deccc(count, 8);
1776 __ stx(O3, end_to, 8);
1777 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
1778 __ delayed()->stx(O4, end_to, 0);
1779 __ inc(count, 8);
1780
1781 // copy 1 element (2 bytes) at a time
1782 __ BIND(L_copy_2_bytes);
1783 __ br_zero(Assembler::zero, false, Assembler::pt, count, L_exit);
1784 __ delayed()->nop();
1785 __ BIND(L_copy_2_bytes_loop);
1786 __ dec(end_from, 2);
1787 __ dec(end_to, 2);
1788 __ lduh(end_from, 0, O4);
1789 __ deccc(count);
1790 __ brx(Assembler::greater, false, Assembler::pt, L_copy_2_bytes_loop);
1791 __ delayed()->sth(O4, end_to, 0);
1792
1793 __ BIND(L_exit);
1794 // O3, O4 are used as temp registers
1795 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr, O3, O4);
1796 __ retl();
1797 __ delayed()->mov(G0, O0); // return 0
1798 return start;
1799 }
1800
1801 //
1802 // Generate core code for disjoint int copy (and oop copy on 32-bit).
1803 // If "aligned" is true, the "from" and "to" addresses are assumed
1804 // to be heapword aligned.
1805 //
1806 // Arguments:
1807 // from: O0
1808 // to: O1
1809 // count: O2 treated as signed
1810 //
1811 void generate_disjoint_int_copy_core(bool aligned) {
1812
1813 Label L_skip_alignment, L_aligned_copy;
1814 Label L_copy_16_bytes, L_copy_4_bytes, L_copy_4_bytes_loop, L_exit;
1815
1816 const Register from = O0; // source array address
1817 const Register to = O1; // destination array address
1818 const Register count = O2; // elements count
1819 const Register offset = O5; // offset from start of arrays
1820 // O3, O4, G3, G4 are used as temp registers
1821
1822 // 'aligned' == true when it is known statically during compilation
1823 // of this arraycopy call site that both 'from' and 'to' addresses
1824 // are HeapWordSize aligned (see LibraryCallKit::basictype2arraycopy()).
1825 //
1826 // Aligned arrays have 4 bytes alignment in 32-bits VM
1827 // and 8 bytes - in 64-bits VM.
1828 //
1829 #ifdef _LP64
1830 if (!aligned)
1831 #endif
1832 {
1833 // The next check could be put under 'ifndef' since the code in
1834 // generate_disjoint_long_copy_core() has own checks and set 'offset'.
1835
1836 // for short arrays, just do single element copy
1837 __ cmp(count, 5); // 4 + 1 (20 bytes)
1838 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_4_bytes);
1839 __ delayed()->mov(G0, offset);
1840
1841 // copy 1 element to align 'to' on an 8 byte boundary
1842 __ andcc(to, 7, G0);
1843 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1844 __ delayed()->ld(from, 0, O3);
1845 __ inc(from, 4);
1846 __ inc(to, 4);
1847 __ dec(count);
1848 __ st(O3, to, -4);
1849 __ BIND(L_skip_alignment);
1850
1851 // if arrays have same alignment mod 8, do 4 elements copy
1852 __ andcc(from, 7, G0);
1853 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1854 __ delayed()->ld(from, 0, O3);
1855
1856 //
1857 // Load 2 aligned 8-bytes chunks and use one from previous iteration
1858 // to form 2 aligned 8-bytes chunks to store.
1859 //
1860 // copy_16_bytes_forward_with_shift() is not used here since this
1861 // code is more optimal.
1862
1863 // copy with shift 4 elements (16 bytes) at a time
1864 __ dec(count, 4); // The cmp at the beginning guaranty count >= 4
1865
1866 __ align(16);
1867 __ BIND(L_copy_16_bytes);
1868 __ ldx(from, 4, O4);
1869 __ deccc(count, 4); // Can we do next iteration after this one?
1870 __ ldx(from, 12, G4);
1871 __ inc(to, 16);
1872 __ inc(from, 16);
1873 __ sllx(O3, 32, O3);
1874 __ srlx(O4, 32, G3);
1875 __ bset(G3, O3);
1876 __ stx(O3, to, -16);
1877 __ sllx(O4, 32, O4);
1878 __ srlx(G4, 32, G3);
1879 __ bset(G3, O4);
1880 __ stx(O4, to, -8);
1881 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_copy_16_bytes);
1882 __ delayed()->mov(G4, O3);
1883
1884 __ br(Assembler::always, false, Assembler::pt, L_copy_4_bytes);
1885 __ delayed()->inc(count, 4); // restore 'count'
1886
1887 __ BIND(L_aligned_copy);
1888 }
1889 // copy 4 elements (16 bytes) at a time
1890 __ and3(count, 1, G4); // Save
1891 __ srl(count, 1, count);
1892 generate_disjoint_long_copy_core(aligned);
1893 __ mov(G4, count); // Restore
1894
1895 // copy 1 element at a time
1896 __ BIND(L_copy_4_bytes);
1897 __ br_zero(Assembler::zero, false, Assembler::pt, count, L_exit);
1898 __ delayed()->nop();
1899 __ BIND(L_copy_4_bytes_loop);
1900 __ ld(from, offset, O3);
1901 __ deccc(count);
1902 __ st(O3, to, offset);
1903 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_4_bytes_loop);
1904 __ delayed()->inc(offset, 4);
1905 __ BIND(L_exit);
1906 }
1907
1908 //
1909 // Generate stub for disjoint int copy. If "aligned" is true, the
1910 // "from" and "to" addresses are assumed to be heapword aligned.
1911 //
1912 // Arguments for generated stub:
1913 // from: O0
1914 // to: O1
1915 // count: O2 treated as signed
1916 //
1917 address generate_disjoint_int_copy(bool aligned, const char * name) {
1918 __ align(CodeEntryAlignment);
1919 StubCodeMark mark(this, "StubRoutines", name);
1920 address start = __ pc();
1921
1922 const Register count = O2;
1923 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1924
1925 if (!aligned) disjoint_int_copy_entry = __ pc();
1926 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1927 if (!aligned) BLOCK_COMMENT("Entry:");
1928
1929 generate_disjoint_int_copy_core(aligned);
1930
1931 // O3, O4 are used as temp registers
1932 inc_counter_np(SharedRuntime::_jint_array_copy_ctr, O3, O4);
1933 __ retl();
1934 __ delayed()->mov(G0, O0); // return 0
1935 return start;
1936 }
1937
1938 //
1939 // Generate core code for conjoint int copy (and oop copy on 32-bit).
1940 // If "aligned" is true, the "from" and "to" addresses are assumed
1941 // to be heapword aligned.
1942 //
1943 // Arguments:
1944 // from: O0
1945 // to: O1
1946 // count: O2 treated as signed
1947 //
1948 void generate_conjoint_int_copy_core(bool aligned) {
1949 // Do reverse copy.
1950
1951 Label L_skip_alignment, L_aligned_copy;
1952 Label L_copy_16_bytes, L_copy_4_bytes, L_copy_4_bytes_loop, L_exit;
1953
1954 const Register from = O0; // source array address
1955 const Register to = O1; // destination array address
1956 const Register count = O2; // elements count
1957 const Register end_from = from; // source array end address
1958 const Register end_to = to; // destination array end address
1959 // O3, O4, O5, G3 are used as temp registers
1960
1961 const Register byte_count = O3; // bytes count to copy
1962
1963 __ sllx(count, LogBytesPerInt, byte_count);
1964 __ add(to, byte_count, end_to); // offset after last copied element
1965
1966 __ cmp(count, 5); // for short arrays, just do single element copy
1967 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_4_bytes);
1968 __ delayed()->add(from, byte_count, end_from);
1969
1970 // copy 1 element to align 'to' on an 8 byte boundary
1971 __ andcc(end_to, 7, G0);
1972 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1973 __ delayed()->nop();
1974 __ dec(count);
1975 __ dec(end_from, 4);
1976 __ dec(end_to, 4);
1977 __ ld(end_from, 0, O4);
1978 __ st(O4, end_to, 0);
1979 __ BIND(L_skip_alignment);
1980
1981 // Check if 'end_from' and 'end_to' has the same alignment.
1982 __ andcc(end_from, 7, G0);
1983 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1984 __ delayed()->dec(count, 4); // The cmp at the start guaranty cnt >= 4
1985
1986 // copy with shift 4 elements (16 bytes) at a time
1987 //
1988 // Load 2 aligned 8-bytes chunks and use one from previous iteration
1989 // to form 2 aligned 8-bytes chunks to store.
1990 //
1991 __ ldx(end_from, -4, O3);
1992 __ align(16);
1993 __ BIND(L_copy_16_bytes);
1994 __ ldx(end_from, -12, O4);
1995 __ deccc(count, 4);
1996 __ ldx(end_from, -20, O5);
1997 __ dec(end_to, 16);
1998 __ dec(end_from, 16);
1999 __ srlx(O3, 32, O3);
2000 __ sllx(O4, 32, G3);
2001 __ bset(G3, O3);
2002 __ stx(O3, end_to, 8);
2003 __ srlx(O4, 32, O4);
2004 __ sllx(O5, 32, G3);
2005 __ bset(O4, G3);
2006 __ stx(G3, end_to, 0);
2007 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_copy_16_bytes);
2008 __ delayed()->mov(O5, O3);
2009
2010 __ br(Assembler::always, false, Assembler::pt, L_copy_4_bytes);
2011 __ delayed()->inc(count, 4);
2012
2013 // copy 4 elements (16 bytes) at a time
2014 __ align(16);
2015 __ BIND(L_aligned_copy);
2016 __ dec(end_from, 16);
2017 __ ldx(end_from, 8, O3);
2018 __ ldx(end_from, 0, O4);
2019 __ dec(end_to, 16);
2020 __ deccc(count, 4);
2021 __ stx(O3, end_to, 8);
2022 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
2023 __ delayed()->stx(O4, end_to, 0);
2024 __ inc(count, 4);
2025
2026 // copy 1 element (4 bytes) at a time
2027 __ BIND(L_copy_4_bytes);
2028 __ br_zero(Assembler::zero, false, Assembler::pt, count, L_exit);
2029 __ delayed()->nop();
2030 __ BIND(L_copy_4_bytes_loop);
2031 __ dec(end_from, 4);
2032 __ dec(end_to, 4);
2033 __ ld(end_from, 0, O4);
2034 __ deccc(count);
2035 __ brx(Assembler::greater, false, Assembler::pt, L_copy_4_bytes_loop);
2036 __ delayed()->st(O4, end_to, 0);
2037 __ BIND(L_exit);
2038 }
2039
2040 //
2041 // Generate stub for conjoint int copy. If "aligned" is true, the
2042 // "from" and "to" addresses are assumed to be heapword aligned.
2043 //
2044 // Arguments for generated stub:
2045 // from: O0
2046 // to: O1
2047 // count: O2 treated as signed
2048 //
2049 address generate_conjoint_int_copy(bool aligned, const char * name) {
2050 __ align(CodeEntryAlignment);
2051 StubCodeMark mark(this, "StubRoutines", name);
2052 address start = __ pc();
2053
2054 address nooverlap_target = aligned ?
2055 StubRoutines::arrayof_jint_disjoint_arraycopy() :
2056 disjoint_int_copy_entry;
2057
2058 assert_clean_int(O2, O3); // Make sure 'count' is clean int.
2059
2060 if (!aligned) int_copy_entry = __ pc();
2061 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2062 if (!aligned) BLOCK_COMMENT("Entry:");
2063
2064 array_overlap_test(nooverlap_target, 2);
2065
2066 generate_conjoint_int_copy_core(aligned);
2067
2068 // O3, O4 are used as temp registers
2069 inc_counter_np(SharedRuntime::_jint_array_copy_ctr, O3, O4);
2070 __ retl();
2071 __ delayed()->mov(G0, O0); // return 0
2072 return start;
2073 }
2074
2075 //
2076 // Generate core code for disjoint long copy (and oop copy on 64-bit).
2077 // "aligned" is ignored, because we must make the stronger
2078 // assumption that both addresses are always 64-bit aligned.
2079 //
2080 // Arguments:
2081 // from: O0
2082 // to: O1
2083 // count: O2 treated as signed
2084 //
2085 void generate_disjoint_long_copy_core(bool aligned) {
2086 Label L_copy_8_bytes, L_copy_16_bytes, L_exit;
2087 const Register from = O0; // source array address
2088 const Register to = O1; // destination array address
2089 const Register count = O2; // elements count
2090 const Register offset0 = O4; // element offset
2091 const Register offset8 = O5; // next element offset
2092
2093 __ deccc(count, 2);
2094 __ mov(G0, offset0); // offset from start of arrays (0)
2095 __ brx(Assembler::negative, false, Assembler::pn, L_copy_8_bytes );
2096 __ delayed()->add(offset0, 8, offset8);
2097 __ align(16);
2098 __ BIND(L_copy_16_bytes);
2099 __ ldx(from, offset0, O3);
2100 __ ldx(from, offset8, G3);
2101 __ deccc(count, 2);
2102 __ stx(O3, to, offset0);
2103 __ inc(offset0, 16);
2104 __ stx(G3, to, offset8);
2105 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_copy_16_bytes);
2106 __ delayed()->inc(offset8, 16);
2107
2108 __ BIND(L_copy_8_bytes);
2109 __ inccc(count, 2);
2110 __ brx(Assembler::zero, true, Assembler::pn, L_exit );
2111 __ delayed()->mov(offset0, offset8); // Set O5 used by other stubs
2112 __ ldx(from, offset0, O3);
2113 __ stx(O3, to, offset0);
2114 __ BIND(L_exit);
2115 }
2116
2117 //
2118 // Generate stub for disjoint long copy.
2119 // "aligned" is ignored, because we must make the stronger
2120 // assumption that both addresses are always 64-bit aligned.
2121 //
2122 // Arguments for generated stub:
2123 // from: O0
2124 // to: O1
2125 // count: O2 treated as signed
2126 //
2127 address generate_disjoint_long_copy(bool aligned, const char * name) {
2128 __ align(CodeEntryAlignment);
2129 StubCodeMark mark(this, "StubRoutines", name);
2130 address start = __ pc();
2131
2132 assert_clean_int(O2, O3); // Make sure 'count' is clean int.
2133
2134 if (!aligned) disjoint_long_copy_entry = __ pc();
2135 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2136 if (!aligned) BLOCK_COMMENT("Entry:");
2137
2138 generate_disjoint_long_copy_core(aligned);
2139
2140 // O3, O4 are used as temp registers
2141 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr, O3, O4);
2142 __ retl();
2143 __ delayed()->mov(G0, O0); // return 0
2144 return start;
2145 }
2146
2147 //
2148 // Generate core code for conjoint long copy (and oop copy on 64-bit).
2149 // "aligned" is ignored, because we must make the stronger
2150 // assumption that both addresses are always 64-bit aligned.
2151 //
2152 // Arguments:
2153 // from: O0
2154 // to: O1
2155 // count: O2 treated as signed
2156 //
2157 void generate_conjoint_long_copy_core(bool aligned) {
2158 // Do reverse copy.
2159 Label L_copy_8_bytes, L_copy_16_bytes, L_exit;
2160 const Register from = O0; // source array address
2161 const Register to = O1; // destination array address
2162 const Register count = O2; // elements count
2163 const Register offset8 = O4; // element offset
2164 const Register offset0 = O5; // previous element offset
2165
2166 __ subcc(count, 1, count);
2167 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_8_bytes );
2168 __ delayed()->sllx(count, LogBytesPerLong, offset8);
2169 __ sub(offset8, 8, offset0);
2170 __ align(16);
2171 __ BIND(L_copy_16_bytes);
2172 __ ldx(from, offset8, O2);
2173 __ ldx(from, offset0, O3);
2174 __ stx(O2, to, offset8);
2175 __ deccc(offset8, 16); // use offset8 as counter
2176 __ stx(O3, to, offset0);
2177 __ brx(Assembler::greater, false, Assembler::pt, L_copy_16_bytes);
2178 __ delayed()->dec(offset0, 16);
2179
2180 __ BIND(L_copy_8_bytes);
2181 __ brx(Assembler::negative, false, Assembler::pn, L_exit );
2182 __ delayed()->nop();
2183 __ ldx(from, 0, O3);
2184 __ stx(O3, to, 0);
2185 __ BIND(L_exit);
2186 }
2187
2188 // Generate stub for conjoint long copy.
2189 // "aligned" is ignored, because we must make the stronger
2190 // assumption that both addresses are always 64-bit aligned.
2191 //
2192 // Arguments for generated stub:
2193 // from: O0
2194 // to: O1
2195 // count: O2 treated as signed
2196 //
2197 address generate_conjoint_long_copy(bool aligned, const char * name) {
2198 __ align(CodeEntryAlignment);
2199 StubCodeMark mark(this, "StubRoutines", name);
2200 address start = __ pc();
2201
2202 assert(!aligned, "usage");
2203 address nooverlap_target = disjoint_long_copy_entry;
2204
2205 assert_clean_int(O2, O3); // Make sure 'count' is clean int.
2206
2207 if (!aligned) long_copy_entry = __ pc();
2208 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2209 if (!aligned) BLOCK_COMMENT("Entry:");
2210
2211 array_overlap_test(nooverlap_target, 3);
2212
2213 generate_conjoint_long_copy_core(aligned);
2214
2215 // O3, O4 are used as temp registers
2216 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr, O3, O4);
2217 __ retl();
2218 __ delayed()->mov(G0, O0); // return 0
2219 return start;
2220 }
2221
2222 // Generate stub for disjoint oop copy. If "aligned" is true, the
2223 // "from" and "to" addresses are assumed to be heapword aligned.
2224 //
2225 // Arguments for generated stub:
2226 // from: O0
2227 // to: O1
2228 // count: O2 treated as signed
2229 //
2230 address generate_disjoint_oop_copy(bool aligned, const char * name) {
2231
2232 const Register from = O0; // source array address
2233 const Register to = O1; // destination array address
2234 const Register count = O2; // elements count
2235
2236 __ align(CodeEntryAlignment);
2237 StubCodeMark mark(this, "StubRoutines", name);
2238 address start = __ pc();
2239
2240 assert_clean_int(count, O3); // Make sure 'count' is clean int.
2241
2242 if (!aligned) disjoint_oop_copy_entry = __ pc();
2243 // caller can pass a 64-bit byte count here
2244 if (!aligned) BLOCK_COMMENT("Entry:");
2245
2246 // save arguments for barrier generation
2247 __ mov(to, G1);
2248 __ mov(count, G5);
2249 gen_write_ref_array_pre_barrier(G1, G5);
2250 #ifdef _LP64
2251 assert_clean_int(count, O3); // Make sure 'count' is clean int.
2252 if (UseCompressedOops) {
2253 generate_disjoint_int_copy_core(aligned);
2254 } else {
2255 generate_disjoint_long_copy_core(aligned);
2256 }
2257 #else
2258 generate_disjoint_int_copy_core(aligned);
2259 #endif
2260 // O0 is used as temp register
2261 gen_write_ref_array_post_barrier(G1, G5, O0);
2262
2263 // O3, O4 are used as temp registers
2264 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4);
2265 __ retl();
2266 __ delayed()->mov(G0, O0); // return 0
2267 return start;
2268 }
2269
2270 // Generate stub for conjoint oop copy. If "aligned" is true, the
2271 // "from" and "to" addresses are assumed to be heapword aligned.
2272 //
2273 // Arguments for generated stub:
2274 // from: O0
2275 // to: O1
2276 // count: O2 treated as signed
2277 //
2278 address generate_conjoint_oop_copy(bool aligned, const char * name) {
2279
2280 const Register from = O0; // source array address
2281 const Register to = O1; // destination array address
2282 const Register count = O2; // elements count
2283
2284 __ align(CodeEntryAlignment);
2285 StubCodeMark mark(this, "StubRoutines", name);
2286 address start = __ pc();
2287
2288 assert_clean_int(count, O3); // Make sure 'count' is clean int.
2289
2290 if (!aligned) oop_copy_entry = __ pc();
2291 // caller can pass a 64-bit byte count here
2292 if (!aligned) BLOCK_COMMENT("Entry:");
2293
2294 // save arguments for barrier generation
2295 __ mov(to, G1);
2296 __ mov(count, G5);
2297
2298 gen_write_ref_array_pre_barrier(G1, G5);
2299
2300 address nooverlap_target = aligned ?
2301 StubRoutines::arrayof_oop_disjoint_arraycopy() :
2302 disjoint_oop_copy_entry;
2303
2304 array_overlap_test(nooverlap_target, LogBytesPerHeapOop);
2305
2306 #ifdef _LP64
2307 if (UseCompressedOops) {
2308 generate_conjoint_int_copy_core(aligned);
2309 } else {
2310 generate_conjoint_long_copy_core(aligned);
2311 }
2312 #else
2313 generate_conjoint_int_copy_core(aligned);
2314 #endif
2315
2316 // O0 is used as temp register
2317 gen_write_ref_array_post_barrier(G1, G5, O0);
2318
2319 // O3, O4 are used as temp registers
2320 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4);
2321 __ retl();
2322 __ delayed()->mov(G0, O0); // return 0
2323 return start;
2324 }
2325
2326
2327 // Helper for generating a dynamic type check.
2328 // Smashes only the given temp registers.
2329 void generate_type_check(Register sub_klass,
2330 Register super_check_offset,
2331 Register super_klass,
2332 Register temp,
2333 Label& L_success,
2334 Register deccc_hack = noreg) {
2335 assert_different_registers(sub_klass, super_check_offset, super_klass, temp);
2336
2337 BLOCK_COMMENT("type_check:");
2338
2339 Label L_miss;
2340
2341 assert_clean_int(super_check_offset, temp);
2342
2343 // maybe decrement caller's trip count:
2344 #define DELAY_SLOT delayed(); \
2345 { if (deccc_hack == noreg) __ nop(); else __ deccc(deccc_hack); }
2346
2347 // if the pointers are equal, we are done (e.g., String[] elements)
2348 __ cmp(sub_klass, super_klass);
2349 __ brx(Assembler::equal, true, Assembler::pt, L_success);
2350 __ DELAY_SLOT;
2351
2352 // check the supertype display:
2353 __ ld_ptr(sub_klass, super_check_offset, temp); // query the super type
2354 __ cmp(super_klass, temp); // test the super type
2355 __ brx(Assembler::equal, true, Assembler::pt, L_success);
2356 __ DELAY_SLOT;
2357
2358 int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
2359 Klass::secondary_super_cache_offset_in_bytes());
2360 __ cmp(super_klass, sc_offset);
2361 __ brx(Assembler::notEqual, true, Assembler::pt, L_miss);
2362 __ delayed()->nop();
2363
2364 __ save_frame(0);
2365 __ mov(sub_klass->after_save(), O1);
2366 // mov(super_klass->after_save(), O2); //fill delay slot
2367 assert(StubRoutines::Sparc::_partial_subtype_check != NULL, "order of generation");
2368 __ call(StubRoutines::Sparc::_partial_subtype_check);
2369 __ delayed()->mov(super_klass->after_save(), O2);
2370 __ restore();
2371
2372 // Upon return, the condition codes are already set.
2373 __ brx(Assembler::equal, true, Assembler::pt, L_success);
2374 __ DELAY_SLOT;
2375
2376 #undef DELAY_SLOT
2377
2378 // Fall through on failure!
2379 __ BIND(L_miss);
2380 }
2381
2382
2383 // Generate stub for checked oop copy.
2384 //
2385 // Arguments for generated stub:
2386 // from: O0
2387 // to: O1
2388 // count: O2 treated as signed
2389 // ckoff: O3 (super_check_offset)
2390 // ckval: O4 (super_klass)
2391 // ret: O0 zero for success; (-1^K) where K is partial transfer count
2392 //
2393 address generate_checkcast_copy(const char* name) {
2394
2395 const Register O0_from = O0; // source array address
2396 const Register O1_to = O1; // destination array address
2397 const Register O2_count = O2; // elements count
2398 const Register O3_ckoff = O3; // super_check_offset
2399 const Register O4_ckval = O4; // super_klass
2400
2401 const Register O5_offset = O5; // loop var, with stride wordSize
2402 const Register G1_remain = G1; // loop var, with stride -1
2403 const Register G3_oop = G3; // actual oop copied
2404 const Register G4_klass = G4; // oop._klass
2405 const Register G5_super = G5; // oop._klass._primary_supers[ckval]
2406
2407 __ align(CodeEntryAlignment);
2408 StubCodeMark mark(this, "StubRoutines", name);
2409 address start = __ pc();
2410
2411 gen_write_ref_array_pre_barrier(O1, O2);
2412
2413 #ifdef ASSERT
2414 // We sometimes save a frame (see partial_subtype_check below).
2415 // If this will cause trouble, let's fail now instead of later.
2416 __ save_frame(0);
2417 __ restore();
2418 #endif
2419
2420 #ifdef ASSERT
2421 // caller guarantees that the arrays really are different
2422 // otherwise, we would have to make conjoint checks
2423 { Label L;
2424 __ mov(O3, G1); // spill: overlap test smashes O3
2425 __ mov(O4, G4); // spill: overlap test smashes O4
2426 array_overlap_test(L, LogBytesPerHeapOop);
2427 __ stop("checkcast_copy within a single array");
2428 __ bind(L);
2429 __ mov(G1, O3);
2430 __ mov(G4, O4);
2431 }
2432 #endif //ASSERT
2433
2434 assert_clean_int(O2_count, G1); // Make sure 'count' is clean int.
2435
2436 checkcast_copy_entry = __ pc();
2437 // caller can pass a 64-bit byte count here (from generic stub)
2438 BLOCK_COMMENT("Entry:");
2439
2440 Label load_element, store_element, do_card_marks, fail, done;
2441 __ addcc(O2_count, 0, G1_remain); // initialize loop index, and test it
2442 __ brx(Assembler::notZero, false, Assembler::pt, load_element);
2443 __ delayed()->mov(G0, O5_offset); // offset from start of arrays
2444
2445 // Empty array: Nothing to do.
2446 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, O3, O4);
2447 __ retl();
2448 __ delayed()->set(0, O0); // return 0 on (trivial) success
2449
2450 // ======== begin loop ========
2451 // (Loop is rotated; its entry is load_element.)
2452 // Loop variables:
2453 // (O5 = 0; ; O5 += wordSize) --- offset from src, dest arrays
2454 // (O2 = len; O2 != 0; O2--) --- number of oops *remaining*
2455 // G3, G4, G5 --- current oop, oop.klass, oop.klass.super
2456 __ align(16);
2457
2458 __ bind(store_element);
2459 // deccc(G1_remain); // decrement the count (hoisted)
2460 __ store_heap_oop(G3_oop, O1_to, O5_offset); // store the oop
2461 __ inc(O5_offset, heapOopSize); // step to next offset
2462 __ brx(Assembler::zero, true, Assembler::pt, do_card_marks);
2463 __ delayed()->set(0, O0); // return -1 on success
2464
2465 // ======== loop entry is here ========
2466 __ bind(load_element);
2467 __ load_heap_oop(O0_from, O5_offset, G3_oop); // load the oop
2468 __ br_null(G3_oop, true, Assembler::pt, store_element);
2469 __ delayed()->deccc(G1_remain); // decrement the count
2470
2471 __ load_klass(G3_oop, G4_klass); // query the object klass
2472
2473 generate_type_check(G4_klass, O3_ckoff, O4_ckval, G5_super,
2474 // branch to this on success:
2475 store_element,
2476 // decrement this on success:
2477 G1_remain);
2478 // ======== end loop ========
2479
2480 // It was a real error; we must depend on the caller to finish the job.
2481 // Register G1 has number of *remaining* oops, O2 number of *total* oops.
2482 // Emit GC store barriers for the oops we have copied (O2 minus G1),
2483 // and report their number to the caller.
2484 __ bind(fail);
2485 __ subcc(O2_count, G1_remain, O2_count);
2486 __ brx(Assembler::zero, false, Assembler::pt, done);
2487 __ delayed()->not1(O2_count, O0); // report (-1^K) to caller
2488
2489 __ bind(do_card_marks);
2490 gen_write_ref_array_post_barrier(O1_to, O2_count, O3); // store check on O1[0..O2]
2491
2492 __ bind(done);
2493 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, O3, O4);
2494 __ retl();
2495 __ delayed()->nop(); // return value in 00
2496
2497 return start;
2498 }
2499
2500
2501 // Generate 'unsafe' array copy stub
2502 // Though just as safe as the other stubs, it takes an unscaled
2503 // size_t argument instead of an element count.
2504 //
2505 // Arguments for generated stub:
2506 // from: O0
2507 // to: O1
2508 // count: O2 byte count, treated as ssize_t, can be zero
2509 //
2510 // Examines the alignment of the operands and dispatches
2511 // to a long, int, short, or byte copy loop.
2512 //
2513 address generate_unsafe_copy(const char* name) {
2514
2515 const Register O0_from = O0; // source array address
2516 const Register O1_to = O1; // destination array address
2517 const Register O2_count = O2; // elements count
2518
2519 const Register G1_bits = G1; // test copy of low bits
2520
2521 __ align(CodeEntryAlignment);
2522 StubCodeMark mark(this, "StubRoutines", name);
2523 address start = __ pc();
2524
2525 // bump this on entry, not on exit:
2526 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr, G1, G3);
2527
2528 __ or3(O0_from, O1_to, G1_bits);
2529 __ or3(O2_count, G1_bits, G1_bits);
2530
2531 __ btst(BytesPerLong-1, G1_bits);
2532 __ br(Assembler::zero, true, Assembler::pt,
2533 long_copy_entry, relocInfo::runtime_call_type);
2534 // scale the count on the way out:
2535 __ delayed()->srax(O2_count, LogBytesPerLong, O2_count);
2536
2537 __ btst(BytesPerInt-1, G1_bits);
2538 __ br(Assembler::zero, true, Assembler::pt,
2539 int_copy_entry, relocInfo::runtime_call_type);
2540 // scale the count on the way out:
2541 __ delayed()->srax(O2_count, LogBytesPerInt, O2_count);
2542
2543 __ btst(BytesPerShort-1, G1_bits);
2544 __ br(Assembler::zero, true, Assembler::pt,
2545 short_copy_entry, relocInfo::runtime_call_type);
2546 // scale the count on the way out:
2547 __ delayed()->srax(O2_count, LogBytesPerShort, O2_count);
2548
2549 __ br(Assembler::always, false, Assembler::pt,
2550 byte_copy_entry, relocInfo::runtime_call_type);
2551 __ delayed()->nop();
2552
2553 return start;
2554 }
2555
2556
2557 // Perform range checks on the proposed arraycopy.
2558 // Kills the two temps, but nothing else.
2559 // Also, clean the sign bits of src_pos and dst_pos.
2560 void arraycopy_range_checks(Register src, // source array oop (O0)
2561 Register src_pos, // source position (O1)
2562 Register dst, // destination array oo (O2)
2563 Register dst_pos, // destination position (O3)
2564 Register length, // length of copy (O4)
2565 Register temp1, Register temp2,
2566 Label& L_failed) {
2567 BLOCK_COMMENT("arraycopy_range_checks:");
2568
2569 // if (src_pos + length > arrayOop(src)->length() ) FAIL;
2570
2571 const Register array_length = temp1; // scratch
2572 const Register end_pos = temp2; // scratch
2573
2574 // Note: This next instruction may be in the delay slot of a branch:
2575 __ add(length, src_pos, end_pos); // src_pos + length
2576 __ lduw(src, arrayOopDesc::length_offset_in_bytes(), array_length);
2577 __ cmp(end_pos, array_length);
2578 __ br(Assembler::greater, false, Assembler::pn, L_failed);
2579
2580 // if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
2581 __ delayed()->add(length, dst_pos, end_pos); // dst_pos + length
2582 __ lduw(dst, arrayOopDesc::length_offset_in_bytes(), array_length);
2583 __ cmp(end_pos, array_length);
2584 __ br(Assembler::greater, false, Assembler::pn, L_failed);
2585
2586 // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2587 // Move with sign extension can be used since they are positive.
2588 __ delayed()->signx(src_pos, src_pos);
2589 __ signx(dst_pos, dst_pos);
2590
2591 BLOCK_COMMENT("arraycopy_range_checks done");
2592 }
2593
2594
2595 //
2596 // Generate generic array copy stubs
2597 //
2598 // Input:
2599 // O0 - src oop
2600 // O1 - src_pos
2601 // O2 - dst oop
2602 // O3 - dst_pos
2603 // O4 - element count
2604 //
2605 // Output:
2606 // O0 == 0 - success
2607 // O0 == -1 - need to call System.arraycopy
2608 //
2609 address generate_generic_copy(const char *name) {
2610
2611 Label L_failed, L_objArray;
2612
2613 // Input registers
2614 const Register src = O0; // source array oop
2615 const Register src_pos = O1; // source position
2616 const Register dst = O2; // destination array oop
2617 const Register dst_pos = O3; // destination position
2618 const Register length = O4; // elements count
2619
2620 // registers used as temp
2621 const Register G3_src_klass = G3; // source array klass
2622 const Register G4_dst_klass = G4; // destination array klass
2623 const Register G5_lh = G5; // layout handler
2624 const Register O5_temp = O5;
2625
2626 __ align(CodeEntryAlignment);
2627 StubCodeMark mark(this, "StubRoutines", name);
2628 address start = __ pc();
2629
2630 // bump this on entry, not on exit:
2631 inc_counter_np(SharedRuntime::_generic_array_copy_ctr, G1, G3);
2632
2633 // In principle, the int arguments could be dirty.
2634 //assert_clean_int(src_pos, G1);
2635 //assert_clean_int(dst_pos, G1);
2636 //assert_clean_int(length, G1);
2637
2638 //-----------------------------------------------------------------------
2639 // Assembler stubs will be used for this call to arraycopy
2640 // if the following conditions are met:
2641 //
2642 // (1) src and dst must not be null.
2643 // (2) src_pos must not be negative.
2644 // (3) dst_pos must not be negative.
2645 // (4) length must not be negative.
2646 // (5) src klass and dst klass should be the same and not NULL.
2647 // (6) src and dst should be arrays.
2648 // (7) src_pos + length must not exceed length of src.
2649 // (8) dst_pos + length must not exceed length of dst.
2650 BLOCK_COMMENT("arraycopy initial argument checks");
2651
2652 // if (src == NULL) return -1;
2653 __ br_null(src, false, Assembler::pn, L_failed);
2654
2655 // if (src_pos < 0) return -1;
2656 __ delayed()->tst(src_pos);
2657 __ br(Assembler::negative, false, Assembler::pn, L_failed);
2658 __ delayed()->nop();
2659
2660 // if (dst == NULL) return -1;
2661 __ br_null(dst, false, Assembler::pn, L_failed);
2662
2663 // if (dst_pos < 0) return -1;
2664 __ delayed()->tst(dst_pos);
2665 __ br(Assembler::negative, false, Assembler::pn, L_failed);
2666
2667 // if (length < 0) return -1;
2668 __ delayed()->tst(length);
2669 __ br(Assembler::negative, false, Assembler::pn, L_failed);
2670
2671 BLOCK_COMMENT("arraycopy argument klass checks");
2672 // get src->klass()
2673 if (UseCompressedOops) {
2674 __ delayed()->nop(); // ??? not good
2675 __ load_klass(src, G3_src_klass);
2676 } else {
2677 __ delayed()->ld_ptr(src, oopDesc::klass_offset_in_bytes(), G3_src_klass);
2678 }
2679
2680 #ifdef ASSERT
2681 // assert(src->klass() != NULL);
2682 BLOCK_COMMENT("assert klasses not null");
2683 { Label L_a, L_b;
2684 __ br_notnull(G3_src_klass, false, Assembler::pt, L_b); // it is broken if klass is NULL
2685 __ delayed()->nop();
2686 __ bind(L_a);
2687 __ stop("broken null klass");
2688 __ bind(L_b);
2689 __ load_klass(dst, G4_dst_klass);
2690 __ br_null(G4_dst_klass, false, Assembler::pn, L_a); // this would be broken also
2691 __ delayed()->mov(G0, G4_dst_klass); // scribble the temp
2692 BLOCK_COMMENT("assert done");
2693 }
2694 #endif
2695
2696 // Load layout helper
2697 //
2698 // |array_tag| | header_size | element_type | |log2_element_size|
2699 // 32 30 24 16 8 2 0
2700 //
2701 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2702 //
2703
2704 int lh_offset = klassOopDesc::header_size() * HeapWordSize +
2705 Klass::layout_helper_offset_in_bytes();
2706
2707 // Load 32-bits signed value. Use br() instruction with it to check icc.
2708 __ lduw(G3_src_klass, lh_offset, G5_lh);
2709
2710 if (UseCompressedOops) {
2711 __ load_klass(dst, G4_dst_klass);
2712 }
2713 // Handle objArrays completely differently...
2714 juint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2715 __ set(objArray_lh, O5_temp);
2716 __ cmp(G5_lh, O5_temp);
2717 __ br(Assembler::equal, false, Assembler::pt, L_objArray);
2718 if (UseCompressedOops) {
2719 __ delayed()->nop();
2720 } else {
2721 __ delayed()->ld_ptr(dst, oopDesc::klass_offset_in_bytes(), G4_dst_klass);
2722 }
2723
2724 // if (src->klass() != dst->klass()) return -1;
2725 __ cmp(G3_src_klass, G4_dst_klass);
2726 __ brx(Assembler::notEqual, false, Assembler::pn, L_failed);
2727 __ delayed()->nop();
2728
2729 // if (!src->is_Array()) return -1;
2730 __ cmp(G5_lh, Klass::_lh_neutral_value); // < 0
2731 __ br(Assembler::greaterEqual, false, Assembler::pn, L_failed);
2732
2733 // At this point, it is known to be a typeArray (array_tag 0x3).
2734 #ifdef ASSERT
2735 __ delayed()->nop();
2736 { Label L;
2737 jint lh_prim_tag_in_place = (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2738 __ set(lh_prim_tag_in_place, O5_temp);
2739 __ cmp(G5_lh, O5_temp);
2740 __ br(Assembler::greaterEqual, false, Assembler::pt, L);
2741 __ delayed()->nop();
2742 __ stop("must be a primitive array");
2743 __ bind(L);
2744 }
2745 #else
2746 __ delayed(); // match next insn to prev branch
2747 #endif
2748
2749 arraycopy_range_checks(src, src_pos, dst, dst_pos, length,
2750 O5_temp, G4_dst_klass, L_failed);
2751
2752 // typeArrayKlass
2753 //
2754 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2755 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2756 //
2757
2758 const Register G4_offset = G4_dst_klass; // array offset
2759 const Register G3_elsize = G3_src_klass; // log2 element size
2760
2761 __ srl(G5_lh, Klass::_lh_header_size_shift, G4_offset);
2762 __ and3(G4_offset, Klass::_lh_header_size_mask, G4_offset); // array_offset
2763 __ add(src, G4_offset, src); // src array offset
2764 __ add(dst, G4_offset, dst); // dst array offset
2765 __ and3(G5_lh, Klass::_lh_log2_element_size_mask, G3_elsize); // log2 element size
2766
2767 // next registers should be set before the jump to corresponding stub
2768 const Register from = O0; // source array address
2769 const Register to = O1; // destination array address
2770 const Register count = O2; // elements count
2771
2772 // 'from', 'to', 'count' registers should be set in this order
2773 // since they are the same as 'src', 'src_pos', 'dst'.
2774
2775 BLOCK_COMMENT("scale indexes to element size");
2776 __ sll_ptr(src_pos, G3_elsize, src_pos);
2777 __ sll_ptr(dst_pos, G3_elsize, dst_pos);
2778 __ add(src, src_pos, from); // src_addr
2779 __ add(dst, dst_pos, to); // dst_addr
2780
2781 BLOCK_COMMENT("choose copy loop based on element size");
2782 __ cmp(G3_elsize, 0);
2783 __ br(Assembler::equal,true,Assembler::pt,StubRoutines::_jbyte_arraycopy);
2784 __ delayed()->signx(length, count); // length
2785
2786 __ cmp(G3_elsize, LogBytesPerShort);
2787 __ br(Assembler::equal,true,Assembler::pt,StubRoutines::_jshort_arraycopy);
2788 __ delayed()->signx(length, count); // length
2789
2790 __ cmp(G3_elsize, LogBytesPerInt);
2791 __ br(Assembler::equal,true,Assembler::pt,StubRoutines::_jint_arraycopy);
2792 __ delayed()->signx(length, count); // length
2793 #ifdef ASSERT
2794 { Label L;
2795 __ cmp(G3_elsize, LogBytesPerLong);
2796 __ br(Assembler::equal, false, Assembler::pt, L);
2797 __ delayed()->nop();
2798 __ stop("must be long copy, but elsize is wrong");
2799 __ bind(L);
2800 }
2801 #endif
2802 __ br(Assembler::always,false,Assembler::pt,StubRoutines::_jlong_arraycopy);
2803 __ delayed()->signx(length, count); // length
2804
2805 // objArrayKlass
2806 __ BIND(L_objArray);
2807 // live at this point: G3_src_klass, G4_dst_klass, src[_pos], dst[_pos], length
2808
2809 Label L_plain_copy, L_checkcast_copy;
2810 // test array classes for subtyping
2811 __ cmp(G3_src_klass, G4_dst_klass); // usual case is exact equality
2812 __ brx(Assembler::notEqual, true, Assembler::pn, L_checkcast_copy);
2813 __ delayed()->lduw(G4_dst_klass, lh_offset, O5_temp); // hoisted from below
2814
2815 // Identically typed arrays can be copied without element-wise checks.
2816 arraycopy_range_checks(src, src_pos, dst, dst_pos, length,
2817 O5_temp, G5_lh, L_failed);
2818
2819 __ add(src, arrayOopDesc::base_offset_in_bytes(T_OBJECT), src); //src offset
2820 __ add(dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT), dst); //dst offset
2821 __ sll_ptr(src_pos, LogBytesPerHeapOop, src_pos);
2822 __ sll_ptr(dst_pos, LogBytesPerHeapOop, dst_pos);
2823 __ add(src, src_pos, from); // src_addr
2824 __ add(dst, dst_pos, to); // dst_addr
2825 __ BIND(L_plain_copy);
2826 __ br(Assembler::always, false, Assembler::pt,StubRoutines::_oop_arraycopy);
2827 __ delayed()->signx(length, count); // length
2828
2829 __ BIND(L_checkcast_copy);
2830 // live at this point: G3_src_klass, G4_dst_klass
2831 {
2832 // Before looking at dst.length, make sure dst is also an objArray.
2833 // lduw(G4_dst_klass, lh_offset, O5_temp); // hoisted to delay slot
2834 __ cmp(G5_lh, O5_temp);
2835 __ br(Assembler::notEqual, false, Assembler::pn, L_failed);
2836
2837 // It is safe to examine both src.length and dst.length.
2838 __ delayed(); // match next insn to prev branch
2839 arraycopy_range_checks(src, src_pos, dst, dst_pos, length,
2840 O5_temp, G5_lh, L_failed);
2841
2842 // Marshal the base address arguments now, freeing registers.
2843 __ add(src, arrayOopDesc::base_offset_in_bytes(T_OBJECT), src); //src offset
2844 __ add(dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT), dst); //dst offset
2845 __ sll_ptr(src_pos, LogBytesPerHeapOop, src_pos);
2846 __ sll_ptr(dst_pos, LogBytesPerHeapOop, dst_pos);
2847 __ add(src, src_pos, from); // src_addr
2848 __ add(dst, dst_pos, to); // dst_addr
2849 __ signx(length, count); // length (reloaded)
2850
2851 Register sco_temp = O3; // this register is free now
2852 assert_different_registers(from, to, count, sco_temp,
2853 G4_dst_klass, G3_src_klass);
2854
2855 // Generate the type check.
2856 int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
2857 Klass::super_check_offset_offset_in_bytes());
2858 __ lduw(G4_dst_klass, sco_offset, sco_temp);
2859 generate_type_check(G3_src_klass, sco_temp, G4_dst_klass,
2860 O5_temp, L_plain_copy);
2861
2862 // Fetch destination element klass from the objArrayKlass header.
2863 int ek_offset = (klassOopDesc::header_size() * HeapWordSize +
2864 objArrayKlass::element_klass_offset_in_bytes());
2865
2866 // the checkcast_copy loop needs two extra arguments:
2867 __ ld_ptr(G4_dst_klass, ek_offset, O4); // dest elem klass
2868 // lduw(O4, sco_offset, O3); // sco of elem klass
2869
2870 __ br(Assembler::always, false, Assembler::pt, checkcast_copy_entry);
2871 __ delayed()->lduw(O4, sco_offset, O3);
2872 }
2873
2874 __ BIND(L_failed);
2875 __ retl();
2876 __ delayed()->sub(G0, 1, O0); // return -1
2877 return start;
2878 }
2879
2880 void generate_arraycopy_stubs() {
2881
2882 // Note: the disjoint stubs must be generated first, some of
2883 // the conjoint stubs use them.
2884 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, "jbyte_disjoint_arraycopy");
2885 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, "jshort_disjoint_arraycopy");
2886 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, "jint_disjoint_arraycopy");
2887 StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_copy(false, "jlong_disjoint_arraycopy");
2888 StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_oop_copy(false, "oop_disjoint_arraycopy");
2889 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, "arrayof_jbyte_disjoint_arraycopy");
2890 StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, "arrayof_jshort_disjoint_arraycopy");
2891 StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, "arrayof_jint_disjoint_arraycopy");
2892 StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, "arrayof_jlong_disjoint_arraycopy");
2893 StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_disjoint_oop_copy(true, "arrayof_oop_disjoint_arraycopy");
2894
2895 StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, "jbyte_arraycopy");
2896 StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, "jshort_arraycopy");
2897 StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, "jint_arraycopy");
2898 StubRoutines::_jlong_arraycopy = generate_conjoint_long_copy(false, "jlong_arraycopy");
2899 StubRoutines::_oop_arraycopy = generate_conjoint_oop_copy(false, "oop_arraycopy");
2900 StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, "arrayof_jbyte_arraycopy");
2901 StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_short_copy(true, "arrayof_jshort_arraycopy");
2902 #ifdef _LP64
2903 // since sizeof(jint) < sizeof(HeapWord), there's a different flavor:
2904 StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, "arrayof_jint_arraycopy");
2905 #else
2906 StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy;
2907 #endif
2908 StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;
2909 StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy;
2910
2911 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy");
2912 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy");
2913 StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy");
2914 }
2915
2916 void generate_initial() {
2917 // Generates all stubs and initializes the entry points
2918
2919 //------------------------------------------------------------------------------------------------------------------------
2920 // entry points that exist in all platforms
2921 // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
2922 // the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
2923 StubRoutines::_forward_exception_entry = generate_forward_exception();
2924
2925 StubRoutines::_call_stub_entry = generate_call_stub(StubRoutines::_call_stub_return_address);
2926 StubRoutines::_catch_exception_entry = generate_catch_exception();
2927
2928 //------------------------------------------------------------------------------------------------------------------------
2929 // entry points that are platform specific
2930 StubRoutines::Sparc::_test_stop_entry = generate_test_stop();
2931
2932 StubRoutines::Sparc::_stop_subroutine_entry = generate_stop_subroutine();
2933 StubRoutines::Sparc::_flush_callers_register_windows_entry = generate_flush_callers_register_windows();
2934
2935 #if !defined(COMPILER2) && !defined(_LP64)
2936 StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
2937 StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg();
2938 StubRoutines::_atomic_add_entry = generate_atomic_add();
2939 StubRoutines::_atomic_xchg_ptr_entry = StubRoutines::_atomic_xchg_entry;
2940 StubRoutines::_atomic_cmpxchg_ptr_entry = StubRoutines::_atomic_cmpxchg_entry;
2941 StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
2942 StubRoutines::_atomic_add_ptr_entry = StubRoutines::_atomic_add_entry;
2943 StubRoutines::_fence_entry = generate_fence();
2944 #endif // COMPILER2 !=> _LP64
2945
2946 StubRoutines::Sparc::_partial_subtype_check = generate_partial_subtype_check();
2947 }
2948
2949
2950 void generate_all() {
2951 // Generates all stubs and initializes the entry points
2952
2953 // These entry points require SharedInfo::stack0 to be set up in non-core builds
2954 StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError), false);
2955 StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError), false);
2956 StubRoutines::_throw_ArithmeticException_entry = generate_throw_exception("ArithmeticException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_ArithmeticException), true);
2957 StubRoutines::_throw_NullPointerException_entry = generate_throw_exception("NullPointerException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException), true);
2958 StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call), false);
2959 StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError), false);
2960
2961 StubRoutines::_handler_for_unsafe_access_entry =
2962 generate_handler_for_unsafe_access();
2963
2964 // support for verify_oop (must happen after universe_init)
2965 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop_subroutine();
2966
2967 // arraycopy stubs used by compilers
2968 generate_arraycopy_stubs();
2969 }
2970
2971
2972 public:
2973 StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
2974 // replace the standard masm with a special one:
2975 _masm = new MacroAssembler(code);
2976
2977 _stub_count = !all ? 0x100 : 0x200;
2978 if (all) {
2979 generate_all();
2980 } else {
2981 generate_initial();
2982 }
2983
2984 // make sure this stub is available for all local calls
2985 if (_atomic_add_stub.is_unbound()) {
2986 // generate a second time, if necessary
2987 (void) generate_atomic_add();
2988 }
2989 }
2990
2991
2992 private:
2993 int _stub_count;
2994 void stub_prolog(StubCodeDesc* cdesc) {
2995 # ifdef ASSERT
2996 // put extra information in the stub code, to make it more readable
2997 #ifdef _LP64
2998 // Write the high part of the address
2999 // [RGV] Check if there is a dependency on the size of this prolog
3000 __ emit_data((intptr_t)cdesc >> 32, relocInfo::none);
3001 #endif
3002 __ emit_data((intptr_t)cdesc, relocInfo::none);
3003 __ emit_data(++_stub_count, relocInfo::none);
3004 # endif
3005 align(true);
3006 }
3007
3008 void align(bool at_header = false) {
3009 // %%%%% move this constant somewhere else
3010 // UltraSPARC cache line size is 8 instructions:
3011 const unsigned int icache_line_size = 32;
3012 const unsigned int icache_half_line_size = 16;
3013
3014 if (at_header) {
3015 while ((intptr_t)(__ pc()) % icache_line_size != 0) {
3016 __ emit_data(0, relocInfo::none);
3017 }
3018 } else {
3019 while ((intptr_t)(__ pc()) % icache_half_line_size != 0) {
3020 __ nop();
3021 }
3022 }
3023 }
3024
3025 }; // end class declaration
3026
3027
3028 address StubGenerator::disjoint_byte_copy_entry = NULL;
3029 address StubGenerator::disjoint_short_copy_entry = NULL;
3030 address StubGenerator::disjoint_int_copy_entry = NULL;
3031 address StubGenerator::disjoint_long_copy_entry = NULL;
3032 address StubGenerator::disjoint_oop_copy_entry = NULL;
3033
3034 address StubGenerator::byte_copy_entry = NULL;
3035 address StubGenerator::short_copy_entry = NULL;
3036 address StubGenerator::int_copy_entry = NULL;
3037 address StubGenerator::long_copy_entry = NULL;
3038 address StubGenerator::oop_copy_entry = NULL;
3039
3040 address StubGenerator::checkcast_copy_entry = NULL;
3041
3042 void StubGenerator_generate(CodeBuffer* code, bool all) {
3043 StubGenerator g(code, all);
3044 }