--- old/src/cpu/sparc/vm/assembler_sparc.cpp	Wed May 14 20:04:25 2008
+++ new/src/cpu/sparc/vm/assembler_sparc.cpp	Wed May 14 20:04:24 2008
@@ -1523,7 +1523,22 @@
   return Address(d, address(obj), oop_Relocation::spec(oop_index));
 }
 
+void  MacroAssembler::load_narrow_oop_con(jobject obj, Register d) {
+  assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
+  int oop_index = oop_recorder()->find_index(obj);
+  RelocationHolder rspec = oop_Relocation::spec(oop_index);
 
+  assert_not_delayed();
+  // Relocation with special format (see relocInfo_sparc.hpp).
+  relocate(rspec, 1);
+  // Assembler::sethi(0x3fffff, d);
+  emit_long( op(branch_op) | rd(d) | op2(sethi_op2) | hi22(0x3fffff) );
+  // Don't add relocation for 'add'. Do patching during 'sethi' processing.
+  add(d, 0x3ff, d);
+
+}
+
+
 void MacroAssembler::align(int modulus) {
   while (offset() % modulus != 0) nop();
 }
@@ -3537,28 +3552,27 @@
   }
 }
 
-void MacroAssembler::load_klass(Register s, Register d) {
+void MacroAssembler::load_klass(Register src_oop, Register klass) {
   // The number of bytes in this code is used by
   // MachCallDynamicJavaNode::ret_addr_offset()
   // if this changes, change that.
   if (UseCompressedOops) {
-    lduw(s, oopDesc::klass_offset_in_bytes(), d);
-    decode_heap_oop_not_null(d);
+    lduw(src_oop, oopDesc::klass_offset_in_bytes(), klass);
+    decode_heap_oop_not_null(klass);
   } else {
-    ld_ptr(s, oopDesc::klass_offset_in_bytes(), d);
+    ld_ptr(src_oop, oopDesc::klass_offset_in_bytes(), klass);
   }
 }
 
 // ??? figure out src vs. dst!
-void MacroAssembler::store_klass(Register d, Register s1) {
+void MacroAssembler::store_klass(Register klass, Register dst_oop) {
   if (UseCompressedOops) {
-    assert(s1 != d, "not enough registers");
-    encode_heap_oop_not_null(d);
-    // Zero out entire klass field first.
-    st_ptr(G0, s1, oopDesc::klass_offset_in_bytes());
-    st(d, s1, oopDesc::klass_offset_in_bytes());
+    assert(dst_oop != klass, "not enough registers");
+    encode_heap_oop_not_null(klass);
+    sllx(klass, BitsPerInt, klass);
+    stx(klass, dst_oop, oopDesc::klass_offset_in_bytes());
   } else {
-    st_ptr(d, s1, oopDesc::klass_offset_in_bytes());
+    st_ptr(klass, dst_oop, oopDesc::klass_offset_in_bytes());
   }
 }
 
--- old/src/cpu/sparc/vm/assembler_sparc.hpp	Wed May 14 20:04:26 2008
+++ new/src/cpu/sparc/vm/assembler_sparc.hpp	Wed May 14 20:04:26 2008
@@ -1977,8 +1977,8 @@
   inline void movbool( bool boolconst, Register d) { mov( (int) boolconst, d); }
 
   // klass oop manipulations if compressed
-  void load_klass(Register  src_oop, Register dst);
-  void store_klass(Register dst_oop, Register s1);
+  void load_klass(Register src_oop, Register klass);
+  void store_klass(Register klass, Register dst_oop);
 
    // oop manipulations
   void load_heap_oop(const Address& s, Register d, int offset = 0);
@@ -2103,6 +2103,8 @@
   inline void set_oop_constant( jobject obj, Register d ); // uses constant_oop_address
   inline void set_oop         ( Address obj_addr );        // same as load_address
 
+  void load_narrow_oop_con( jobject obj, Register d );
+
   // nop padding
   void align(int modulus);
 
--- old/src/cpu/sparc/vm/relocInfo_sparc.cpp	Wed May 14 20:04:28 2008
+++ new/src/cpu/sparc/vm/relocInfo_sparc.cpp	Wed May 14 20:04:27 2008
@@ -87,6 +87,17 @@
 #ifdef _LP64
     jint inst2;
     guarantee(Assembler::inv_op2(inst)==Assembler::sethi_op2, "must be sethi");
+    if (format() != 0) {
+      assert(type() == relocInfo::oop_type, "only narrow oops case");
+      jint np = oopDesc::encode_heap_oop((oop)x);
+      inst &= ~Assembler::hi22(-1);
+      inst |=  Assembler::hi22((intptr_t)np);
+      ip->set_long_at(0, inst);
+      inst2 = ip->long_at( NativeInstruction::nop_instruction_size );
+      guarantee(Assembler::inv_op(inst2)==Assembler::arith_op, "arith op");
+      ip->set_long_at(NativeInstruction::nop_instruction_size, ip->set_data32_simm13( inst2, (intptr_t)np));
+      break;
+    }
     ip->set_data64_sethi( ip->addr_at(0), (intptr_t)x );
 #ifdef COMPILER2
     // [RGV] Someone must have missed putting in a reloc entry for the
--- old/src/cpu/sparc/vm/relocInfo_sparc.hpp	Wed May 14 20:04:29 2008
+++ new/src/cpu/sparc/vm/relocInfo_sparc.hpp	Wed May 14 20:04:29 2008
@@ -30,8 +30,12 @@
     offset_unit        =  4,
 
     // There is no need for format bits; the instructions are
-    // sufficiently self-identifying.
+    // sufficiently self-identifying. Except narrow oops in 64-bits VM.
+#ifdef _LP64
+    format_width       =  1
+#else
     format_width       =  0
+#endif
   };
 
 
--- old/src/cpu/sparc/vm/sharedRuntime_sparc.cpp	Wed May 14 20:04:30 2008
+++ new/src/cpu/sparc/vm/sharedRuntime_sparc.cpp	Wed May 14 20:04:30 2008
@@ -2556,7 +2556,6 @@
   int total_strings = 0;
   int first_arg_to_pass = 0;
   int total_c_args = 0;
-  int box_offset = java_lang_boxing_object::value_offset_in_bytes();
 
   // Skip the receiver as dtrace doesn't want to see it
   if( !method->is_static() ) {
@@ -2778,7 +2777,9 @@
             __ br_null(in_reg, true, Assembler::pn, skipUnbox);
             __ delayed()->mov(G0, tmp);
 
-            switch (out_sig_bt[c_arg]) {
+            BasicType bt = out_sig_bt[c_arg];
+            int box_offset = java_lang_boxing_object::value_offset_in_bytes(bt);
+            switch (bt) {
                 case T_BYTE:
                   __ ldub(in_reg, box_offset, tmp); break;
                 case T_SHORT:
--- old/src/cpu/sparc/vm/sparc.ad	Wed May 14 20:04:32 2008
+++ new/src/cpu/sparc/vm/sparc.ad	Wed May 14 20:04:31 2008
@@ -5471,7 +5471,7 @@
 // Load Klass Pointer
 instruct loadKlass(iRegP dst, memory mem) %{
   match(Set dst (LoadKlass mem));
-  predicate(!n->in(MemNode::Address)->bottom_type()->is_narrow());
+  predicate(!n->in(MemNode::Address)->bottom_type()->is_ptr_to_narrowoop());
   ins_cost(MEMORY_REF_COST);
   size(4);
 
@@ -5486,10 +5486,10 @@
   ins_pipe(iload_mem);
 %}
 
-// Load Klass Pointer
-instruct loadKlassComp(iRegP dst, memory mem) %{
-  match(Set dst (LoadKlass mem));
-  predicate(n->in(MemNode::Address)->bottom_type()->is_narrow());
+// Load narrow Klass Pointer
+instruct loadNKlass(iRegN dst, memory mem) %{
+  match(Set dst (LoadNKlass mem));
+  predicate(n->in(MemNode::Address)->bottom_type()->is_ptr_to_narrowoop());
   ins_cost(MEMORY_REF_COST);
 
   format %{ "LDUW   $mem,$dst\t! compressed klass ptr" %}
@@ -5503,9 +5503,6 @@
      } else {
        __ lduw(base, $mem$$disp, dst);
      }
-     // klass oop never null but this is generated for nonheader klass loads
-     // too which can be null.
-     __ decode_heap_oop(dst);
   %}
   ins_pipe(iload_mem);
 %}
@@ -5609,22 +5606,24 @@
   ins_pipe(loadConP_poll);
 %}
 
+instruct loadConN0(iRegN dst, immN0 src) %{
+  match(Set dst src);
+
+  size(4);
+  format %{ "CLR    $dst\t! compressed NULL ptr" %}
+  ins_encode( SetNull( dst ) );
+  ins_pipe(ialu_imm);
+%}
+
 instruct loadConN(iRegN dst, immN src) %{
   match(Set dst src);
-  ins_cost(DEFAULT_COST * 2);
-  format %{ "SET    $src,$dst\t!ptr" %}
+  ins_cost(DEFAULT_COST * 3/2);
+  format %{ "SET    $src,$dst\t! compressed ptr" %}
   ins_encode %{
-    address con = (address)$src$$constant;
     Register dst = $dst$$Register;
-    if (con == NULL) {
-      __ mov(G0, dst);
-    } else {
-      __ set_oop((jobject)$src$$constant, dst);
-      __ encode_heap_oop(dst);
-    }
+    __ load_narrow_oop_con((jobject)$src$$constant, dst);
   %}
-  ins_pipe(loadConP);
-
+  ins_pipe(ialu_hi_lo_reg);
 %}
 
 instruct loadConL(iRegL dst, immL src, o7RegL tmp) %{
@@ -6258,6 +6257,34 @@
   ins_pipe(ialu_imm);
 %}
 
+// Conditional move ofr RegN. Only cmov(reg,reg).
+instruct cmovNP_reg(cmpOpP cmp, flagsRegP pcc, iRegN dst, iRegN src) %{
+  match(Set dst (CMoveN (Binary cmp pcc) (Binary dst src)));
+  ins_cost(150);
+  format %{ "MOV$cmp $pcc,$src,$dst" %}
+  ins_encode( enc_cmov_reg(cmp,dst,src, (Assembler::ptr_cc)) );
+  ins_pipe(ialu_reg);
+%}
+
+// This instruction works also for CmpN so we don't need cmovNN_reg.
+instruct cmovNI_reg(cmpOp cmp, flagsReg icc, iRegN dst, iRegN src) %{
+  match(Set dst (CMoveN (Binary cmp icc) (Binary dst src)));
+  ins_cost(150);
+  size(4);
+  format %{ "MOV$cmp  $icc,$src,$dst" %}
+  ins_encode( enc_cmov_reg(cmp,dst,src, (Assembler::icc)) );
+  ins_pipe(ialu_reg);
+%}
+
+instruct cmovNF_reg(cmpOpF cmp, flagsRegF fcc, iRegN dst, iRegN src) %{
+  match(Set dst (CMoveN (Binary cmp fcc) (Binary dst src)));
+  ins_cost(150);
+  size(4);
+  format %{ "MOV$cmp $fcc,$src,$dst" %}
+  ins_encode( enc_cmov_reg_f(cmp,dst,src, fcc) );
+  ins_pipe(ialu_reg);
+%}
+
 // Conditional move
 instruct cmovPP_reg(cmpOpP cmp, flagsRegP pcc, iRegP dst, iRegP src) %{
   match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
@@ -6417,7 +6444,6 @@
 %}
 
 
-
 //----------OS and Locking Instructions----------------------------------------
 
 // This name is KNOWN by the ADLC and cannot be changed.
@@ -8265,6 +8291,27 @@
   ins_pipe(ialu_cconly_reg_imm);
 %}
 
+// Compare Narrow oops
+instruct compN_iRegN(flagsReg icc, iRegN op1, iRegN op2 ) %{
+  match(Set icc (CmpN op1 op2));
+
+  size(4);
+  format %{ "CMP    $op1,$op2\t! compressed ptr" %}
+  opcode(Assembler::subcc_op3, Assembler::arith_op);
+  ins_encode( form3_rs1_rs2_rd( op1, op2, R_G0 ) );
+  ins_pipe(ialu_cconly_reg_reg);
+%}
+
+instruct compN_iRegN_immN0(flagsReg icc, iRegN op1, immN0 op2 ) %{
+  match(Set icc (CmpN op1 op2));
+
+  size(4);
+  format %{ "CMP    $op1,$op2\t! compressed ptr" %}
+  opcode(Assembler::subcc_op3, Assembler::arith_op);
+  ins_encode( form3_rs1_simm13_rd( op1, op2, R_G0 ) );
+  ins_pipe(ialu_cconly_reg_imm);
+%}
+
 //----------Max and Min--------------------------------------------------------
 // Min Instructions
 // Conditional move for min
@@ -8595,6 +8642,14 @@
   ins_pipe(ialu_imm);
 %}
 
+instruct cmovNL_reg(cmpOp cmp, flagsRegL xcc, iRegN dst, iRegN src) %{
+  match(Set dst (CMoveN (Binary cmp xcc) (Binary dst src)));
+  ins_cost(150);
+  format %{ "MOV$cmp  $xcc,$src,$dst" %}
+  ins_encode( enc_cmov_reg(cmp,dst,src, (Assembler::xcc)) );
+  ins_pipe(ialu_reg);
+%}
+
 instruct cmovPL_reg(cmpOp cmp, flagsRegL xcc, iRegP dst, iRegP src) %{
   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
   ins_cost(150);
@@ -8826,16 +8881,6 @@
 %}
 
 
-instruct compP_iRegN_immN0(flagsRegP pcc, iRegN op1, immN0 op2 ) %{
-  match(Set pcc (CmpN op1 op2));
-
-  size(4);
-  format %{ "CMP    $op1,$op2\t! ptr" %}
-  opcode(Assembler::subcc_op3, Assembler::arith_op);
-  ins_encode( form3_rs1_simm13_rd( op1, op2, R_G0 ) );
-  ins_pipe(ialu_cconly_reg_imm);
-%}
-
 // ============================================================================
 // inlined locking and unlocking
 
--- old/src/cpu/x86/vm/assembler_x86_64.cpp	Wed May 14 20:04:34 2008
+++ new/src/cpu/x86/vm/assembler_x86_64.cpp	Wed May 14 20:04:34 2008
@@ -683,7 +683,8 @@
 
   case REP8(0xB8): // movl/q r, #32/#64(oop?)
     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
-    assert((which == call32_operand || which == imm64_operand) && is_64bit, "");
+    assert((which == call32_operand || which == imm64_operand) && is_64bit ||
+           which == narrow_oop_operand && !is_64bit, "");
     return ip;
 
   case 0x69: // imul r, a, #32
@@ -909,7 +910,8 @@
   } else if (r->is_call() || format == call32_operand) {
     opnd = locate_operand(inst, call32_operand);
   } else if (r->is_data()) {
-    assert(format == imm64_operand || format == disp32_operand, "format ok");
+    assert(format == imm64_operand || format == disp32_operand ||
+           format == narrow_oop_operand, "format ok");
     opnd = locate_operand(inst, (WhichOperand) format);
   } else {
     assert(format == 0, "cannot specify a format");
@@ -5157,12 +5159,9 @@
 void MacroAssembler::store_klass(Register dst, Register src) {
   if (UseCompressedOops) {
     encode_heap_oop_not_null(src);
-    // zero the entire klass field first as the gap needs to be zeroed too.
-    movptr(Address(dst, oopDesc::klass_offset_in_bytes()), NULL_WORD);
-    movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
-  } else {
-    movq(Address(dst, oopDesc::klass_offset_in_bytes()), src);
+    // Store to the wide klass field to zero the gap.
   }
+  movq(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 }
 
 void MacroAssembler::load_heap_oop(Register dst, Address src) {
@@ -5277,6 +5276,18 @@
   leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 }
 
+void  MacroAssembler::load_narrow_oop_con(Register dst, jobject obj) {
+  assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
+  int oop_index = oop_recorder()->find_index(obj);
+  RelocationHolder rspec = oop_Relocation::spec(oop_index);
+
+  InstructionMark im(this);
+  int encode = prefix_and_encode(dst->encoding());
+  emit_byte(0xB8 | encode);
+  emit_data(oop_index, rspec, narrow_oop_operand);
+}
+
+
 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
   switch (cond) {
     // Note some conditions are synonyms for others
--- old/src/cpu/x86/vm/assembler_x86_64.hpp	Wed May 14 20:04:36 2008
+++ new/src/cpu/x86/vm/assembler_x86_64.hpp	Wed May 14 20:04:36 2008
@@ -490,7 +490,12 @@
     imm64_operand  = 0,          // embedded 64-bit immediate operand
     disp32_operand = 1,          // embedded 32-bit displacement
     call32_operand = 2,          // embedded 32-bit self-relative displacement
+#ifdef AMD64
+    narrow_oop_operand = 3,      // embedded 32-bit immediate narrow oop
+    _WhichOperand_limit = 4
+#else
     _WhichOperand_limit = 3
+#endif // AMD64
   };
 
   public:
@@ -1114,6 +1119,8 @@
   void encode_heap_oop_not_null(Register dst, Register src);
   void decode_heap_oop_not_null(Register dst, Register src);
 
+  void load_narrow_oop_con(Register dst, jobject obj);
+
   // Stack frame creation/removal
   void enter();
   void leave();
--- old/src/cpu/x86/vm/relocInfo_x86.cpp	Wed May 14 20:04:37 2008
+++ new/src/cpu/x86/vm/relocInfo_x86.cpp	Wed May 14 20:04:37 2008
@@ -30,11 +30,15 @@
 #ifdef AMD64
   x += o;
   typedef Assembler::WhichOperand WhichOperand;
-  WhichOperand which = (WhichOperand) format(); // that is, disp32 or imm64, call32
+  WhichOperand which = (WhichOperand) format(); // that is, disp32 or imm64, call32, narrow oop
   assert(which == Assembler::disp32_operand ||
+         which == Assembler::narrow_oop_operand ||
          which == Assembler::imm64_operand, "format unpacks ok");
   if (which == Assembler::imm64_operand) {
     *pd_address_in_code() = x;
+  } else if (which == Assembler::narrow_oop_operand) {
+    address disp = Assembler::locate_operand(addr(), which);
+    *(int32_t*) disp = oopDesc::encode_heap_oop((oop)x);
   } else {
     // Note:  Use runtime_call_type relocations for call32_operand.
     address ip = addr();
--- old/src/cpu/x86/vm/relocInfo_x86.hpp	Wed May 14 20:04:38 2008
+++ new/src/cpu/x86/vm/relocInfo_x86.hpp	Wed May 14 20:04:38 2008
@@ -28,6 +28,12 @@
     // Intel instructions are byte-aligned.
     offset_unit        =  1,
 
+#ifdef _LP64
+    // Encodes Assembler::disp32_operand vs. Assembler::imm64_operand
+    // vs Assembler::narrow_oop_operand.
+    format_width       =  2
+#else
     // Encodes Assembler::disp32_operand vs. Assembler::imm32_operand.
     format_width       =  1
+#endif
   };
--- old/src/cpu/x86/vm/sharedRuntime_x86_32.cpp	Wed May 14 20:04:40 2008
+++ new/src/cpu/x86/vm/sharedRuntime_x86_32.cpp	Wed May 14 20:04:39 2008
@@ -1920,7 +1920,6 @@
   int total_strings = 0;
   int first_arg_to_pass = 0;
   int total_c_args = 0;
-  int box_offset = java_lang_boxing_object::value_offset_in_bytes();
 
   if( !method->is_static() ) {  // Pass in receiver first
     in_sig_bt[i++] = T_OBJECT;
@@ -2131,7 +2130,10 @@
           assert(dst.first()->is_stack() &&
                  (!dst.second()->is_valid() || dst.second()->is_stack()),
                  "value(s) must go into stack slots");
-          if ( out_sig_bt[c_arg] == T_LONG ) {
+
+          BasicType bt = out_sig_bt[c_arg];
+          int box_offset = java_lang_boxing_object::value_offset_in_bytes(bt);
+          if ( bt == T_LONG ) {
             __ movl(rbx, Address(in_reg,
                                  box_offset + VMRegImpl::stack_slot_size));
             __ movl(Address(rsp, reg2offset_out(dst.second())), rbx);
--- old/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Wed May 14 20:04:41 2008
+++ new/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Wed May 14 20:04:41 2008
@@ -1950,7 +1950,6 @@
   int total_strings = 0;
   int first_arg_to_pass = 0;
   int total_c_args = 0;
-  int box_offset = java_lang_boxing_object::value_offset_in_bytes();
 
   // Skip the receiver as dtrace doesn't want to see it
   if( !method->is_static() ) {
@@ -2197,8 +2196,10 @@
               __ testq(in_reg, in_reg);
               __ jcc(Assembler::zero, skipUnbox);
 
+              BasicType bt = out_sig_bt[c_arg];
+              int box_offset = java_lang_boxing_object::value_offset_in_bytes(bt);
               Address src1(in_reg, box_offset);
-              if ( out_sig_bt[c_arg] == T_LONG ) {
+              if ( bt == T_LONG ) {
                 __ movq(in_reg,  src1);
                 __ movq(stack_dst, in_reg);
                 assert(out_sig_bt[c_arg+1] == T_VOID, "must be");
@@ -2460,8 +2461,10 @@
           Label skip;
           __ testq(r, r);
           __ jcc(Assembler::equal, skip);
+          BasicType bt = out_sig_bt[c_arg];
+          int box_offset = java_lang_boxing_object::value_offset_in_bytes(bt);
           Address src1(r, box_offset);
-          if ( out_sig_bt[c_arg] == T_LONG ) {
+          if ( bt == T_LONG ) {
             __ movq(r, src1);
           } else {
             __ movl(r, src1);
--- old/src/cpu/x86/vm/x86_64.ad	Wed May 14 20:04:43 2008
+++ new/src/cpu/x86/vm/x86_64.ad	Wed May 14 20:04:42 2008
@@ -6064,7 +6064,7 @@
 instruct loadKlass(rRegP dst, memory mem)
 %{
   match(Set dst (LoadKlass mem));
-  predicate(!n->in(MemNode::Address)->bottom_type()->is_narrow());
+  predicate(!n->in(MemNode::Address)->bottom_type()->is_ptr_to_narrowoop());
 
   ins_cost(125); // XXX
   format %{ "movq    $dst, $mem\t# class" %}
@@ -6073,22 +6073,18 @@
   ins_pipe(ialu_reg_mem); // XXX
 %}
 
-// Load Klass Pointer
-instruct loadKlassComp(rRegP dst, memory mem)
+// Load narrow Klass Pointer
+instruct loadNKlass(rRegN dst, memory mem)
 %{
-  match(Set dst (LoadKlass mem));
-  predicate(n->in(MemNode::Address)->bottom_type()->is_narrow());
+  match(Set dst (LoadNKlass mem));
+  predicate(n->in(MemNode::Address)->bottom_type()->is_ptr_to_narrowoop());
 
   ins_cost(125); // XXX
-  format %{ "movl    $dst, $mem\t# compressed class\n\t"
-            "decode_heap_oop $dst,$dst" %}
+  format %{ "movl    $dst, $mem\t# compressed class\n\t" %}
   ins_encode %{
     Address addr = build_address($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
     Register dst = as_Register($dst$$reg);
     __ movl(dst, addr);
-    // klass is never null in the header but this is generated for all
-    // klass loads not just the _klass field in the header.
-    __ decode_heap_oop(dst);
   %}
   ins_pipe(ialu_reg_mem); // XXX
 %}
@@ -6362,8 +6358,7 @@
   match(Set dst src);
 
   ins_cost(125);
-  format %{ "movq    $dst, $src\t# compressed ptr\n\t"
-            "encode_heap_oop_not_null $dst,$dst" %}
+  format %{ "movl    $dst, $src\n# compressed ptr" %}
   ins_encode %{
     address con = (address)$src$$constant;
     Register dst = $dst$$Register;
@@ -6370,8 +6365,7 @@
     if (con == NULL) {
       ShouldNotReachHere();
     } else {
-      __ movoop(dst, (jobject)$src$$constant);
-      __ encode_heap_oop_not_null(dst);
+      __ load_narrow_oop_con(dst, (jobject)$src$$constant);
     }
   %}
   ins_pipe(ialu_reg_fat); // XXX
@@ -7143,6 +7137,30 @@
 %}
 
 // Conditional move
+instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
+%{
+  match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
+
+  ins_cost(200); // XXX
+  format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
+  opcode(0x0F, 0x40);
+  ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
+  ins_pipe(pipe_cmov_reg);
+%}
+
+// Conditional move
+instruct cmovN_regU(rRegN dst, rRegN src, rFlagsRegU cr, cmpOpU cop)
+%{
+  match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
+
+  ins_cost(200); // XXX
+  format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
+  opcode(0x0F, 0x40);
+  ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
+  ins_pipe(pipe_cmov_reg);
+%}
+
+// Conditional move
 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 %{
   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
@@ -11055,14 +11073,50 @@
   ins_pipe(ialu_cr_reg_imm);
 %}
 
+
+instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
+%{
+  match(Set cr (CmpN op1 op2));
+
+  format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
+  ins_encode %{ __ cmpl(as_Register($op1$$reg), as_Register($op2$$reg)); %}
+  ins_pipe(ialu_cr_reg_reg);
+%}
+
+instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
+%{
+  match(Set cr (CmpN src (LoadN mem)));
+
+  ins_cost(500); // XXX
+  format %{ "cmpl    $src, mem\t# compressed ptr" %}
+  ins_encode %{
+    Address adr = build_address($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
+    __ cmpl(as_Register($src$$reg), adr);
+  %}
+  ins_pipe(ialu_cr_reg_mem);
+%}
+
 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
   match(Set cr (CmpN src zero));
 
-  format %{ "testl   $src, $src" %}
+  format %{ "testl   $src, $src\t# compressed ptr" %}
   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
   ins_pipe(ialu_cr_reg_imm);
 %}
 
+instruct testN_reg_mem(rFlagsReg cr, memory mem, immN0 zero)
+%{
+  match(Set cr (CmpN (LoadN mem) zero));
+
+  ins_cost(500); // XXX
+  format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
+  ins_encode %{
+    Address addr = build_address($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
+    __ cmpl(addr, (int)0xFFFFFFFF);
+  %}
+  ins_pipe(ialu_cr_reg_mem);
+%}
+
 // Yanked all unsigned pointer compare operations.
 // Pointer compares are done with CmpP which is already unsigned.
 
--- old/src/share/vm/adlc/forms.cpp	Wed May 14 20:04:46 2008
+++ new/src/share/vm/adlc/forms.cpp	Wed May 14 20:04:45 2008
@@ -258,6 +258,7 @@
   if( strcmp(opType,"LoadLLocked")==0 )  return Form::idealL;
   if( strcmp(opType,"LoadP")==0 )  return Form::idealP;
   if( strcmp(opType,"LoadN")==0 )  return Form::idealN;
+  if( strcmp(opType,"LoadNKlass")==0 ) return Form::idealN;
   if( strcmp(opType,"LoadRange")==0 )  return Form::idealI;
   if( strcmp(opType,"LoadS")==0 )  return Form::idealS;
   if( strcmp(opType,"Load16B")==0 )  return Form::idealB;
--- old/src/share/vm/adlc/formssel.cpp	Wed May 14 20:04:47 2008
+++ new/src/share/vm/adlc/formssel.cpp	Wed May 14 20:04:47 2008
@@ -3313,7 +3313,7 @@
     "Store8B","Store4B","Store8C","Store4C","Store2C",
     "Load4I" ,"Load2I" ,"Load2L" ,"Load2D" ,"Load4F" ,"Load2F" ,"Load16B" ,
     "Load8B" ,"Load4B" ,"Load8C" ,"Load4C" ,"Load2C" ,"Load8S", "Load4S","Load2S",
-    "LoadRange", "LoadKlass", "LoadL_unaligned", "LoadD_unaligned",
+    "LoadRange", "LoadKlass", "LoadNKlass", "LoadL_unaligned", "LoadD_unaligned",
     "LoadPLocked", "LoadLLocked",
     "StorePConditional", "StoreLConditional",
     "CompareAndSwapI", "CompareAndSwapL", "CompareAndSwapP", "CompareAndSwapN",
--- old/src/share/vm/ci/ciInstanceKlass.cpp	Wed May 14 20:04:49 2008
+++ new/src/share/vm/ci/ciInstanceKlass.cpp	Wed May 14 20:04:48 2008
@@ -392,12 +392,12 @@
   assert(!is_java_lang_Object(), "bootstrap OK");
 
   // Size in bytes of my fields, including inherited fields.
-  int fsize = nonstatic_field_size() * wordSize;
+  int fsize = nonstatic_field_size() * heapOopSize;
 
   ciInstanceKlass* super = this->super();
   GrowableArray<ciField*>* super_fields = NULL;
   if (super != NULL && super->has_nonstatic_fields()) {
-    int super_fsize  = super->nonstatic_field_size() * wordSize;
+    int super_fsize  = super->nonstatic_field_size() * heapOopSize;
     int super_flen   = super->nof_nonstatic_fields();
     super_fields = super->_nonstatic_fields;
     assert(super_flen == 0 || super_fields != NULL, "first get nof_fields");
@@ -438,7 +438,7 @@
     // This is a minor inefficiency classFileParser.cpp.
     last_offset = offset + size;
   }
-  assert(last_offset <= (int)sizeof(oopDesc) + fsize, "no overflow");
+  assert(last_offset <= (int)instanceOopDesc::base_offset_in_bytes() + fsize, "no overflow");
 #endif
 
   _nonstatic_fields = fields;
--- old/src/share/vm/classfile/classFileParser.cpp	Wed May 14 20:04:50 2008
+++ new/src/share/vm/classfile/classFileParser.cpp	Wed May 14 20:04:50 2008
@@ -2664,8 +2664,8 @@
                                   fac.static_byte_count ), wordSize );
     static_field_size           = (next_static_type_offset -
                                   next_static_oop_offset) / wordSize;
-    first_nonstatic_field_offset = (instanceOopDesc::header_size() +
-                                    nonstatic_field_size) * wordSize;
+    first_nonstatic_field_offset = instanceOopDesc::base_offset_in_bytes() +
+                                   nonstatic_field_size * heapOopSize;
     next_nonstatic_field_offset = first_nonstatic_field_offset;
 
     // Add fake fields for java.lang.Class instances (also see below)
@@ -2734,9 +2734,9 @@
       next_nonstatic_byte_offset  = next_nonstatic_short_offset +
                                     (nonstatic_short_count * BytesPerShort);
       next_nonstatic_type_offset  = align_size_up((next_nonstatic_byte_offset +
-                                    nonstatic_byte_count ), wordSize );
+                                    nonstatic_byte_count ), heapOopSize );
       orig_nonstatic_field_size   = nonstatic_field_size +
-        ((next_nonstatic_type_offset - first_nonstatic_field_offset)/wordSize);
+      ((next_nonstatic_type_offset - first_nonstatic_field_offset)/heapOopSize);
     }
 #endif
     bool compact_fields   = CompactFields;
@@ -2791,18 +2791,8 @@
     int nonstatic_short_space_offset;
     int nonstatic_byte_space_offset;
 
-    bool compact_into_header = (UseCompressedOops &&
-                                allocation_style == 1 && compact_fields &&
-                                !super_has_nonstatic_fields);
-
-    if( compact_into_header || nonstatic_double_count > 0 ) {
-      int offset;
-      // Pack something in with the header if no super klass has done so.
-      if (compact_into_header) {
-        offset = oopDesc::klass_gap_offset_in_bytes();
-      } else {
-        offset = next_nonstatic_double_offset;
-      }
+    if( nonstatic_double_count > 0 ) {
+      int offset = next_nonstatic_double_offset;
       next_nonstatic_double_offset = align_size_up(offset, BytesPerLong);
       if( compact_fields && offset != next_nonstatic_double_offset ) {
         // Allocate available fields into the gap before double field.
@@ -2830,8 +2820,7 @@
         }
         // Allocate oop field in the gap if there are no other fields for that.
         nonstatic_oop_space_offset = offset;
-        if(!compact_into_header && length >= heapOopSize &&
-            nonstatic_oop_count > 0 &&
+        if( length >= heapOopSize && nonstatic_oop_count > 0 &&
             allocation_style != 0 ) { // when oop fields not first
           nonstatic_oop_count      -= 1;
           nonstatic_oop_space_count = 1; // Only one will fit
@@ -2854,14 +2843,13 @@
     } else { // allocation_style == 1
       next_nonstatic_oop_offset = next_nonstatic_byte_offset + nonstatic_byte_count;
       if( nonstatic_oop_count > 0 ) {
-        notaligned_offset = next_nonstatic_oop_offset;
         next_nonstatic_oop_offset = align_size_up(next_nonstatic_oop_offset, heapOopSize);
       }
       notaligned_offset = next_nonstatic_oop_offset + (nonstatic_oop_count * heapOopSize);
     }
-    next_nonstatic_type_offset = align_size_up(notaligned_offset, wordSize );
+    next_nonstatic_type_offset = align_size_up(notaligned_offset, heapOopSize );
     nonstatic_field_size = nonstatic_field_size + ((next_nonstatic_type_offset
-                                      - first_nonstatic_field_offset)/wordSize);
+                                   - first_nonstatic_field_offset)/heapOopSize);
 
     // Iterate over fields again and compute correct offsets.
     // The field allocation type was temporarily stored in the offset slot.
@@ -2962,9 +2950,10 @@
     // Size of instances
     int instance_size;
 
+    next_nonstatic_type_offset = align_size_up(notaligned_offset, wordSize );
     instance_size = align_object_size(next_nonstatic_type_offset / wordSize);
 
-    assert(instance_size == align_object_size(instanceOopDesc::header_size() + nonstatic_field_size), "consistent layout helper value");
+    assert(instance_size == align_object_size(align_size_up((instanceOopDesc::base_offset_in_bytes() + nonstatic_field_size*heapOopSize), wordSize) / wordSize), "consistent layout helper value");
 
     // Size of non-static oop map blocks (in words) allocated at end of klass
     int nonstatic_oop_map_size = compute_oop_map_size(super_klass, nonstatic_oop_map_count, first_nonstatic_oop_offset);
--- old/src/share/vm/classfile/javaClasses.cpp	Wed May 14 20:04:52 2008
+++ new/src/share/vm/classfile/javaClasses.cpp	Wed May 14 20:04:51 2008
@@ -1872,7 +1872,7 @@
       box->float_field_put(value_offset, value->f);
       break;
     case T_DOUBLE:
-      box->double_field_put(value_offset, value->d);
+      box->double_field_put(long_value_offset, value->d);
       break;
     case T_BYTE:
       box->byte_field_put(value_offset, value->b);
@@ -1884,7 +1884,7 @@
       box->int_field_put(value_offset, value->i);
       break;
     case T_LONG:
-      box->long_field_put(value_offset, value->j);
+      box->long_field_put(long_value_offset, value->j);
       break;
     default:
       return NULL;
@@ -1915,7 +1915,7 @@
     value->f = box->float_field(value_offset);
     break;
   case T_DOUBLE:
-    value->d = box->double_field(value_offset);
+    value->d = box->double_field(long_value_offset);
     break;
   case T_BYTE:
     value->b = box->byte_field(value_offset);
@@ -1927,7 +1927,7 @@
     value->i = box->int_field(value_offset);
     break;
   case T_LONG:
-    value->j = box->long_field(value_offset);
+    value->j = box->long_field(long_value_offset);
     break;
   default:
     return T_ILLEGAL;
@@ -1949,7 +1949,7 @@
     box->float_field_put(value_offset, value->f);
     break;
   case T_DOUBLE:
-    box->double_field_put(value_offset, value->d);
+    box->double_field_put(long_value_offset, value->d);
     break;
   case T_BYTE:
     box->byte_field_put(value_offset, value->b);
@@ -1961,7 +1961,7 @@
     box->int_field_put(value_offset, value->i);
     break;
   case T_LONG:
-    box->long_field_put(value_offset, value->j);
+    box->long_field_put(long_value_offset, value->j);
     break;
   default:
     return T_ILLEGAL;
@@ -2163,6 +2163,7 @@
 int java_lang_reflect_Field::signature_offset;
 int java_lang_reflect_Field::annotations_offset;
 int java_lang_boxing_object::value_offset;
+int java_lang_boxing_object::long_value_offset;
 int java_lang_ref_Reference::referent_offset;
 int java_lang_ref_Reference::queue_offset;
 int java_lang_ref_Reference::next_offset;
@@ -2282,10 +2283,7 @@
 // are not available to determine the offset_of_static_fields.
 void JavaClasses::compute_hard_coded_offsets() {
   const int x = heapOopSize;
-  // Objects don't get allocated in the gap in the header with compressed oops
-  // for these special classes because hard coded offsets can't be conditional
-  // so base_offset_in_bytes() is wrong here, allocate after the header.
-  const int header = sizeof(instanceOopDesc);
+  const int header = instanceOopDesc::base_offset_in_bytes();
 
   // Do the String Class
   java_lang_String::value_offset  = java_lang_String::hc_value_offset  * x + header;
@@ -2308,7 +2306,8 @@
   java_lang_Throwable::stackTrace_offset = java_lang_Throwable::hc_stackTrace_offset * x + header;
 
   // java_lang_boxing_object
-  java_lang_boxing_object::value_offset = java_lang_boxing_object::hc_value_offset * x + header;
+  java_lang_boxing_object::value_offset = java_lang_boxing_object::hc_value_offset + header;
+  java_lang_boxing_object::long_value_offset = align_size_up((java_lang_boxing_object::hc_value_offset + header), BytesPerLong);
 
   // java_lang_ref_Reference:
   java_lang_ref_Reference::referent_offset = java_lang_ref_Reference::hc_referent_offset * x + header;
@@ -2322,7 +2321,7 @@
   java_lang_ref_Reference::number_of_fake_oop_fields = 1;
 
   // java_lang_ref_SoftReference Class
-  java_lang_ref_SoftReference::timestamp_offset = java_lang_ref_SoftReference::hc_timestamp_offset * x + header;
+  java_lang_ref_SoftReference::timestamp_offset = align_size_up((java_lang_ref_SoftReference::hc_timestamp_offset * x + header), BytesPerLong);
   // Don't multiply static fields because they are always in wordSize units
   java_lang_ref_SoftReference::static_clock_offset = java_lang_ref_SoftReference::hc_static_clock_offset * x;
 
@@ -2469,6 +2468,9 @@
 #define CHECK_OFFSET(klass_name, cpp_klass_name, field_name, field_sig) \
   valid &= check_offset(klass_name, cpp_klass_name :: field_name ## _offset, #field_name, field_sig)
 
+#define CHECK_LONG_OFFSET(klass_name, cpp_klass_name, field_name, field_sig) \
+  valid &= check_offset(klass_name, cpp_klass_name :: long_ ## field_name ## _offset, #field_name, field_sig)
+
 #define CHECK_STATIC_OFFSET(klass_name, cpp_klass_name, field_name, field_sig) \
   valid &= check_static_offset(klass_name, cpp_klass_name :: static_ ## field_name ## _offset, #field_name, field_sig)
 
@@ -2501,11 +2503,11 @@
   CHECK_OFFSET("java/lang/Boolean",   java_lang_boxing_object, value, "Z");
   CHECK_OFFSET("java/lang/Character", java_lang_boxing_object, value, "C");
   CHECK_OFFSET("java/lang/Float",     java_lang_boxing_object, value, "F");
-  CHECK_OFFSET("java/lang/Double",    java_lang_boxing_object, value, "D");
+  CHECK_LONG_OFFSET("java/lang/Double", java_lang_boxing_object, value, "D");
   CHECK_OFFSET("java/lang/Byte",      java_lang_boxing_object, value, "B");
   CHECK_OFFSET("java/lang/Short",     java_lang_boxing_object, value, "S");
   CHECK_OFFSET("java/lang/Integer",   java_lang_boxing_object, value, "I");
-  CHECK_OFFSET("java/lang/Long",      java_lang_boxing_object, value, "J");
+  CHECK_LONG_OFFSET("java/lang/Long", java_lang_boxing_object, value, "J");
 
   // java.lang.ClassLoader
 
--- old/src/share/vm/classfile/javaClasses.hpp	Wed May 14 20:04:53 2008
+++ new/src/share/vm/classfile/javaClasses.hpp	Wed May 14 20:04:53 2008
@@ -653,6 +653,7 @@
    hc_value_offset = 0
   };
   static int value_offset;
+  static int long_value_offset;
 
   static oop initialize_and_allocate(BasicType type, TRAPS);
  public:
@@ -665,7 +666,10 @@
   static bool is_instance(oop box)                 { return basic_type(box) != T_ILLEGAL; }
   static bool is_instance(oop box, BasicType type) { return basic_type(box) == type; }
 
-  static int value_offset_in_bytes() { return value_offset; }
+  static int value_offset_in_bytes(BasicType type) {
+    return ( type == T_LONG || type == T_DOUBLE ) ? long_value_offset :
+                                                    value_offset;
+  }
 
   // Debugging
   friend class JavaClasses;
@@ -747,7 +751,7 @@
  public:
   enum {
    // The timestamp is a long field and may need to be adjusted for alignment.
-   hc_timestamp_offset    = align_object_offset_(hc_discovered_offset + 1)
+   hc_timestamp_offset  = hc_discovered_offset + 1
   };
   enum {
    hc_static_clock_offset = 0
--- old/src/share/vm/gc_implementation/includeDB_gc_shared	Wed May 14 20:04:54 2008
+++ new/src/share/vm/gc_implementation/includeDB_gc_shared	Wed May 14 20:04:54 2008
@@ -54,6 +54,7 @@
 markSweep.inline.hpp                    psParallelCompact.hpp
 
 mutableNUMASpace.cpp                    mutableNUMASpace.hpp
+mutableNUMASpace.cpp                    oop.inline.hpp
 mutableNUMASpace.cpp                    sharedHeap.hpp
 mutableNUMASpace.cpp                    thread_<os_family>.inline.hpp
 
--- old/src/share/vm/includeDB_core	Wed May 14 20:04:56 2008
+++ new/src/share/vm/includeDB_core	Wed May 14 20:04:55 2008
@@ -3492,6 +3492,7 @@
 relocInfo_<arch>.cpp                    assembler.inline.hpp
 relocInfo_<arch>.cpp                    assembler_<arch_model>.inline.hpp
 relocInfo_<arch>.cpp                    nativeInst_<arch>.hpp
+relocInfo_<arch>.cpp                    oop.inline.hpp
 relocInfo_<arch>.cpp                    relocInfo.hpp
 relocInfo_<arch>.cpp                    safepoint.hpp
 
--- old/src/share/vm/oops/arrayOop.hpp	Wed May 14 20:04:58 2008
+++ new/src/share/vm/oops/arrayOop.hpp	Wed May 14 20:04:57 2008
@@ -43,9 +43,8 @@
   // Returns the aligned header_size_in_bytes.  This is not equivalent to
   // sizeof(arrayOopDesc) which should not appear in the code, except here.
   static int header_size_in_bytes() {
-    size_t hs = UseCompressedOops ?
-            sizeof(arrayOopDesc) :
-            align_size_up(sizeof(arrayOopDesc) + sizeof(int), HeapWordSize);
+    size_t hs = align_size_up(length_offset_in_bytes() + sizeof(int), 
+                              HeapWordSize);
 #ifdef ASSERT
     // make sure it isn't called before UseCompressedOops is initialized.
     static size_t arrayoopdesc_hs = 0;
--- old/src/share/vm/oops/instanceKlass.hpp	Wed May 14 20:04:59 2008
+++ new/src/share/vm/oops/instanceKlass.hpp	Wed May 14 20:04:58 2008
@@ -180,9 +180,8 @@
   // End of the oop block.
   //
 
-  // number of words used by non-static fields in this klass (including
-  // inherited fields but after header_size()).  If fields are compressed into
-  // header, this can be zero so it's not the same as number of static fields.
+  // Number of heapOopSize words used by non-static fields in this klass
+  // (including inherited fields but after header_size()).
   int             _nonstatic_field_size;
   int             _static_field_size;    // number words used by static fields (oop and non-oop) in this klass
   int             _static_oop_field_size;// number of static oop fields in this klass
--- old/src/share/vm/oops/instanceKlassKlass.cpp	Wed May 14 20:05:00 2008
+++ new/src/share/vm/oops/instanceKlassKlass.cpp	Wed May 14 20:05:00 2008
@@ -581,7 +581,7 @@
   OopMapBlock* map     = ik->start_of_nonstatic_oop_maps();
   OopMapBlock* end_map = map + ik->nonstatic_oop_map_size();
   while (map < end_map) {
-    st->print("%d-%d ", map->offset(), map->offset() + oopSize*(map->length() - 1));
+    st->print("%d-%d ", map->offset(), map->offset() + heapOopSize*(map->length() - 1));
     map++;
   }
   st->cr();
--- old/src/share/vm/oops/instanceOop.hpp	Wed May 14 20:05:01 2008
+++ new/src/share/vm/oops/instanceOop.hpp	Wed May 14 20:05:01 2008
@@ -39,14 +39,7 @@
 
   static bool contains_field_offset(int offset, int nonstatic_field_size) {
     int base_in_bytes = base_offset_in_bytes();
-    if (UseCompressedOops) {
-      return (offset >= base_in_bytes &&
-              // field can be embedded in header, or is after header.
-              (offset < (int)sizeof(instanceOopDesc) ||
-              (offset-(int)sizeof(instanceOopDesc))/wordSize < nonstatic_field_size));
-    } else {
-      return (offset >= base_in_bytes &&
-              (offset-base_in_bytes)/wordSize < nonstatic_field_size);
-    }
+    return (offset >= base_in_bytes &&
+            (offset-base_in_bytes) < nonstatic_field_size * heapOopSize);
   }
 };
--- old/src/share/vm/opto/c2_globals.hpp	Wed May 14 20:05:03 2008
+++ new/src/share/vm/opto/c2_globals.hpp	Wed May 14 20:05:03 2008
@@ -390,5 +390,8 @@
                                                                             \
   product(intx, MaxLabelRootDepth, 1100,                                    \
           "Maximum times call Label_Root to prevent stack overflow")        \
+                                                                            \
+  diagnostic(intx, DominatorSearchLimit, 1000,                              \
+          "Iterations limit in Node::dominates")                            \
 
 C2_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_NOTPRODUCT_FLAG)
--- old/src/share/vm/opto/callnode.cpp	Wed May 14 20:05:05 2008
+++ new/src/share/vm/opto/callnode.cpp	Wed May 14 20:05:04 2008
@@ -676,13 +676,19 @@
           at_k->is_loaded() &&
           at_k->is_java_klass() &&
           !at_k->is_interface()) {
-        // If we have found an argument matching addr_t, check if the field
-        // at the specified offset is modified.
-        int at_idx = C->get_alias_index(at_ptr->add_offset(offset)->isa_oopptr());
-        if (base_idx == at_idx &&
+        if (adr_k->is_subclass_of(at_k) &&
             (bcea == NULL ||
              bcea->is_arg_modified(i - TypeFunc::Parms, offset, size))) {
           return true;
+        } else if (at_k->is_subclass_of(adr_k)) {
+          // If we have found an argument matching addr_t, check if the field
+          // at the specified offset is modified.
+          int at_idx = C->get_alias_index(at_ptr->add_offset(offset)->isa_oopptr());
+          if (base_idx == at_idx &&
+              (bcea == NULL ||
+               bcea->is_arg_modified(i - TypeFunc::Parms, offset, size))) {
+            return true;
+          }
         }
       }
     }
--- old/src/share/vm/opto/cfgnode.cpp	Wed May 14 20:05:07 2008
+++ new/src/share/vm/opto/cfgnode.cpp	Wed May 14 20:05:06 2008
@@ -707,8 +707,14 @@
 //------------------------split_out_instance-----------------------------------
 // Split out an instance type from a bottom phi.
 PhiNode* PhiNode::split_out_instance(const TypePtr* at, PhaseIterGVN *igvn) const {
-  assert(type() == Type::MEMORY && (adr_type() == TypePtr::BOTTOM ||
-         adr_type() == TypeRawPtr::BOTTOM) , "bottom or raw memory required");
+  const TypeOopPtr *t_oop = at->isa_oopptr();
+  assert(t_oop != NULL && t_oop->is_instance(), "expecting instance oopptr");
+  const TypePtr *t = adr_type();
+  assert(type() == Type::MEMORY && 
+         (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ||
+          t->isa_oopptr() && !t->is_oopptr()->is_instance() && 
+          t->is_oopptr()->cast_to_instance(t_oop->instance_id()) == t_oop),
+         "bottom or raw memory required");
 
   // Check if an appropriate node already exists.
   Node *region = in(0);
@@ -1342,7 +1348,7 @@
     Node *n = phi->in(i);
     if( !n ) return NULL;
     if( phase->type(n) == Type::TOP ) return NULL;
-    if( n->Opcode() == Op_ConP )
+    if( n->Opcode() == Op_ConP || n->Opcode() == Op_ConN )
       break;
   }
   if( i >= phi->req() )         // Only split for constants
--- old/src/share/vm/opto/classes.hpp	Wed May 14 20:05:08 2008
+++ new/src/share/vm/opto/classes.hpp	Wed May 14 20:05:08 2008
@@ -64,6 +64,7 @@
 macro(CMoveI)
 macro(CMoveL)
 macro(CMoveP)
+macro(CMoveN)
 macro(CmpN)
 macro(CmpD)
 macro(CmpD3)
@@ -139,6 +140,7 @@
 macro(LoadLLocked)
 macro(LoadP)
 macro(LoadN)
+macro(LoadNKlass)
 macro(LoadRange)
 macro(LoadS)
 macro(Lock)
--- old/src/share/vm/opto/compile.cpp	Wed May 14 20:05:09 2008
+++ new/src/share/vm/opto/compile.cpp	Wed May 14 20:05:09 2008
@@ -1065,6 +1065,8 @@
       // No constant oop pointers (such as Strings); they alias with
       // unknown strings.
       tj = to = TypeInstPtr::make(TypePtr::BotPTR,to->klass(),false,0,offset);
+    } else if( to->is_instance_field() ) {
+      tj = to; // Keep NotNull and klass_is_exact
     } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
       // During the 2nd round of IterGVN, NotNull castings are removed.
       // Make sure the Bottom and NotNull variants alias the same.
@@ -1084,7 +1086,7 @@
     } else {
       ciInstanceKlass *canonical_holder = k->get_canonical_holder(offset);
       if (!k->equals(canonical_holder) || tj->offset() != offset) {
-        tj = to = TypeInstPtr::make(TypePtr::BotPTR, canonical_holder, false, NULL, offset, to->instance_id());
+        tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, false, NULL, offset, to->instance_id());
       }
     }
   }
@@ -1967,6 +1969,7 @@
   case Op_LoadLLocked:
   case Op_LoadP:
   case Op_LoadN:
+  case Op_LoadNKlass:
   case Op_LoadRange:
   case Op_LoadS: {
   handle_mem:
@@ -1991,6 +1994,38 @@
     break;
   }
 
+#ifdef _LP64
+  case Op_CmpP:
+    if( n->in(1)->Opcode() == Op_DecodeN ) {
+      Compile* C = Compile::current();
+      Node* in2 = NULL;
+      if( n->in(2)->Opcode() == Op_DecodeN ) {
+        in2 = n->in(2)->in(1);
+      } else if ( n->in(2)->Opcode() == Op_ConP ) {
+        const Type* t = n->in(2)->bottom_type();
+        if (t == TypePtr::NULL_PTR) {
+          Node *in1 = n->in(1);
+          uint i = 0;
+          for (; i < in1->outcnt(); i++) {
+            if (in1->raw_out(i)->is_AddP())
+              break;
+          }
+          if (i >= in1->outcnt()) {
+            // Don't replace CmpP(o ,null) if 'o' is used in AddP
+            // to generate implicit NULL check.
+            in2 = ConNode::make(C, TypeNarrowOop::NULL_PTR);
+          }
+        } else if (t->isa_oopptr()) {
+          in2 = ConNode::make(C, t->is_oopptr()->make_narrowoop());
+        }
+      }
+      if( in2 != NULL ) {
+        Node* cmpN = new (C, 3) CmpNNode(n->in(1)->in(1), in2);
+        n->replace_by( cmpN );
+      }
+    }
+#endif
+
   case Op_ModI:
     if (UseDivMod) {
       // Check if a%b and a/b both exist
--- old/src/share/vm/opto/connode.cpp	Wed May 14 20:05:11 2008
+++ new/src/share/vm/opto/connode.cpp	Wed May 14 20:05:11 2008
@@ -36,9 +36,9 @@
 //------------------------------make-------------------------------------------
 ConNode *ConNode::make( Compile* C, const Type *t ) {
   if (t->isa_narrowoop()) return new (C, 1) ConNNode( t->is_narrowoop() );
+
   switch( t->basic_type() ) {
   case T_INT:       return new (C, 1) ConINode( t->is_int() );
-  case T_ARRAY:     return new (C, 1) ConPNode( t->is_aryptr() );
   case T_LONG:      return new (C, 1) ConLNode( t->is_long() );
   case T_FLOAT:     return new (C, 1) ConFNode( t->is_float_constant() );
   case T_DOUBLE:    return new (C, 1) ConDNode( t->is_double_constant() );
@@ -45,6 +45,7 @@
   case T_VOID:      return new (C, 1) ConNode ( Type::TOP );
   case T_OBJECT:    return new (C, 1) ConPNode( t->is_oopptr() );
   case T_ADDRESS:   return new (C, 1) ConPNode( t->is_ptr() );
+  case T_ARRAY:     return new (C, 1) ConPNode( t->is_aryptr() );
     // Expected cases:  TypePtr::NULL_PTR, any is_rawptr()
     // Also seen: AnyPtr(TopPTR *+top); from command line:
     //   r -XX:+PrintOpto -XX:CIStart=285 -XX:+CompileTheWorld -XX:CompileTheWorldStartAt=660
@@ -185,6 +186,7 @@
   case T_LONG:    return new (C, 4) CMoveLNode( bol, left, right, t->is_long() );
   case T_OBJECT:  return new (C, 4) CMovePNode( c, bol, left, right, t->is_oopptr() );
   case T_ADDRESS: return new (C, 4) CMovePNode( c, bol, left, right, t->is_ptr() );
+  case T_NARROWOOP: return new (C, 4) CMoveNNode( c, bol, left, right, t );
   default:
     ShouldNotReachHere();
     return NULL;
@@ -570,7 +572,7 @@
   return bottom_type();
 }
 
-Node* DecodeNNode::decode(PhaseGVN* phase, Node* value) {
+Node* DecodeNNode::decode(PhaseTransform* phase, Node* value) {
   if (value->Opcode() == Op_EncodeP) {
     // (DecodeN (EncodeP p)) -> p
     return value->in(1);
@@ -578,8 +580,11 @@
   const Type* newtype = value->bottom_type();
   if (newtype == TypeNarrowOop::NULL_PTR) {
     return phase->transform(new (phase->C, 1) ConPNode(TypePtr::NULL_PTR));
-  } else {
+  } else if (newtype->isa_narrowoop()) {
     return phase->transform(new (phase->C, 2) DecodeNNode(value, newtype->is_narrowoop()->make_oopptr()));
+  } else {
+    ShouldNotReachHere();
+    return NULL; // to make C++ compiler happy.
   }
 }
 
@@ -601,7 +606,7 @@
   return bottom_type();
 }
 
-Node* EncodePNode::encode(PhaseGVN* phase, Node* value) {
+Node* EncodePNode::encode(PhaseTransform* phase, Node* value) {
   if (value->Opcode() == Op_DecodeN) {
     // (EncodeP (DecodeN p)) -> p
     return value->in(1);
@@ -617,6 +622,9 @@
   }
 }
 
+Node *EncodePNode::Ideal_DU_postCCP( PhaseCCP *ccp ) {
+  return MemNode::Ideal_common_DU_postCCP(ccp, this, in(1));
+}
 
 //=============================================================================
 //------------------------------Identity---------------------------------------
--- old/src/share/vm/opto/connode.hpp	Wed May 14 20:05:12 2008
+++ new/src/share/vm/opto/connode.hpp	Wed May 14 20:05:12 2008
@@ -70,11 +70,6 @@
     else
       return new (C, 1) ConPNode( TypeRawPtr::make(con) );
   }
-
-  static ConPNode* make( Compile *C, ciObject* con ) {
-    return new (C, 1) ConPNode( TypeOopPtr::make_from_constant(con) );
-  }
-
 };
 
 
@@ -84,11 +79,6 @@
 public:
   ConNNode( const TypeNarrowOop *t ) : ConNode(t) {}
   virtual int Opcode() const;
-
-  static ConNNode* make( Compile *C, ciObject* con ) {
-    return new (C, 1) ConNNode( TypeNarrowOop::make_from_constant(con) );
-  }
-
 };
 
 
@@ -210,7 +200,14 @@
   virtual int Opcode() const;
 };
 
-//------------------------------ConstraintCastNode-------------------------------------
+//------------------------------CMoveNNode-------------------------------------
+class CMoveNNode : public CMoveNode {
+public:
+  CMoveNNode( Node *c, Node *bol, Node *left, Node *right, const Type* t ) : CMoveNode(bol,left,right,t) { init_req(Control,c); }
+  virtual int Opcode() const;
+};
+
+//------------------------------ConstraintCastNode-----------------------------
 // cast to a different range
 class ConstraintCastNode: public TypeNode {
 public:
@@ -282,7 +279,8 @@
   virtual const Type *Value( PhaseTransform *phase ) const;
   virtual uint  ideal_reg() const { return Op_RegN; }
 
-  static Node* encode(PhaseGVN* phase, Node* value);
+  static Node* encode(PhaseTransform* phase, Node* value);
+  virtual Node *Ideal_DU_postCCP( PhaseCCP *ccp );
 };
 
 //------------------------------DecodeN--------------------------------
@@ -301,7 +299,7 @@
   virtual const Type *Value( PhaseTransform *phase ) const;
   virtual uint  ideal_reg() const { return Op_RegP; }
 
-  static Node* decode(PhaseGVN* phase, Node* value);
+  static Node* decode(PhaseTransform* phase, Node* value);
 };
 
 //------------------------------Conv2BNode-------------------------------------
--- old/src/share/vm/opto/doCall.cpp	Wed May 14 20:05:14 2008
+++ new/src/share/vm/opto/doCall.cpp	Wed May 14 20:05:13 2008
@@ -580,7 +580,7 @@
   Node* ex_klass_node = NULL;
   if (has_ex_handler() && !ex_type->klass_is_exact()) {
     Node* p = basic_plus_adr( ex_node, ex_node, oopDesc::klass_offset_in_bytes());
-    ex_klass_node = _gvn.transform(new (C, 3) LoadKlassNode(NULL, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT));
+    ex_klass_node = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT) );
 
     // Compute the exception klass a little more cleverly.
     // Obvious solution is to simple do a LoadKlass from the 'ex_node'.
@@ -592,7 +592,7 @@
       ex_klass_node = new (C, ex_node->req()) PhiNode( ex_node->in(0), TypeKlassPtr::OBJECT );
       for( uint i = 1; i < ex_node->req(); i++ ) {
         Node* p = basic_plus_adr( ex_node->in(i), ex_node->in(i), oopDesc::klass_offset_in_bytes() );
-        Node* k = _gvn.transform(new (C, 3) LoadKlassNode(0, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT));
+        Node* k = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT) );
         ex_klass_node->init_req( i, k );
       }
       _gvn.set_type(ex_klass_node, TypeKlassPtr::OBJECT);
--- old/src/share/vm/opto/escape.cpp	Wed May 14 20:05:15 2008
+++ new/src/share/vm/opto/escape.cpp	Wed May 14 20:05:15 2008
@@ -417,11 +417,18 @@
   //       | |
   //       AddP  ( base == address )
   //
+  // case #8. narrow Klass's field reference.
+  //      LoadNKlass
+  //       |
+  //      DecodeN
+  //       | |
+  //       AddP  ( base == address )
+  //
   Node *base = addp->in(AddPNode::Base)->uncast();
   if (base->is_top()) { // The AddP case #3 and #6.
     base = addp->in(AddPNode::Address)->uncast();
     assert(base->Opcode() == Op_ConP || base->Opcode() == Op_ThreadLocal ||
-           base->Opcode() == Op_CastX2P ||
+           base->Opcode() == Op_CastX2P || base->Opcode() == Op_DecodeN ||
            (base->is_Mem() && base->bottom_type() == TypeRawPtr::NOTNULL) ||
            (base->is_Proj() && base->in(0)->is_Allocate()), "sanity");
   }
@@ -888,6 +895,23 @@
       record_for_optimizer(n);
       if (alloc->is_Allocate() && ptn->_scalar_replaceable &&
           (t->isa_instptr() || t->isa_aryptr())) {
+
+        // First, put on the worklist all Field edges from Connection Graph
+        // which is more accurate then putting immediate users from Ideal Graph.
+        for (uint e = 0; e < ptn->edge_count(); e++) {
+          Node *use = _nodes->adr_at(ptn->edge_target(e))->_node;
+          assert(ptn->edge_type(e) == PointsToNode::FieldEdge && use->is_AddP(),
+                 "only AddP nodes are Field edges in CG");
+          if (use->outcnt() > 0) { // Don't process dead nodes
+            Node* addp2 = find_second_addp(use, use->in(AddPNode::Base));
+            if (addp2 != NULL) {
+              assert(alloc->is_AllocateArray(),"array allocation was expected");
+              alloc_worklist.append_if_missing(addp2);
+            }
+            alloc_worklist.append_if_missing(use);
+          }
+        }
+
         // An allocation may have an Initialize which has raw stores. Scan
         // the users of the raw allocation result and push AddP users
         // on alloc_worklist.
@@ -919,6 +943,8 @@
       tinst = igvn->type(base)->isa_oopptr();
     } else if (n->is_Phi() ||
                n->is_CheckCastPP() ||
+               n->Opcode() == Op_EncodeP ||
+               n->Opcode() == Op_DecodeN ||
                (n->is_ConstraintCast() && n->Opcode() == Op_CastPP)) {
       if (visited.test_set(n->_idx)) {
         assert(n->is_Phi(), "loops only through Phi's");
@@ -935,13 +961,25 @@
         tinst = igvn->type(val)->isa_oopptr();
         assert(tinst != NULL && tinst->is_instance() &&
                tinst->instance_id() == elem , "instance type expected.");
-        const TypeOopPtr *tn_t = igvn->type(tn)->isa_oopptr();
 
+        const TypeOopPtr *tn_t = NULL;
+        const Type *tn_type = igvn->type(tn);
+        if (tn_type->isa_narrowoop()) {
+          tn_t = tn_type->is_narrowoop()->make_oopptr()->isa_oopptr();
+        } else {
+          tn_t = tn_type->isa_oopptr();
+        }
+
         if (tn_t != NULL &&
  tinst->cast_to_instance(TypeOopPtr::UNKNOWN_INSTANCE)->higher_equal(tn_t)) {
+          if (tn_type->isa_narrowoop()) {
+            tn_type = tinst->make_narrowoop();
+          } else {
+            tn_type = tinst;
+          }
           igvn->hash_delete(tn);
-          igvn->set_type(tn, tinst);
-          tn->set_type(tinst);
+          igvn->set_type(tn, tn_type);
+          tn->set_type(tn_type);
           igvn->hash_insert(tn);
           record_for_optimizer(n);
         }
@@ -978,6 +1016,8 @@
         alloc_worklist.append_if_missing(use);
       } else if (use->is_Phi() ||
                  use->is_CheckCastPP() ||
+                 use->Opcode() == Op_EncodeP ||
+                 use->Opcode() == Op_DecodeN ||
                  (use->is_ConstraintCast() && use->Opcode() == Op_CastPP)) {
         alloc_worklist.append_if_missing(use);
       }
@@ -1199,7 +1239,7 @@
 
 void ConnectionGraph::compute_escape() {
 
-  // 1. Populate Connection Graph with Ideal nodes.
+  // 1. Populate Connection Graph (CG) with Ideal nodes.
 
   Unique_Node_List worklist_init;
   worklist_init.map(_compile->unique(), NULL);  // preallocate space
@@ -1281,11 +1321,13 @@
       remove_deferred(ni, &deferred_edges, &visited);
       if (n->is_AddP()) {
         // If this AddP computes an address which may point to more that one
-        // object, nothing the address points to can be scalar replaceable.
+        // object or more then one field (array's element), nothing the address
+        // points to can be scalar replaceable.
         Node *base = get_addp_base(n);
         ptset.Clear();
         PointsTo(ptset, base, igvn);
-        if (ptset.Size() > 1) {
+        if (ptset.Size() > 1 ||
+            (ptset.Size() != 0 && ptn->offset() == Type::OffsetBot)) {
           for( VectorSetI j(&ptset); j.test(); ++j ) {
             uint pt = j.elem;
             ptnode_adr(pt)->_scalar_replaceable = false;
@@ -1538,6 +1580,7 @@
       if (k->Opcode() == Op_LoadKlass) {
         kt = k->as_Load()->type()->isa_klassptr();
       } else {
+        // Also works for DecodeN(LoadNKlass).
         kt = k->as_Type()->type()->isa_klassptr();
       }
       assert(kt != NULL, "TypeKlassPtr  required.");
@@ -1776,6 +1819,7 @@
       break;
     }
     case Op_LoadKlass:
+    case Op_LoadNKlass:
     {
       add_node(n, PointsToNode::JavaObject, PointsToNode::GlobalEscape, true);
       break;
@@ -1979,6 +2023,11 @@
       assert(false, "Op_ConP");
       break;
     }
+    case Op_ConN:
+    {
+      assert(false, "Op_ConN");
+      break;
+    }
     case Op_CreateEx:
     {
       assert(false, "Op_CreateEx");
@@ -1985,6 +2034,7 @@
       break;
     }
     case Op_LoadKlass:
+    case Op_LoadNKlass:
     {
       assert(false, "Op_LoadKlass");
       break;
--- old/src/share/vm/opto/graphKit.cpp	Wed May 14 20:05:16 2008
+++ new/src/share/vm/opto/graphKit.cpp	Wed May 14 20:05:16 2008
@@ -532,7 +532,7 @@
         C->log()->elem("hot_throw preallocated='1' reason='%s'",
                        Deoptimization::trap_reason_name(reason));
       const TypeInstPtr* ex_con  = TypeInstPtr::make(ex_obj);
-      Node*              ex_node = _gvn.transform(new (C, 1) ConPNode(ex_con));
+      Node*              ex_node = _gvn.transform( ConNode::make(C, ex_con) );
 
       // Clear the detail message of the preallocated exception object.
       // Weblogic sometimes mutates the detail message of exceptions
@@ -1043,7 +1043,7 @@
   Node* akls = AllocateNode::Ideal_klass(obj, &_gvn);
   if (akls != NULL)  return akls;
   Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
-  return _gvn.transform( new (C, 3) LoadKlassNode(0, immutable_memory(), k_adr, TypeInstPtr::KLASS) );
+  return _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), k_adr, TypeInstPtr::KLASS) );
 }
 
 //-------------------------load_array_length-----------------------------------
@@ -2210,7 +2210,7 @@
   // cache which is mutable so can't use immutable memory.  Other
   // types load from the super-class display table which is immutable.
   Node *kmem = might_be_cache ? memory(p2) : immutable_memory();
-  Node *nkls = _gvn.transform( new (C, 3) LoadKlassNode( NULL, kmem, p2, _gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL ) );
+  Node *nkls = _gvn.transform( LoadKlassNode::make( _gvn, kmem, p2, _gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL ) );
 
   // Compile speed common case: ARE a subtype and we canNOT fail
   if( superklass == nkls )
@@ -2801,7 +2801,6 @@
     // initialization, and source them from the new InitializeNode.
     // This will allow us to observe initializations when they occur,
     // and link them properly (as a group) to the InitializeNode.
-    Node* klass_node = alloc->in(AllocateNode::KlassNode);
     assert(init->in(InitializeNode::Memory) == malloc, "");
     MergeMemNode* minit_in = MergeMemNode::make(C, malloc);
     init->set_req(InitializeNode::Memory, minit_in);
--- old/src/share/vm/opto/lcm.cpp	Wed May 14 20:05:18 2008
+++ new/src/share/vm/opto/lcm.cpp	Wed May 14 20:05:18 2008
@@ -111,6 +111,7 @@
     case Op_LoadL:
     case Op_LoadP:
     case Op_LoadN:
+    case Op_LoadNKlass:
     case Op_LoadS:
     case Op_LoadKlass:
     case Op_LoadRange:
--- old/src/share/vm/opto/library_call.cpp	Wed May 14 20:05:19 2008
+++ new/src/share/vm/opto/library_call.cpp	Wed May 14 20:05:19 2008
@@ -896,7 +896,7 @@
   Node* sourcea       = basic_plus_adr(string_object, string_object, value_offset);
   Node* source        = make_load(no_ctrl, sourcea, source_type, T_OBJECT, string_type->add_offset(value_offset));
 
-  Node* target = _gvn.transform(ConPNode::make(C, target_array));
+  Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array)) );
   jint target_length = target_array->length();
   const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
   const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
@@ -2168,7 +2168,7 @@
     // (They don't if CAS fails, but it isn't worth checking.)
     pre_barrier(control(), base, adr, alias_idx, newval, value_type, T_OBJECT);
 #ifdef _LP64
-    if (adr->bottom_type()->is_narrow()) {
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
       cas = _gvn.transform(new (C, 5) CompareAndSwapNNode(control(), mem, adr,
                                                            EncodePNode::encode(&_gvn, newval),
                                                            EncodePNode::encode(&_gvn, oldval)));
@@ -2454,7 +2454,7 @@
   if (region == NULL)  never_see_null = true;
   Node* p = basic_plus_adr(mirror, offset);
   const TypeKlassPtr*  kls_type = TypeKlassPtr::OBJECT_OR_NULL;
-  Node* kls = _gvn.transform(new (C, 3) LoadKlassNode(0, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
+  Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) );
   _sp += nargs; // any deopt will start just before call to enclosing method
   Node* null_ctl = top();
   kls = null_check_oop(kls, &null_ctl, never_see_null);
@@ -2634,7 +2634,7 @@
       phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
     // If we fall through, it's a plain class.  Get its _super.
     p = basic_plus_adr(kls, Klass::super_offset_in_bytes() + sizeof(oopDesc));
-    kls = _gvn.transform(new (C, 3) LoadKlassNode(0, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
+    kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) );
     null_ctl = top();
     kls = null_check_oop(kls, &null_ctl);
     if (null_ctl != top()) {
@@ -2720,7 +2720,7 @@
     args[which_arg] = _gvn.transform(arg);
 
     Node* p = basic_plus_adr(arg, class_klass_offset);
-    Node* kls = new (C, 3) LoadKlassNode(0, immutable_memory(), p, adr_type, kls_type);
+    Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type);
     klasses[which_arg] = _gvn.transform(kls);
   }
 
@@ -2838,6 +2838,8 @@
   _sp += nargs;  // set original stack for use by uncommon_trap
   mirror = do_null_check(mirror, T_OBJECT);
   _sp -= nargs;
+  // If mirror or obj is dead, only null-path is taken.
+  if (stopped())  return true;
 
   enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
   RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
@@ -3827,24 +3829,22 @@
   if (!stopped()) {
     // Copy the fastest available way.
     // (No need for PreserveJVMState, since we're using it all up now.)
+    // TODO: generate fields/elements copies for small objects instead.
     Node* src  = obj;
     Node* dest = raw_obj;
-    Node* end  = dest;
     Node* size = _gvn.transform(alloc_siz);
 
     // Exclude the header.
     int base_off = instanceOopDesc::base_offset_in_bytes();
     if (UseCompressedOops) {
-      // copy the header gap though.
-      Node* sptr = basic_plus_adr(src,  base_off);
-      Node* dptr = basic_plus_adr(dest, base_off);
-      Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, raw_adr_type);
-      store_to_memory(control(), dptr, sval, T_INT, raw_adr_type);
-      base_off += sizeof(int);
+      assert(base_off % BytesPerLong != 0, "base with compressed oops");
+      // With compressed oops base_offset_in_bytes is 12 which creates
+      // the gap since countx is rounded by 8 bytes below.
+      // Copy klass and the gap.
+      base_off = instanceOopDesc::klass_offset_in_bytes();
     }
     src  = basic_plus_adr(src,  base_off);
     dest = basic_plus_adr(dest, base_off);
-    end  = basic_plus_adr(end,  size);
 
     // Compute the length also, if needed:
     Node* countx = size;
@@ -4388,7 +4388,7 @@
       // (At this point we can assume disjoint_bases, since types differ.)
       int ek_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc);
       Node* p1 = basic_plus_adr(dest_klass, ek_offset);
-      Node* n1 = new (C, 3) LoadKlassNode(0, immutable_memory(), p1, TypeRawPtr::BOTTOM);
+      Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
       Node* dest_elem_klass = _gvn.transform(n1);
       Node* cv = generate_checkcast_arraycopy(adr_type,
                                               dest_elem_klass,
--- old/src/share/vm/opto/loopopts.cpp	Wed May 14 20:05:21 2008
+++ new/src/share/vm/opto/loopopts.cpp	Wed May 14 20:05:21 2008
@@ -464,6 +464,7 @@
     case T_FLOAT:
     case T_DOUBLE:
     case T_ADDRESS:             // (RawPtr)
+    case T_NARROWOOP:
       cost++;
       break;
     case T_OBJECT: {            // Base oops are OK, but not derived oops
--- old/src/share/vm/opto/macro.cpp	Wed May 14 20:05:23 2008
+++ new/src/share/vm/opto/macro.cpp	Wed May 14 20:05:22 2008
@@ -1282,12 +1282,6 @@
   }
   rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
 
-  if (UseCompressedOops) {
-    Node *zeronode = makecon(TypeInt::ZERO);
-    // store uncompressed 0 into klass ptr to zero out gap.  The gap is
-    // used for primitive fields and has to be zeroed.
-    rawmem = make_store(control, rawmem, object, oopDesc::klass_gap_offset_in_bytes(), zeronode, T_INT);
-  }
   rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_OBJECT);
   int header_size = alloc->minimum_header_size();  // conservatively small
 
--- old/src/share/vm/opto/matcher.cpp	Wed May 14 20:05:24 2008
+++ new/src/share/vm/opto/matcher.cpp	Wed May 14 20:05:24 2008
@@ -880,7 +880,7 @@
         Node *m = n->in(i);          // Get input
         int op = m->Opcode();
         assert((op == Op_BoxLock) == jvms->is_monitor_use(i), "boxes only at monitor sites");
-        if( op == Op_ConI || op == Op_ConP ||
+        if( op == Op_ConI || op == Op_ConP || op == Op_ConN ||
             op == Op_ConF || op == Op_ConD || op == Op_ConL
             // || op == Op_BoxLock  // %%%% enable this and remove (+++) in chaitin.cpp
             ) {
@@ -1726,6 +1726,14 @@
         }
         break;
       }
+      case Op_ConN: {  // Convert narrow pointers above the centerline to NUL
+        TypeNode *tn = n->as_Type(); // Constants derive from type nodes
+        const TypePtr* tp = tn->type()->is_narrowoop()->make_oopptr();
+        if (tp->_ptr == TypePtr::AnyNull) {
+          tn->set_type(TypeNarrowOop::NULL_PTR);
+        }
+        break;
+      }
       case Op_Binary:         // These are introduced in the Post_Visit state.
         ShouldNotReachHere();
         break;
@@ -1764,6 +1772,7 @@
       case Op_LoadS:
       case Op_LoadP:
       case Op_LoadN:
+      case Op_LoadNKlass:
       case Op_LoadRange:
       case Op_LoadD_unaligned:
       case Op_LoadL_unaligned:
@@ -1891,6 +1900,7 @@
       case Op_CMoveF:
       case Op_CMoveI:
       case Op_CMoveL:
+      case Op_CMoveN:
       case Op_CMoveP: {
         // Restructure into a binary tree for Matching.  It's possible that
         // we could move this code up next to the graph reshaping for IfNodes
--- old/src/share/vm/opto/memnode.cpp	Wed May 14 20:05:26 2008
+++ new/src/share/vm/opto/memnode.cpp	Wed May 14 20:05:25 2008
@@ -133,7 +133,9 @@
     PhiNode *mphi = result->as_Phi();
     assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
     const TypePtr *t = mphi->adr_type();
-    if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM) {
+    if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ||
+        t->isa_oopptr() && !t->is_oopptr()->is_instance() && 
+        t->is_oopptr()->cast_to_instance(t_oop->instance_id()) == t_oop) {
       // clone the Phi with our address type
       result = mphi->split_out_instance(t_adr, igvn);
     } else {
@@ -256,7 +258,7 @@
   if (dom == NULL || dom->is_top())
     return false; // Conservative answer for dead code
 
-  if (dom->is_Start() || dom->is_Root() || dom == sub)
+  if (dom->is_Con() || dom->is_Start() || dom->is_Root() || dom == sub)
     return true;
 
   // 'dom' dominates 'sub' if its control edge and control edges
@@ -263,7 +265,10 @@
   // of all its inputs dominate or equal to sub's control edge.
 
   // Currently 'sub' is either Allocate, Initialize or Start nodes.
-  assert(sub->is_Allocate() || sub->is_Initialize() || sub->is_Start(), "expecting only these nodes");
+  // Or Region for the check in LoadNode::Ideal();
+  // 'sub' should have sub->in(0) != NULL.
+  assert(sub->is_Allocate() || sub->is_Initialize() || sub->is_Start() ||
+         sub->is_Region(), "expecting only these nodes");
 
   // Get control edge of 'sub'.
   sub = sub->find_exact_control(sub->in(0));
@@ -298,7 +303,7 @@
           return false; // Conservative answer for dead code
         assert(n->is_CFG(), "expecting control");
       }
-      if (n->is_Start() || n->is_Root()) {
+      if (n->is_Con() || n->is_Start() || n->is_Root()) {
         only_dominating_controls = true;
       } else if (n->is_CFG()) {
         if (n->dominates(sub, nlist))
@@ -308,12 +313,11 @@
       } else {
         // First, own control edge.
         Node* m = n->find_exact_control(n->in(0));
-        if (m == NULL)
-          continue;
-        if (m->is_top())
-          return false; // Conservative answer for dead code
-        dom_list.push(m);
-
+        if (m != NULL) {
+          if (m->is_top())
+            return false; // Conservative answer for dead code
+          dom_list.push(m);
+        }
         // Now, the rest of edges.
         uint cnt = n->req();
         for (uint i = 1; i < cnt; i++) {
@@ -577,6 +581,12 @@
 // Find any cast-away of null-ness and keep its control.  Null cast-aways are
 // going away in this pass and we need to make this memory op depend on the
 // gating null check.
+Node *MemNode::Ideal_DU_postCCP( PhaseCCP *ccp ) {
+  // Need a null check?  Regular static accesses do not because they are
+  // from constant addresses.  Array ops are gated by the range check (which
+  // always includes a NULL check).  Just check field ops.
+  return Ideal_common_DU_postCCP(ccp, this, in(MemNode::Address));
+}
 
 // I tried to leave the CastPP's in.  This makes the graph more accurate in
 // some sense; we get to keep around the knowledge that an oop is not-null
@@ -586,15 +596,9 @@
 // some of the more trivial cases in the optimizer.  Removing more useless
 // Phi's started allowing Loads to illegally float above null checks.  I gave
 // up on this approach.  CNC 10/20/2000
-Node *MemNode::Ideal_DU_postCCP( PhaseCCP *ccp ) {
-  Node *ctr = in(MemNode::Control);
-  Node *mem = in(MemNode::Memory);
-  Node *adr = in(MemNode::Address);
+Node *MemNode::Ideal_common_DU_postCCP( PhaseCCP *ccp, Node* n, Node* adr ) {
   Node *skipped_cast = NULL;
-  // Need a null check?  Regular static accesses do not because they are
-  // from constant addresses.  Array ops are gated by the range check (which
-  // always includes a NULL check).  Just check field ops.
-  if( !ctr ) {
+  if( n->in(MemNode::Control) == NULL ) {
     // Scan upwards for the highest location we can place this memory op.
     while( true ) {
       switch( adr->Opcode() ) {
@@ -619,10 +623,10 @@
         }
         // CastPP is going away in this pass!  We need this memory op to be
         // control-dependent on the test that is guarding the CastPP.
-        ccp->hash_delete(this);
-        set_req(MemNode::Control, adr->in(0));
-        ccp->hash_insert(this);
-        return this;
+        ccp->hash_delete(n);
+        n->set_req(MemNode::Control, adr->in(0));
+        ccp->hash_insert(n);
+        return n;
 
       case Op_Phi:
         // Attempt to float above a Phi to some dominating point.
@@ -653,10 +657,10 @@
           adr = adr->in(1);
           continue;
         }
-        ccp->hash_delete(this);
-        set_req(MemNode::Control, adr->in(0));
-        ccp->hash_insert(this);
-        return this;
+        ccp->hash_delete(n);
+        n->set_req(MemNode::Control, adr->in(0));
+        ccp->hash_insert(n);
+        return n;
 
         // List of "safe" opcodes; those that implicitly block the memory
         // op below any null check.
@@ -665,10 +669,13 @@
       case Op_LoadP:            // Loading from within a klass
       case Op_LoadN:            // Loading from within a klass
       case Op_LoadKlass:        // Loading from within a klass
+      case Op_LoadNKlass:       // Loading from within a klass
       case Op_ConP:             // Loading from a klass
+      case Op_ConN:             // Reading from a klass
       case Op_CreateEx:         // Sucking up the guts of an exception oop
       case Op_Con:              // Reading from TLS
       case Op_CMoveP:           // CMoveP is pinned
+      case Op_CMoveN:           // CMoveN is pinned
         break;                  // No progress
 
       case Op_Proj:             // Direct call to an allocation routine
@@ -677,8 +684,8 @@
         {
           assert(adr->as_Proj()->_con == TypeFunc::Parms, "must be return value");
           const Node* call = adr->in(0);
-          if (call->is_CallStaticJava()) {
-            const CallStaticJavaNode* call_java = call->as_CallStaticJava();
+          if (call->is_CallJava()) {
+            const CallJavaNode* call_java = call->as_CallJava();
             const TypeTuple *r = call_java->tf()->range();
             assert(r->cnt() > TypeFunc::Parms, "must return value");
             const Type* ret_type = r->field_at(TypeFunc::Parms);
@@ -750,7 +757,7 @@
   case T_ADDRESS: return new (C, 3) LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr()    );
   case T_OBJECT:
 #ifdef _LP64
-    if (adr->bottom_type()->is_narrow()) {
+    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
       const TypeNarrowOop* narrowtype;
       if (rt->isa_narrowoop()) {
         narrowtype = rt->is_narrowoop();
@@ -762,10 +769,10 @@
       return DecodeNNode::decode(&gvn, load);
     } else
 #endif
-      {
-        assert(!adr->bottom_type()->is_narrow(), "should have got back a narrow oop");
-        return new (C, 3) LoadPNode(ctl, mem, adr, adr_type, rt->is_oopptr());
-      }
+    {
+      assert(!adr->bottom_type()->is_ptr_to_narrowoop(), "should have got back a narrow oop");
+      return new (C, 3) LoadPNode(ctl, mem, adr, adr_type, rt->is_oopptr());
+    }
   }
   ShouldNotReachHere();
   return (LoadNode*)NULL;
@@ -1206,6 +1213,11 @@
         if ( !phase->eqv(this, this->Identity(phase)) )
           return NULL;
 
+        // Skip the split if the loop head dominates some control edges 
+        // of the address.
+        if (cnt == 3 && !MemNode::all_controls_dominate(address, region))
+          goto find_store;
+
         const Type* this_type = this->bottom_type();
         int this_index  = phase->C->get_alias_index(addr_t);
         int this_offset = addr_t->offset();
@@ -1284,7 +1296,7 @@
       }
     }
   }
-
+find_store:
   // Check for prior store with a different base or offset; make Load
   // independent.  Skip through any number of them.  Bail out if the stores
   // are in an endless dead cycle and report no progress.  This is a key
@@ -1585,8 +1597,35 @@
 }
 
 //=============================================================================
+//----------------------------LoadKlassNode::make------------------------------
+// Polymorphic factory method:
+Node *LoadKlassNode::make( PhaseGVN& gvn, Node *mem, Node *adr, const TypePtr* at, const TypeKlassPtr *tk ) {
+  Compile* C = gvn.C;
+  Node *ctl = NULL;
+  // sanity check the alias category against the created node type
+  const TypeOopPtr *adr_type = adr->bottom_type()->isa_oopptr();
+  assert(adr_type != NULL, "expecting TypeOopPtr");
+#ifdef _LP64
+  if (adr_type->is_ptr_to_narrowoop()) {
+    const TypeNarrowOop* narrowtype = tk->is_oopptr()->make_narrowoop();
+    Node* load_klass = gvn.transform(new (C, 3) LoadNKlassNode(ctl, mem, adr, at, narrowtype));
+    return DecodeNNode::decode(&gvn, load_klass);
+  } else
+#endif
+  {
+    assert(!adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
+    return new (C, 3) LoadKlassNode(ctl, mem, adr, at, tk);
+  }
+  ShouldNotReachHere();
+  return (LoadKlassNode*)NULL;
+}
+
 //------------------------------Value------------------------------------------
 const Type *LoadKlassNode::Value( PhaseTransform *phase ) const {
+  return klass_value_common(phase);
+}
+
+const Type *LoadNode::klass_value_common( PhaseTransform *phase ) const {
   // Either input is TOP ==> the result is TOP
   const Type *t1 = phase->type( in(MemNode::Memory) );
   if (t1 == Type::TOP)  return Type::TOP;
@@ -1718,6 +1757,10 @@
 // To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
 // Also feed through the klass in Allocate(...klass...)._klass.
 Node* LoadKlassNode::Identity( PhaseTransform *phase ) {
+  return klass_identity_common(phase);
+}
+
+Node* LoadNode::klass_identity_common(PhaseTransform *phase ) {
   Node* x = LoadNode::Identity(phase);
   if (x != this)  return x;
 
@@ -1776,6 +1819,34 @@
   return this;
 }
 
+
+//------------------------------Value------------------------------------------
+const Type *LoadNKlassNode::Value( PhaseTransform *phase ) const {
+  const Type *t = klass_value_common(phase);
+
+  if (t == TypePtr::NULL_PTR) {
+    return TypeNarrowOop::NULL_PTR;
+  }
+  if (t != Type::TOP && !t->isa_narrowoop()) {
+    assert(t->is_oopptr(), "sanity");
+    t = t->is_oopptr()->make_narrowoop();
+  }
+  return t;
+}
+
+//------------------------------Identity---------------------------------------
+// To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
+// Also feed through the klass in Allocate(...klass...)._klass.
+Node* LoadNKlassNode::Identity( PhaseTransform *phase ) {
+  Node *x = klass_identity_common(phase);
+
+  const Type *t = phase->type( x );
+  if( t == Type::TOP ) return x;
+  if( t->isa_narrowoop()) return x;
+
+  return EncodePNode::encode(phase, x);
+}
+
 //------------------------------Value-----------------------------------------
 const Type *LoadRangeNode::Value( PhaseTransform *phase ) const {
   // Either input is TOP ==> the result is TOP
@@ -1836,7 +1907,7 @@
   case T_ADDRESS:
   case T_OBJECT:
 #ifdef _LP64
-    if (adr->bottom_type()->is_narrow() ||
+    if (adr->bottom_type()->is_ptr_to_narrowoop() ||
         (UseCompressedOops && val->bottom_type()->isa_klassptr() &&
          adr->bottom_type()->isa_rawptr())) {
       const TypePtr* type = val->bottom_type()->is_ptr();
--- old/src/share/vm/opto/memnode.hpp	Wed May 14 20:05:28 2008
+++ new/src/share/vm/opto/memnode.hpp	Wed May 14 20:05:27 2008
@@ -72,7 +72,8 @@
   // This one should probably be a phase-specific function:
   static bool all_controls_dominate(Node* dom, Node* sub);
 
-  // Is this Node a MemNode or some descendent?  Default is YES.
+  // Find any cast-away of null-ness and keep its control.
+  static  Node *Ideal_common_DU_postCCP( PhaseCCP *ccp, Node* n, Node* adr );
   virtual Node *Ideal_DU_postCCP( PhaseCCP *ccp );
 
   virtual const class TypePtr *adr_type() const;  // returns bottom_type of address
@@ -157,6 +158,10 @@
   // then call the virtual add() to set the type.
   virtual const Type *Value( PhaseTransform *phase ) const;
 
+  // Common methods for LoadKlass and LoadNKlass nodes.
+  const Type *klass_value_common( PhaseTransform *phase ) const;
+  Node *klass_identity_common( PhaseTransform *phase );
+
   virtual uint ideal_reg() const;
   virtual const Type *bottom_type() const;
   // Following method is copied from TypeNode:
@@ -358,14 +363,35 @@
 // Load a Klass from an object
 class LoadKlassNode : public LoadPNode {
 public:
-  LoadKlassNode( Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeKlassPtr *tk = TypeKlassPtr::OBJECT )
+  LoadKlassNode( Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeKlassPtr *tk )
     : LoadPNode(c,mem,adr,at,tk) {}
   virtual int Opcode() const;
   virtual const Type *Value( PhaseTransform *phase ) const;
   virtual Node *Identity( PhaseTransform *phase );
   virtual bool depends_only_on_test() const { return true; }
+
+  // Polymorphic factory method:
+  static Node* make( PhaseGVN& gvn, Node *mem, Node *adr, const TypePtr* at,
+                     const TypeKlassPtr *tk = TypeKlassPtr::OBJECT );
 };
 
+//------------------------------LoadNKlassNode---------------------------------
+// Load a narrow Klass from an object.
+class LoadNKlassNode : public LoadNNode {
+public:
+  LoadNKlassNode( Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeNarrowOop *tk )
+    : LoadNNode(c,mem,adr,at,tk) {}
+  virtual int Opcode() const;
+  virtual uint ideal_reg() const { return Op_RegN; }
+  virtual int store_Opcode() const { return Op_StoreN; }
+  virtual BasicType memory_type() const { return T_NARROWOOP; }
+
+  virtual const Type *Value( PhaseTransform *phase ) const;
+  virtual Node *Identity( PhaseTransform *phase );
+  virtual bool depends_only_on_test() const { return true; }
+};
+
+
 //------------------------------LoadSNode--------------------------------------
 // Load a short (16bits signed) from memory
 class LoadSNode : public LoadNode {
--- old/src/share/vm/opto/node.cpp	Wed May 14 20:05:29 2008
+++ new/src/share/vm/opto/node.cpp	Wed May 14 20:05:29 2008
@@ -1043,50 +1043,84 @@
   assert(this->is_CFG(), "expecting control");
   assert(sub != NULL && sub->is_CFG(), "expecting control");
 
+  // detect dead cycle without regions
+  int iterations_without_region_limit = DominatorSearchLimit;
+
   Node* orig_sub = sub;
   nlist.clear();
   bool this_dominates = false;
   uint region_input = 0;
+  bool result = false; // Conservative answer 
+
   while (sub != NULL) {        // walk 'sub' up the chain to 'this'
     if (sub == this) {
       if (nlist.size() == 0) {
         // No Region nodes except loops were visited before and the EntryControl
         // path was taken for loops: it did not walk in a cycle.
-        return true;
-      } else if (!this_dominates) {
+        result = true;
+        break;
+      } else if (this_dominates) {
+        result = false;          // already met before: walk in a cycle
+        break;
+      } else {
         // Region nodes were visited. Continue walk up to Start or Root
         // to make sure that it did not walk in a cycle.
         this_dominates = true; // first time meet
-      } else {
-        return false;          // already met before: walk in a cycle
-      }
+        iterations_without_region_limit = DominatorSearchLimit; // Reset
+     }
     }
-    if (sub->is_Start() || sub->is_Root())
-      return this_dominates;
-
+    if (sub->is_Start() || sub->is_Root()) {
+      result = this_dominates;
+      break;
+    }
     Node* up = sub->find_exact_control(sub->in(0));
-    if (up == NULL || up->is_top())
-      return false; // Conservative answer for dead code
-
+    if (up == NULL || up->is_top()) {
+      result = false; // Conservative answer for dead code
+      break;
+    }
+    bool region_was_visited_before = false;
     if (sub == up && sub->is_Loop()) {
-      up = sub->in(0); // in(LoopNode::EntryControl);
-    } else if (sub == up && sub->is_Region()) {
+      up = sub->in(1); // in(LoopNode::EntryControl);
+    } else if (sub == up && sub->is_Region() && sub->req() == 3) {
+      iterations_without_region_limit = DominatorSearchLimit; // Reset
       uint i = 1;
-      if (nlist.size() == 0) {
+      uint size = nlist.size();
+      if (size == 0) {
         // No Region nodes (except Loops) were visited before.
         // Take first valid path on the way up to 'this'.
-      } else if (nlist.at(nlist.size() - 1) == sub) {
+      } else if ((Node*)(((intptr_t)nlist.at(size - 1)) & ~1) == sub) {
         // This Region node was just visited. Take other path.
         i = region_input + 1;
         nlist.pop();
+        region_was_visited_before = true;
       } else {
         // Was this Region node visited before?
-        uint size = nlist.size();
-        for (uint j = 0; j < size; j++) {
-          if (nlist.at(j) == sub) {
-            return false; // The Region node was visited before. Give up.
+        intptr_t ni;
+        uint j = 0;
+        for (; j < size; j++) {
+          ni = (intptr_t)nlist.at(j);
+          if ((Node*)(ni & ~1) == sub) {
+            if ((ni & 1) != 0) {
+              break; // Visited 2 paths. Give up.
+            } else {
+              // The Region node was visited before once.
+              nlist.remove(j);
+              region_was_visited_before = true;
+              for (; i < sub->req(); i++) {
+                Node* in = sub->in(i);
+                if (in != NULL && !in->is_top() && in != sub) {
+                  break;
+                }
+              }
+              i++; // Take other path.
+              break;
+            }
           }
         }
+        if (j < size && (ni & 1) != 0) {
+          result = false; // Visited 2 paths. Give up.
+          break;
+        }
         // The Region node was not visited before.
         // Take first valid path on the way up to 'this'.
       }
@@ -1097,19 +1131,25 @@
         }
       }
       if (i < sub->req()) {
-        nlist.push(sub);
         up = sub->in(i);
         region_input = i;
+        if (region_was_visited_before && sub != up)
+          nlist.push((Node*)((intptr_t)sub + 1));
+        else
+          nlist.push(sub);
       }
     }
-    if (sub == up)
-      return false;    // some kind of tight cycle
-    if (orig_sub == up)
-      return false;    // walk in a cycle
-
+    if (sub == up) {
+      result = false;    // some kind of tight cycle
+      break;
+    }
+    if (--iterations_without_region_limit < 0) {
+      result = false;    // dead cycle
+      break;
+    }
     sub = up;
   }
-  return false;
+  return result;
 }
 
 //------------------------------remove_dead_region-----------------------------
--- old/src/share/vm/opto/parse1.cpp	Wed May 14 20:05:31 2008
+++ new/src/share/vm/opto/parse1.cpp	Wed May 14 20:05:30 2008
@@ -1901,7 +1901,7 @@
   // finalization.  In general this will fold up since the concrete
   // class is often visible so the access flags are constant.
   Node* klass_addr = basic_plus_adr( receiver, receiver, oopDesc::klass_offset_in_bytes() );
-  Node* klass = _gvn.transform(new (C, 3) LoadKlassNode(NULL, immutable_memory(), klass_addr, TypeInstPtr::KLASS));
+  Node* klass = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), klass_addr, TypeInstPtr::KLASS) );
 
   Node* access_flags_addr = basic_plus_adr(klass, klass, Klass::access_flags_offset_in_bytes() + sizeof(oopDesc));
   Node* access_flags = make_load(NULL, access_flags_addr, TypeInt::INT, T_INT);
--- old/src/share/vm/opto/parseHelper.cpp	Wed May 14 20:05:32 2008
+++ new/src/share/vm/opto/parseHelper.cpp	Wed May 14 20:05:32 2008
@@ -38,7 +38,7 @@
 
   // Get method
   const TypeInstPtr* method_type = TypeInstPtr::make(TypePtr::Constant, method->klass(), true, method, 0);
-  Node *method_node = _gvn.transform( new (C, 1) ConPNode(method_type) );
+  Node *method_node = _gvn.transform( ConNode::make(C, method_type) );
 
   kill_dead_locals();
 
@@ -143,7 +143,7 @@
   int klass_offset = oopDesc::klass_offset_in_bytes();
   Node* p = basic_plus_adr( ary, ary, klass_offset );
   // p's type is array-of-OOPS plus klass_offset
-  Node* array_klass = _gvn.transform(new (C, 3) LoadKlassNode(0, immutable_memory(), p, TypeInstPtr::KLASS));
+  Node* array_klass = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeInstPtr::KLASS) );
   // Get the array klass
   const TypeKlassPtr *tak = _gvn.type(array_klass)->is_klassptr();
 
@@ -189,7 +189,7 @@
   // Extract the array element class
   int element_klass_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc);
   Node *p2 = basic_plus_adr(array_klass, array_klass, element_klass_offset);
-  Node *a_e_klass = _gvn.transform(new (C, 3) LoadKlassNode(0, immutable_memory(), p2, tak));
+  Node *a_e_klass = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p2, tak) );
 
   // Check (the hard way) and throw if not a subklass.
   // Result is ignored, we just need the CFG effects.
--- old/src/share/vm/opto/type.cpp	Wed May 14 20:05:33 2008
+++ new/src/share/vm/opto/type.cpp	Wed May 14 20:05:33 2008
@@ -311,8 +311,18 @@
   mreg2type[Op_RegFlags] = TypeInt::CC;
 
   TypeAryPtr::RANGE   = TypeAryPtr::make( TypePtr::BotPTR, TypeAry::make(Type::BOTTOM,TypeInt::POS), current->env()->Object_klass(), false, arrayOopDesc::length_offset_in_bytes());
-  // There is no shared klass for Object[].  See note in TypeAryPtr::klass().
-  TypeAryPtr::OOPS    = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS), NULL /*ciArrayKlass::make(o)*/,  false,  Type::OffsetBot);
+
+  TypeAryPtr::NARROWOOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeNarrowOop::BOTTOM, TypeInt::POS), NULL /*ciArrayKlass::make(o)*/,  false,  Type::OffsetBot);
+
+#ifdef _LP64
+  if (UseCompressedOops) {
+    TypeAryPtr::OOPS  = TypeAryPtr::NARROWOOPS;
+  } else
+#endif
+  {
+    // There is no shared klass for Object[].  See note in TypeAryPtr::klass().
+    TypeAryPtr::OOPS  = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS), NULL /*ciArrayKlass::make(o)*/,  false,  Type::OffsetBot);
+  }
   TypeAryPtr::BYTES   = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE      ,TypeInt::POS), ciTypeArrayKlass::make(T_BYTE),   true,  Type::OffsetBot);
   TypeAryPtr::SHORTS  = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::SHORT     ,TypeInt::POS), ciTypeArrayKlass::make(T_SHORT),  true,  Type::OffsetBot);
   TypeAryPtr::CHARS   = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::CHAR      ,TypeInt::POS), ciTypeArrayKlass::make(T_CHAR),   true,  Type::OffsetBot);
@@ -321,9 +331,9 @@
   TypeAryPtr::FLOATS  = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::FLOAT        ,TypeInt::POS), ciTypeArrayKlass::make(T_FLOAT),  true,  Type::OffsetBot);
   TypeAryPtr::DOUBLES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::DOUBLE       ,TypeInt::POS), ciTypeArrayKlass::make(T_DOUBLE), true,  Type::OffsetBot);
 
-  TypeAryPtr::_array_body_type[T_NARROWOOP] = NULL; // what should this be?
+  TypeAryPtr::_array_body_type[T_NARROWOOP] = TypeAryPtr::NARROWOOPS;
   TypeAryPtr::_array_body_type[T_OBJECT]  = TypeAryPtr::OOPS;
-  TypeAryPtr::_array_body_type[T_ARRAY]   = TypeAryPtr::OOPS;   // arrays are stored in oop arrays
+  TypeAryPtr::_array_body_type[T_ARRAY]   = TypeAryPtr::OOPS; // arrays are stored in oop arrays
   TypeAryPtr::_array_body_type[T_BYTE]    = TypeAryPtr::BYTES;
   TypeAryPtr::_array_body_type[T_BOOLEAN] = TypeAryPtr::BYTES;  // boolean[] is a byte array
   TypeAryPtr::_array_body_type[T_SHORT]   = TypeAryPtr::SHORTS;
@@ -696,7 +706,7 @@
   ResourceMark rm;
   Dict d(cmpkey,hashkey);       // Stop recursive type dumping
   dump2(d,1, st);
-  if (isa_ptr() && is_ptr()->is_narrow()) {
+  if (is_ptr_to_narrowoop()) {
     st->print(" [narrow]");
   }
 }
@@ -2146,6 +2156,67 @@
 // Convenience common pre-built type.
 const TypeOopPtr *TypeOopPtr::BOTTOM;
 
+//------------------------------TypeOopPtr-------------------------------------
+TypeOopPtr::TypeOopPtr( TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id )
+  : TypePtr(t, ptr, offset),
+    _const_oop(o), _klass(k),
+    _klass_is_exact(xk),
+    _is_ptr_to_narrowoop(false),
+    _instance_id(instance_id) {
+#ifdef _LP64
+  if (UseCompressedOops && _offset != 0) {
+    if (klass() == NULL) {
+      assert(this->isa_aryptr(), "only arrays without klass");
+      _is_ptr_to_narrowoop = true;
+    } else if (_offset == oopDesc::klass_offset_in_bytes()) {
+      _is_ptr_to_narrowoop = true;
+    } else if (this->isa_aryptr()) {
+      _is_ptr_to_narrowoop = (klass()->is_obj_array_klass() &&
+                             _offset != arrayOopDesc::length_offset_in_bytes());
+    } else if (klass() == ciEnv::current()->Class_klass() &&
+               (_offset == java_lang_Class::klass_offset_in_bytes() ||
+                _offset == java_lang_Class::array_klass_offset_in_bytes())) {
+      // Special hidden fields from the Class.
+      assert(this->isa_instptr(), "must be an instance ptr.");
+      _is_ptr_to_narrowoop = true;
+    } else if (klass()->is_instance_klass()) {
+      ciInstanceKlass* ik = klass()->as_instance_klass();
+      ciField* field = NULL;
+      if (this->isa_klassptr()) {
+        // Perm objects don't use compressed references, except for
+        // static fields which are currently compressed.
+        field = ik->get_field_by_offset(_offset, true);
+        if (field != NULL) {
+          BasicType basic_elem_type = field->layout_type();
+          _is_ptr_to_narrowoop = (basic_elem_type == T_OBJECT ||
+                                  basic_elem_type == T_ARRAY);
+        }
+      } else if (_offset == OffsetBot || _offset == OffsetTop) {
+        // unsafe access
+        _is_ptr_to_narrowoop = true;
+      } else { // exclude unsafe ops
+        assert(this->isa_instptr(), "must be an instance ptr.");
+        // Field which contains a compressed oop references.
+        field = ik->get_field_by_offset(_offset, false);
+        if (field != NULL) {
+          BasicType basic_elem_type = field->layout_type();
+          _is_ptr_to_narrowoop = (basic_elem_type == T_OBJECT ||
+                                  basic_elem_type == T_ARRAY);
+        } else if (klass()->equals(ciEnv::current()->Object_klass())) {
+          // Compile::find_alias_type() cast exactness on all types to verify
+          // that it does not affect alias type. 
+          _is_ptr_to_narrowoop = true;
+        } else {
+          // Type for the copy start in LibraryCallKit::inline_native_clone().
+          assert(!klass_is_exact(), "only non-exact klass");
+          _is_ptr_to_narrowoop = true;
+        }
+      }
+    }
+  }
+#endif
+}
+
 //------------------------------make-------------------------------------------
 const TypeOopPtr *TypeOopPtr::make(PTR ptr,
                                    int offset) {
@@ -2593,9 +2664,13 @@
 //-----------------------------cast_to_instance-------------------------------
 const TypeOopPtr *TypeInstPtr::cast_to_instance(int instance_id) const {
   if( instance_id == _instance_id) return this;
-  bool exact = (instance_id == UNKNOWN_INSTANCE) ? _klass_is_exact : true;
-
-  return make(ptr(), klass(), exact, const_oop(), _offset, instance_id);
+  bool exact = true;
+  PTR  ptr_t = NotNull;
+  if (instance_id == UNKNOWN_INSTANCE) {
+    exact = _klass_is_exact;
+    ptr_t = _ptr;
+  }
+  return make(ptr_t, klass(), exact, const_oop(), _offset, instance_id);
 }
 
 //------------------------------xmeet_unloaded---------------------------------
@@ -3014,6 +3089,7 @@
 // Convenience common pre-built types.
 const TypeAryPtr *TypeAryPtr::RANGE;
 const TypeAryPtr *TypeAryPtr::OOPS;
+const TypeAryPtr *TypeAryPtr::NARROWOOPS;
 const TypeAryPtr *TypeAryPtr::BYTES;
 const TypeAryPtr *TypeAryPtr::SHORTS;
 const TypeAryPtr *TypeAryPtr::CHARS;
@@ -3063,8 +3139,13 @@
 //-----------------------------cast_to_instance-------------------------------
 const TypeOopPtr *TypeAryPtr::cast_to_instance(int instance_id) const {
   if( instance_id == _instance_id) return this;
-  bool exact = (instance_id == UNKNOWN_INSTANCE) ? _klass_is_exact : true;
-  return make(ptr(), const_oop(), _ary, klass(), exact, _offset, instance_id);
+  bool exact = true;
+  PTR  ptr_t = NotNull;
+  if (instance_id == UNKNOWN_INSTANCE) {
+    exact = _klass_is_exact;
+    ptr_t = _ptr;
+  }
+  return make(ptr_t, const_oop(), _ary, klass(), exact, _offset, instance_id);
 }
 
 //-----------------------------narrow_size_type-------------------------------
@@ -3547,7 +3628,7 @@
     k_ary = ciTypeArrayKlass::make(el->basic_type());
   }
 
-  if( this != TypeAryPtr::OOPS )
+  if( this != TypeAryPtr::OOPS ) {
     // The _klass field acts as a cache of the underlying
     // ciKlass for this array type.  In order to set the field,
     // we need to cast away const-ness.
@@ -3562,6 +3643,11 @@
     // a bit less efficient than caching, but calls to
     // TypeAryPtr::OOPS->klass() are not common enough to matter.
     ((TypeAryPtr*)this)->_klass = k_ary;
+    if (UseCompressedOops && k_ary != NULL && k_ary->is_obj_array_klass() &&
+        _offset != 0 && _offset != arrayOopDesc::length_offset_in_bytes()) {
+      ((TypeAryPtr*)this)->_is_ptr_to_narrowoop = true;
+    }
+  }
   return k_ary;
 }
 
--- old/src/share/vm/opto/type.hpp	Wed May 14 20:05:35 2008
+++ new/src/share/vm/opto/type.hpp	Wed May 14 20:05:35 2008
@@ -191,9 +191,8 @@
   virtual const Type *filter( const Type *kills ) const;
 
   // Returns true if this pointer points at memory which contains a
-  // compressed oop references.  In 32-bit builds it's non-virtual
-  // since we don't support compressed oops at all in the mode.
-  LP64_ONLY(virtual) bool is_narrow() const { return false; }
+  // compressed oop references.
+  bool is_ptr_to_narrowoop() const;
 
   // Convenience access
   float getf() const;
@@ -213,8 +212,8 @@
   const TypePtr    *isa_ptr() const;             // Returns NULL if not ptr type
   const TypeRawPtr *isa_rawptr() const;          // NOT Java oop
   const TypeRawPtr *is_rawptr() const;           // Asserts is rawptr
-  const TypeNarrowOop  *is_narrowoop() const;        // Java-style GC'd pointer
-  const TypeNarrowOop  *isa_narrowoop() const;       // Returns NULL if not oop ptr type
+  const TypeNarrowOop  *is_narrowoop() const;    // Java-style GC'd pointer
+  const TypeNarrowOop  *isa_narrowoop() const;   // Returns NULL if not oop ptr type
   const TypeOopPtr   *isa_oopptr() const;        // Returns NULL if not oop ptr type
   const TypeOopPtr   *is_oopptr() const;         // Java-style GC'd pointer
   const TypeKlassPtr *isa_klassptr() const;      // Returns NULL if not KlassPtr
@@ -643,7 +642,7 @@
 // Some kind of oop (Java pointer), either klass or instance or array.
 class TypeOopPtr : public TypePtr {
 protected:
-  TypeOopPtr( TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id ) : TypePtr(t, ptr, offset), _const_oop(o), _klass(k), _klass_is_exact(xk), _instance_id(instance_id) { }
+  TypeOopPtr( TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id );
 public:
   virtual bool eq( const Type *t ) const;
   virtual int  hash() const;             // Type specific hashing
@@ -660,8 +659,9 @@
   ciKlass*      _klass;       // Klass object
   // Does the type exclude subclasses of the klass?  (Inexact == polymorphic.)
   bool          _klass_is_exact;
+  bool          _is_ptr_to_narrowoop;
 
-  int          _instance_id;   // if not UNKNOWN_INSTANCE, indicates that this is a particular instance
+  int           _instance_id;  // if not UNKNOWN_INSTANCE, indicates that this is a particular instance
                                // of this type which is distinct.  This is the  the node index of the
                                // node creating this instance
 
@@ -696,6 +696,11 @@
   ciObject* const_oop()    const { return _const_oop; }
   virtual ciKlass* klass() const { return _klass;     }
   bool klass_is_exact()    const { return _klass_is_exact; }
+
+  // Returns true if this pointer points at memory which contains a
+  // compressed oop references.
+  bool points_to_narrowoop() const { return _is_ptr_to_narrowoop; }
+
   bool is_instance()       const { return _instance_id != UNKNOWN_INSTANCE; }
   uint instance_id()       const { return _instance_id; }
   bool is_instance_field() const { return _instance_id != UNKNOWN_INSTANCE && _offset >= 0; }
@@ -716,12 +721,6 @@
   // returns the equivalent compressed version of this pointer type
   virtual const TypeNarrowOop* make_narrowoop() const;
 
-#ifdef _LP64
-  virtual bool is_narrow() const {
-    return (UseCompressedOops && _offset != 0);
-  }
-#endif
-
   virtual const Type *xmeet( const Type *t ) const;
   virtual const Type *xdual() const;    // Compute dual right now.
 
@@ -843,15 +842,10 @@
   virtual const Type *xmeet( const Type *t ) const;
   virtual const Type *xdual() const;    // Compute dual right now.
 
-#ifdef _LP64
-  virtual bool is_narrow() const {
-    return (UseCompressedOops && klass() != NULL && _offset != 0);
-  }
-#endif
-
   // Convenience common pre-built types.
   static const TypeAryPtr *RANGE;
   static const TypeAryPtr *OOPS;
+  static const TypeAryPtr *NARROWOOPS;
   static const TypeAryPtr *BYTES;
   static const TypeAryPtr *SHORTS;
   static const TypeAryPtr *CHARS;
@@ -901,18 +895,6 @@
   virtual const Type    *xmeet( const Type *t ) const;
   virtual const Type    *xdual() const;      // Compute dual right now.
 
-#ifdef _LP64
-  // Perm objects don't use compressed references, except for static fields
-  // which are currently compressed
-  virtual bool is_narrow() const {
-    if (UseCompressedOops && _offset != 0 && _klass->is_instance_klass()) {
-      ciInstanceKlass* ik = _klass->as_instance_klass();
-      return ik != NULL && ik->get_field_by_offset(_offset, true) != NULL;
-    }
-    return false;
-  }
-#endif
-
   // Convenience common pre-built types.
   static const TypeKlassPtr* OBJECT; // Not-null object klass or below
   static const TypeKlassPtr* OBJECT_OR_NULL; // Maybe-null version of same
@@ -921,7 +903,7 @@
 #endif
 };
 
-//------------------------------TypeNarrowOop----------------------------------------
+//------------------------------TypeNarrowOop----------------------------------
 // A compressed reference to some kind of Oop.  This type wraps around
 // a preexisting TypeOopPtr and forwards most of it's operations to
 // the underlying type.  It's only real purpose is to track the
@@ -1013,6 +995,14 @@
 };
 
 //------------------------------accessors--------------------------------------
+inline bool Type::is_ptr_to_narrowoop() const {
+#ifdef _LP64
+  return (isa_oopptr() != NULL && is_oopptr()->points_to_narrowoop());
+#else
+  return false;
+#endif
+}
+
 inline float Type::getf() const {
   assert( _base == FloatCon, "Not a FloatCon" );
   return ((TypeF*)this)->_f;
--- old/src/share/vm/runtime/globals.hpp	Wed May 14 20:05:37 2008
+++ new/src/share/vm/runtime/globals.hpp	Wed May 14 20:05:36 2008
@@ -2506,7 +2506,7 @@
   develop(intx, MaxRecursiveInlineLevel, 1,                                 \
           "maximum number of nested recursive calls that are inlined")      \
                                                                             \
-  develop(intx, InlineSmallCode, 1000,                                      \
+  product(intx, InlineSmallCode, 1000,                                      \
           "Only inline already compiled methods if their code size is "     \
           "less than this")                                                 \
                                                                             \
--- /dev/null	Wed May 14 20:05:38 2008
+++ new/test/compiler/6695810/Test.java	Wed May 14 20:05:38 2008
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+/*
+ * @test
+ * @bug 6695810
+ * @summary null oop passed to encode_heap_oop_not_null
+ * @run main/othervm -Xbatch Test
+ */
+
+public class Test {
+    Test _t;
+
+    static void test(Test t1, Test t2) {
+        if (t2 != null)
+            t1._t = t2;
+
+        if (t2 != null)
+            t1._t = t2;
+    }
+
+    public static void main(String[] args) {
+        Test t = new Test();
+        for (int i = 0; i < 50; i++) {
+            for (int j = 0; j < 100; j++) {
+                test(t, t);
+            }
+            test(t, null);
+        }
+        for (int i = 0; i < 10000; i++) {
+            test(t, t);
+        }
+        test(t, null);
+    }
+}