src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp

*** 188,198 ****
    // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
    // offsets match. The ability to tell free chunks from objects
    // depends on this property.
    debug_only(
      FreeChunk* junk = NULL;
!     assert(junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
             "Offset of FreeChunk::_prev within FreeChunk must match"
             "  that of OopDesc::_klass within OopDesc");
    )
    if (ParallelGCThreads > 0) {
      typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
--- 188,199 ----
    // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
    // offsets match. The ability to tell free chunks from objects
    // depends on this property.
    debug_only(
      FreeChunk* junk = NULL;
!     assert(UseCompressedOops ||
!            junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
             "Offset of FreeChunk::_prev within FreeChunk must match"
             "  that of OopDesc::_klass within OopDesc");
    )
    if (ParallelGCThreads > 0) {
      typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
*** 1037,1047 ****
      _markBitMap.mark(start + 1);      // object is potentially uninitialized?
      _markBitMap.mark(start + size - 1);
                                        // mark end of object
    }
    // check that oop looks uninitialized
!   assert(oop(start)->klass() == NULL, "_klass should be NULL");
  }
  
  void CMSCollector::promoted(bool par, HeapWord* start,
                              bool is_obj_array, size_t obj_size) {
    assert(_markBitMap.covers(start), "Out of bounds");
--- 1038,1048 ----
      _markBitMap.mark(start + 1);      // object is potentially uninitialized?
      _markBitMap.mark(start + size - 1);
                                        // mark end of object
    }
    // check that oop looks uninitialized
!   assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");
  }
  
  void CMSCollector::promoted(bool par, HeapWord* start,
                              bool is_obj_array, size_t obj_size) {
    assert(_markBitMap.covers(start), "Out of bounds");
*** 1307,1335 ****
       if (obj_ptr == NULL) {
         return NULL;
       }
    }
    oop obj = oop(obj_ptr);
!   assert(obj->klass() == NULL, "Object should be uninitialized here.");
    // Otherwise, copy the object.  Here we must be careful to insert the
    // klass pointer last, since this marks the block as an allocated object.
    HeapWord* old_ptr = (HeapWord*)old;
    if (word_sz > (size_t)oopDesc::header_size()) {
      Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
                                   obj_ptr + oopDesc::header_size(),
                                   word_sz - oopDesc::header_size());
    }
    // Restore the mark word copied above.
    obj->set_mark(m);
    // Now we can track the promoted object, if necessary.  We take care
    // To delay the transition from uninitialized to full object
    // (i.e., insertion of klass pointer) until after, so that it
    // atomically becomes a promoted object.
    if (promoInfo->tracking()) {
      promoInfo->track((PromotedObject*)obj, old->klass());
    }
!   // Finally, install the klass pointer.
    obj->set_klass(old->klass());
  
    assert(old->is_oop(), "Will dereference klass ptr below");
    collector()->promoted(true,          // parallel
                          obj_ptr, old->is_objArray(), word_sz);
--- 1308,1345 ----
       if (obj_ptr == NULL) {
         return NULL;
       }
    }
    oop obj = oop(obj_ptr);
!   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
    // Otherwise, copy the object.  Here we must be careful to insert the
    // klass pointer last, since this marks the block as an allocated object.
+   // Except with compressed oops it's the mark word.
    HeapWord* old_ptr = (HeapWord*)old;
    if (word_sz > (size_t)oopDesc::header_size()) {
      Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
                                   obj_ptr + oopDesc::header_size(),
                                   word_sz - oopDesc::header_size());
    }
+ 
+   if (UseCompressedOops) {
+     // Copy gap missed by (aligned) header size calculation above
+     obj->set_klass_gap(old->klass_gap());
+   }
+ 
    // Restore the mark word copied above.
    obj->set_mark(m);
+ 
    // Now we can track the promoted object, if necessary.  We take care
    // To delay the transition from uninitialized to full object
    // (i.e., insertion of klass pointer) until after, so that it
    // atomically becomes a promoted object.
    if (promoInfo->tracking()) {
      promoInfo->track((PromotedObject*)obj, old->klass());
    }
! 
!   // Finally, install the klass pointer (this should be volatile).
    obj->set_klass(old->klass());
  
    assert(old->is_oop(), "Will dereference klass ptr below");
    collector()->promoted(true,          // parallel
                          obj_ptr, old->is_objArray(), word_sz);
*** 6163,6173 ****
  }
  
  HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
    size_t sz = 0;
    oop p = (oop)addr;
!   if (p->klass() != NULL && p->is_parsable()) {
      sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
    } else {
      sz = block_size_using_printezis_bits(addr);
    }
    assert(sz > 0, "size must be nonzero");
--- 6173,6183 ----
  }
  
  HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
    size_t sz = 0;
    oop p = (oop)addr;
!   if (p->klass_or_null() != NULL && p->is_parsable()) {
      sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
    } else {
      sz = block_size_using_printezis_bits(addr);
    }
    assert(sz > 0, "size must be nonzero");
*** 6600,6610 ****
      // and we have been asked to abort this ongoing preclean cycle.
      return 0;
    }
    if (_bitMap->isMarked(addr)) {
      // it's marked; is it potentially uninitialized?
!     if (p->klass() != NULL) {
        if (CMSPermGenPrecleaningEnabled && !p->is_parsable()) {
          // Signal precleaning to redirty the card since
          // the klass pointer is already installed.
          assert(size == 0, "Initial value");
        } else {
--- 6610,6620 ----
      // and we have been asked to abort this ongoing preclean cycle.
      return 0;
    }
    if (_bitMap->isMarked(addr)) {
      // it's marked; is it potentially uninitialized?
!     if (p->klass_or_null() != NULL) {
        if (CMSPermGenPrecleaningEnabled && !p->is_parsable()) {
          // Signal precleaning to redirty the card since
          // the klass pointer is already installed.
          assert(size == 0, "Initial value");
        } else {
*** 6613,6627 ****
          // since we are running concurrent with mutators
          assert(p->is_oop(true), "should be an oop");
          if (p->is_objArray()) {
            // objArrays are precisely marked; restrict scanning
            // to dirty cards only.
!           size = p->oop_iterate(_scanningClosure, mr);
!           assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
!                  "adjustObjectSize should be the identity for array sizes, "
!                  "which are necessarily larger than minimum object size of "
!                  "two heap words");
          } else {
            // A non-array may have been imprecisely marked; we need
            // to scan object in its entirety.
            size = CompactibleFreeListSpace::adjustObjectSize(
                     p->oop_iterate(_scanningClosure));
--- 6623,6634 ----
          // since we are running concurrent with mutators
          assert(p->is_oop(true), "should be an oop");
          if (p->is_objArray()) {
            // objArrays are precisely marked; restrict scanning
            // to dirty cards only.
!           size = CompactibleFreeListSpace::adjustObjectSize(
!                    p->oop_iterate(_scanningClosure, mr));
          } else {
            // A non-array may have been imprecisely marked; we need
            // to scan object in its entirety.
            size = CompactibleFreeListSpace::adjustObjectSize(
                     p->oop_iterate(_scanningClosure));
*** 6651,6661 ****
        // will dirty the card when the klass pointer is installed in the
        // object (signalling the completion of initialization).
      }
    } else {
      // Either a not yet marked object or an uninitialized object
!     if (p->klass() == NULL || !p->is_parsable()) {
        // An uninitialized object, skip to the next card, since
        // we may not be able to read its P-bits yet.
        assert(size == 0, "Initial value");
      } else {
        // An object not (yet) reached by marking: we merely need to
--- 6658,6668 ----
        // will dirty the card when the klass pointer is installed in the
        // object (signalling the completion of initialization).
      }
    } else {
      // Either a not yet marked object or an uninitialized object
!     if (p->klass_or_null() == NULL || !p->is_parsable()) {
        // An uninitialized object, skip to the next card, since
        // we may not be able to read its P-bits yet.
        assert(size == 0, "Initial value");
      } else {
        // An object not (yet) reached by marking: we merely need to
*** 6708,6718 ****
  size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
  
    HeapWord* addr = (HeapWord*)p;
    DEBUG_ONLY(_collector->verify_work_stacks_empty();)
    assert(!_span.contains(addr), "we are scanning the survivor spaces");
!   assert(p->klass() != NULL, "object should be initializd");
    assert(p->is_parsable(), "must be parsable.");
    // an initialized object; ignore mark word in verification below
    // since we are running concurrent with mutators
    assert(p->is_oop(true), "should be an oop");
    // Note that we do not yield while we iterate over
--- 6715,6725 ----
  size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
  
    HeapWord* addr = (HeapWord*)p;
    DEBUG_ONLY(_collector->verify_work_stacks_empty();)
    assert(!_span.contains(addr), "we are scanning the survivor spaces");
!   assert(p->klass_or_null() != NULL, "object should be initializd");
    assert(p->is_parsable(), "must be parsable.");
    // an initialized object; ignore mark word in verification below
    // since we are running concurrent with mutators
    assert(p->is_oop(true), "should be an oop");
    // Note that we do not yield while we iterate over
*** 6866,6876 ****
    if (_bitMap->isMarked(addr+1)) {
      // this is an allocated but not yet initialized object
      assert(_skipBits == 0, "tautology");
      _skipBits = 2;  // skip next two marked bits ("Printezis-marks")
      oop p = oop(addr);
!     if (p->klass() == NULL || !p->is_parsable()) {
        DEBUG_ONLY(if (!_verifying) {)
          // We re-dirty the cards on which this object lies and increase
          // the _threshold so that we'll come back to scan this object
          // during the preclean or remark phase. (CMSCleanOnEnter)
          if (CMSCleanOnEnter) {
--- 6873,6883 ----
    if (_bitMap->isMarked(addr+1)) {
      // this is an allocated but not yet initialized object
      assert(_skipBits == 0, "tautology");
      _skipBits = 2;  // skip next two marked bits ("Printezis-marks")
      oop p = oop(addr);
!     if (p->klass_or_null() == NULL || !p->is_parsable()) {
        DEBUG_ONLY(if (!_verifying) {)
          // We re-dirty the cards on which this object lies and increase
          // the _threshold so that we'll come back to scan this object
          // during the preclean or remark phase. (CMSCleanOnEnter)
          if (CMSCleanOnEnter) {
*** 6888,6898 ****
            assert(_threshold <= end_card_addr,
                   "Because we are just scanning into this object");
            if (_threshold < end_card_addr) {
              _threshold = end_card_addr;
            }
!           if (p->klass() != NULL) {
              // Redirty the range of cards...
              _mut->mark_range(redirty_range);
            } // ...else the setting of klass will dirty the card anyway.
          }
        DEBUG_ONLY(})
--- 6895,6905 ----
            assert(_threshold <= end_card_addr,
                   "Because we are just scanning into this object");
            if (_threshold < end_card_addr) {
              _threshold = end_card_addr;
            }
!           if (p->klass_or_null() != NULL) {
              // Redirty the range of cards...
              _mut->mark_range(redirty_range);
            } // ...else the setting of klass will dirty the card anyway.
          }
        DEBUG_ONLY(})
*** 7046,7056 ****
    if (_bit_map->isMarked(addr+1)) {
      // this is an allocated object that might not yet be initialized
      assert(_skip_bits == 0, "tautology");
      _skip_bits = 2;  // skip next two marked bits ("Printezis-marks")
      oop p = oop(addr);
!     if (p->klass() == NULL || !p->is_parsable()) {
        // in the case of Clean-on-Enter optimization, redirty card
        // and avoid clearing card by increasing  the threshold.
        return;
      }
    }
--- 7053,7063 ----
    if (_bit_map->isMarked(addr+1)) {
      // this is an allocated object that might not yet be initialized
      assert(_skip_bits == 0, "tautology");
      _skip_bits = 2;  // skip next two marked bits ("Printezis-marks")
      oop p = oop(addr);
!     if (p->klass_or_null() == NULL || !p->is_parsable()) {
        // in the case of Clean-on-Enter optimization, redirty card
        // and avoid clearing card by increasing  the threshold.
        return;
      }
    }
*** 8021,8031 ****
      size = pointer_delta(nextOneAddr + 1, addr);
      assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
             "alignment problem");
  
      #ifdef DEBUG
!       if (oop(addr)->klass() != NULL &&
            (   !_collector->should_unload_classes()
             || oop(addr)->is_parsable())) {
          // Ignore mark word because we are running concurrent with mutators
          assert(oop(addr)->is_oop(true), "live block should be an oop");
          assert(size ==
--- 8028,8038 ----
      size = pointer_delta(nextOneAddr + 1, addr);
      assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
             "alignment problem");
  
      #ifdef DEBUG
!       if (oop(addr)->klass_or_null() != NULL &&
            (   !_collector->should_unload_classes()
             || oop(addr)->is_parsable())) {
          // Ignore mark word because we are running concurrent with mutators
          assert(oop(addr)->is_oop(true), "live block should be an oop");
          assert(size ==
*** 8034,8044 ****
        }
      #endif
  
    } else {
      // This should be an initialized object that's alive.
!     assert(oop(addr)->klass() != NULL &&
             (!_collector->should_unload_classes()
              || oop(addr)->is_parsable()),
             "Should be an initialized object");
      // Ignore mark word because we are running concurrent with mutators
      assert(oop(addr)->is_oop(true), "live block should be an oop");
--- 8041,8051 ----
        }
      #endif
  
    } else {
      // This should be an initialized object that's alive.
!     assert(oop(addr)->klass_or_null() != NULL &&
             (!_collector->should_unload_classes()
              || oop(addr)->is_parsable()),
             "Should be an initialized object");
      // Ignore mark word because we are running concurrent with mutators
      assert(oop(addr)->is_oop(true), "live block should be an oop");