173 _numWordsAllocated = 0;
174 )
175
176 _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
177 use_adaptive_freelists,
178 dictionaryChoice);
179 NOT_PRODUCT(debug_cms_space = _cmsSpace;)
180 if (_cmsSpace == NULL) {
181 vm_exit_during_initialization(
182 "CompactibleFreeListSpace allocation failure");
183 }
184 _cmsSpace->_gen = this;
185
186 _gc_stats = new CMSGCStats();
187
188 // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
189 // offsets match. The ability to tell free chunks from objects
190 // depends on this property.
191 debug_only(
192 FreeChunk* junk = NULL;
193 assert(junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
194 "Offset of FreeChunk::_prev within FreeChunk must match"
195 " that of OopDesc::_klass within OopDesc");
196 )
197 if (ParallelGCThreads > 0) {
198 typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
199 _par_gc_thread_states =
200 NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads);
201 if (_par_gc_thread_states == NULL) {
202 vm_exit_during_initialization("Could not allocate par gc structs");
203 }
204 for (uint i = 0; i < ParallelGCThreads; i++) {
205 _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());
206 if (_par_gc_thread_states[i] == NULL) {
207 vm_exit_during_initialization("Could not allocate par gc structs");
208 }
209 }
210 } else {
211 _par_gc_thread_states = NULL;
212 }
213 _incremental_collection_failed = false;
1022 // Note that if this object contains references, the writing
1023 // of those references will dirty the card containing this object
1024 // allowing the object to be blackened (and its references scanned)
1025 // either during a preclean phase or at the final checkpoint.
1026 void CMSCollector::direct_allocated(HeapWord* start, size_t size) {
1027 assert(_markBitMap.covers(start, size), "Out of bounds");
1028 if (_collectorState >= Marking) {
1029 MutexLockerEx y(_markBitMap.lock(),
1030 Mutex::_no_safepoint_check_flag);
1031 // [see comments preceding SweepClosure::do_blk() below for details]
1032 // 1. need to mark the object as live so it isn't collected
1033 // 2. need to mark the 2nd bit to indicate the object may be uninitialized
1034 // 3. need to mark the end of the object so sweeper can skip over it
1035 // if it's uninitialized when the sweeper reaches it.
1036 _markBitMap.mark(start); // object is live
1037 _markBitMap.mark(start + 1); // object is potentially uninitialized?
1038 _markBitMap.mark(start + size - 1);
1039 // mark end of object
1040 }
1041 // check that oop looks uninitialized
1042 assert(oop(start)->klass() == NULL, "_klass should be NULL");
1043 }
1044
1045 void CMSCollector::promoted(bool par, HeapWord* start,
1046 bool is_obj_array, size_t obj_size) {
1047 assert(_markBitMap.covers(start), "Out of bounds");
1048 // See comment in direct_allocated() about when objects should
1049 // be allocated live.
1050 if (_collectorState >= Marking) {
1051 // we already hold the marking bit map lock, taken in
1052 // the prologue
1053 if (par) {
1054 _markBitMap.par_mark(start);
1055 } else {
1056 _markBitMap.mark(start);
1057 }
1058 // We don't need to mark the object as uninitialized (as
1059 // in direct_allocated above) because this is being done with the
1060 // world stopped and the object will be initialized by the
1061 // time the sweeper gets to look at it.
1062 assert(SafepointSynchronize::is_at_safepoint(),
1292 // then allocate and copy, then track promoted info if needed.
1293 // When tracking (see PromotionInfo::track()), the mark word may
1294 // be displaced and in this case restoration of the mark word
1295 // occurs in the (oop_since_save_marks_)iterate phase.
1296 if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1297 // Out of space for allocating spooling buffers;
1298 // try expanding and allocating spooling buffers.
1299 if (!expand_and_ensure_spooling_space(promoInfo)) {
1300 return NULL;
1301 }
1302 }
1303 assert(promoInfo->has_spooling_space(), "Control point invariant");
1304 HeapWord* obj_ptr = ps->lab.alloc(word_sz);
1305 if (obj_ptr == NULL) {
1306 obj_ptr = expand_and_par_lab_allocate(ps, word_sz);
1307 if (obj_ptr == NULL) {
1308 return NULL;
1309 }
1310 }
1311 oop obj = oop(obj_ptr);
1312 assert(obj->klass() == NULL, "Object should be uninitialized here.");
1313 // Otherwise, copy the object. Here we must be careful to insert the
1314 // klass pointer last, since this marks the block as an allocated object.
1315 HeapWord* old_ptr = (HeapWord*)old;
1316 if (word_sz > (size_t)oopDesc::header_size()) {
1317 Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
1318 obj_ptr + oopDesc::header_size(),
1319 word_sz - oopDesc::header_size());
1320 }
1321 // Restore the mark word copied above.
1322 obj->set_mark(m);
1323 // Now we can track the promoted object, if necessary. We take care
1324 // To delay the transition from uninitialized to full object
1325 // (i.e., insertion of klass pointer) until after, so that it
1326 // atomically becomes a promoted object.
1327 if (promoInfo->tracking()) {
1328 promoInfo->track((PromotedObject*)obj, old->klass());
1329 }
1330 // Finally, install the klass pointer.
1331 obj->set_klass(old->klass());
1332
1333 assert(old->is_oop(), "Will dereference klass ptr below");
1334 collector()->promoted(true, // parallel
1335 obj_ptr, old->is_objArray(), word_sz);
1336
1337 NOT_PRODUCT(
1338 Atomic::inc(&_numObjectsPromoted);
1339 Atomic::add((jint)CompactibleFreeListSpace::adjustObjectSize(obj->size()),
1340 &_numWordsPromoted);
1341 )
1342
1343 return obj;
1344 }
1345
1346 void
1347 ConcurrentMarkSweepGeneration::
1348 par_promote_alloc_undo(int thread_num,
1349 HeapWord* obj, size_t word_sz) {
1350 // CMS does not support promotion undo.
6148 // A variant of the above (block_size_using_printezis_bits()) except
6149 // that we return 0 if the P-bits are not yet set.
6150 size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const {
6151 if (_markBitMap.isMarked(addr)) {
6152 assert(_markBitMap.isMarked(addr + 1), "Missing Printezis bit?");
6153 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
6154 size_t size = pointer_delta(nextOneAddr + 1, addr);
6155 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6156 "alignment problem");
6157 assert(size >= 3, "Necessary for Printezis marks to work");
6158 return size;
6159 } else {
6160 assert(!_markBitMap.isMarked(addr + 1), "Bit map inconsistency?");
6161 return 0;
6162 }
6163 }
6164
6165 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
6166 size_t sz = 0;
6167 oop p = (oop)addr;
6168 if (p->klass() != NULL && p->is_parsable()) {
6169 sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
6170 } else {
6171 sz = block_size_using_printezis_bits(addr);
6172 }
6173 assert(sz > 0, "size must be nonzero");
6174 HeapWord* next_block = addr + sz;
6175 HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
6176 CardTableModRefBS::card_size);
6177 assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
6178 round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
6179 "must be different cards");
6180 return next_card;
6181 }
6182
6183
6184 // CMS Bit Map Wrapper /////////////////////////////////////////
6185
6186 // Construct a CMS bit map infrastructure, but don't create the
6187 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
6188 // further below.
6585 void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6586 void Par_MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6587
6588 // This closure is used to rescan the marked objects on the dirty cards
6589 // in the mod union table and the card table proper.
6590 size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m(
6591 oop p, MemRegion mr) {
6592
6593 size_t size = 0;
6594 HeapWord* addr = (HeapWord*)p;
6595 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6596 assert(_span.contains(addr), "we are scanning the CMS generation");
6597 // check if it's time to yield
6598 if (do_yield_check()) {
6599 // We yielded for some foreground stop-world work,
6600 // and we have been asked to abort this ongoing preclean cycle.
6601 return 0;
6602 }
6603 if (_bitMap->isMarked(addr)) {
6604 // it's marked; is it potentially uninitialized?
6605 if (p->klass() != NULL) {
6606 if (CMSPermGenPrecleaningEnabled && !p->is_parsable()) {
6607 // Signal precleaning to redirty the card since
6608 // the klass pointer is already installed.
6609 assert(size == 0, "Initial value");
6610 } else {
6611 assert(p->is_parsable(), "must be parsable.");
6612 // an initialized object; ignore mark word in verification below
6613 // since we are running concurrent with mutators
6614 assert(p->is_oop(true), "should be an oop");
6615 if (p->is_objArray()) {
6616 // objArrays are precisely marked; restrict scanning
6617 // to dirty cards only.
6618 size = p->oop_iterate(_scanningClosure, mr);
6619 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6620 "adjustObjectSize should be the identity for array sizes, "
6621 "which are necessarily larger than minimum object size of "
6622 "two heap words");
6623 } else {
6624 // A non-array may have been imprecisely marked; we need
6625 // to scan object in its entirety.
6626 size = CompactibleFreeListSpace::adjustObjectSize(
6627 p->oop_iterate(_scanningClosure));
6628 }
6629 #ifdef DEBUG
6630 size_t direct_size =
6631 CompactibleFreeListSpace::adjustObjectSize(p->size());
6632 assert(size == direct_size, "Inconsistency in size");
6633 assert(size >= 3, "Necessary for Printezis marks to work");
6634 if (!_bitMap->isMarked(addr+1)) {
6635 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size);
6636 } else {
6637 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1);
6638 assert(_bitMap->isMarked(addr+size-1),
6639 "inconsistent Printezis mark");
6640 }
6641 #endif // DEBUG
6642 }
6643 } else {
6644 // an unitialized object
6645 assert(_bitMap->isMarked(addr+1), "missing Printezis mark?");
6646 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
6647 size = pointer_delta(nextOneAddr + 1, addr);
6648 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6649 "alignment problem");
6650 // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass()
6651 // will dirty the card when the klass pointer is installed in the
6652 // object (signalling the completion of initialization).
6653 }
6654 } else {
6655 // Either a not yet marked object or an uninitialized object
6656 if (p->klass() == NULL || !p->is_parsable()) {
6657 // An uninitialized object, skip to the next card, since
6658 // we may not be able to read its P-bits yet.
6659 assert(size == 0, "Initial value");
6660 } else {
6661 // An object not (yet) reached by marking: we merely need to
6662 // compute its size so as to go look at the next block.
6663 assert(p->is_oop(true), "should be an oop");
6664 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6665 }
6666 }
6667 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6668 return size;
6669 }
6670
6671 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6672 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6673 "CMS thread should hold CMS token");
6674 assert_lock_strong(_freelistLock);
6675 assert_lock_strong(_bitMap->lock());
6676 // relinquish the free_list_lock and bitMaplock()
6693 ConcurrentMarkSweepThread::acknowledge_yield_request();
6694 }
6695
6696 ConcurrentMarkSweepThread::synchronize(true);
6697 _freelistLock->lock_without_safepoint_check();
6698 _bitMap->lock()->lock_without_safepoint_check();
6699 _collector->startTimer();
6700 }
6701
6702
6703 //////////////////////////////////////////////////////////////////
6704 // SurvivorSpacePrecleanClosure
6705 //////////////////////////////////////////////////////////////////
6706 // This (single-threaded) closure is used to preclean the oops in
6707 // the survivor spaces.
6708 size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
6709
6710 HeapWord* addr = (HeapWord*)p;
6711 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6712 assert(!_span.contains(addr), "we are scanning the survivor spaces");
6713 assert(p->klass() != NULL, "object should be initializd");
6714 assert(p->is_parsable(), "must be parsable.");
6715 // an initialized object; ignore mark word in verification below
6716 // since we are running concurrent with mutators
6717 assert(p->is_oop(true), "should be an oop");
6718 // Note that we do not yield while we iterate over
6719 // the interior oops of p, pushing the relevant ones
6720 // on our marking stack.
6721 size_t size = p->oop_iterate(_scanning_closure);
6722 do_yield_check();
6723 // Observe that below, we do not abandon the preclean
6724 // phase as soon as we should; rather we empty the
6725 // marking stack before returning. This is to satisfy
6726 // some existing assertions. In general, it may be a
6727 // good idea to abort immediately and complete the marking
6728 // from the grey objects at a later time.
6729 while (!_mark_stack->isEmpty()) {
6730 oop new_oop = _mark_stack->pop();
6731 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6732 assert(new_oop->is_parsable(), "Found unparsable oop");
6733 assert(_bit_map->isMarked((HeapWord*)new_oop),
6851 (intptr_t)_finger, CardTableModRefBS::card_size);
6852 }
6853
6854 // Should revisit to see if this should be restructured for
6855 // greater efficiency.
6856 void MarkFromRootsClosure::do_bit(size_t offset) {
6857 if (_skipBits > 0) {
6858 _skipBits--;
6859 return;
6860 }
6861 // convert offset into a HeapWord*
6862 HeapWord* addr = _bitMap->startWord() + offset;
6863 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6864 "address out of range");
6865 assert(_bitMap->isMarked(addr), "tautology");
6866 if (_bitMap->isMarked(addr+1)) {
6867 // this is an allocated but not yet initialized object
6868 assert(_skipBits == 0, "tautology");
6869 _skipBits = 2; // skip next two marked bits ("Printezis-marks")
6870 oop p = oop(addr);
6871 if (p->klass() == NULL || !p->is_parsable()) {
6872 DEBUG_ONLY(if (!_verifying) {)
6873 // We re-dirty the cards on which this object lies and increase
6874 // the _threshold so that we'll come back to scan this object
6875 // during the preclean or remark phase. (CMSCleanOnEnter)
6876 if (CMSCleanOnEnter) {
6877 size_t sz = _collector->block_size_using_printezis_bits(addr);
6878 HeapWord* start_card_addr = (HeapWord*)round_down(
6879 (intptr_t)addr, CardTableModRefBS::card_size);
6880 HeapWord* end_card_addr = (HeapWord*)round_to(
6881 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6882 MemRegion redirty_range = MemRegion(start_card_addr, end_card_addr);
6883 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6884 // Bump _threshold to end_card_addr; note that
6885 // _threshold cannot possibly exceed end_card_addr, anyhow.
6886 // This prevents future clearing of the card as the scan proceeds
6887 // to the right.
6888 assert(_threshold <= end_card_addr,
6889 "Because we are just scanning into this object");
6890 if (_threshold < end_card_addr) {
6891 _threshold = end_card_addr;
6892 }
6893 if (p->klass() != NULL) {
6894 // Redirty the range of cards...
6895 _mut->mark_range(redirty_range);
6896 } // ...else the setting of klass will dirty the card anyway.
6897 }
6898 DEBUG_ONLY(})
6899 return;
6900 }
6901 }
6902 scanOopsInOop(addr);
6903 }
6904
6905 // We take a break if we've been at this for a while,
6906 // so as to avoid monopolizing the locks involved.
6907 void MarkFromRootsClosure::do_yield_work() {
6908 // First give up the locks, then yield, then re-lock
6909 // We should probably use a constructor/destructor idiom to
6910 // do this unlock/lock or modify the MutexUnlocker class to
6911 // serve our purpose. XXX
6912 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6913 "CMS thread should hold CMS token");
7031 assert(_span.contains(_finger), "Out of bounds _finger?");
7032 }
7033
7034 // Should revisit to see if this should be restructured for
7035 // greater efficiency.
7036 void Par_MarkFromRootsClosure::do_bit(size_t offset) {
7037 if (_skip_bits > 0) {
7038 _skip_bits--;
7039 return;
7040 }
7041 // convert offset into a HeapWord*
7042 HeapWord* addr = _bit_map->startWord() + offset;
7043 assert(_bit_map->endWord() && addr < _bit_map->endWord(),
7044 "address out of range");
7045 assert(_bit_map->isMarked(addr), "tautology");
7046 if (_bit_map->isMarked(addr+1)) {
7047 // this is an allocated object that might not yet be initialized
7048 assert(_skip_bits == 0, "tautology");
7049 _skip_bits = 2; // skip next two marked bits ("Printezis-marks")
7050 oop p = oop(addr);
7051 if (p->klass() == NULL || !p->is_parsable()) {
7052 // in the case of Clean-on-Enter optimization, redirty card
7053 // and avoid clearing card by increasing the threshold.
7054 return;
7055 }
7056 }
7057 scan_oops_in_oop(addr);
7058 }
7059
7060 void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
7061 assert(_bit_map->isMarked(ptr), "expected bit to be set");
7062 // Should we assert that our work queue is empty or
7063 // below some drain limit?
7064 assert(_work_queue->size() == 0,
7065 "should drain stack to limit stack usage");
7066 // convert ptr to an oop preparatory to scanning
7067 oop obj = oop(ptr);
7068 // Ignore mark word in verification below, since we
7069 // may be running concurrent with mutators.
7070 assert(obj->is_oop(true), "should be an oop");
7071 assert(_finger <= ptr, "_finger runneth ahead");
8006 }
8007
8008 // Common code path for original and adaptive free lists.
8009
8010 // this object is live: we'd normally expect this to be
8011 // an oop, and like to assert the following:
8012 // assert(oop(addr)->is_oop(), "live block should be an oop");
8013 // However, as we commented above, this may be an object whose
8014 // header hasn't yet been initialized.
8015 size_t size;
8016 assert(_bitMap->isMarked(addr), "Tautology for this control point");
8017 if (_bitMap->isMarked(addr + 1)) {
8018 // Determine the size from the bit map, rather than trying to
8019 // compute it from the object header.
8020 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
8021 size = pointer_delta(nextOneAddr + 1, addr);
8022 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
8023 "alignment problem");
8024
8025 #ifdef DEBUG
8026 if (oop(addr)->klass() != NULL &&
8027 ( !_collector->should_unload_classes()
8028 || oop(addr)->is_parsable())) {
8029 // Ignore mark word because we are running concurrent with mutators
8030 assert(oop(addr)->is_oop(true), "live block should be an oop");
8031 assert(size ==
8032 CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()),
8033 "P-mark and computed size do not agree");
8034 }
8035 #endif
8036
8037 } else {
8038 // This should be an initialized object that's alive.
8039 assert(oop(addr)->klass() != NULL &&
8040 (!_collector->should_unload_classes()
8041 || oop(addr)->is_parsable()),
8042 "Should be an initialized object");
8043 // Ignore mark word because we are running concurrent with mutators
8044 assert(oop(addr)->is_oop(true), "live block should be an oop");
8045 // Verify that the bit map has no bits marked between
8046 // addr and purported end of this block.
8047 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
8048 assert(size >= 3, "Necessary for Printezis marks to work");
8049 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
8050 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
8051 }
8052 return size;
8053 }
8054
8055 void SweepClosure::doPostIsFreeOrGarbageChunk(FreeChunk* fc,
8056 size_t chunkSize) {
8057 // doPostIsFreeOrGarbageChunk() should only be called in the smart allocation
8058 // scheme.
8059 bool fcInFreeLists = fc->isFree();
|
173 _numWordsAllocated = 0;
174 )
175
176 _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
177 use_adaptive_freelists,
178 dictionaryChoice);
179 NOT_PRODUCT(debug_cms_space = _cmsSpace;)
180 if (_cmsSpace == NULL) {
181 vm_exit_during_initialization(
182 "CompactibleFreeListSpace allocation failure");
183 }
184 _cmsSpace->_gen = this;
185
186 _gc_stats = new CMSGCStats();
187
188 // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
189 // offsets match. The ability to tell free chunks from objects
190 // depends on this property.
191 debug_only(
192 FreeChunk* junk = NULL;
193 assert(UseCompressedOops ||
194 junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
195 "Offset of FreeChunk::_prev within FreeChunk must match"
196 " that of OopDesc::_klass within OopDesc");
197 )
198 if (ParallelGCThreads > 0) {
199 typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
200 _par_gc_thread_states =
201 NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads);
202 if (_par_gc_thread_states == NULL) {
203 vm_exit_during_initialization("Could not allocate par gc structs");
204 }
205 for (uint i = 0; i < ParallelGCThreads; i++) {
206 _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());
207 if (_par_gc_thread_states[i] == NULL) {
208 vm_exit_during_initialization("Could not allocate par gc structs");
209 }
210 }
211 } else {
212 _par_gc_thread_states = NULL;
213 }
214 _incremental_collection_failed = false;
1023 // Note that if this object contains references, the writing
1024 // of those references will dirty the card containing this object
1025 // allowing the object to be blackened (and its references scanned)
1026 // either during a preclean phase or at the final checkpoint.
1027 void CMSCollector::direct_allocated(HeapWord* start, size_t size) {
1028 assert(_markBitMap.covers(start, size), "Out of bounds");
1029 if (_collectorState >= Marking) {
1030 MutexLockerEx y(_markBitMap.lock(),
1031 Mutex::_no_safepoint_check_flag);
1032 // [see comments preceding SweepClosure::do_blk() below for details]
1033 // 1. need to mark the object as live so it isn't collected
1034 // 2. need to mark the 2nd bit to indicate the object may be uninitialized
1035 // 3. need to mark the end of the object so sweeper can skip over it
1036 // if it's uninitialized when the sweeper reaches it.
1037 _markBitMap.mark(start); // object is live
1038 _markBitMap.mark(start + 1); // object is potentially uninitialized?
1039 _markBitMap.mark(start + size - 1);
1040 // mark end of object
1041 }
1042 // check that oop looks uninitialized
1043 assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");
1044 }
1045
1046 void CMSCollector::promoted(bool par, HeapWord* start,
1047 bool is_obj_array, size_t obj_size) {
1048 assert(_markBitMap.covers(start), "Out of bounds");
1049 // See comment in direct_allocated() about when objects should
1050 // be allocated live.
1051 if (_collectorState >= Marking) {
1052 // we already hold the marking bit map lock, taken in
1053 // the prologue
1054 if (par) {
1055 _markBitMap.par_mark(start);
1056 } else {
1057 _markBitMap.mark(start);
1058 }
1059 // We don't need to mark the object as uninitialized (as
1060 // in direct_allocated above) because this is being done with the
1061 // world stopped and the object will be initialized by the
1062 // time the sweeper gets to look at it.
1063 assert(SafepointSynchronize::is_at_safepoint(),
1293 // then allocate and copy, then track promoted info if needed.
1294 // When tracking (see PromotionInfo::track()), the mark word may
1295 // be displaced and in this case restoration of the mark word
1296 // occurs in the (oop_since_save_marks_)iterate phase.
1297 if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1298 // Out of space for allocating spooling buffers;
1299 // try expanding and allocating spooling buffers.
1300 if (!expand_and_ensure_spooling_space(promoInfo)) {
1301 return NULL;
1302 }
1303 }
1304 assert(promoInfo->has_spooling_space(), "Control point invariant");
1305 HeapWord* obj_ptr = ps->lab.alloc(word_sz);
1306 if (obj_ptr == NULL) {
1307 obj_ptr = expand_and_par_lab_allocate(ps, word_sz);
1308 if (obj_ptr == NULL) {
1309 return NULL;
1310 }
1311 }
1312 oop obj = oop(obj_ptr);
1313 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1314 // Otherwise, copy the object. Here we must be careful to insert the
1315 // klass pointer last, since this marks the block as an allocated object.
1316 // Except with compressed oops it's the mark word.
1317 HeapWord* old_ptr = (HeapWord*)old;
1318 if (word_sz > (size_t)oopDesc::header_size()) {
1319 Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
1320 obj_ptr + oopDesc::header_size(),
1321 word_sz - oopDesc::header_size());
1322 }
1323
1324 if (UseCompressedOops) {
1325 // Copy gap missed by (aligned) header size calculation above
1326 obj->set_klass_gap(old->klass_gap());
1327 }
1328
1329 // Restore the mark word copied above.
1330 obj->set_mark(m);
1331
1332 // Now we can track the promoted object, if necessary. We take care
1333 // To delay the transition from uninitialized to full object
1334 // (i.e., insertion of klass pointer) until after, so that it
1335 // atomically becomes a promoted object.
1336 if (promoInfo->tracking()) {
1337 promoInfo->track((PromotedObject*)obj, old->klass());
1338 }
1339
1340 // Finally, install the klass pointer (this should be volatile).
1341 obj->set_klass(old->klass());
1342
1343 assert(old->is_oop(), "Will dereference klass ptr below");
1344 collector()->promoted(true, // parallel
1345 obj_ptr, old->is_objArray(), word_sz);
1346
1347 NOT_PRODUCT(
1348 Atomic::inc(&_numObjectsPromoted);
1349 Atomic::add((jint)CompactibleFreeListSpace::adjustObjectSize(obj->size()),
1350 &_numWordsPromoted);
1351 )
1352
1353 return obj;
1354 }
1355
1356 void
1357 ConcurrentMarkSweepGeneration::
1358 par_promote_alloc_undo(int thread_num,
1359 HeapWord* obj, size_t word_sz) {
1360 // CMS does not support promotion undo.
6158 // A variant of the above (block_size_using_printezis_bits()) except
6159 // that we return 0 if the P-bits are not yet set.
6160 size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const {
6161 if (_markBitMap.isMarked(addr)) {
6162 assert(_markBitMap.isMarked(addr + 1), "Missing Printezis bit?");
6163 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
6164 size_t size = pointer_delta(nextOneAddr + 1, addr);
6165 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6166 "alignment problem");
6167 assert(size >= 3, "Necessary for Printezis marks to work");
6168 return size;
6169 } else {
6170 assert(!_markBitMap.isMarked(addr + 1), "Bit map inconsistency?");
6171 return 0;
6172 }
6173 }
6174
6175 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
6176 size_t sz = 0;
6177 oop p = (oop)addr;
6178 if (p->klass_or_null() != NULL && p->is_parsable()) {
6179 sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
6180 } else {
6181 sz = block_size_using_printezis_bits(addr);
6182 }
6183 assert(sz > 0, "size must be nonzero");
6184 HeapWord* next_block = addr + sz;
6185 HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
6186 CardTableModRefBS::card_size);
6187 assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
6188 round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
6189 "must be different cards");
6190 return next_card;
6191 }
6192
6193
6194 // CMS Bit Map Wrapper /////////////////////////////////////////
6195
6196 // Construct a CMS bit map infrastructure, but don't create the
6197 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
6198 // further below.
6595 void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6596 void Par_MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6597
6598 // This closure is used to rescan the marked objects on the dirty cards
6599 // in the mod union table and the card table proper.
6600 size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m(
6601 oop p, MemRegion mr) {
6602
6603 size_t size = 0;
6604 HeapWord* addr = (HeapWord*)p;
6605 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6606 assert(_span.contains(addr), "we are scanning the CMS generation");
6607 // check if it's time to yield
6608 if (do_yield_check()) {
6609 // We yielded for some foreground stop-world work,
6610 // and we have been asked to abort this ongoing preclean cycle.
6611 return 0;
6612 }
6613 if (_bitMap->isMarked(addr)) {
6614 // it's marked; is it potentially uninitialized?
6615 if (p->klass_or_null() != NULL) {
6616 if (CMSPermGenPrecleaningEnabled && !p->is_parsable()) {
6617 // Signal precleaning to redirty the card since
6618 // the klass pointer is already installed.
6619 assert(size == 0, "Initial value");
6620 } else {
6621 assert(p->is_parsable(), "must be parsable.");
6622 // an initialized object; ignore mark word in verification below
6623 // since we are running concurrent with mutators
6624 assert(p->is_oop(true), "should be an oop");
6625 if (p->is_objArray()) {
6626 // objArrays are precisely marked; restrict scanning
6627 // to dirty cards only.
6628 size = CompactibleFreeListSpace::adjustObjectSize(
6629 p->oop_iterate(_scanningClosure, mr));
6630 } else {
6631 // A non-array may have been imprecisely marked; we need
6632 // to scan object in its entirety.
6633 size = CompactibleFreeListSpace::adjustObjectSize(
6634 p->oop_iterate(_scanningClosure));
6635 }
6636 #ifdef DEBUG
6637 size_t direct_size =
6638 CompactibleFreeListSpace::adjustObjectSize(p->size());
6639 assert(size == direct_size, "Inconsistency in size");
6640 assert(size >= 3, "Necessary for Printezis marks to work");
6641 if (!_bitMap->isMarked(addr+1)) {
6642 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size);
6643 } else {
6644 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1);
6645 assert(_bitMap->isMarked(addr+size-1),
6646 "inconsistent Printezis mark");
6647 }
6648 #endif // DEBUG
6649 }
6650 } else {
6651 // an unitialized object
6652 assert(_bitMap->isMarked(addr+1), "missing Printezis mark?");
6653 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
6654 size = pointer_delta(nextOneAddr + 1, addr);
6655 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6656 "alignment problem");
6657 // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass()
6658 // will dirty the card when the klass pointer is installed in the
6659 // object (signalling the completion of initialization).
6660 }
6661 } else {
6662 // Either a not yet marked object or an uninitialized object
6663 if (p->klass_or_null() == NULL || !p->is_parsable()) {
6664 // An uninitialized object, skip to the next card, since
6665 // we may not be able to read its P-bits yet.
6666 assert(size == 0, "Initial value");
6667 } else {
6668 // An object not (yet) reached by marking: we merely need to
6669 // compute its size so as to go look at the next block.
6670 assert(p->is_oop(true), "should be an oop");
6671 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6672 }
6673 }
6674 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6675 return size;
6676 }
6677
6678 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6679 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6680 "CMS thread should hold CMS token");
6681 assert_lock_strong(_freelistLock);
6682 assert_lock_strong(_bitMap->lock());
6683 // relinquish the free_list_lock and bitMaplock()
6700 ConcurrentMarkSweepThread::acknowledge_yield_request();
6701 }
6702
6703 ConcurrentMarkSweepThread::synchronize(true);
6704 _freelistLock->lock_without_safepoint_check();
6705 _bitMap->lock()->lock_without_safepoint_check();
6706 _collector->startTimer();
6707 }
6708
6709
6710 //////////////////////////////////////////////////////////////////
6711 // SurvivorSpacePrecleanClosure
6712 //////////////////////////////////////////////////////////////////
6713 // This (single-threaded) closure is used to preclean the oops in
6714 // the survivor spaces.
6715 size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
6716
6717 HeapWord* addr = (HeapWord*)p;
6718 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6719 assert(!_span.contains(addr), "we are scanning the survivor spaces");
6720 assert(p->klass_or_null() != NULL, "object should be initializd");
6721 assert(p->is_parsable(), "must be parsable.");
6722 // an initialized object; ignore mark word in verification below
6723 // since we are running concurrent with mutators
6724 assert(p->is_oop(true), "should be an oop");
6725 // Note that we do not yield while we iterate over
6726 // the interior oops of p, pushing the relevant ones
6727 // on our marking stack.
6728 size_t size = p->oop_iterate(_scanning_closure);
6729 do_yield_check();
6730 // Observe that below, we do not abandon the preclean
6731 // phase as soon as we should; rather we empty the
6732 // marking stack before returning. This is to satisfy
6733 // some existing assertions. In general, it may be a
6734 // good idea to abort immediately and complete the marking
6735 // from the grey objects at a later time.
6736 while (!_mark_stack->isEmpty()) {
6737 oop new_oop = _mark_stack->pop();
6738 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6739 assert(new_oop->is_parsable(), "Found unparsable oop");
6740 assert(_bit_map->isMarked((HeapWord*)new_oop),
6858 (intptr_t)_finger, CardTableModRefBS::card_size);
6859 }
6860
6861 // Should revisit to see if this should be restructured for
6862 // greater efficiency.
6863 void MarkFromRootsClosure::do_bit(size_t offset) {
6864 if (_skipBits > 0) {
6865 _skipBits--;
6866 return;
6867 }
6868 // convert offset into a HeapWord*
6869 HeapWord* addr = _bitMap->startWord() + offset;
6870 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6871 "address out of range");
6872 assert(_bitMap->isMarked(addr), "tautology");
6873 if (_bitMap->isMarked(addr+1)) {
6874 // this is an allocated but not yet initialized object
6875 assert(_skipBits == 0, "tautology");
6876 _skipBits = 2; // skip next two marked bits ("Printezis-marks")
6877 oop p = oop(addr);
6878 if (p->klass_or_null() == NULL || !p->is_parsable()) {
6879 DEBUG_ONLY(if (!_verifying) {)
6880 // We re-dirty the cards on which this object lies and increase
6881 // the _threshold so that we'll come back to scan this object
6882 // during the preclean or remark phase. (CMSCleanOnEnter)
6883 if (CMSCleanOnEnter) {
6884 size_t sz = _collector->block_size_using_printezis_bits(addr);
6885 HeapWord* start_card_addr = (HeapWord*)round_down(
6886 (intptr_t)addr, CardTableModRefBS::card_size);
6887 HeapWord* end_card_addr = (HeapWord*)round_to(
6888 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6889 MemRegion redirty_range = MemRegion(start_card_addr, end_card_addr);
6890 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6891 // Bump _threshold to end_card_addr; note that
6892 // _threshold cannot possibly exceed end_card_addr, anyhow.
6893 // This prevents future clearing of the card as the scan proceeds
6894 // to the right.
6895 assert(_threshold <= end_card_addr,
6896 "Because we are just scanning into this object");
6897 if (_threshold < end_card_addr) {
6898 _threshold = end_card_addr;
6899 }
6900 if (p->klass_or_null() != NULL) {
6901 // Redirty the range of cards...
6902 _mut->mark_range(redirty_range);
6903 } // ...else the setting of klass will dirty the card anyway.
6904 }
6905 DEBUG_ONLY(})
6906 return;
6907 }
6908 }
6909 scanOopsInOop(addr);
6910 }
6911
6912 // We take a break if we've been at this for a while,
6913 // so as to avoid monopolizing the locks involved.
6914 void MarkFromRootsClosure::do_yield_work() {
6915 // First give up the locks, then yield, then re-lock
6916 // We should probably use a constructor/destructor idiom to
6917 // do this unlock/lock or modify the MutexUnlocker class to
6918 // serve our purpose. XXX
6919 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6920 "CMS thread should hold CMS token");
7038 assert(_span.contains(_finger), "Out of bounds _finger?");
7039 }
7040
7041 // Should revisit to see if this should be restructured for
7042 // greater efficiency.
7043 void Par_MarkFromRootsClosure::do_bit(size_t offset) {
7044 if (_skip_bits > 0) {
7045 _skip_bits--;
7046 return;
7047 }
7048 // convert offset into a HeapWord*
7049 HeapWord* addr = _bit_map->startWord() + offset;
7050 assert(_bit_map->endWord() && addr < _bit_map->endWord(),
7051 "address out of range");
7052 assert(_bit_map->isMarked(addr), "tautology");
7053 if (_bit_map->isMarked(addr+1)) {
7054 // this is an allocated object that might not yet be initialized
7055 assert(_skip_bits == 0, "tautology");
7056 _skip_bits = 2; // skip next two marked bits ("Printezis-marks")
7057 oop p = oop(addr);
7058 if (p->klass_or_null() == NULL || !p->is_parsable()) {
7059 // in the case of Clean-on-Enter optimization, redirty card
7060 // and avoid clearing card by increasing the threshold.
7061 return;
7062 }
7063 }
7064 scan_oops_in_oop(addr);
7065 }
7066
7067 void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
7068 assert(_bit_map->isMarked(ptr), "expected bit to be set");
7069 // Should we assert that our work queue is empty or
7070 // below some drain limit?
7071 assert(_work_queue->size() == 0,
7072 "should drain stack to limit stack usage");
7073 // convert ptr to an oop preparatory to scanning
7074 oop obj = oop(ptr);
7075 // Ignore mark word in verification below, since we
7076 // may be running concurrent with mutators.
7077 assert(obj->is_oop(true), "should be an oop");
7078 assert(_finger <= ptr, "_finger runneth ahead");
8013 }
8014
8015 // Common code path for original and adaptive free lists.
8016
8017 // this object is live: we'd normally expect this to be
8018 // an oop, and like to assert the following:
8019 // assert(oop(addr)->is_oop(), "live block should be an oop");
8020 // However, as we commented above, this may be an object whose
8021 // header hasn't yet been initialized.
8022 size_t size;
8023 assert(_bitMap->isMarked(addr), "Tautology for this control point");
8024 if (_bitMap->isMarked(addr + 1)) {
8025 // Determine the size from the bit map, rather than trying to
8026 // compute it from the object header.
8027 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
8028 size = pointer_delta(nextOneAddr + 1, addr);
8029 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
8030 "alignment problem");
8031
8032 #ifdef DEBUG
8033 if (oop(addr)->klass_or_null() != NULL &&
8034 ( !_collector->should_unload_classes()
8035 || oop(addr)->is_parsable())) {
8036 // Ignore mark word because we are running concurrent with mutators
8037 assert(oop(addr)->is_oop(true), "live block should be an oop");
8038 assert(size ==
8039 CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()),
8040 "P-mark and computed size do not agree");
8041 }
8042 #endif
8043
8044 } else {
8045 // This should be an initialized object that's alive.
8046 assert(oop(addr)->klass_or_null() != NULL &&
8047 (!_collector->should_unload_classes()
8048 || oop(addr)->is_parsable()),
8049 "Should be an initialized object");
8050 // Ignore mark word because we are running concurrent with mutators
8051 assert(oop(addr)->is_oop(true), "live block should be an oop");
8052 // Verify that the bit map has no bits marked between
8053 // addr and purported end of this block.
8054 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
8055 assert(size >= 3, "Necessary for Printezis marks to work");
8056 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
8057 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
8058 }
8059 return size;
8060 }
8061
8062 void SweepClosure::doPostIsFreeOrGarbageChunk(FreeChunk* fc,
8063 size_t chunkSize) {
8064 // doPostIsFreeOrGarbageChunk() should only be called in the smart allocation
8065 // scheme.
8066 bool fcInFreeLists = fc->isFree();
|