788 }
789 return NULL;
790 }
791
792
793 HeapWord* CompactibleFreeListSpace::block_start(const void* p) const {
794 NOT_PRODUCT(verify_objects_initialized());
795 return _bt.block_start(p);
796 }
797
798 HeapWord* CompactibleFreeListSpace::block_start_careful(const void* p) const {
799 return _bt.block_start_careful(p);
800 }
801
802 size_t CompactibleFreeListSpace::block_size(const HeapWord* p) const {
803 NOT_PRODUCT(verify_objects_initialized());
804 assert(MemRegion(bottom(), end()).contains(p), "p not in space");
805 // This must be volatile, or else there is a danger that the compiler
806 // will compile the code below into a sometimes-infinite loop, by keeping
807 // the value read the first time in a register.
808 oop o = (oop)p;
809 volatile oop* second_word_addr = o->klass_addr();
810 while (true) {
811 klassOop k = (klassOop)(*second_word_addr);
812 // We must do this until we get a consistent view of the object.
813 if (FreeChunk::secondWordIndicatesFreeChunk((intptr_t)k)) {
814 FreeChunk* fc = (FreeChunk*)p;
815 volatile size_t* sz_addr = (volatile size_t*)(fc->size_addr());
816 size_t res = (*sz_addr);
817 klassOop k2 = (klassOop)(*second_word_addr); // Read to confirm.
818 if (k == k2) {
819 assert(res != 0, "Block size should not be 0");
820 return res;
821 }
822 } else if (k != NULL) {
823 assert(k->is_oop(true /* ignore mark word */), "Should really be klass oop.");
824 assert(o->is_parsable(), "Should be parsable");
825 assert(o->is_oop(true /* ignore mark word */), "Should be an oop.");
826 size_t res = o->size_given_klass(k->klass_part());
827 res = adjustObjectSize(res);
828 assert(res != 0, "Block size should not be 0");
829 return res;
830 }
831 }
832 }
833
834 // A variant of the above that uses the Printezis bits for
835 // unparsable but allocated objects. This avoids any possible
836 // stalls waiting for mutators to initialize objects, and is
837 // thus potentially faster than the variant above. However,
838 // this variant may return a zero size for a block that is
839 // under mutation and for which a consistent size cannot be
840 // inferred without stalling; see CMSCollector::block_size_if_printezis_bits().
841 size_t CompactibleFreeListSpace::block_size_no_stall(HeapWord* p,
842 const CMSCollector* c)
843 const {
844 assert(MemRegion(bottom(), end()).contains(p), "p not in space");
845 // This must be volatile, or else there is a danger that the compiler
846 // will compile the code below into a sometimes-infinite loop, by keeping
847 // the value read the first time in a register.
848 oop o = (oop)p;
849 volatile oop* second_word_addr = o->klass_addr();
850 DEBUG_ONLY(uint loops = 0;)
851 while (true) {
852 klassOop k = (klassOop)(*second_word_addr);
853 // We must do this until we get a consistent view of the object.
854 if (FreeChunk::secondWordIndicatesFreeChunk((intptr_t)k)) {
855 FreeChunk* fc = (FreeChunk*)p;
856 volatile size_t* sz_addr = (volatile size_t*)(fc->size_addr());
857 size_t res = (*sz_addr);
858 klassOop k2 = (klassOop)(*second_word_addr); // Read to confirm.
859 if (k == k2) {
860 assert(res != 0, "Block size should not be 0");
861 assert(loops == 0, "Should be 0");
862 return res;
863 }
864 } else if (k != NULL && o->is_parsable()) {
865 assert(k->is_oop(), "Should really be klass oop.");
866 assert(o->is_oop(), "Should be an oop");
867 size_t res = o->size_given_klass(k->klass_part());
868 res = adjustObjectSize(res);
869 assert(res != 0, "Block size should not be 0");
870 return res;
871 } else {
872 return c->block_size_if_printezis_bits(p);
873 }
874 assert(loops == 0, "Can loop at most once");
875 DEBUG_ONLY(loops++;)
876 }
877 }
878
879 size_t CompactibleFreeListSpace::block_size_nopar(const HeapWord* p) const {
880 NOT_PRODUCT(verify_objects_initialized());
881 assert(MemRegion(bottom(), end()).contains(p), "p not in space");
882 FreeChunk* fc = (FreeChunk*)p;
883 if (fc->isFree()) {
884 return fc->size();
885 } else {
886 // Ignore mark word because this may be a recently promoted
887 // object whose mark word is used to chain together grey
888 // objects (the last one would have a null value).
889 assert(oop(p)->is_oop(true), "Should be an oop");
890 return adjustObjectSize(oop(p)->size());
891 }
892 }
893
894 // This implementation assumes that the property of "being an object" is
895 // stable. But being a free chunk may not be (because of parallel
896 // promotion.)
897 bool CompactibleFreeListSpace::block_is_obj(const HeapWord* p) const {
898 FreeChunk* fc = (FreeChunk*)p;
899 assert(is_in_reserved(p), "Should be in space");
900 // When doing a mark-sweep-compact of the CMS generation, this
901 // assertion may fail because prepare_for_compaction() uses
902 // space that is garbage to maintain information on ranges of
903 // live objects so that these live ranges can be moved as a whole.
904 // Comment out this assertion until that problem can be solved
905 // (i.e., that the block start calculation may look at objects
906 // at address below "p" in finding the object that contains "p"
907 // and those objects (if garbage) may have been modified to hold
908 // live range information.
909 // assert(ParallelGCThreads > 0 || _bt.block_start(p) == p, "Should be a block boundary");
910 klassOop k = oop(p)->klass();
911 intptr_t ki = (intptr_t)k;
912 if (FreeChunk::secondWordIndicatesFreeChunk(ki)) return false;
913 if (k != NULL) {
914 // Ignore mark word because it may have been used to
915 // chain together promoted objects (the last one
916 // would have a null value).
917 assert(oop(p)->is_oop(true), "Should be an oop");
918 return true;
919 } else {
920 return false; // Was not an object at the start of collection.
921 }
922 }
923
924 // Check if the object is alive. This fact is checked either by consulting
925 // the main marking bitmap in the sweeping phase or, if it's a permanent
926 // generation and we're not in the sweeping phase, by checking the
927 // perm_gen_verify_bit_map where we store the "deadness" information if
928 // we did not sweep the perm gen in the most recent previous GC cycle.
929 bool CompactibleFreeListSpace::obj_is_alive(const HeapWord* p) const {
930 assert (block_is_obj(p), "The address should point to an object");
931
932 // If we're sweeping, we use object liveness information from the main bit map
1010 HeapWord* CompactibleFreeListSpace::allocate(size_t size) {
1011 assert_lock_strong(freelistLock());
1012 HeapWord* res = NULL;
1013 assert(size == adjustObjectSize(size),
1014 "use adjustObjectSize() before calling into allocate()");
1015
1016 if (_adaptive_freelists) {
1017 res = allocate_adaptive_freelists(size);
1018 } else { // non-adaptive free lists
1019 res = allocate_non_adaptive_freelists(size);
1020 }
1021
1022 if (res != NULL) {
1023 // check that res does lie in this space!
1024 assert(is_in_reserved(res), "Not in this space!");
1025 assert(is_aligned((void*)res), "alignment check");
1026
1027 FreeChunk* fc = (FreeChunk*)res;
1028 fc->markNotFree();
1029 assert(!fc->isFree(), "shouldn't be marked free");
1030 assert(oop(fc)->klass() == NULL, "should look uninitialized");
1031 // Verify that the block offset table shows this to
1032 // be a single block, but not one which is unallocated.
1033 _bt.verify_single_block(res, size);
1034 _bt.verify_not_unallocated(res, size);
1035 // mangle a just allocated object with a distinct pattern.
1036 debug_only(fc->mangleAllocated(size));
1037 }
1038
1039 return res;
1040 }
1041
1042 HeapWord* CompactibleFreeListSpace::allocate_non_adaptive_freelists(size_t size) {
1043 HeapWord* res = NULL;
1044 // try and use linear allocation for smaller blocks
1045 if (size < _smallLinearAllocBlock._allocation_size_limit) {
1046 // if successful, the following also adjusts block offset table
1047 res = getChunkFromSmallLinearAllocBlock(size);
1048 }
1049 // Else triage to indexed lists for smaller sizes
1050 if (res == NULL) {
2576 // This locking manages sync with other large object allocations.
2577 MutexLockerEx x(_cfls->parDictionaryAllocLock(),
2578 Mutex::_no_safepoint_check_flag);
2579 res = _cfls->getChunkFromDictionaryExact(word_sz);
2580 if (res == NULL) return NULL;
2581 } else {
2582 FreeList* fl = &_indexedFreeList[word_sz];
2583 bool filled = false; //TRAP
2584 if (fl->count() == 0) {
2585 bool filled = true; //TRAP
2586 // Attempt to refill this local free list.
2587 _cfls->par_get_chunk_of_blocks(word_sz, _blocks_to_claim, fl);
2588 // If it didn't work, give up.
2589 if (fl->count() == 0) return NULL;
2590 }
2591 res = fl->getChunkAtHead();
2592 assert(res != NULL, "Why was count non-zero?");
2593 }
2594 res->markNotFree();
2595 assert(!res->isFree(), "shouldn't be marked free");
2596 assert(oop(res)->klass() == NULL, "should look uninitialized");
2597 // mangle a just allocated object with a distinct pattern.
2598 debug_only(res->mangleAllocated(word_sz));
2599 return (HeapWord*)res;
2600 }
2601
2602 void CFLS_LAB::retire() {
2603 for (size_t i = CompactibleFreeListSpace::IndexSetStart;
2604 i < CompactibleFreeListSpace::IndexSetSize;
2605 i += CompactibleFreeListSpace::IndexSetStride) {
2606 if (_indexedFreeList[i].count() > 0) {
2607 MutexLockerEx x(_cfls->_indexedFreeListParLocks[i],
2608 Mutex::_no_safepoint_check_flag);
2609 _cfls->_indexedFreeList[i].prepend(&_indexedFreeList[i]);
2610 // Reset this list.
2611 _indexedFreeList[i] = FreeList();
2612 _indexedFreeList[i].set_size(i);
2613 }
2614 }
2615 }
2616
|
788 }
789 return NULL;
790 }
791
792
793 HeapWord* CompactibleFreeListSpace::block_start(const void* p) const {
794 NOT_PRODUCT(verify_objects_initialized());
795 return _bt.block_start(p);
796 }
797
798 HeapWord* CompactibleFreeListSpace::block_start_careful(const void* p) const {
799 return _bt.block_start_careful(p);
800 }
801
802 size_t CompactibleFreeListSpace::block_size(const HeapWord* p) const {
803 NOT_PRODUCT(verify_objects_initialized());
804 assert(MemRegion(bottom(), end()).contains(p), "p not in space");
805 // This must be volatile, or else there is a danger that the compiler
806 // will compile the code below into a sometimes-infinite loop, by keeping
807 // the value read the first time in a register.
808 while (true) {
809 // We must do this until we get a consistent view of the object.
810 if (FreeChunk::indicatesFreeChunk(p)) {
811 volatile FreeChunk* fc = (volatile FreeChunk*)p;
812 size_t res = fc->size();
813 // If the object is still a free chunk, return the size, else it
814 // has been allocated so try again.
815 if (FreeChunk::indicatesFreeChunk(p)) {
816 assert(res != 0, "Block size should not be 0");
817 return res;
818 }
819 } else {
820 // must read from what 'p' points to in each loop.
821 klassOop k = ((volatile oopDesc*)p)->klass_or_null();
822 if (k != NULL) {
823 assert(k->is_oop(true /* ignore mark word */), "Should really be klass oop.");
824 oop o = (oop)p;
825 assert(o->is_parsable(), "Should be parsable");
826 assert(o->is_oop(true /* ignore mark word */), "Should be an oop.");
827 size_t res = o->size_given_klass(k->klass_part());
828 res = adjustObjectSize(res);
829 assert(res != 0, "Block size should not be 0");
830 return res;
831 }
832 }
833 }
834 }
835
836 // A variant of the above that uses the Printezis bits for
837 // unparsable but allocated objects. This avoids any possible
838 // stalls waiting for mutators to initialize objects, and is
839 // thus potentially faster than the variant above. However,
840 // this variant may return a zero size for a block that is
841 // under mutation and for which a consistent size cannot be
842 // inferred without stalling; see CMSCollector::block_size_if_printezis_bits().
843 size_t CompactibleFreeListSpace::block_size_no_stall(HeapWord* p,
844 const CMSCollector* c)
845 const {
846 assert(MemRegion(bottom(), end()).contains(p), "p not in space");
847 // This must be volatile, or else there is a danger that the compiler
848 // will compile the code below into a sometimes-infinite loop, by keeping
849 // the value read the first time in a register.
850 DEBUG_ONLY(uint loops = 0;)
851 while (true) {
852 // We must do this until we get a consistent view of the object.
853 if (FreeChunk::indicatesFreeChunk(p)) {
854 volatile FreeChunk* fc = (volatile FreeChunk*)p;
855 size_t res = fc->size();
856 if (FreeChunk::indicatesFreeChunk(p)) {
857 assert(res != 0, "Block size should not be 0");
858 assert(loops == 0, "Should be 0");
859 return res;
860 }
861 } else {
862 // must read from what 'p' points to in each loop.
863 klassOop k = ((volatile oopDesc*)p)->klass_or_null();
864 if (k != NULL && ((oopDesc*)p)->is_parsable()) {
865 assert(k->is_oop(), "Should really be klass oop.");
866 oop o = (oop)p;
867 assert(o->is_oop(), "Should be an oop");
868 size_t res = o->size_given_klass(k->klass_part());
869 res = adjustObjectSize(res);
870 assert(res != 0, "Block size should not be 0");
871 return res;
872 } else {
873 return c->block_size_if_printezis_bits(p);
874 }
875 }
876 assert(loops == 0, "Can loop at most once");
877 DEBUG_ONLY(loops++;)
878 }
879 }
880
881 size_t CompactibleFreeListSpace::block_size_nopar(const HeapWord* p) const {
882 NOT_PRODUCT(verify_objects_initialized());
883 assert(MemRegion(bottom(), end()).contains(p), "p not in space");
884 FreeChunk* fc = (FreeChunk*)p;
885 if (fc->isFree()) {
886 return fc->size();
887 } else {
888 // Ignore mark word because this may be a recently promoted
889 // object whose mark word is used to chain together grey
890 // objects (the last one would have a null value).
891 assert(oop(p)->is_oop(true), "Should be an oop");
892 return adjustObjectSize(oop(p)->size());
893 }
894 }
895
896 // This implementation assumes that the property of "being an object" is
897 // stable. But being a free chunk may not be (because of parallel
898 // promotion.)
899 bool CompactibleFreeListSpace::block_is_obj(const HeapWord* p) const {
900 FreeChunk* fc = (FreeChunk*)p;
901 assert(is_in_reserved(p), "Should be in space");
902 // When doing a mark-sweep-compact of the CMS generation, this
903 // assertion may fail because prepare_for_compaction() uses
904 // space that is garbage to maintain information on ranges of
905 // live objects so that these live ranges can be moved as a whole.
906 // Comment out this assertion until that problem can be solved
907 // (i.e., that the block start calculation may look at objects
908 // at address below "p" in finding the object that contains "p"
909 // and those objects (if garbage) may have been modified to hold
910 // live range information.
911 // assert(ParallelGCThreads > 0 || _bt.block_start(p) == p, "Should be a block boundary");
912 if (FreeChunk::indicatesFreeChunk(p)) return false;
913 klassOop k = oop(p)->klass_or_null();
914 if (k != NULL) {
915 // Ignore mark word because it may have been used to
916 // chain together promoted objects (the last one
917 // would have a null value).
918 assert(oop(p)->is_oop(true), "Should be an oop");
919 return true;
920 } else {
921 return false; // Was not an object at the start of collection.
922 }
923 }
924
925 // Check if the object is alive. This fact is checked either by consulting
926 // the main marking bitmap in the sweeping phase or, if it's a permanent
927 // generation and we're not in the sweeping phase, by checking the
928 // perm_gen_verify_bit_map where we store the "deadness" information if
929 // we did not sweep the perm gen in the most recent previous GC cycle.
930 bool CompactibleFreeListSpace::obj_is_alive(const HeapWord* p) const {
931 assert (block_is_obj(p), "The address should point to an object");
932
933 // If we're sweeping, we use object liveness information from the main bit map
1011 HeapWord* CompactibleFreeListSpace::allocate(size_t size) {
1012 assert_lock_strong(freelistLock());
1013 HeapWord* res = NULL;
1014 assert(size == adjustObjectSize(size),
1015 "use adjustObjectSize() before calling into allocate()");
1016
1017 if (_adaptive_freelists) {
1018 res = allocate_adaptive_freelists(size);
1019 } else { // non-adaptive free lists
1020 res = allocate_non_adaptive_freelists(size);
1021 }
1022
1023 if (res != NULL) {
1024 // check that res does lie in this space!
1025 assert(is_in_reserved(res), "Not in this space!");
1026 assert(is_aligned((void*)res), "alignment check");
1027
1028 FreeChunk* fc = (FreeChunk*)res;
1029 fc->markNotFree();
1030 assert(!fc->isFree(), "shouldn't be marked free");
1031 assert(oop(fc)->klass_or_null() == NULL, "should look uninitialized");
1032 // Verify that the block offset table shows this to
1033 // be a single block, but not one which is unallocated.
1034 _bt.verify_single_block(res, size);
1035 _bt.verify_not_unallocated(res, size);
1036 // mangle a just allocated object with a distinct pattern.
1037 debug_only(fc->mangleAllocated(size));
1038 }
1039
1040 return res;
1041 }
1042
1043 HeapWord* CompactibleFreeListSpace::allocate_non_adaptive_freelists(size_t size) {
1044 HeapWord* res = NULL;
1045 // try and use linear allocation for smaller blocks
1046 if (size < _smallLinearAllocBlock._allocation_size_limit) {
1047 // if successful, the following also adjusts block offset table
1048 res = getChunkFromSmallLinearAllocBlock(size);
1049 }
1050 // Else triage to indexed lists for smaller sizes
1051 if (res == NULL) {
2577 // This locking manages sync with other large object allocations.
2578 MutexLockerEx x(_cfls->parDictionaryAllocLock(),
2579 Mutex::_no_safepoint_check_flag);
2580 res = _cfls->getChunkFromDictionaryExact(word_sz);
2581 if (res == NULL) return NULL;
2582 } else {
2583 FreeList* fl = &_indexedFreeList[word_sz];
2584 bool filled = false; //TRAP
2585 if (fl->count() == 0) {
2586 bool filled = true; //TRAP
2587 // Attempt to refill this local free list.
2588 _cfls->par_get_chunk_of_blocks(word_sz, _blocks_to_claim, fl);
2589 // If it didn't work, give up.
2590 if (fl->count() == 0) return NULL;
2591 }
2592 res = fl->getChunkAtHead();
2593 assert(res != NULL, "Why was count non-zero?");
2594 }
2595 res->markNotFree();
2596 assert(!res->isFree(), "shouldn't be marked free");
2597 assert(oop(res)->klass_or_null() == NULL, "should look uninitialized");
2598 // mangle a just allocated object with a distinct pattern.
2599 debug_only(res->mangleAllocated(word_sz));
2600 return (HeapWord*)res;
2601 }
2602
2603 void CFLS_LAB::retire() {
2604 for (size_t i = CompactibleFreeListSpace::IndexSetStart;
2605 i < CompactibleFreeListSpace::IndexSetSize;
2606 i += CompactibleFreeListSpace::IndexSetStride) {
2607 if (_indexedFreeList[i].count() > 0) {
2608 MutexLockerEx x(_cfls->_indexedFreeListParLocks[i],
2609 Mutex::_no_safepoint_check_flag);
2610 _cfls->_indexedFreeList[i].prepend(&_indexedFreeList[i]);
2611 // Reset this list.
2612 _indexedFreeList[i] = FreeList();
2613 _indexedFreeList[i].set_size(i);
2614 }
2615 }
2616 }
2617
|