private unsafe void CompactHeapObjects(UIntPtr previousEnd) { while (!this.relocationQueue.IsEmpty) { UIntPtr sourceAddress = this.relocationQueue.Read(); UIntPtr destinationAddress = this.relocationQueue.Read(); UIntPtr runLength = this.relocationQueue.Read(); if (previousEnd != destinationAddress) { VTable.Assert(previousEnd < destinationAddress); if (PageTable.Page(destinationAddress) != PageTable.Page(previousEnd + PreHeader.Size)) { if (!PageTable.PageAligned(previousEnd)) { UIntPtr pageLimit = PageTable.PagePad(previousEnd); BumpAllocator.WriteUnusedMarker(previousEnd); previousEnd += UIntPtr.Size; Util.MemClear(previousEnd, pageLimit - previousEnd); } if (!PageTable.PageAligned(destinationAddress)) { // This only happens before pinned objects and // large objects UIntPtr start = PageTable.PageAlign(destinationAddress); VTable.Assert(previousEnd <= start); while (start < destinationAddress) { Allocator.WriteAlignment(start); start += UIntPtr.Size; } } UIntPtr objAddr = destinationAddress + PreHeader.Size; InteriorPtrTable.SetFirst(objAddr); } else { VTable.Assert(previousEnd < destinationAddress); UIntPtr start = previousEnd; while (start < destinationAddress) { Allocator.WriteAlignment(start); start += UIntPtr.Size; } } } Util.MemCopy(destinationAddress, sourceAddress, runLength); previousEnd = destinationAddress + runLength; } // Zero out the end of the allocation page if (!PageTable.PageAligned(previousEnd)) { UIntPtr pageLimit = PageTable.PagePad(previousEnd); Util.MemClear(previousEnd, pageLimit - previousEnd); } this.relocationQueue.Cleanup(true); }
private UIntPtr FreshAlloc(UIntPtr bytes, uint alignment, Thread currentThread) { #if SINGULARITY_KERNEL Kernel.Waypoint(702); #endif this.Truncate(); UIntPtr paddedBytes = PageTable.PagePad(bytes + alignment - UIntPtr.Size); BaseCollector.IncrementNewBytesSinceGC(paddedBytes); UIntPtr pages = PageTable.PageCount(paddedBytes); bool fCleanPages = CLEAR_POOL_PAGES(); // We may eventually want to ask for specific pages // between asking if any pages are reusable and asking the // OS for any possible page. UIntPtr startPage = PageManager.EnsurePages(currentThread, pages, this.pageType, ref fCleanPages); UIntPtr startAddr = PageTable.PageAddr(startPage); UIntPtr limitAddr = PageTable.PageAddr(startPage + pages); startAddr = Allocator.AlignedAllocationPtr(startAddr, limitAddr, alignment); this.allocNew = startAddr; this.allocPtr = startAddr + bytes; if (fCleanPages) { this.zeroedLimit = limitAddr; } else { Util.MemClear(startAddr, bytes); this.zeroedLimit = this.allocPtr; } this.reserveLimit = limitAddr; UIntPtr resultAddr = startAddr + PreHeader.Size; InteriorPtrTable.SetFirst(resultAddr); #if !SINGULARITY || SEMISPACE_COLLECTOR || ADAPTIVE_COPYING_COLLECTOR || SLIDING_COLLECTOR if (GC.remsetType == RemSetType.Cards) { UIntPtr nextPageAddr = startAddr + PageTable.PageSize; VTable.Assert(resultAddr < nextPageAddr); if (this.allocPtr > nextPageAddr) { #if DONT_RECORD_OBJALLOC_IN_OFFSETTABLE #else OffsetTable.SetLast(resultAddr); #endif } } #endif #if SINGULARITY_KERNEL Kernel.Waypoint(703); #endif return(resultAddr); }
// Finds the object base for an interior pointer. In the case of a // pointer to the tail of an object and the head of another, it will // return the former object (the one whose tail we point at). To // get the base pointer for a pointer into the pre-header, you should // add PreHeader.Size before calling this. internal static UIntPtr Find(UIntPtr addr) { UIntPtr page = PageTable.Page(addr); UIntPtr currAddr = InteriorPtrTable.First(page); // Look out for the unused space token: this page may not // have been completely allocated: its "first" object might not // be valid. if (BumpAllocator.IsUnusedSpace(currAddr) || currAddr > addr) { // Back up to the previous object. Should be fast // since First updated the InteriorPtrTable entries. currAddr = Before(PageTable.PageAddr(page)); } VTable.Assert(!BumpAllocator.IsUnusedSpace(currAddr), "InteriorPtrTable.Find 0"); VTable.Assert(currAddr <= addr, "InteriorPtrTable.Find 1"); while (true) { // Watch out for alignment padding; advance the pointer if // it points to a syncblock index rather than a vtable // pointer. Note that we must do this before scrolling, // since the page table value was set before we knew the // required alignment. if (Allocator.IsAlignment(currAddr)) { currAddr += UIntPtr.Size; } else if (BumpAllocator.IsUnusedSpace(currAddr)) { UIntPtr postAddr = PageTable.PagePad(currAddr) + PreHeader.Size; VTable.Assert(postAddr <= addr, "InteriorPtrTable.Find 2"); currAddr = postAddr; } else { VTable.Assert(currAddr <= addr, "InteriorPtrTable.Find 3"); UIntPtr size = ObjectSize(currAddr); VTable.Assert(size >= UIntPtr.Zero, "InteriorPtrTable.Find 4"); UIntPtr postAddr = currAddr + size; if (postAddr > addr) { return(currAddr); } else { currAddr = postAddr; } } } }
internal void ProcessPinnedPages(ReferenceVisitor ptrVisitor) { if (pinnedPageList == null || pinnedPageList.Count == 0) { return; } pinnedPageList.Sort(comparer); int limit = pinnedPageList.Count; for (int i = 0; i < limit; i++) { UIntPtr page = (UIntPtr)pinnedPageList[i]; PageType fromSpaceType = PageTable.Type(page); VTable.Assert(PageTable.IsZombiePage(fromSpaceType), "Semispace:RegisterPinnedReference:2"); PageType toSpaceType = PageTable.ZombieToLive(fromSpaceType); PageTable.SetType(page, toSpaceType); } int pageIndex = 0; while (pageIndex < limit) { UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex]; UIntPtr endPage = startPage + 1; pageIndex++; while (pageIndex < limit && (UIntPtr)pinnedPageList[pageIndex] == endPage) { pageIndex++; endPage++; } UIntPtr objectAddr = FirstPinnedObjectAddr(startPage); UIntPtr pastAddr = PostPinnedObjectAddr(endPage); while (objectAddr < pastAddr) { if (Allocator.IsAlignment(objectAddr)) { objectAddr += UIntPtr.Size; } else if (BumpAllocator.IsUnusedSpace(objectAddr)) { objectAddr = (PageTable.PagePad(objectAddr) + PreHeader.Size); } else { Object obj = Magic.fromAddress(objectAddr); objectAddr += ptrVisitor.VisitReferenceFields(obj); } } } }
internal static void Truncate() { UIntPtr allocLimit = PageTable.PagePad(allocPtr); UIntPtr unusedSize = limitPtr - allocLimit; if (GC.gcType != GCType.NullCollector) { PageManager.ReleaseUnusedPages(PageTable.Page(allocLimit), PageTable.PageCount(unusedSize), true); } limitPtr = allocLimit; }
private unsafe void SkipDestinationAreas(ref UIntPtr destPage, UIntPtr destCursor, ref UIntPtr destLimit, UIntPtr sourceCursor) { UIntPtr cursorPage = PageTable.Page(destCursor); UIntPtr sourcePage = PageTable.Page(sourceCursor); if (cursorPage != sourcePage) { UIntPtr destPageLimit = PageTable.PagePad(destCursor); if (destPageLimit != destCursor) { cursorPage++; } VTable.Assert(PageTable.PageAligned(destLimit)); UIntPtr limitPage = PageTable.Page(destLimit); while (destPage < sourcePage) { if (cursorPage < limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } do { destPage++; } while (!IsMyZombiePage(destPage)); cursorPage = destPage; do { destPage++; } while (IsMyZombiePage(destPage)); limitPage = destPage; } destLimit = PageTable.PageAddr(limitPage); VTable.Assert(destPage > sourcePage); VTable.Assert(cursorPage <= sourcePage); if (cursorPage < sourcePage) { this.RegisterSkippedPages(cursorPage, sourcePage); cursorPage = sourcePage; } InteriorPtrTable.ClearFirst(cursorPage, destPage); InteriorPtrTable.SetFirst(sourceCursor + PreHeader.Size); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(cursorPage), PageTable.PageAddr(destPage) - 1); } } }
internal static unsafe bool IsRestOfPageZero(UIntPtr lowAddr) { UIntPtr *initial = (UIntPtr *)lowAddr; UIntPtr *pageLimit = (UIntPtr *)PageTable.PagePad(lowAddr); for (UIntPtr *addr = initial; addr < pageLimit; addr++) { if (*addr != UIntPtr.Zero) { return(false); } } return(true); }
internal static unsafe void MarkThreadStack(Thread thread) { int stackVariable; UIntPtr stackBase = PageTable.PagePad(new UIntPtr(&stackVariable)); CallStack.SetStackBase(thread, stackBase); UIntPtr topPageAddr = PageTable.PageAlign(CallStack.StackBase(thread) - 1); SetStackPages(topPageAddr, CallStack.StackBase(thread), thread); UIntPtr regionAddr, regionSize; bool fUsed = MemoryManager.QueryMemory(topPageAddr, out regionAddr, out regionSize); VTable.Assert(fUsed); SetStackPages(regionAddr, topPageAddr, thread); }
internal static void Initialize() { for (int section = 0; section < sectionCount; section++) { UIntPtr startAddr = (UIntPtr)dataSectionBase[section]; startAddr = PageTable.PageAlign(startAddr); UIntPtr size = (UIntPtr)dataSectionEnd[section] - startAddr; size = PageTable.PagePad(size); PageManager.SetStaticDataPages(startAddr, size); } for (int section = 0; section < sectionCount; section++) { UIntPtr startAddr = (UIntPtr)roDataSectionBase[section]; startAddr = PageTable.PageAlign(startAddr); UIntPtr size = (UIntPtr)roDataSectionEnd[section] - startAddr; size = PageTable.PagePad(size); PageManager.SetStaticDataPages(startAddr, size); } }
/* * Returns a pointer to the first object on the given page. * N.B. If called on a page with no ~allocated~ first object it may * return a pointer to the unused space token. */ internal static UIntPtr First(UIntPtr page) { uint offset = PageTable.Extra(page); UIntPtr pageAddr = PageTable.PageAddr(page); UIntPtr currAddr; if (offset != OFFSET_NO_DATA) { currAddr = pageAddr + (offset - OFFSET_SKEW); } else { currAddr = Before(pageAddr); VTable.Assert(currAddr <= pageAddr); UIntPtr nextPageStart = PageTable.PagePad(currAddr + 1); while (currAddr < pageAddr) { if (Allocator.IsAlignment(currAddr)) { currAddr += UIntPtr.Size; } else if (BumpAllocator.IsUnusedSpace(currAddr)) { currAddr = PageTable.PagePad(currAddr) + PreHeader.Size; } else { if (currAddr >= nextPageStart) { InteriorPtrTable.SetFirst(currAddr); nextPageStart = PageTable.PagePad(currAddr + 1); } currAddr += ObjectSize(currAddr); } } } currAddr = Allocator.SkipAlignment(currAddr); return(currAddr); }
private void CompactPhaseCleanup(Thread currentThread, PageType generation, UIntPtr newLimitPtr) { VTable.Assert(IsValidGeneration((int)generation)); registerThreadReferenceVisitor.Cleanup(); // Free up skipped pages while (!this.skippedPageQueue.IsEmpty) { UIntPtr start = this.skippedPageQueue.Read(); UIntPtr finish = this.skippedPageQueue.Read(); InteriorPtrTable.ClearFirst(start, finish); PageManager.FreePageRange(start, finish); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(start), PageTable.PageAddr(finish) - 1); } } this.skippedPageQueue.Cleanup(true); // Release the queue standby pages UnmanagedPageList.ReleaseStandbyPages(); // Update the ownership information for the copied data PageType destGeneration = (generation == MAX_GENERATION) ? MAX_GENERATION : (PageType)(generation + 1); UIntPtr limitPage = PageTable.Page(PageTable.PagePad(newLimitPtr)); for (UIntPtr i = UIntPtr.Zero; i < limitPage; i++) { if (IsMyZombiePage(i)) { PageTable.SetType(i, (PageType)destGeneration); } } }
internal static unsafe UIntPtr SkipNonObjectData(UIntPtr objectAddr, UIntPtr limit) { // Skip any UNUSED_SPACE_TOKEN and ALIGNMENT_TOKEN areas while (true) { if (objectAddr >= limit) { break; } if (Allocator.IsAlignment(objectAddr)) { objectAddr += UIntPtr.Size; } else if (IsUnusedSpace(objectAddr)) { UIntPtr wordAfterUnused = objectAddr - PreHeader.Size + UIntPtr.Size; UIntPtr pageLimit = PageTable.PagePad(wordAfterUnused); CheckMemoryClear(wordAfterUnused, pageLimit); objectAddr = pageLimit + PreHeader.Size; } else if (Allocator.IsZeroVTable(objectAddr)) { // The rest of the page must not be used UIntPtr pageLimit = PageTable.PagePad(objectAddr); CheckMemoryClear(objectAddr, pageLimit); objectAddr = pageLimit + PreHeader.Size; } else { break; } } return(objectAddr); }
private unsafe void FindDestinationArea(ref UIntPtr destPage, ref UIntPtr destCursor, ref UIntPtr destLimit, UIntPtr objectSize, PageType destGeneration) { VTable.Assert(IsValidGeneration((int)destGeneration)); UIntPtr cursorPage = PageTable.Page(destCursor); UIntPtr limitPage = PageTable.Page(destLimit); UIntPtr pageAddr = PageTable.PagePad(destCursor); UIntPtr testPage = limitPage; UIntPtr endTestPage = PageTable.PageCount(destCursor + objectSize); if (destCursor > UIntPtr.Zero && IsMyZombiePage(PageTable.Page(destCursor - 1))) { VTable.Assert(destPage == limitPage); while (IsMyZombiePage(testPage) || (testPage < endTestPage && (PageTable.IsUnusedPage(testPage)))) { testPage++; } if (testPage >= endTestPage) { // We can expand the current region endTestPage = testPage; VTable.Assert(PageTable.PageAligned(destLimit)); InteriorPtrTable.ClearFirst(limitPage, testPage); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(limitPage), PageTable.PageAddr(testPage) - 1); } while (limitPage != endTestPage) { VTable.Assert(PageTable.IsUnusedPage(destPage)); do { destPage++; } while (destPage < endTestPage && PageTable.IsUnusedPage(destPage)); bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, limitPage, destPage - limitPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(limitPage, destPage - limitPage, destGeneration); while (destPage < endTestPage && IsMyZombiePage(destPage)) { destPage++; } limitPage = destPage; } destLimit = PageTable.PageAddr(limitPage); return; } } if (destCursor != pageAddr) { cursorPage++; } if (cursorPage != limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } // Find new region big enough to contain object UIntPtr neededPages = PageTable.PageCount(objectSize); UIntPtr prefixPage; while (true) { do { destPage++; } while (!IsMyZombiePage(destPage)); cursorPage = destPage; prefixPage = cursorPage; do { destPage++; } while (IsMyZombiePage(destPage)); limitPage = destPage; if (neededPages <= limitPage - cursorPage) { break; } // Check for following unused pages endTestPage = cursorPage + neededPages; VTable.Assert(endTestPage <= PageTable.pageTableCount); while (destPage < endTestPage && (PageTable.IsUnusedPage(destPage) || (IsMyZombiePage(destPage)))) { destPage++; } if (destPage == endTestPage) { break; } // Check for preceding unused pages if (destPage >= neededPages) { endTestPage = destPage - neededPages; prefixPage = cursorPage - 1; while (prefixPage >= UIntPtr.Zero && PageTable.IsUnusedPage(prefixPage)) { prefixPage--; } prefixPage++; if (prefixPage == endTestPage) { break; } } // Register any skipped regions of pages this.RegisterSkippedPages(cursorPage, limitPage); while (limitPage < destPage) { VTable.Assert(PageTable.IsUnusedPage(limitPage)); do { limitPage++; } while (limitPage < destPage && PageTable.IsUnusedPage(limitPage)); cursorPage = limitPage; while (limitPage < destPage && IsMyZombiePage(limitPage)) { limitPage++; } if (cursorPage != limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } } } // We found an area big enough. Commit the pre- and // postfix areas of unused pages if (prefixPage != cursorPage) { bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, prefixPage, cursorPage - prefixPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(prefixPage, cursorPage - prefixPage, destGeneration); } while (destPage != limitPage) { // Mark the region of unused pages as fromspace UIntPtr unusedPage = limitPage; VTable.Assert(PageTable.IsUnusedPage(unusedPage)); do { unusedPage++; } while (unusedPage < destPage && PageTable.IsUnusedPage(unusedPage)); bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, limitPage, unusedPage - limitPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(limitPage, unusedPage - limitPage, destGeneration); // Skip any sections of pages already marked as fromspace limitPage = unusedPage; while (limitPage < destPage && IsMyZombiePage(limitPage)) { limitPage++; } } destCursor = PageTable.PageAddr(prefixPage); destLimit = PageTable.PageAddr(limitPage); // Take ownership of the new pages InteriorPtrTable.ClearFirst(prefixPage, limitPage); InteriorPtrTable.SetFirst(destCursor + PreHeader.Size); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(prefixPage), PageTable.PageAddr(limitPage) - 1); } }
// Reference updates and object relocation private unsafe UIntPtr ForwardReferences(PageType generation, out UIntPtr oldAllocPtr) { VTable.Assert(IsValidGeneration((int)generation)); UIntPtr destPage = UIntPtr.Zero; UIntPtr destCursor; UIntPtr destLimit; PageType destGeneration; if (generation < MAX_GENERATION) { destGeneration = generation + 1; } else { destGeneration = MAX_GENERATION; } destCursor = UIntPtr.Zero; destLimit = UIntPtr.Zero; oldAllocPtr = destCursor; UIntPtr runLength = UIntPtr.Zero; for (UIntPtr i = UIntPtr.Zero; i < PageTable.pageTableCount; i++) { if (!IsMyZombiePage(i)) { continue; } UIntPtr deltaBytes = (UIntPtr)0x80000000; UIntPtr sourceCursor = PageTable.PageAddr(i); do { i++; } while (i < PageTable.pageTableCount && IsMyZombiePage(i)); UIntPtr sourceLimit = PageTable.PageAddr(i); while (true) { if (sourceCursor >= sourceLimit) { break; } if (Allocator.IsAlignmentMarkerAddr(sourceCursor)) { sourceCursor += UIntPtr.Size; deltaBytes += UIntPtr.Size; continue; } if (BumpAllocator.IsUnusedMarkerAddr(sourceCursor)) { sourceCursor += UIntPtr.Size; sourceCursor = PageTable.PagePad(sourceCursor); deltaBytes = (UIntPtr)0x80000000; continue; } UIntPtr objectAddr = sourceCursor + PreHeader.Size; UIntPtr vtableOrMarker = Allocator.GetObjectVTable(objectAddr); if (vtableOrMarker == UIntPtr.Zero) { // We found the end of an allocation page sourceCursor = PageTable.PagePad(sourceCursor); deltaBytes = (UIntPtr)0x80000000; continue; } UIntPtr vtableAddr; if ((vtableOrMarker & 1) != 0) { UIntPtr temp = *(UIntPtr *)(vtableOrMarker - 1); while ((temp & 1) != 0) { temp = *(UIntPtr *)(temp - 1); } VTable.Assert(PageTable.IsNonGcPage(PageTable.Type(PageTable.Page(temp)))); vtableAddr = temp; if ((temp & 2) != 0) { // Found pinned object SkipDestinationAreas(ref destPage, destCursor, ref destLimit, sourceCursor); deltaBytes -= (sourceCursor - destCursor); destCursor = sourceCursor; vtableAddr -= 2; // Remove "pinned" bit } Allocator.SetObjectVTable(objectAddr, vtableAddr); } else { vtableAddr = vtableOrMarker; } VTable vtable = Magic.toVTable(Magic.fromAddress(vtableAddr)); UIntPtr objectSize = ObjectLayout.ObjectSize(objectAddr, vtable); VTable.Assert(objectSize > 0); if ((vtableOrMarker & 1) != 0) { if (GenerationalCollector.IsLargeObjectSize (objectSize)) { // Don't move large objects SkipDestinationAreas(ref destPage, destCursor, ref destLimit, sourceCursor); UIntPtr localDelta = sourceCursor - destCursor; deltaBytes -= localDelta; if (deltaBytes == UIntPtr.Zero && runLength != UIntPtr.Zero) { runLength += localDelta; } destCursor = sourceCursor; UIntPtr objLimit = sourceCursor + objectSize; UIntPtr pageEndAddr = PageTable.PagePad(objLimit); objectSize = (pageEndAddr - sourceCursor); } else if (destCursor + objectSize > destLimit) { UIntPtr oldDestCursor = destCursor; FindDestinationArea(ref destPage, ref destCursor, ref destLimit, objectSize, destGeneration); VTable.Assert(destCursor <= sourceCursor); VTable.Assert(destCursor + objectSize <= destLimit); deltaBytes -= (destCursor - oldDestCursor); } else if (vtable.baseAlignment > UIntPtr.Size) { uint alignmentMask = vtable.baseAlignment - 1; int offset = PreHeader.Size + UIntPtr.Size; while (((destCursor + offset) & alignmentMask) != 0) { destCursor += UIntPtr.Size; deltaBytes -= UIntPtr.Size; if (deltaBytes == UIntPtr.Zero && runLength != UIntPtr.Zero) { runLength += UIntPtr.Size; } } } if (runLength == UIntPtr.Zero || deltaBytes != UIntPtr.Zero) { if (runLength != UIntPtr.Zero) { RegisterRelocationEnd(runLength); } RegisterRelocationStart(sourceCursor, destCursor); deltaBytes = UIntPtr.Zero; runLength = UIntPtr.Zero; } UIntPtr newObjectAddr = destCursor + PreHeader.Size; do { UIntPtr *ptrAddr = (UIntPtr *)(vtableOrMarker - 1); vtableOrMarker = *ptrAddr; *ptrAddr = newObjectAddr; } while ((vtableOrMarker & 1) != 0); destCursor += objectSize; runLength += objectSize; } else { deltaBytes += objectSize; if (runLength != UIntPtr.Zero) { RegisterRelocationEnd(runLength); } runLength = UIntPtr.Zero; } sourceCursor += objectSize; } } if (runLength != UIntPtr.Zero) { RegisterRelocationEnd(runLength); } return(destCursor); }
private UIntPtr ExtendAlloc(UIntPtr bytes, uint alignment, Thread currentThread) { if (this.reserveLimit == UIntPtr.Zero) { return(UIntPtr.Zero); } #if SINGULARITY_KERNEL Kernel.Waypoint(700); #endif UIntPtr neededBytes = bytes + // Bytes required for object + alignment - UIntPtr.Size - // worst case alignment overhead + (this.reserveLimit - this.allocPtr); // bytes already available UIntPtr paddedNeed = PageTable.PagePad(neededBytes); UIntPtr pageCount = PageTable.PageCount(paddedNeed); UIntPtr startPage = PageTable.Page(this.reserveLimit); bool fCleanPages = CLEAR_POOL_PAGES(); bool gotPages = PageManager.TryReserveUnusedPages(currentThread, startPage, pageCount, this.pageType, ref fCleanPages); if (!gotPages) { // We can't indiscriminately ask for more memory if we have // unused pages already available. return(UIntPtr.Zero); } if (this.reserveLimit == UIntPtr.Zero) { // A collection occurred, so there is no region to extend PageManager.ReleaseUnusedPages(startPage, pageCount, fCleanPages); return(UIntPtr.Zero); } BaseCollector.IncrementNewBytesSinceGC(paddedNeed); this.allocNew = this.reserveLimit; // Pad alignment space if necessary. NB: a prior call to // AllocateFast may have started generating alignment tokens, // but we may need to finish the job here if the residual space // was insufficient for a multi-word alignment. UIntPtr oldReserveLimit = this.reserveLimit; this.reserveLimit += paddedNeed; this.allocPtr = Allocator.AlignedAllocationPtr(this.allocPtr, this.reserveLimit, alignment); if (this.zeroedLimit < this.allocPtr) { this.zeroedLimit = this.allocPtr; } UIntPtr objectAddr = this.allocPtr + PreHeader.Size; this.allocPtr += bytes; if (fCleanPages) { if (this.zeroedLimit < oldReserveLimit) { Util.MemClear(this.zeroedLimit, oldReserveLimit - this.zeroedLimit); } this.zeroedLimit = this.reserveLimit; } else { Util.MemClear(this.zeroedLimit, this.allocPtr - this.zeroedLimit); this.zeroedLimit = this.allocPtr; } VTable.Assert(this.allocPtr <= this.zeroedLimit); VTable.Assert(PageTable.PageAligned(this.reserveLimit)); if (objectAddr >= oldReserveLimit) { // Object is first on new page InteriorPtrTable.SetFirst(objectAddr); } else if (objectAddr + bytes < this.reserveLimit) { // The object does not end on new limit // N.B. The next object may not be allocated at exactly // (objectAddr + bytes) due to alignment considerations. It // also might not ever be allocated. These cases are handled // by InteriorPtrTable.First skipping over alignment tokens // and callers of First watching out for unused space tokens. InteriorPtrTable.SetFirst(objectAddr + bytes); } // We know an object is located as the last one in a page // when it extends through the page to the next. // Otherwise, it is totally before or below the page, and // we are not sure whether it is the last object or not. // So record only such an object for the last card in that // page. Many objects may have been omitted due to // this coarse-grain recording. But we should be able // to incrementally update the offset table and find them. // I believe this is a better choice than simply recording // any object to the offset table, because most objects // may just die and need not to record. #if !SINGULARITY || SEMISPACE_COLLECTOR || ADAPTIVE_COPYING_COLLECTOR || SLIDING_COLLECTOR if (GC.remsetType == RemSetType.Cards) { if (objectAddr < oldReserveLimit && allocPtr + bytes > oldReserveLimit) { #if DONT_RECORD_OBJALLOC_IN_OFFSETTABLE #else OffsetTable.SetLast(objectAddr); #endif } } #endif #if SINGULARITY_KERNEL Kernel.Waypoint(701); #endif return(objectAddr); }
private static UIntPtr FirstPtrFromInteriorTable(UIntPtr c) { UIntPtr cardAddr = CardTable.CardAddr(c); UIntPtr nextCardAddr = CardTable.NextCardAddr(c); UIntPtr page = PageTable.Page(cardAddr); UIntPtr pageAddr = PageTable.PageAddr(page); UIntPtr currAddr; if (page == 0) { currAddr = PtrToNextObject(pageAddr, (UIntPtr)PreHeader.Size, nextCardAddr); } else { short offset = PageTable.Extra(page); currAddr = UIntPtr.Zero; if (offset != InteriorPtrTable.OFFSET_NO_DATA) { currAddr = pageAddr + (offset - InteriorPtrTable.OFFSET_SKEW); } // In general, we expect currAddr <= cardAddr. Or in the extreme // case, when the object starts from the page boundary, // currAddr - Object.HEADER_BYTES <= cardAddr. The contrary // cases has to be handled by searching previous pages. if (currAddr == UIntPtr.Zero || (currAddr > cardAddr && currAddr - PreHeader.Size > cardAddr)) { // look from previous pages, in case that an object on // them spans to the current page. In that case, we should // should use that object's ptr. currAddr = InteriorPtrTable.Last(page - 1); // Usually, Last() returns a pointer before or at the page // boundary. However, there is one exception: when an object // exactly fits to the last byte of the previous page, and the next // object starts right from the page boundary (the first byte of // the next page), then the pointer to this next object is returned. // Example found: objPtr =3d09fa8, size=60, pageboundary= // 3d0a000, next objPtr=3d0a008. Then returned pointer is // 3d0a008, which is beyond the page boundary. VTable.Assert(currAddr <= pageAddr || currAddr - PreHeader.Size <= pageAddr, "object is expected before page or right at the beginning of it"); } } VTable.Assert(currAddr < nextCardAddr, "object is expected before next card"); while (currAddr < nextCardAddr) { if (Allocator.IsAlignment(currAddr)) { currAddr += UIntPtr.Size; } else if (BumpAllocator.IsUnusedSpace(currAddr)) { currAddr = PageTable.PagePad(currAddr) + PreHeader.Size; } else { UIntPtr size = InteriorPtrTable.ObjectSize(currAddr); if (currAddr + size - PreHeader.Size > cardAddr) { return(currAddr); } currAddr += size; } } VTable.Assert(false, "No obj ptr found by looking at interior table"); return(UIntPtr.Zero); }
internal void CleanPinnedPages() { if (pinnedPageList == null || pinnedPageList.Count == 0) { return; } int pageIndex = 0; int limit = pinnedPageList.Count; UIntPtr lastPostPinnedAddr = UIntPtr.Zero; while (pageIndex < limit) { UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex]; UIntPtr endPage = startPage + 1; pageIndex++; while (pageIndex < limit && (UIntPtr)pinnedPageList[pageIndex] == endPage) { pageIndex++; endPage++; } // Zero out the area between the start of the page and // the first object on the page UIntPtr firstObjectAddr = FirstPinnedObjectAddr(startPage); UIntPtr firstAddr = firstObjectAddr - PreHeader.Size; UIntPtr trashAddr = PageTable.PageAddr(startPage); if (firstAddr < trashAddr) { // The first object "spills" into the previous page, // presumably by no more than HEADER_BYTES bytes VTable.Assert( PageTable.Page(firstAddr) == startPage - 1, "Semispace:RegisterPinnedReference:3"); // Prepare to zero the preceding page unless it also // had pinned data on it trashAddr = PageTable.PageAddr(startPage - 1); InteriorPtrTable.ClearFirst(startPage - 1); if (trashAddr >= lastPostPinnedAddr) { // Need to mark the spilled-onto page live to // keep the spilled data around PageType fromSpaceType = PageTable.Type(startPage - 1); VTable.Assert( PageTable.IsZombiePage(fromSpaceType), "Semispace:RegisterPinnedReference:4"); PageType toSpaceType = PageTable.ZombieToLive(fromSpaceType); PageTable.SetType(startPage - 1, toSpaceType); } } // If lastPostPinnedAddr is on the page that trashAddr // starts, pinned data from the last run of pinned pages // and pinned data from this run of pinned data are on the // same page, so just write alignment tokens from // lastPostPinnedAddr to the first pinned object. // Otherwise, write an unused marker at lastPostPinnedAddr // since the rest of its page must be copied or dead. if (trashAddr < lastPostPinnedAddr) { trashAddr = lastPostPinnedAddr; } else { CleanPageTail(lastPostPinnedAddr); } if (GC.remsetType == RemSetType.Cards && trashAddr < firstAddr) { UIntPtr firstCard = CardTable.CardNo(trashAddr); UIntPtr lastCard = CardTable.CardNo(firstAddr - 1); if (!OffsetTable.NoObjectPtrToTheCard(firstCard)) { UIntPtr offset = OffsetTable.GetOffset(firstCard); UIntPtr objPtr = CardTable.CardAddr(firstCard) + offset; UIntPtr size = OffsetTable.ObjectSize(objPtr); VTable.Assert ((objPtr + size - PreHeader.Size <= trashAddr) || (objPtr >= trashAddr), "Object should be totally " + "above or below trashAddr"); if (objPtr >= trashAddr) { // The offset in this card needs to be updated OffsetTable.ClearCards(firstCard, firstCard); } } OffsetTable.ClearCards(firstCard + 1, lastCard - 1); if (lastCard != CardTable.CardNo(firstObjectAddr)) { OffsetTable.ClearCards(lastCard, lastCard); } else { VTable.Assert(OffsetTable.GetOffset(lastCard) >= (firstObjectAddr - CardTable.CardAddr(lastCard)), "wrong offset"); } } { // trashAddr should go back at most one page. UIntPtr trashPage = PageTable.Page(trashAddr); UIntPtr firstObjectAddrPage = PageTable.Page(firstObjectAddr); VTable.Assert((trashPage == firstObjectAddrPage - 1) || (trashPage == firstObjectAddrPage)); } // If the InteriorPtrTable already had a value, then this is // redundant, but if the call to First above has to compute // the value, then (since it won't store it in the table) we // should store it. Why? At this point the previous page // would be "connected" to this one. After this collection // the previous page will be unused or re-used and unrelated // to this page and subsequent calls to First would then // rely on it making the leap between unrelated pages. InteriorPtrTable.SetFirst(firstObjectAddr); while (trashAddr < firstAddr) { Allocator.WriteAlignment(trashAddr); trashAddr += UIntPtr.Size; } // Zero out the area between the last whole object on // the last page and the end of the last page UIntPtr pastAddr = PostPinnedObjectAddr(endPage); UIntPtr newEndPage = PageTable.Page(PageTable.PagePad(pastAddr)); while (endPage < newEndPage) { // The last object spills into the next page(s), so // mark those page(s) live PageType fromPageType = PageTable.Type(endPage); if (PageTable.IsZombiePage(fromPageType)) { PageType toSpaceType = PageTable.ZombieToLive(fromPageType); PageTable.SetType(endPage, toSpaceType); } else { // final page might be live already because // something else on it was pinned. // pageIndex has already been incremented, // so it points to the start of the next // set of contiguous pages VTable.Assert( PageTable.IsLiveGcPage(fromPageType) && pageIndex < limit && endPage == (UIntPtr)pinnedPageList[pageIndex], "Semispace:RegisterPinnedReference:5"); } ++endPage; } lastPostPinnedAddr = pastAddr; } CleanPageTail(lastPostPinnedAddr); pinnedPageList = null; comparer = null; }
private static void CleanPageTail(UIntPtr postPinnedAddr) { if (!PageTable.PageAligned(postPinnedAddr)) { // If postPinnedAddr points to the first object on its page, // then we are removing all objects (specifically the part // of the object that the InteriorPtrTable tracks, the // vtables) from the page, so we should clear the page's // entry in the InteriorPtrTable. UIntPtr page = PageTable.Page(postPinnedAddr); UIntPtr firstObjPtr = InteriorPtrTable.First(page); if (firstObjPtr > postPinnedAddr) { VTable.Assert (firstObjPtr - PreHeader.Size >= postPinnedAddr, "postPinnedAddr should not point to the " + "interior of an object (1)"); InteriorPtrTable.ClearFirst(page); } else if (!BumpAllocator.IsUnusedSpace(firstObjPtr)) { UIntPtr firstObjSize = InteriorPtrTable.ObjectSize(firstObjPtr); VTable.Assert (firstObjPtr + firstObjSize - PreHeader.Size <= postPinnedAddr, "postPinnedAddr should not point to the " + "interior of an object (2)"); } UIntPtr byteCount = PageTable.PagePad(postPinnedAddr) - postPinnedAddr; Util.MemClear(postPinnedAddr, byteCount); BumpAllocator.WriteUnusedMarker(postPinnedAddr); if (GC.remsetType == RemSetType.Cards && byteCount > 0) { UIntPtr firstCard = CardTable.CardNo(postPinnedAddr); UIntPtr lastCard = CardTable.CardNo(postPinnedAddr + byteCount - 1); if (!OffsetTable.NoObjectPtrToTheCard(firstCard)) { UIntPtr offset = OffsetTable.GetOffset(firstCard); UIntPtr objPtr = CardTable.CardAddr(firstCard) + offset; UIntPtr size = OffsetTable.ObjectSize(objPtr); VTable.Assert ((objPtr + size - PreHeader.Size <= postPinnedAddr) || (objPtr >= postPinnedAddr), "Object should be totally " + "above or below postPinnedAddr"); if (objPtr >= postPinnedAddr) { OffsetTable.ClearCards(firstCard, firstCard); } } OffsetTable.ClearCards(firstCard + 1, lastCard); } } }
/* * Returns a pointer past the last object _that_fits_completely_ on * the given page. Note that the "last" object on a page may * actually start on a previous page. */ internal static UIntPtr Last(UIntPtr page) { UIntPtr currAddr = InteriorPtrTable.First(page); UIntPtr endAddr = PageTable.PageAddr(page + 1); // Look out for the unused space token: this page may not // have been completely allocated: its "first" object might not // be valid. if (BumpAllocator.IsUnusedSpace(currAddr) || currAddr >= endAddr) { // Back up to the previous object. Should be fast // since First updated the InteriorPtrTable entries. currAddr = Before(PageTable.PageAddr(page)); } // REVIEW this is very similar to Find(addr) below. VTable.Assert(currAddr <= endAddr); while (true) { // Watch out for alignment padding; advance the pointer if // it points to a syncblock index rather than a vtable // pointer. Note that we must do this before scrolling, // since the page table value was set before we knew the // required alignment. if (Allocator.IsAlignment(currAddr)) { currAddr += UIntPtr.Size; } else if (BumpAllocator.IsUnusedSpace(currAddr)) { UIntPtr nextAddr = PageTable.PagePad(currAddr) + PreHeader.Size; if (nextAddr >= endAddr) { return(currAddr); } else { currAddr = nextAddr; } } else { VTable.Assert(currAddr <= endAddr); UIntPtr size = ObjectSize(currAddr); UIntPtr postAddr = currAddr + size; if (postAddr > endAddr) { if (postAddr - PreHeader.Size > endAddr) { // The object spills over onto the next page return(currAddr); } else { // The object ended at or before the page boundary return(postAddr); } } else { currAddr = postAddr; } } } }