internal static void ReclaimZombiePages(UIntPtr heapPageCount, int generation) { // to indicate if we want to release pages back to the OS bool releasePages = true; UIntPtr reservePages = UIntPtr.Zero; if (generation == (int)nurseryGeneration) { // don't bother when we do nursery collection since // nursery size is small. releasePages = false; } else { reservePages = heapPageCount; UIntPtr alreadyReservedPages = PageManager.TotalUnusedPages(); if (reservePages > alreadyReservedPages) { reservePages = reservePages - alreadyReservedPages; } else { reservePages = UIntPtr.Zero; } } // MarkZombiePages updates the range for this generation, so we do // not need to take the union ranges of all target generations UIntPtr minZombiePage = MinGenPage[generation]; UIntPtr maxZombiePage = MaxGenPage[generation]; for (UIntPtr i = minZombiePage; i <= maxZombiePage; i++) { if (IsMyZombiePage(i)) { UIntPtr startPage = i; UIntPtr endPage = startPage; do { endPage++; } while (IsMyZombiePage(endPage)); InteriorPtrTable.ClearFirst(startPage, endPage); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(startPage), PageTable.PageAddr(endPage) - 1); } if (!releasePages) { // Don't need to worry about giving the pages back // Zero out the memory for reuse UIntPtr pageCount = endPage - startPage; PageManager.ReleaseUnusedPages(startPage, pageCount, false); } else if (reservePages > UIntPtr.Zero) { // Keep sufficient pages for the new nursery UIntPtr pageCount = endPage - startPage; if (pageCount > reservePages) { // Zero out the memory for reuse PageManager.ReleaseUnusedPages(startPage, reservePages, false); startPage += reservePages; PageManager.FreePageRange(startPage, endPage); reservePages = UIntPtr.Zero; } else { // Zero out the memory for reuse PageManager.ReleaseUnusedPages(startPage, pageCount, false); reservePages = reservePages - pageCount; } } else { PageManager.FreePageRange(startPage, endPage); } i = endPage - 1; } } }
// Interface with the compiler! internal static unsafe UIntPtr AllocateBig(UIntPtr numBytes, uint alignment, Thread currentThread) { // Pretenure Trigger pretenuredSinceLastFullGC += numBytes; if (pretenuredSinceLastFullGC > PretenureHardGCTrigger) { GC.InvokeMajorCollection(currentThread); } // Potentially Join a collection GC.CheckForNeededGCWork(currentThread); int maxAlignmentOverhead = unchecked ((int)alignment) - UIntPtr.Size; UIntPtr pageCount = PageTable.PageCount(numBytes + maxAlignmentOverhead); bool fCleanPages = true; UIntPtr page = PageManager.EnsurePages(currentThread, pageCount, largeObjectGeneration, ref fCleanPages); int unusedBytes = unchecked ((int)(PageTable.RegionSize(pageCount) - numBytes)); int unusedCacheLines = unchecked ((int)(unusedBytes - maxAlignmentOverhead)) >> 5; int pageOffset = 0; if (unusedCacheLines != 0) { pageOffset = (bigOffset % unusedCacheLines) << 5; bigOffset++; } UIntPtr pageStart = PageTable.PageAddr(page); for (int i = 0; i < pageOffset; i += UIntPtr.Size) { Allocator.WriteAlignment(pageStart + i); } UIntPtr unalignedStartAddr = pageStart + pageOffset; UIntPtr startAddr = Allocator.AlignedAllocationPtr(unalignedStartAddr, pageStart + unusedBytes, alignment); pageOffset += unchecked ((int)(uint)(startAddr - unalignedStartAddr)); if (pageOffset < unusedBytes) { BumpAllocator.WriteUnusedMarker(pageStart + pageOffset + numBytes); } UIntPtr resultAddr = startAddr + PreHeader.Size; InteriorPtrTable.SetFirst(resultAddr); VTable.Assert(PageTable.Page(resultAddr) < PageTable.Page(startAddr + numBytes - 1), "Big object should cross pages"); if (GC.remsetType == RemSetType.Cards) { #if DONT_RECORD_OBJALLOC_IN_OFFSETTABLE #else OffsetTable.SetLast(resultAddr); #endif } return(resultAddr); }
internal static unsafe bool AccumulateRCUpdates(String methodName, int methodIndex, uint maxIndex, AcctRecord rec) { VTable.Assert(RCCollector.ProfilingMode, @"RCCollector.ProfilingMode"); // Return if the page table hasn't been set up yet. if (PageTable.pageTableCount == UIntPtr.Zero) { return(false); } if (methods == null) { // Allocate up front storage for the accounting records. // // This is requisitioned directly from the memory // manager. Care should be taken to ensure that // AccumulateRCUpdates does not indirectly call // methods that may have compiler-inserted RC updates. VTable vtable = ((RuntimeType)typeof(AcctRecord[])).classVtable; UIntPtr size = ObjectLayout.ArraySize(vtable, maxIndex + 1); BumpAllocator profileData = new BumpAllocator(PageType.NonGC); UIntPtr profileDataStart = MemoryManager.AllocateMemory(size); profileData.SetRange(profileDataStart, size); PageManager.SetStaticDataPages(profileDataStart, size); methods = (AcctRecord[])Allocate(ref profileData, vtable, size); VTable.Assert(methods != null, @"methods != null"); *(uint *)(Magic.addressOf(methods) + PostHeader.Size) = maxIndex + 1; } VTable.Assert(methods.Length == maxIndex + 1, @"methods.Length == maxIndex+1"); if (methods[methodIndex].methodName == null) { methodNames[methodIndex].methodName = methodName; } // Not "methodNames[methodIndex].methodName == methodName" // because the Equality operator carries compiler-inserted // RC updates! VTable.Assert(Magic.addressOf(methodNames[methodIndex]. methodName) == Magic.addressOf(methodName), @"Magic.addressOf(methodNames[methodIndex]. methodName) == Magic.addressOf(methodName)"); methods[methodIndex] += rec; return(true); }
void VerifyPages(ObjectLayout.ObjectVisitor objectVisitor) { UIntPtr page = UIntPtr.Zero; while (page < PageTable.pageTableCount) { UIntPtr startPage = page; if (!PageTable.IsMyPage(startPage)) { page++; continue; } PageType pageType = PageTable.Type(page); uint pageProcess = PageTable.Process(page); do { page++; } while (page < PageTable.pageTableCount && PageTable.Type(page) == pageType && PageTable.Process(page) == pageProcess); UIntPtr endPage = page; switch (pageType) { case PageType.Unallocated: case PageType.Unknown: case PageType.Shared: { // The region does not belong to us, so there is // nothing to check. break; } case PageType.UnusedClean: case PageType.UnusedDirty: { PageManager.VerifyUnusedRegion(startPage, endPage); break; } case PageType.System: { // We have looked at the region, but it is off-limits // for the verifier. break; } case PageType.NonGC: { // Since there may be non-objects in the static data // pages, we cannot apply the heapVerifier to the // region. break; } case PageType.Stack: { // The page contains (part of) the activation record // stack for one or more threads. break; } default: { // We have found a data region VTable.Assert(PageTable.IsGcPage(startPage)); UIntPtr startAddr = PageTable.PageAddr(startPage); UIntPtr endAddr = PageTable.PageAddr(endPage); GC.installedGC.VisitObjects(objectVisitor, startAddr, endAddr); break; } } } }
private unsafe void FindDestinationArea(ref UIntPtr destPage, ref UIntPtr destCursor, ref UIntPtr destLimit, UIntPtr objectSize, PageType destGeneration) { VTable.Assert(IsValidGeneration((int)destGeneration)); UIntPtr cursorPage = PageTable.Page(destCursor); UIntPtr limitPage = PageTable.Page(destLimit); UIntPtr pageAddr = PageTable.PagePad(destCursor); UIntPtr testPage = limitPage; UIntPtr endTestPage = PageTable.PageCount(destCursor + objectSize); if (destCursor > UIntPtr.Zero && IsMyZombiePage(PageTable.Page(destCursor - 1))) { VTable.Assert(destPage == limitPage); while (IsMyZombiePage(testPage) || (testPage < endTestPage && (PageTable.IsUnusedPage(testPage)))) { testPage++; } if (testPage >= endTestPage) { // We can expand the current region endTestPage = testPage; VTable.Assert(PageTable.PageAligned(destLimit)); InteriorPtrTable.ClearFirst(limitPage, testPage); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(limitPage), PageTable.PageAddr(testPage) - 1); } while (limitPage != endTestPage) { VTable.Assert(PageTable.IsUnusedPage(destPage)); do { destPage++; } while (destPage < endTestPage && PageTable.IsUnusedPage(destPage)); bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, limitPage, destPage - limitPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(limitPage, destPage - limitPage, destGeneration); while (destPage < endTestPage && IsMyZombiePage(destPage)) { destPage++; } limitPage = destPage; } destLimit = PageTable.PageAddr(limitPage); return; } } if (destCursor != pageAddr) { cursorPage++; } if (cursorPage != limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } // Find new region big enough to contain object UIntPtr neededPages = PageTable.PageCount(objectSize); UIntPtr prefixPage; while (true) { do { destPage++; } while (!IsMyZombiePage(destPage)); cursorPage = destPage; prefixPage = cursorPage; do { destPage++; } while (IsMyZombiePage(destPage)); limitPage = destPage; if (neededPages <= limitPage - cursorPage) { break; } // Check for following unused pages endTestPage = cursorPage + neededPages; VTable.Assert(endTestPage <= PageTable.pageTableCount); while (destPage < endTestPage && (PageTable.IsUnusedPage(destPage) || (IsMyZombiePage(destPage)))) { destPage++; } if (destPage == endTestPage) { break; } // Check for preceding unused pages if (destPage >= neededPages) { endTestPage = destPage - neededPages; prefixPage = cursorPage - 1; while (prefixPage >= UIntPtr.Zero && PageTable.IsUnusedPage(prefixPage)) { prefixPage--; } prefixPage++; if (prefixPage == endTestPage) { break; } } // Register any skipped regions of pages this.RegisterSkippedPages(cursorPage, limitPage); while (limitPage < destPage) { VTable.Assert(PageTable.IsUnusedPage(limitPage)); do { limitPage++; } while (limitPage < destPage && PageTable.IsUnusedPage(limitPage)); cursorPage = limitPage; while (limitPage < destPage && IsMyZombiePage(limitPage)) { limitPage++; } if (cursorPage != limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } } } // We found an area big enough. Commit the pre- and // postfix areas of unused pages if (prefixPage != cursorPage) { bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, prefixPage, cursorPage - prefixPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(prefixPage, cursorPage - prefixPage, destGeneration); } while (destPage != limitPage) { // Mark the region of unused pages as fromspace UIntPtr unusedPage = limitPage; VTable.Assert(PageTable.IsUnusedPage(unusedPage)); do { unusedPage++; } while (unusedPage < destPage && PageTable.IsUnusedPage(unusedPage)); bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, limitPage, unusedPage - limitPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(limitPage, unusedPage - limitPage, destGeneration); // Skip any sections of pages already marked as fromspace limitPage = unusedPage; while (limitPage < destPage && IsMyZombiePage(limitPage)) { limitPage++; } } destCursor = PageTable.PageAddr(prefixPage); destLimit = PageTable.PageAddr(limitPage); // Take ownership of the new pages InteriorPtrTable.ClearFirst(prefixPage, limitPage); InteriorPtrTable.SetFirst(destCursor + PreHeader.Size); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(prefixPage), PageTable.PageAddr(limitPage) - 1); } }
// Use this method to free heap pages. // There must be no contention regarding ownership of the pages. internal static void FreePageRange(UIntPtr startPage, UIntPtr endPage) { if (VTable.enableDebugPrint) { VTable.DebugPrint("FreePageRange({0}, {1})\n", __arglist(startPage, endPage)); } UIntPtr startAddr = PageTable.PageAddr(startPage); UIntPtr endAddr = PageTable.PageAddr(endPage); UIntPtr rangeSize = endAddr - startAddr; #if SINGULARITY // Singularity doesn't care if you free pages in different // chunks than you acquired them in MemoryManager.FreeMemory(startAddr, rangeSize); #else // We cannot simply release the memory range, as MEM_RELEASE // requires the pointer to be the same as the original call // to VirtualAlloc, and the length to be zero. UIntPtr regionAddr, regionSize; bool fUsed = MemoryManager.QueryMemory(startAddr, out regionAddr, out regionSize); if (VTable.enableDebugPrint) { VTable.DebugPrint(" 1 Query({0}, {1}, {2}) -> {3}\n", __arglist(startAddr, regionAddr, regionSize, fUsed)); } VTable.Assert(fUsed, "Memory to be freed isn't used"); UIntPtr endRegion = regionAddr + regionSize; if (regionAddr < startAddr) { // startAddr is in the middle of an allocation region -> skip if (endRegion >= endAddr) { // [startAddr, endAddr] is fully contained in a region PageManager.Clear(startAddr, endAddr - startAddr); return; } // Part of the address range falls into the next region PageManager.Clear(startAddr, endRegion - startAddr); fUsed = MemoryManager.QueryMemory(endRegion, out regionAddr, out regionSize); if (VTable.enableDebugPrint) { VTable.DebugPrint(" 2 Query({0}, {1}, {2}) -> {3}\n", __arglist(endRegion, regionAddr, regionSize, fUsed)); } VTable.Assert(fUsed, "Area to be freed isn't used"); endRegion = regionAddr + regionSize; } // [regionAddr, endRegion] is contained in [startAddr, endAddr] while (endRegion < endAddr) { if (VTable.enableDebugPrint) { VTable.DebugPrint("Freeing region [{0}, {1}]\n", __arglist(regionAddr, regionAddr + regionSize)); } SetUnallocatedPages(regionAddr, regionSize); MemoryManager.FreeMemory(regionAddr, regionSize); fUsed = MemoryManager.QueryMemory(endRegion, out regionAddr, out regionSize); if (VTable.enableDebugPrint) { VTable.DebugPrint(" 3 Query({0}, {1}, {2}) -> {3}\n", __arglist(endRegion, regionAddr, regionSize, fUsed)); } VTable.Assert(fUsed, "Region to be freed isn't used"); endRegion = regionAddr + regionSize; } if (endRegion == endAddr) { if (VTable.enableDebugPrint) { VTable.DebugPrint("Freeing final region [{0}, {1}]\n", __arglist(regionAddr, regionAddr + regionSize)); } SetUnallocatedPages(regionAddr, regionSize); MemoryManager.FreeMemory(regionAddr, regionSize); } else { PageManager.Clear(regionAddr, endAddr - regionAddr); } if (VTable.enableDebugPrint) { VTable.DebugPrint(" --> FreePageRange({0},{1})\n", __arglist(startPage, endPage)); } #endif // SINGULARITY }
private UIntPtr ExtendAlloc(UIntPtr bytes, uint alignment, Thread currentThread) { if (this.reserveLimit == UIntPtr.Zero) { return(UIntPtr.Zero); } #if SINGULARITY_KERNEL Kernel.Waypoint(700); #endif UIntPtr neededBytes = bytes + // Bytes required for object + alignment - UIntPtr.Size - // worst case alignment overhead + (this.reserveLimit - this.allocPtr); // bytes already available UIntPtr paddedNeed = PageTable.PagePad(neededBytes); UIntPtr pageCount = PageTable.PageCount(paddedNeed); UIntPtr startPage = PageTable.Page(this.reserveLimit); bool fCleanPages = CLEAR_POOL_PAGES(); bool gotPages = PageManager.TryReserveUnusedPages(currentThread, startPage, pageCount, this.pageType, ref fCleanPages); if (!gotPages) { // We can't indiscriminately ask for more memory if we have // unused pages already available. return(UIntPtr.Zero); } if (this.reserveLimit == UIntPtr.Zero) { // A collection occurred, so there is no region to extend PageManager.ReleaseUnusedPages(startPage, pageCount, fCleanPages); return(UIntPtr.Zero); } BaseCollector.IncrementNewBytesSinceGC(paddedNeed); this.allocNew = this.reserveLimit; // Pad alignment space if necessary. NB: a prior call to // AllocateFast may have started generating alignment tokens, // but we may need to finish the job here if the residual space // was insufficient for a multi-word alignment. UIntPtr oldReserveLimit = this.reserveLimit; this.reserveLimit += paddedNeed; this.allocPtr = Allocator.AlignedAllocationPtr(this.allocPtr, this.reserveLimit, alignment); if (this.zeroedLimit < this.allocPtr) { this.zeroedLimit = this.allocPtr; } UIntPtr objectAddr = this.allocPtr + PreHeader.Size; this.allocPtr += bytes; if (fCleanPages) { if (this.zeroedLimit < oldReserveLimit) { Util.MemClear(this.zeroedLimit, oldReserveLimit - this.zeroedLimit); } this.zeroedLimit = this.reserveLimit; } else { Util.MemClear(this.zeroedLimit, this.allocPtr - this.zeroedLimit); this.zeroedLimit = this.allocPtr; } VTable.Assert(this.allocPtr <= this.zeroedLimit); VTable.Assert(PageTable.PageAligned(this.reserveLimit)); if (objectAddr >= oldReserveLimit) { // Object is first on new page InteriorPtrTable.SetFirst(objectAddr); } else if (objectAddr + bytes < this.reserveLimit) { // The object does not end on new limit // N.B. The next object may not be allocated at exactly // (objectAddr + bytes) due to alignment considerations. It // also might not ever be allocated. These cases are handled // by InteriorPtrTable.First skipping over alignment tokens // and callers of First watching out for unused space tokens. InteriorPtrTable.SetFirst(objectAddr + bytes); } // We know an object is located as the last one in a page // when it extends through the page to the next. // Otherwise, it is totally before or below the page, and // we are not sure whether it is the last object or not. // So record only such an object for the last card in that // page. Many objects may have been omitted due to // this coarse-grain recording. But we should be able // to incrementally update the offset table and find them. // I believe this is a better choice than simply recording // any object to the offset table, because most objects // may just die and need not to record. #if !SINGULARITY || SEMISPACE_COLLECTOR || ADAPTIVE_COPYING_COLLECTOR || SLIDING_COLLECTOR if (GC.remsetType == RemSetType.Cards) { if (objectAddr < oldReserveLimit && allocPtr + bytes > oldReserveLimit) { #if DONT_RECORD_OBJALLOC_IN_OFFSETTABLE #else OffsetTable.SetLast(objectAddr); #endif } } #endif #if SINGULARITY_KERNEL Kernel.Waypoint(701); #endif return(objectAddr); }