private unsafe void SkipDestinationAreas(ref UIntPtr destPage, UIntPtr destCursor, ref UIntPtr destLimit, UIntPtr sourceCursor) { UIntPtr cursorPage = PageTable.Page(destCursor); UIntPtr sourcePage = PageTable.Page(sourceCursor); if (cursorPage != sourcePage) { UIntPtr destPageLimit = PageTable.PagePad(destCursor); if (destPageLimit != destCursor) { cursorPage++; } VTable.Assert(PageTable.PageAligned(destLimit)); UIntPtr limitPage = PageTable.Page(destLimit); while (destPage < sourcePage) { if (cursorPage < limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } do { destPage++; } while (!IsMyZombiePage(destPage)); cursorPage = destPage; do { destPage++; } while (IsMyZombiePage(destPage)); limitPage = destPage; } destLimit = PageTable.PageAddr(limitPage); VTable.Assert(destPage > sourcePage); VTable.Assert(cursorPage <= sourcePage); if (cursorPage < sourcePage) { this.RegisterSkippedPages(cursorPage, sourcePage); cursorPage = sourcePage; } InteriorPtrTable.ClearFirst(cursorPage, destPage); InteriorPtrTable.SetFirst(sourceCursor + PreHeader.Size); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(cursorPage), PageTable.PageAddr(destPage) - 1); } } }
private void CompactPhaseCleanup(Thread currentThread, PageType generation, UIntPtr newLimitPtr) { VTable.Assert(IsValidGeneration((int)generation)); registerThreadReferenceVisitor.Cleanup(); // Free up skipped pages while (!this.skippedPageQueue.IsEmpty) { UIntPtr start = this.skippedPageQueue.Read(); UIntPtr finish = this.skippedPageQueue.Read(); InteriorPtrTable.ClearFirst(start, finish); PageManager.FreePageRange(start, finish); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(start), PageTable.PageAddr(finish) - 1); } } this.skippedPageQueue.Cleanup(true); // Release the queue standby pages UnmanagedPageList.ReleaseStandbyPages(); // Update the ownership information for the copied data PageType destGeneration = (generation == MAX_GENERATION) ? MAX_GENERATION : (PageType)(generation + 1); UIntPtr limitPage = PageTable.Page(PageTable.PagePad(newLimitPtr)); for (UIntPtr i = UIntPtr.Zero; i < limitPage; i++) { if (IsMyZombiePage(i)) { PageTable.SetType(i, (PageType)destGeneration); } } }
internal static void ReclaimZombiePages(UIntPtr heapPageCount, int generation) { // to indicate if we want to release pages back to the OS bool releasePages = true; UIntPtr reservePages = UIntPtr.Zero; if (generation == (int)nurseryGeneration) { // don't bother when we do nursery collection since // nursery size is small. releasePages = false; } else { reservePages = heapPageCount; UIntPtr alreadyReservedPages = PageManager.TotalUnusedPages(); if (reservePages > alreadyReservedPages) { reservePages = reservePages - alreadyReservedPages; } else { reservePages = UIntPtr.Zero; } } // MarkZombiePages updates the range for this generation, so we do // not need to take the union ranges of all target generations UIntPtr minZombiePage = MinGenPage[generation]; UIntPtr maxZombiePage = MaxGenPage[generation]; for (UIntPtr i = minZombiePage; i <= maxZombiePage; i++) { if (IsMyZombiePage(i)) { UIntPtr startPage = i; UIntPtr endPage = startPage; do { endPage++; } while (IsMyZombiePage(endPage)); InteriorPtrTable.ClearFirst(startPage, endPage); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(startPage), PageTable.PageAddr(endPage) - 1); } if (!releasePages) { // Don't need to worry about giving the pages back // Zero out the memory for reuse UIntPtr pageCount = endPage - startPage; PageManager.ReleaseUnusedPages(startPage, pageCount, false); } else if (reservePages > UIntPtr.Zero) { // Keep sufficient pages for the new nursery UIntPtr pageCount = endPage - startPage; if (pageCount > reservePages) { // Zero out the memory for reuse PageManager.ReleaseUnusedPages(startPage, reservePages, false); startPage += reservePages; PageManager.FreePageRange(startPage, endPage); reservePages = UIntPtr.Zero; } else { // Zero out the memory for reuse PageManager.ReleaseUnusedPages(startPage, pageCount, false); reservePages = reservePages - pageCount; } } else { PageManager.FreePageRange(startPage, endPage); } i = endPage - 1; } } }
private unsafe void FindDestinationArea(ref UIntPtr destPage, ref UIntPtr destCursor, ref UIntPtr destLimit, UIntPtr objectSize, PageType destGeneration) { VTable.Assert(IsValidGeneration((int)destGeneration)); UIntPtr cursorPage = PageTable.Page(destCursor); UIntPtr limitPage = PageTable.Page(destLimit); UIntPtr pageAddr = PageTable.PagePad(destCursor); UIntPtr testPage = limitPage; UIntPtr endTestPage = PageTable.PageCount(destCursor + objectSize); if (destCursor > UIntPtr.Zero && IsMyZombiePage(PageTable.Page(destCursor - 1))) { VTable.Assert(destPage == limitPage); while (IsMyZombiePage(testPage) || (testPage < endTestPage && (PageTable.IsUnusedPage(testPage)))) { testPage++; } if (testPage >= endTestPage) { // We can expand the current region endTestPage = testPage; VTable.Assert(PageTable.PageAligned(destLimit)); InteriorPtrTable.ClearFirst(limitPage, testPage); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(limitPage), PageTable.PageAddr(testPage) - 1); } while (limitPage != endTestPage) { VTable.Assert(PageTable.IsUnusedPage(destPage)); do { destPage++; } while (destPage < endTestPage && PageTable.IsUnusedPage(destPage)); bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, limitPage, destPage - limitPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(limitPage, destPage - limitPage, destGeneration); while (destPage < endTestPage && IsMyZombiePage(destPage)) { destPage++; } limitPage = destPage; } destLimit = PageTable.PageAddr(limitPage); return; } } if (destCursor != pageAddr) { cursorPage++; } if (cursorPage != limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } // Find new region big enough to contain object UIntPtr neededPages = PageTable.PageCount(objectSize); UIntPtr prefixPage; while (true) { do { destPage++; } while (!IsMyZombiePage(destPage)); cursorPage = destPage; prefixPage = cursorPage; do { destPage++; } while (IsMyZombiePage(destPage)); limitPage = destPage; if (neededPages <= limitPage - cursorPage) { break; } // Check for following unused pages endTestPage = cursorPage + neededPages; VTable.Assert(endTestPage <= PageTable.pageTableCount); while (destPage < endTestPage && (PageTable.IsUnusedPage(destPage) || (IsMyZombiePage(destPage)))) { destPage++; } if (destPage == endTestPage) { break; } // Check for preceding unused pages if (destPage >= neededPages) { endTestPage = destPage - neededPages; prefixPage = cursorPage - 1; while (prefixPage >= UIntPtr.Zero && PageTable.IsUnusedPage(prefixPage)) { prefixPage--; } prefixPage++; if (prefixPage == endTestPage) { break; } } // Register any skipped regions of pages this.RegisterSkippedPages(cursorPage, limitPage); while (limitPage < destPage) { VTable.Assert(PageTable.IsUnusedPage(limitPage)); do { limitPage++; } while (limitPage < destPage && PageTable.IsUnusedPage(limitPage)); cursorPage = limitPage; while (limitPage < destPage && IsMyZombiePage(limitPage)) { limitPage++; } if (cursorPage != limitPage) { this.RegisterSkippedPages(cursorPage, limitPage); } } } // We found an area big enough. Commit the pre- and // postfix areas of unused pages if (prefixPage != cursorPage) { bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, prefixPage, cursorPage - prefixPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(prefixPage, cursorPage - prefixPage, destGeneration); } while (destPage != limitPage) { // Mark the region of unused pages as fromspace UIntPtr unusedPage = limitPage; VTable.Assert(PageTable.IsUnusedPage(unusedPage)); do { unusedPage++; } while (unusedPage < destPage && PageTable.IsUnusedPage(unusedPage)); bool fCleanPages = true; bool status = PageManager.TryReserveUnusedPages(null, limitPage, unusedPage - limitPage, nurseryGeneration, ref fCleanPages); VTable.Assert(status); MakeZombiePages(limitPage, unusedPage - limitPage, destGeneration); // Skip any sections of pages already marked as fromspace limitPage = unusedPage; while (limitPage < destPage && IsMyZombiePage(limitPage)) { limitPage++; } } destCursor = PageTable.PageAddr(prefixPage); destLimit = PageTable.PageAddr(limitPage); // Take ownership of the new pages InteriorPtrTable.ClearFirst(prefixPage, limitPage); InteriorPtrTable.SetFirst(destCursor + PreHeader.Size); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(prefixPage), PageTable.PageAddr(limitPage) - 1); } }
private static void CleanPageTail(UIntPtr postPinnedAddr) { if (!PageTable.PageAligned(postPinnedAddr)) { // If postPinnedAddr points to the first object on its page, // then we are removing all objects (specifically the part // of the object that the InteriorPtrTable tracks, the // vtables) from the page, so we should clear the page's // entry in the InteriorPtrTable. UIntPtr page = PageTable.Page(postPinnedAddr); UIntPtr firstObjPtr = InteriorPtrTable.First(page); if (firstObjPtr > postPinnedAddr) { VTable.Assert (firstObjPtr - PreHeader.Size >= postPinnedAddr, "postPinnedAddr should not point to the " + "interior of an object (1)"); InteriorPtrTable.ClearFirst(page); } else if (!BumpAllocator.IsUnusedSpace(firstObjPtr)) { UIntPtr firstObjSize = InteriorPtrTable.ObjectSize(firstObjPtr); VTable.Assert (firstObjPtr + firstObjSize - PreHeader.Size <= postPinnedAddr, "postPinnedAddr should not point to the " + "interior of an object (2)"); } UIntPtr byteCount = PageTable.PagePad(postPinnedAddr) - postPinnedAddr; Util.MemClear(postPinnedAddr, byteCount); BumpAllocator.WriteUnusedMarker(postPinnedAddr); if (GC.remsetType == RemSetType.Cards && byteCount > 0) { UIntPtr firstCard = CardTable.CardNo(postPinnedAddr); UIntPtr lastCard = CardTable.CardNo(postPinnedAddr + byteCount - 1); if (!OffsetTable.NoObjectPtrToTheCard(firstCard)) { UIntPtr offset = OffsetTable.GetOffset(firstCard); UIntPtr objPtr = CardTable.CardAddr(firstCard) + offset; UIntPtr size = OffsetTable.ObjectSize(objPtr); VTable.Assert ((objPtr + size - PreHeader.Size <= postPinnedAddr) || (objPtr >= postPinnedAddr), "Object should be totally " + "above or below postPinnedAddr"); if (objPtr >= postPinnedAddr) { OffsetTable.ClearCards(firstCard, firstCard); } } OffsetTable.ClearCards(firstCard + 1, lastCard); } } }
internal void CleanPinnedPages() { if (pinnedPageList == null || pinnedPageList.Count == 0) { return; } int pageIndex = 0; int limit = pinnedPageList.Count; UIntPtr lastPostPinnedAddr = UIntPtr.Zero; while (pageIndex < limit) { UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex]; UIntPtr endPage = startPage + 1; pageIndex++; while (pageIndex < limit && (UIntPtr)pinnedPageList[pageIndex] == endPage) { pageIndex++; endPage++; } // Zero out the area between the start of the page and // the first object on the page UIntPtr firstObjectAddr = FirstPinnedObjectAddr(startPage); UIntPtr firstAddr = firstObjectAddr - PreHeader.Size; UIntPtr trashAddr = PageTable.PageAddr(startPage); if (firstAddr < trashAddr) { // The first object "spills" into the previous page, // presumably by no more than HEADER_BYTES bytes VTable.Assert( PageTable.Page(firstAddr) == startPage - 1, "Semispace:RegisterPinnedReference:3"); // Prepare to zero the preceding page unless it also // had pinned data on it trashAddr = PageTable.PageAddr(startPage - 1); InteriorPtrTable.ClearFirst(startPage - 1); if (trashAddr >= lastPostPinnedAddr) { // Need to mark the spilled-onto page live to // keep the spilled data around PageType fromSpaceType = PageTable.Type(startPage - 1); VTable.Assert( PageTable.IsZombiePage(fromSpaceType), "Semispace:RegisterPinnedReference:4"); PageType toSpaceType = PageTable.ZombieToLive(fromSpaceType); PageTable.SetType(startPage - 1, toSpaceType); } } // If lastPostPinnedAddr is on the page that trashAddr // starts, pinned data from the last run of pinned pages // and pinned data from this run of pinned data are on the // same page, so just write alignment tokens from // lastPostPinnedAddr to the first pinned object. // Otherwise, write an unused marker at lastPostPinnedAddr // since the rest of its page must be copied or dead. if (trashAddr < lastPostPinnedAddr) { trashAddr = lastPostPinnedAddr; } else { CleanPageTail(lastPostPinnedAddr); } if (GC.remsetType == RemSetType.Cards && trashAddr < firstAddr) { UIntPtr firstCard = CardTable.CardNo(trashAddr); UIntPtr lastCard = CardTable.CardNo(firstAddr - 1); if (!OffsetTable.NoObjectPtrToTheCard(firstCard)) { UIntPtr offset = OffsetTable.GetOffset(firstCard); UIntPtr objPtr = CardTable.CardAddr(firstCard) + offset; UIntPtr size = OffsetTable.ObjectSize(objPtr); VTable.Assert ((objPtr + size - PreHeader.Size <= trashAddr) || (objPtr >= trashAddr), "Object should be totally " + "above or below trashAddr"); if (objPtr >= trashAddr) { // The offset in this card needs to be updated OffsetTable.ClearCards(firstCard, firstCard); } } OffsetTable.ClearCards(firstCard + 1, lastCard - 1); if (lastCard != CardTable.CardNo(firstObjectAddr)) { OffsetTable.ClearCards(lastCard, lastCard); } else { VTable.Assert(OffsetTable.GetOffset(lastCard) >= (firstObjectAddr - CardTable.CardAddr(lastCard)), "wrong offset"); } } { // trashAddr should go back at most one page. UIntPtr trashPage = PageTable.Page(trashAddr); UIntPtr firstObjectAddrPage = PageTable.Page(firstObjectAddr); VTable.Assert((trashPage == firstObjectAddrPage - 1) || (trashPage == firstObjectAddrPage)); } // If the InteriorPtrTable already had a value, then this is // redundant, but if the call to First above has to compute // the value, then (since it won't store it in the table) we // should store it. Why? At this point the previous page // would be "connected" to this one. After this collection // the previous page will be unused or re-used and unrelated // to this page and subsequent calls to First would then // rely on it making the leap between unrelated pages. InteriorPtrTable.SetFirst(firstObjectAddr); while (trashAddr < firstAddr) { Allocator.WriteAlignment(trashAddr); trashAddr += UIntPtr.Size; } // Zero out the area between the last whole object on // the last page and the end of the last page UIntPtr pastAddr = PostPinnedObjectAddr(endPage); UIntPtr newEndPage = PageTable.Page(PageTable.PagePad(pastAddr)); while (endPage < newEndPage) { // The last object spills into the next page(s), so // mark those page(s) live PageType fromPageType = PageTable.Type(endPage); if (PageTable.IsZombiePage(fromPageType)) { PageType toSpaceType = PageTable.ZombieToLive(fromPageType); PageTable.SetType(endPage, toSpaceType); } else { // final page might be live already because // something else on it was pinned. // pageIndex has already been incremented, // so it points to the start of the next // set of contiguous pages VTable.Assert( PageTable.IsLiveGcPage(fromPageType) && pageIndex < limit && endPage == (UIntPtr)pinnedPageList[pageIndex], "Semispace:RegisterPinnedReference:5"); } ++endPage; } lastPostPinnedAddr = pastAddr; } CleanPageTail(lastPostPinnedAddr); pinnedPageList = null; comparer = null; }