internal static unsafe void SetStaticDataPages(UIntPtr startAddr, UIntPtr size) { #if SINGULARITY // It's perfectly fine to be given memory outside the page table region, so // long as we intend to treat that memory as NonGC anyway. if (startAddr < PageTable.baseAddr) { if (startAddr + size < PageTable.baseAddr) { // nothing to do. All pages are below the base covered by the page table return; } // The range overlaps with the region covered by the page table size -= (PageTable.baseAddr - startAddr); startAddr = PageTable.baseAddr; } UIntPtr endAddr = startAddr + size; if (endAddr > PageTable.limitAddr) { if (startAddr > PageTable.limitAddr) { // nothing to do. All pages are above the limit covered by the page table return; } // The range overlaps with the region covered by the page table size -= (PageTable.limitAddr - endAddr); } #endif UIntPtr startIndex = PageTable.Page(startAddr); UIntPtr pageCount = PageTable.PageCount(size); PageTable.SetType(startIndex, pageCount, PageType.NonGC); }
internal unsafe static void MakeZombiePage(UIntPtr page, PageType destGeneration) { VTable.Assert(destGeneration >= nurseryGeneration && destGeneration <= MAX_GENERATION); PageType pageType = PageTable.LiveToZombie(destGeneration); PageTable.SetType(page, pageType); }
internal void ProcessPinnedPages(ReferenceVisitor ptrVisitor) { if (pinnedPageList == null || pinnedPageList.Count == 0) { return; } pinnedPageList.Sort(comparer); int limit = pinnedPageList.Count; for (int i = 0; i < limit; i++) { UIntPtr page = (UIntPtr)pinnedPageList[i]; PageType fromSpaceType = PageTable.Type(page); VTable.Assert(PageTable.IsZombiePage(fromSpaceType), "Semispace:RegisterPinnedReference:2"); PageType toSpaceType = PageTable.ZombieToLive(fromSpaceType); PageTable.SetType(page, toSpaceType); } int pageIndex = 0; while (pageIndex < limit) { UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex]; UIntPtr endPage = startPage + 1; pageIndex++; while (pageIndex < limit && (UIntPtr)pinnedPageList[pageIndex] == endPage) { pageIndex++; endPage++; } UIntPtr objectAddr = FirstPinnedObjectAddr(startPage); UIntPtr pastAddr = PostPinnedObjectAddr(endPage); while (objectAddr < pastAddr) { if (Allocator.IsAlignment(objectAddr)) { objectAddr += UIntPtr.Size; } else if (BumpAllocator.IsUnusedSpace(objectAddr)) { objectAddr = (PageTable.PagePad(objectAddr) + PreHeader.Size); } else { Object obj = Magic.fromAddress(objectAddr); objectAddr += ptrVisitor.VisitReferenceFields(obj); } } } }
private static void MarkUnusedPages(Thread currentThread, UIntPtr startPage, UIntPtr pageCount, bool fCleanPages) { Trace.Log(Trace.Area.Page, "MarkUnusedPages start={0:x} count={1:x}", __arglist(startPage, pageCount)); UIntPtr endPage = startPage + pageCount; if (avoidDirtyPages && !fCleanPages) { UIntPtr dirtyStartAddr = PageTable.PageAddr(startPage); UIntPtr dirtySize = PageTable.RegionSize(pageCount); Util.MemClear(dirtyStartAddr, dirtySize); fCleanPages = true; } bool iflag = EnterMutex(currentThread); try { if (endPage < PageTable.pageTableCount) { if (PageTable.IsUnusedPage(endPage) && PageTable.IsMyPage(endPage)) { UIntPtr regionSize = UnlinkUnusedPages(endPage); endPage += regionSize; } } UIntPtr queryStartPage = startPage - 1; UIntPtr newStartPage = startPage; if (PageTable.IsUnusedPage(queryStartPage) && PageTable.IsMyPage(queryStartPage)) { UnusedBlockHeader *tailUnused = (UnusedBlockHeader *) PageTable.PageAddr(queryStartPage); UIntPtr newStartAddr = (UIntPtr)tailUnused->curr; newStartPage = PageTable.Page(newStartAddr); UIntPtr regionSize = UnlinkUnusedPages(newStartPage); VTable.Assert(newStartPage + regionSize == startPage); } PageType pageType = fCleanPages ? PageType.UnusedClean : PageType.UnusedDirty; PageTable.SetType(startPage, pageCount, pageType); LinkUnusedPages(newStartPage, endPage - newStartPage, false); } finally { LeaveMutex(currentThread, iflag); } }
// Bartok uses the extra field in the PageTable to easily // map from a stack address to the thread that owns that stack. // This is used to implement GetCurrentThread. // Singularity uses a different mechanism that does not involve // the extra data at all. private static unsafe void SetStackPages(UIntPtr startAddr, UIntPtr endAddr, Thread thread) { UIntPtr startPage = PageTable.Page(startAddr); UIntPtr endPage = PageTable.Page(endAddr); UIntPtr pageCount = endPage - startPage; PageTable.VerifyType(startPage, pageCount, PageType.Unallocated); PageTable.VerifyExtra(startPage, pageCount, 0); PageTable.SetType(startPage, pageCount, PageType.Stack); PageTable.SetExtra(startPage, pageCount, (uint)thread.threadIndex); }
// Tell the GC that a certain data area is no longer allowed to // contain off-limits, non-moveable data. private static unsafe void SetUnallocatedPages(UIntPtr startAddr, UIntPtr size) { UIntPtr startPage = PageTable.Page(startAddr); UIntPtr endAddr = startAddr + size; UIntPtr endPage = PageTable.Page(endAddr); UIntPtr pageCount = endPage - startPage; if (pageCount == 1) { PageTable.SetExtra(startPage, 0); PageTable.SetType(startPage, PageType.Unallocated); PageTable.SetProcess(startPage, 0); } else { PageTable.SetExtra(startPage, pageCount, 0); PageTable.SetType(startPage, pageCount, PageType.Unallocated); PageTable.SetProcess(startPage, pageCount, 0); } }
private void CompactPhaseCleanup(Thread currentThread, PageType generation, UIntPtr newLimitPtr) { VTable.Assert(IsValidGeneration((int)generation)); registerThreadReferenceVisitor.Cleanup(); // Free up skipped pages while (!this.skippedPageQueue.IsEmpty) { UIntPtr start = this.skippedPageQueue.Read(); UIntPtr finish = this.skippedPageQueue.Read(); InteriorPtrTable.ClearFirst(start, finish); PageManager.FreePageRange(start, finish); if (GC.remsetType == RemSetType.Cards) { OffsetTable.ClearLast(PageTable.PageAddr(start), PageTable.PageAddr(finish) - 1); } } this.skippedPageQueue.Cleanup(true); // Release the queue standby pages UnmanagedPageList.ReleaseStandbyPages(); // Update the ownership information for the copied data PageType destGeneration = (generation == MAX_GENERATION) ? MAX_GENERATION : (PageType)(generation + 1); UIntPtr limitPage = PageTable.Page(PageTable.PagePad(newLimitPtr)); for (UIntPtr i = UIntPtr.Zero; i < limitPage; i++) { if (IsMyZombiePage(i)) { PageTable.SetType(i, (PageType)destGeneration); } } }
//=============================== // Routines to mark special pages private static unsafe void SetNonheapPages(UIntPtr startAddr, UIntPtr size) { UIntPtr startPage = PageTable.Page(startAddr); UIntPtr endAddr = startAddr + size; UIntPtr endPage = PageTable.Page(endAddr); if (!PageTable.PageAligned(endAddr)) { endPage++; } UIntPtr pageCount = endPage - startPage; if (pageCount == 1) { PageTable.SetType(startPage, PageType.System); } else { PageTable.SetType(startPage, pageCount, PageType.System); } }
private Object CopyLarge(Object obj, UIntPtr size) { // Don't copy large objects. // Change the page type instead. UIntPtr startAddr = Magic.addressOf(obj) - PreHeader.Size; UIntPtr startPage = PageTable.Page(startAddr); UIntPtr endPage = PageTable.Page(startAddr + size - 1) + 1; VTable.Assert(this.pageType == PageTable.ZombieToLive(PageTable.Type(startPage))); PageTable.SetType(startPage, (endPage - startPage), this.pageType); UIntPtr sizeOfPages = PageTable.PageSize * (endPage - startPage); Trace.Log(Trace.Area.Pointer, "FwdRef: large object: {0} gen: {1} pagesize: {2}", __arglist(Magic.addressOf(obj), this.pageType, sizeOfPages)); GenerationalGCData.gcPromotedTable[(int)this.pageType - 1] += sizeOfPages; this.AddWork(startAddr, startAddr + size); return(obj); }
private static void SetPageTypeClean(UIntPtr startPage, UIntPtr pageCount, PageType newType) { UIntPtr *tableAddr = (UIntPtr *)PageTable.PageAddr(startPage); UIntPtr *tableCursor = tableAddr + 1; UIntPtr dirtyCount = UIntPtr.Zero; UIntPtr endPage = startPage + pageCount; for (UIntPtr i = startPage; i < endPage; i++) { PageType pageType = PageTable.Type(i); if (pageType == PageType.UnusedDirty) { PageTable.SetType(i, newType); UIntPtr j = i + 1; while (j < endPage && PageTable.Type(j) == PageType.UnusedDirty) { PageTable.SetType(j, newType); j++; } UIntPtr dirtyStartAddr = PageTable.PageAddr(i); UIntPtr dirtyEndAddr = PageTable.PageAddr(j); * tableCursor++ = dirtyStartAddr; * tableCursor++ = dirtyEndAddr - dirtyStartAddr; dirtyCount++; i = j - 1; } else { PageTable.SetType(i, newType); } } *tableAddr = dirtyCount; PageTable.SetProcess(startPage, pageCount); }
internal static UIntPtr EnsurePages(Thread currentThread, UIntPtr pageCount, PageType newType, ref bool fCleanPages) { if (currentThread != null) { GC.CheckForNeededGCWork(currentThread); } VTable.Deny(PageTable.IsUnusedPageType(newType)); // Try to find already allocated but unused pages UIntPtr foundPages = FindUnusedPages(currentThread, pageCount, newType); if (foundPages != UIntPtr.Zero) { if (fCleanPages) { CleanFoundPages(foundPages); } else { fCleanPages = FoundOnlyCleanPages(foundPages); } return(foundPages); } // We need to allocate new pages bool iflag = EnterMutex(currentThread); try { UIntPtr bytesNeeded = PageTable.RegionSize(pageCount); UIntPtr allocSize = Util.Pad(bytesNeeded, heap_commit_size); UIntPtr startAddr = MemoryManager.AllocateMemory(allocSize); if (startAddr == UIntPtr.Zero) { if (heap_commit_size > os_commit_size) { allocSize = Util.Pad(bytesNeeded, os_commit_size); startAddr = MemoryManager.AllocateMemory(allocSize); } } if (startAddr == UIntPtr.Zero) { // BUGBUG: if in CMS, should wait on one complete GC cycle and // the retry. for STW, we may get here even if the collector // hasn't triggered just prior. PageTable.Dump("Out of memory"); throw outOfMemoryException; } UIntPtr startPage = PageTable.Page(startAddr); PageTable.SetType(startPage, pageCount, newType); PageTable.SetProcess(startPage, pageCount); UIntPtr extraPages = PageTable.PageCount(allocSize) - pageCount; if (extraPages > 0) { // Mark the new memory pages as allocated-but-unused MarkUnusedPages(/* avoid recursive locking */ null, startPage + pageCount, extraPages, true); } return(startPage); } finally { LeaveMutex(currentThread, iflag); } }
internal void CleanPinnedPages() { if (pinnedPageList == null || pinnedPageList.Count == 0) { return; } int pageIndex = 0; int limit = pinnedPageList.Count; UIntPtr lastPostPinnedAddr = UIntPtr.Zero; while (pageIndex < limit) { UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex]; UIntPtr endPage = startPage + 1; pageIndex++; while (pageIndex < limit && (UIntPtr)pinnedPageList[pageIndex] == endPage) { pageIndex++; endPage++; } // Zero out the area between the start of the page and // the first object on the page UIntPtr firstObjectAddr = FirstPinnedObjectAddr(startPage); UIntPtr firstAddr = firstObjectAddr - PreHeader.Size; UIntPtr trashAddr = PageTable.PageAddr(startPage); if (firstAddr < trashAddr) { // The first object "spills" into the previous page, // presumably by no more than HEADER_BYTES bytes VTable.Assert( PageTable.Page(firstAddr) == startPage - 1, "Semispace:RegisterPinnedReference:3"); // Prepare to zero the preceding page unless it also // had pinned data on it trashAddr = PageTable.PageAddr(startPage - 1); InteriorPtrTable.ClearFirst(startPage - 1); if (trashAddr >= lastPostPinnedAddr) { // Need to mark the spilled-onto page live to // keep the spilled data around PageType fromSpaceType = PageTable.Type(startPage - 1); VTable.Assert( PageTable.IsZombiePage(fromSpaceType), "Semispace:RegisterPinnedReference:4"); PageType toSpaceType = PageTable.ZombieToLive(fromSpaceType); PageTable.SetType(startPage - 1, toSpaceType); } } // If lastPostPinnedAddr is on the page that trashAddr // starts, pinned data from the last run of pinned pages // and pinned data from this run of pinned data are on the // same page, so just write alignment tokens from // lastPostPinnedAddr to the first pinned object. // Otherwise, write an unused marker at lastPostPinnedAddr // since the rest of its page must be copied or dead. if (trashAddr < lastPostPinnedAddr) { trashAddr = lastPostPinnedAddr; } else { CleanPageTail(lastPostPinnedAddr); } if (GC.remsetType == RemSetType.Cards && trashAddr < firstAddr) { UIntPtr firstCard = CardTable.CardNo(trashAddr); UIntPtr lastCard = CardTable.CardNo(firstAddr - 1); if (!OffsetTable.NoObjectPtrToTheCard(firstCard)) { UIntPtr offset = OffsetTable.GetOffset(firstCard); UIntPtr objPtr = CardTable.CardAddr(firstCard) + offset; UIntPtr size = OffsetTable.ObjectSize(objPtr); VTable.Assert ((objPtr + size - PreHeader.Size <= trashAddr) || (objPtr >= trashAddr), "Object should be totally " + "above or below trashAddr"); if (objPtr >= trashAddr) { // The offset in this card needs to be updated OffsetTable.ClearCards(firstCard, firstCard); } } OffsetTable.ClearCards(firstCard + 1, lastCard - 1); if (lastCard != CardTable.CardNo(firstObjectAddr)) { OffsetTable.ClearCards(lastCard, lastCard); } else { VTable.Assert(OffsetTable.GetOffset(lastCard) >= (firstObjectAddr - CardTable.CardAddr(lastCard)), "wrong offset"); } } { // trashAddr should go back at most one page. UIntPtr trashPage = PageTable.Page(trashAddr); UIntPtr firstObjectAddrPage = PageTable.Page(firstObjectAddr); VTable.Assert((trashPage == firstObjectAddrPage - 1) || (trashPage == firstObjectAddrPage)); } // If the InteriorPtrTable already had a value, then this is // redundant, but if the call to First above has to compute // the value, then (since it won't store it in the table) we // should store it. Why? At this point the previous page // would be "connected" to this one. After this collection // the previous page will be unused or re-used and unrelated // to this page and subsequent calls to First would then // rely on it making the leap between unrelated pages. InteriorPtrTable.SetFirst(firstObjectAddr); while (trashAddr < firstAddr) { Allocator.WriteAlignment(trashAddr); trashAddr += UIntPtr.Size; } // Zero out the area between the last whole object on // the last page and the end of the last page UIntPtr pastAddr = PostPinnedObjectAddr(endPage); UIntPtr newEndPage = PageTable.Page(PageTable.PagePad(pastAddr)); while (endPage < newEndPage) { // The last object spills into the next page(s), so // mark those page(s) live PageType fromPageType = PageTable.Type(endPage); if (PageTable.IsZombiePage(fromPageType)) { PageType toSpaceType = PageTable.ZombieToLive(fromPageType); PageTable.SetType(endPage, toSpaceType); } else { // final page might be live already because // something else on it was pinned. // pageIndex has already been incremented, // so it points to the start of the next // set of contiguous pages VTable.Assert( PageTable.IsLiveGcPage(fromPageType) && pageIndex < limit && endPage == (UIntPtr)pinnedPageList[pageIndex], "Semispace:RegisterPinnedReference:5"); } ++endPage; } lastPostPinnedAddr = pastAddr; } CleanPageTail(lastPostPinnedAddr); pinnedPageList = null; comparer = null; }