internal unsafe override void Visit(UIntPtr *loc) { UIntPtr addr = *loc; // Ignore pointers out of our memory area if (PageTable.IsForeignAddr(addr)) { return; } UIntPtr page = PageTable.Page(addr); PageType pageType = PageTable.Type(page); if (!PageTable.IsGcPage(pageType)) { VTable.Assert((PageTable.IsNonGcPage(pageType) && PageTable.IsMyPage(page)) || PageTable.IsStackPage(pageType)); return; } VTable.Assert(PageTable.IsMyPage(page)); Object obj = Magic.fromAddress(addr); VTable.Assert(IsPossiblyObject(obj), "Bad Object/VTable"); if (obj.GcMark() == UIntPtr.Zero) { // The object was not live *loc = UIntPtr.Zero; } }
internal unsafe override void Visit(UIntPtr *loc) { UIntPtr addr = *loc; // Ignore pointers out of our memory area if (PageTable.IsForeignAddr(addr)) { return; } UIntPtr page = PageTable.Page(addr); PageType pageType = PageTable.Type(page); if (!PageTable.IsGcPage(pageType)) { VTable.Assert((PageTable.IsNonGcPage(pageType) && PageTable.IsMyPage(page)) || PageTable.IsStackPage(pageType)); return; } VTable.Assert(PageTable.IsMyPage(page)); Object obj = Magic.fromAddress(addr); VTable.Assert(IsPossiblyObject(obj), "Bad object/vtable"); if (obj.GcMark((UIntPtr)1)) { // We changed the color of the object, so we // have to mark the objects reachable from the fields workList.Write(addr); } }
internal static unsafe void Verify(UnusedBlockHeader *header) { VTable.Assert(header->magic == (UIntPtr)magicNumber, "Bad magic number in UnusedBlockHeader"); VTable.Assert(header->count > 0, "Count <= 0 in UnusedBlockHeader"); VTable.Assert(header->prev->next == header, "UnusedBlockHeader not linked properly (1)"); if (header->next != null) { VTable.Assert(header->next->prev == header, "UnusedBlockHeader not linked properly (2)"); } UIntPtr count = header->count; UnusedBlockHeader *tailBlock = (UnusedBlockHeader *) (((UIntPtr)header) + PageTable.RegionSize(count - 1)); VTable.Assert(tailBlock->curr == header, "UnusedBlockHeader tail->curr is incorrect"); if (PageManager.SlowDebug) { UIntPtr page = PageTable.Page((UIntPtr)header); for (UIntPtr i = UIntPtr.Zero; i < count; i++) { VTable.Assert(PageTable.IsUnusedPage(page + i) && PageTable.IsMyPage(page + i), "Incorrect page in unused region"); } } }
internal unsafe override void Visit(UIntPtr *loc) { UIntPtr addr = *loc; // Ignore pointers out of our memory area if (PageTable.IsForeignAddr(addr)) { return; } UIntPtr page = PageTable.Page(addr); if (!PageTable.IsMyGcPage(page)) { PageType pageType = PageTable.Type(page); #if SINGULARITY_PROCESS // We have to allow reference pointers to the // ThreadContext, which lives in the kernel space. VTable.Assert((PageTable.IsNonGcPage(pageType) && PageTable.IsMyPage(page)) || PageTable.IsStackPage(pageType) || PageTable.IsSharedPage(pageType) || (PageTable.IsGcPage(pageType) && PageTable.IsKernelPage(page))); #else VTable.Assert((PageTable.IsNonGcPage(pageType) && PageTable.IsMyPage(page)) || PageTable.IsStackPage(pageType) || PageTable.IsSharedPage(pageType)); #endif return; } UIntPtr objectAddr = SegregatedFreeList.Find(addr); markAndProcessReferenceVisitor.Visit(&objectAddr); }
internal static void VerifyUnusedRegion(UIntPtr startPage, UIntPtr endPage) { // Verify that all of the pages are of the same Clean/Dirty type. PageType startType = PageTable.Type(startPage); for (UIntPtr page = startPage; page < endPage; ++page) { VTable.Assert(startType == PageTable.Type(page), "Unused page types don't match in region"); } if (startPage > UIntPtr.Zero && PageTable.IsUnusedPage(startPage - 1) && PageTable.IsMyPage(startPage - 1)) { // We have already checked the region return; } UIntPtr regionAddr = PageTable.PageAddr(startPage); UnusedBlockHeader *regionHeader = (UnusedBlockHeader *)regionAddr; UIntPtr pageCount = regionHeader->count; VTable.Assert (pageCount >= (endPage - startPage), "Region-to-verify is larger than its header specifies"); endPage = startPage + pageCount; for (UIntPtr page = startPage; page < endPage; ++page) { VTable.Assert(PageTable.IsUnusedPage(page) && PageTable.IsMyPage(page), "Non my-unused page in unused region"); PageManager.VerifyUnusedPage (page, (page == startPage) || (page == (endPage - 1))); } VTable.Assert(!(endPage < PageTable.pageTableCount && PageTable.IsUnusedPage(endPage) && PageTable.IsMyPage(endPage)), "My-unused page immediately after unused region"); // Verify that the region is correctly linked into the // list of unused memory blocks int slot = SlotFromCount(pageCount); UnusedBlockHeader *header = unusedMemoryBlocks[slot].next; UnusedBlockHeader.Verify(header); while (regionAddr != (UIntPtr)header) { header = header->next; VTable.Assert(header != null, "Unused region not list for its slot number"); UnusedBlockHeader.Verify(header); } }
internal static bool IsMyLiveGcCard(UIntPtr c) { VTable.Assert(IsValidCardNo(c), "IsMyLiveGcCard invalid"); UIntPtr page = PageTable.Page(CardAddr(c)); return(PageTable.IsMyPage(page) && PageTable.IsLiveGcPage(PageTable.Type(page))); }
internal static bool TryReservePages(Thread currentThread, UIntPtr startPage, UIntPtr pageCount, PageType newType, ref bool fCleanPages) { Trace.Log(Trace.Area.Page, "TryReservePages start={0:x} count={1:x}", __arglist(startPage, pageCount)); VTable.Deny(PageTable.IsUnusedPageType(newType)); VTable.Assert(pageCount > UIntPtr.Zero); VTable.Deny(startPage != UIntPtr.Zero && PageTable.IsUnusedPage(startPage - 1) && PageTable.IsMyPage(startPage - 1)); UIntPtr endPage = startPage + pageCount; UIntPtr index = startPage; while (index < endPage && PageTable.IsUnusedPage(index) && PageTable.IsMyPage(index)) { index++; } if (PageTable.IsUnallocatedPage(PageTable.Type(index))) { // We should try to extend the region of allocated pages UIntPtr pagesNeeded = pageCount - (index - startPage); UIntPtr bytesNeeded = PageTable.RegionSize(pagesNeeded); UIntPtr allocSize = Util.Pad(bytesNeeded, heap_commit_size); UIntPtr startAddr = PageTable.PageAddr(index); bool gotMemory = false; bool iflag = EnterMutex(currentThread); try { gotMemory = MemoryManager.AllocateMemory(startAddr, allocSize); if (gotMemory) { UIntPtr allocPages = PageTable.PageCount(allocSize); MarkUnusedPages(/* avoid recursive locking */ null, index, allocPages, true); } } finally { LeaveMutex(currentThread, iflag); } if (gotMemory) { bool success = TryReserveUnusedPages(currentThread, startPage, pageCount, newType, ref fCleanPages); Trace.Log(Trace.Area.Page, "TryReservePages success={0}", __arglist(success)); return(success); } } return(false); }
internal static void MarkIfNecessaryInline(UIntPtr value, Thread thread) { #if !SINGULARITY || CONCURRENT_MS_COLLECTOR UIntPtr marked = markedColor; if (ThreadHeaderQueue.GcMark(Magic.fromAddress(value)) != marked) { VTable.Assert(PageTable.IsMyPage(PageTable.Page(value))); UIntPtr unmarked = unmarkedColor; ThreadHeaderQueue.Push(thread, value, marked, unmarked); } #endif // CONCURRENT_MS_COLLECTOR }
private static void MarkUnusedPages(Thread currentThread, UIntPtr startPage, UIntPtr pageCount, bool fCleanPages) { Trace.Log(Trace.Area.Page, "MarkUnusedPages start={0:x} count={1:x}", __arglist(startPage, pageCount)); UIntPtr endPage = startPage + pageCount; if (avoidDirtyPages && !fCleanPages) { UIntPtr dirtyStartAddr = PageTable.PageAddr(startPage); UIntPtr dirtySize = PageTable.RegionSize(pageCount); Util.MemClear(dirtyStartAddr, dirtySize); fCleanPages = true; } bool iflag = EnterMutex(currentThread); try { if (endPage < PageTable.pageTableCount) { if (PageTable.IsUnusedPage(endPage) && PageTable.IsMyPage(endPage)) { UIntPtr regionSize = UnlinkUnusedPages(endPage); endPage += regionSize; } } UIntPtr queryStartPage = startPage - 1; UIntPtr newStartPage = startPage; if (PageTable.IsUnusedPage(queryStartPage) && PageTable.IsMyPage(queryStartPage)) { UnusedBlockHeader *tailUnused = (UnusedBlockHeader *) PageTable.PageAddr(queryStartPage); UIntPtr newStartAddr = (UIntPtr)tailUnused->curr; newStartPage = PageTable.Page(newStartAddr); UIntPtr regionSize = UnlinkUnusedPages(newStartPage); VTable.Assert(newStartPage + regionSize == startPage); } PageType pageType = fCleanPages ? PageType.UnusedClean : PageType.UnusedDirty; PageTable.SetType(startPage, pageCount, pageType); LinkUnusedPages(newStartPage, endPage - newStartPage, false); } finally { LeaveMutex(currentThread, iflag); } }
private static void LinkUnusedPages(UIntPtr startPage, UIntPtr pageCount, bool asVictim) { if (PageManager.SlowDebug) { for (UIntPtr i = startPage; i < startPage + pageCount; i++) { VTable.Assert(PageTable.IsUnusedPage(i) && PageTable.IsMyPage(i), "Incorrect page to link into unused region"); } } Trace.Log(Trace.Area.Page, "LinkUnusedPages start={0:x} count={1:x}", __arglist(startPage, pageCount)); VTable.Deny(startPage > UIntPtr.Zero && PageTable.IsUnusedPage(startPage - 1) && PageTable.IsMyPage(startPage - 1)); VTable.Deny(startPage + pageCount > PageTable.pageTableCount); VTable.Deny(startPage + pageCount < PageTable.pageTableCount && PageTable.IsUnusedPage(startPage + pageCount) && PageTable.IsMyPage(startPage + pageCount)); UnusedBlockHeader *header = (UnusedBlockHeader *) PageTable.PageAddr(startPage); UnusedBlockHeader.Initialize(header, pageCount); int slot = SlotFromCount(pageCount); // Unused blocks are linked into the free list either as the result of a collection // or as a result of carving a big block into a smaller allocation and a remainder. // When such a remainder is linked back into the free list, it is identified as a // victim. We favor subsequent allocations from these victims, in an attempt to // reduce fragmentation. This is achieved by keeping victims at the head of the // free list. // // TODO: the long term solution is to perform best fit on the free list. if (asVictim || unusedMemoryBlocks[slot].next == null) { fixed(UnusedBlockHeader *listHeader = &unusedMemoryBlocks[slot]) { UnusedBlockHeader.InsertNext(listHeader, header); } } else { UnusedBlockHeader *listHeader = unusedMemoryBlocks[slot].next; UnusedBlockHeader.InsertNext(listHeader, header); } }
private static UIntPtr UnlinkUnusedPages(UIntPtr startPage) { VTable.Assert(PageTable.IsUnusedPage(startPage) && PageTable.IsMyPage(startPage)); VTable.Deny(startPage > UIntPtr.Zero && PageTable.IsUnusedPage(startPage - 1) && PageTable.IsMyPage(startPage - 1)); UnusedBlockHeader *header = (UnusedBlockHeader *) PageTable.PageAddr(startPage); UIntPtr pageCount = UnusedBlockHeader.Remove(header); Trace.Log(Trace.Area.Page, "UnlinkUnusedPages start={0:x} count={1:x}", __arglist(startPage, pageCount)); return(pageCount); }
internal override unsafe void Visit(UIntPtr *loc) { // <loc> is a traceable pointer; its referent // either resides in the heap, the stack or in // the static data area. UIntPtr addr = *loc; UIntPtr page = PageTable.Page(addr); VTable.Assert(PageTable.IsMyPage(page), "MemoryAccounting: !IsMyPage"); if (!PageTable.IsGcPage(page)) { PageType pageType = PageTable.Type(page); VTable.Assert(pageType == PageType.NonGC || pageType == PageType.Stack || pageType == PageType.Shared, "unexpected page type"); if (pageType == PageType.NonGC) { // A managed pointer into the static data area. managedPtrsToStaticData++; } else { // A managed pointer into the stack area. managedPtrsToStack++; } return; } UIntPtr objAddr = GC.installedGC.FindObjectAddr(addr); if (objAddr != addr) { // A "truly" interior pointer into a heap object. interiorManagedHeapPtrs++; } else { exteriorManagedHeapPtrs++; } }
internal static uint TotalNumPages(PageType kind) { uint pageCount = 0; for (UIntPtr i = UIntPtr.Zero; i < PageTable.pageTableCount; i++) { if (PageTable.IsMyPage(i)) { PageType pageType = PageTable.Type(i); if (pageType == kind) { pageCount++; } } } return(pageCount); }
internal unsafe override void Visit(UIntPtr *loc) { UIntPtr addr = *loc; // Ignore pointers out of our memory area if (PageTable.IsForeignAddr(addr)) { return; } UIntPtr page = PageTable.Page(addr); PageType pageType = PageTable.Type(page); if (PageTable.IsGcPage(pageType)) { Object obj = Magic.fromAddress(addr); VTable.Assert(obj.GcMark() != UIntPtr.Zero); VTable.Assert(PageTable.IsMyPage(page)); } }
internal static bool MarkIfNecessary(UIntPtr value) { #if !SINGULARITY || CONCURRENT_MS_COLLECTOR if (value == 0) { return(false); } UIntPtr marked = markedColor; if (PageTable.IsGcPage(PageTable.Page(value)) && ThreadHeaderQueue.GcMark(Magic.fromAddress(value)) != marked) { VTable.Assert(PageTable.IsMyPage(PageTable.Page(value))); Thread thread = Thread.CurrentThread; UIntPtr unmarked = unmarkedColor; ThreadHeaderQueue.Push(thread, value, marked, unmarked); return(true); } #endif // CONCURRENT_MS_COLLECTOR return(false); }
internal unsafe override void Visit(UIntPtr *loc) { UIntPtr addr = *loc; UIntPtr page = PageTable.Page(addr); PageType pageType = PageTable.Type(page); if (!PageTable.IsGcPage(pageType)) { VTable.Assert(PageTable.IsNonGcPage(pageType) || PageTable.IsStackPage(pageType) || PageTable.IsSharedPage(pageType)); return; } VTable.Assert(PageTable.IsMyPage(page)); UIntPtr objectAddr = GC.installedGC.FindObjectAddr(addr); VTable.Assert(objectAddr <= addr); UIntPtr vtableAddr = Magic.addressOf(Magic.fromAddress(objectAddr).vtable); VTable.Assert(PageTable.IsNonGcPage(PageTable.Type(PageTable.Page(vtableAddr)))); VTable.Assert(Magic.fromAddress(vtableAddr) is VTable); }
internal static bool TryReserveUnusedPages(Thread currentThread, UIntPtr startPage, UIntPtr pageCount, PageType newType, ref bool fCleanPages) { Trace.Log(Trace.Area.Page, "TryReserveUnusedPages start={0:x} count={1:x}", __arglist(startPage, pageCount)); VTable.Deny(PageTable.IsUnusedPageType(newType)); VTable.Assert(pageCount > UIntPtr.Zero); VTable.Deny(startPage != UIntPtr.Zero && PageTable.IsUnusedPage(startPage - 1) && PageTable.IsMyPage(startPage - 1)); UIntPtr endPage = startPage + pageCount; if (endPage > PageTable.pageTableCount) { return(false); } if (currentThread != null) { GC.CheckForNeededGCWork(currentThread); } bool iflag = EnterMutex(currentThread); try { // GC can occur and page can be collected. if (startPage != UIntPtr.Zero && PageTable.IsUnusedPage(startPage - 1)) { return(false); } if (!PageTable.IsUnusedPage(startPage) || !PageTable.IsMyPage(startPage)) { return(false); } UnusedBlockHeader *header = (UnusedBlockHeader *) PageTable.PageAddr(startPage); if (header->count < pageCount) { return(false); } UIntPtr regionPages = UnlinkUnusedPages(startPage); Trace.Log(Trace.Area.Page, "TryReserveUnusedPages found={0:x}", __arglist(regionPages)); SetPageTypeClean(startPage, pageCount, newType); if (regionPages > pageCount) { UIntPtr suffixPages = regionPages - pageCount; LinkUnusedPages(endPage, suffixPages, true); } } finally { LeaveMutex(currentThread, iflag); } // Now that we are outside the Mutex, we should perform the // real cleaning of the gotten pages if (fCleanPages) { CleanFoundPages(startPage); } else { fCleanPages = FoundOnlyCleanPages(startPage); } return(true); }
void VerifyPages(ObjectLayout.ObjectVisitor objectVisitor) { UIntPtr page = UIntPtr.Zero; while (page < PageTable.pageTableCount) { UIntPtr startPage = page; if (!PageTable.IsMyPage(startPage)) { page++; continue; } PageType pageType = PageTable.Type(page); uint pageProcess = PageTable.Process(page); do { page++; } while (page < PageTable.pageTableCount && PageTable.Type(page) == pageType && PageTable.Process(page) == pageProcess); UIntPtr endPage = page; switch (pageType) { case PageType.Unallocated: case PageType.Unknown: case PageType.Shared: { // The region does not belong to us, so there is // nothing to check. break; } case PageType.UnusedClean: case PageType.UnusedDirty: { PageManager.VerifyUnusedRegion(startPage, endPage); break; } case PageType.System: { // We have looked at the region, but it is off-limits // for the verifier. break; } case PageType.NonGC: { // Since there may be non-objects in the static data // pages, we cannot apply the heapVerifier to the // region. break; } case PageType.Stack: { // The page contains (part of) the activation record // stack for one or more threads. break; } default: { // We have found a data region VTable.Assert(PageTable.IsGcPage(startPage)); UIntPtr startAddr = PageTable.PageAddr(startPage); UIntPtr endAddr = PageTable.PageAddr(endPage); GC.installedGC.VisitObjects(objectVisitor, startAddr, endAddr); break; } } } }
internal static bool IsMyGcPage(UIntPtr page) { return(PageTable.IsMyPage(page) && PageTable.IsGcPage(page)); }
internal static bool IsMyZombiePage(UIntPtr page) { return(PageTable.IsMyPage(page) && PageTable.IsZombiePage(PageTable.Type(page))); }