internal unsafe override void Visit(UIntPtr *loc)
{
UIntPtr addr = *loc;
// Ignore pointers out of our memory area
if (PageTable.IsForeignAddr(addr))
{
return;
}
UIntPtr page = PageTable.Page(addr);
PageType pageType = PageTable.Type(page);
if (!PageTable.IsGcPage(pageType))
{
VTable.Assert((PageTable.IsNonGcPage(pageType) &&
PageTable.IsMyPage(page)) ||
PageTable.IsStackPage(pageType));
return;
}
VTable.Assert(PageTable.IsMyPage(page));
Object obj = Magic.fromAddress(addr);
VTable.Assert(IsPossiblyObject(obj), "Bad Object/VTable");
if (obj.GcMark() == UIntPtr.Zero)
{
// The object was not live
*loc = UIntPtr.Zero;
}
}
internal unsafe override void Visit(UIntPtr *loc)
{
UIntPtr addr = *loc;
// Ignore pointers out of our memory area
if (PageTable.IsForeignAddr(addr))
{
return;
}
UIntPtr page = PageTable.Page(addr);
PageType pageType = PageTable.Type(page);
if (!PageTable.IsGcPage(pageType))
{
VTable.Assert((PageTable.IsNonGcPage(pageType) &&
PageTable.IsMyPage(page)) ||
PageTable.IsStackPage(pageType));
return;
}
VTable.Assert(PageTable.IsMyPage(page));
Object obj = Magic.fromAddress(addr);
VTable.Assert(IsPossiblyObject(obj), "Bad object/vtable");
if (obj.GcMark((UIntPtr)1))
{
// We changed the color of the object, so we
// have to mark the objects reachable from the fields
workList.Write(addr);
}
}
internal static unsafe void Verify(UnusedBlockHeader *header)
{
VTable.Assert(header->magic == (UIntPtr)magicNumber,
"Bad magic number in UnusedBlockHeader");
VTable.Assert(header->count > 0,
"Count <= 0 in UnusedBlockHeader");
VTable.Assert(header->prev->next == header,
"UnusedBlockHeader not linked properly (1)");
if (header->next != null)
{
VTable.Assert(header->next->prev == header,
"UnusedBlockHeader not linked properly (2)");
}
UIntPtr count = header->count;
UnusedBlockHeader *tailBlock = (UnusedBlockHeader *)
(((UIntPtr)header) + PageTable.RegionSize(count - 1));
VTable.Assert(tailBlock->curr == header,
"UnusedBlockHeader tail->curr is incorrect");
if (PageManager.SlowDebug)
{
UIntPtr page = PageTable.Page((UIntPtr)header);
for (UIntPtr i = UIntPtr.Zero; i < count; i++)
{
VTable.Assert(PageTable.IsUnusedPage(page + i) &&
PageTable.IsMyPage(page + i),
"Incorrect page in unused region");
}
}
}
internal unsafe override void Visit(UIntPtr *loc)
{
UIntPtr addr = *loc;
// Ignore pointers out of our memory area
if (PageTable.IsForeignAddr(addr))
{
return;
}
UIntPtr page = PageTable.Page(addr);
if (!PageTable.IsMyGcPage(page))
{
PageType pageType = PageTable.Type(page);
#if SINGULARITY_PROCESS
// We have to allow reference pointers to the
// ThreadContext, which lives in the kernel space.
VTable.Assert((PageTable.IsNonGcPage(pageType) &&
PageTable.IsMyPage(page)) ||
PageTable.IsStackPage(pageType) ||
PageTable.IsSharedPage(pageType) ||
(PageTable.IsGcPage(pageType) &&
PageTable.IsKernelPage(page)));
#else
VTable.Assert((PageTable.IsNonGcPage(pageType) &&
PageTable.IsMyPage(page)) ||
PageTable.IsStackPage(pageType) ||
PageTable.IsSharedPage(pageType));
#endif
return;
}
UIntPtr objectAddr = SegregatedFreeList.Find(addr);
markAndProcessReferenceVisitor.Visit(&objectAddr);
}
internal static void VerifyUnusedRegion(UIntPtr startPage,
UIntPtr endPage)
{
// Verify that all of the pages are of the same Clean/Dirty type.
PageType startType = PageTable.Type(startPage);
for (UIntPtr page = startPage; page < endPage; ++page)
{
VTable.Assert(startType == PageTable.Type(page),
"Unused page types don't match in region");
}
if (startPage > UIntPtr.Zero &&
PageTable.IsUnusedPage(startPage - 1) &&
PageTable.IsMyPage(startPage - 1))
{
// We have already checked the region
return;
}
UIntPtr regionAddr = PageTable.PageAddr(startPage);
UnusedBlockHeader *regionHeader = (UnusedBlockHeader *)regionAddr;
UIntPtr pageCount = regionHeader->count;
VTable.Assert
(pageCount >= (endPage - startPage),
"Region-to-verify is larger than its header specifies");
endPage = startPage + pageCount;
for (UIntPtr page = startPage; page < endPage; ++page)
{
VTable.Assert(PageTable.IsUnusedPage(page) &&
PageTable.IsMyPage(page),
"Non my-unused page in unused region");
PageManager.VerifyUnusedPage
(page, (page == startPage) || (page == (endPage - 1)));
}
VTable.Assert(!(endPage < PageTable.pageTableCount &&
PageTable.IsUnusedPage(endPage) &&
PageTable.IsMyPage(endPage)),
"My-unused page immediately after unused region");
// Verify that the region is correctly linked into the
// list of unused memory blocks
int slot = SlotFromCount(pageCount);
UnusedBlockHeader *header = unusedMemoryBlocks[slot].next;
UnusedBlockHeader.Verify(header);
while (regionAddr != (UIntPtr)header)
{
header = header->next;
VTable.Assert(header != null,
"Unused region not list for its slot number");
UnusedBlockHeader.Verify(header);
}
}
internal static bool IsMyLiveGcCard(UIntPtr c)
{
VTable.Assert(IsValidCardNo(c), "IsMyLiveGcCard invalid");
UIntPtr page = PageTable.Page(CardAddr(c));
return(PageTable.IsMyPage(page) &&
PageTable.IsLiveGcPage(PageTable.Type(page)));
}
internal static bool TryReservePages(Thread currentThread,
UIntPtr startPage,
UIntPtr pageCount,
PageType newType,
ref bool fCleanPages)
{
Trace.Log(Trace.Area.Page,
"TryReservePages start={0:x} count={1:x}",
__arglist(startPage, pageCount));
VTable.Deny(PageTable.IsUnusedPageType(newType));
VTable.Assert(pageCount > UIntPtr.Zero);
VTable.Deny(startPage != UIntPtr.Zero &&
PageTable.IsUnusedPage(startPage - 1) &&
PageTable.IsMyPage(startPage - 1));
UIntPtr endPage = startPage + pageCount;
UIntPtr index = startPage;
while (index < endPage &&
PageTable.IsUnusedPage(index) &&
PageTable.IsMyPage(index))
{
index++;
}
if (PageTable.IsUnallocatedPage(PageTable.Type(index)))
{
// We should try to extend the region of allocated pages
UIntPtr pagesNeeded = pageCount - (index - startPage);
UIntPtr bytesNeeded = PageTable.RegionSize(pagesNeeded);
UIntPtr allocSize = Util.Pad(bytesNeeded, heap_commit_size);
UIntPtr startAddr = PageTable.PageAddr(index);
bool gotMemory = false;
bool iflag = EnterMutex(currentThread);
try {
gotMemory =
MemoryManager.AllocateMemory(startAddr, allocSize);
if (gotMemory)
{
UIntPtr allocPages = PageTable.PageCount(allocSize);
MarkUnusedPages(/* avoid recursive locking */ null,
index, allocPages, true);
}
} finally {
LeaveMutex(currentThread, iflag);
}
if (gotMemory)
{
bool success =
TryReserveUnusedPages(currentThread, startPage,
pageCount, newType,
ref fCleanPages);
Trace.Log(Trace.Area.Page,
"TryReservePages success={0}",
__arglist(success));
return(success);
}
}
return(false);
}
internal static void MarkIfNecessaryInline(UIntPtr value,
Thread thread)
{
#if !SINGULARITY || CONCURRENT_MS_COLLECTOR
UIntPtr marked = markedColor;
if (ThreadHeaderQueue.GcMark(Magic.fromAddress(value)) != marked)
{
VTable.Assert(PageTable.IsMyPage(PageTable.Page(value)));
UIntPtr unmarked = unmarkedColor;
ThreadHeaderQueue.Push(thread, value, marked, unmarked);
}
#endif // CONCURRENT_MS_COLLECTOR
}
private static void MarkUnusedPages(Thread currentThread,
UIntPtr startPage,
UIntPtr pageCount,
bool fCleanPages)
{
Trace.Log(Trace.Area.Page,
"MarkUnusedPages start={0:x} count={1:x}",
__arglist(startPage, pageCount));
UIntPtr endPage = startPage + pageCount;
if (avoidDirtyPages && !fCleanPages)
{
UIntPtr dirtyStartAddr = PageTable.PageAddr(startPage);
UIntPtr dirtySize = PageTable.RegionSize(pageCount);
Util.MemClear(dirtyStartAddr, dirtySize);
fCleanPages = true;
}
bool iflag = EnterMutex(currentThread);
try {
if (endPage < PageTable.pageTableCount)
{
if (PageTable.IsUnusedPage(endPage) &&
PageTable.IsMyPage(endPage))
{
UIntPtr regionSize = UnlinkUnusedPages(endPage);
endPage += regionSize;
}
}
UIntPtr queryStartPage = startPage - 1;
UIntPtr newStartPage = startPage;
if (PageTable.IsUnusedPage(queryStartPage) &&
PageTable.IsMyPage(queryStartPage))
{
UnusedBlockHeader *tailUnused = (UnusedBlockHeader *)
PageTable.PageAddr(queryStartPage);
UIntPtr newStartAddr = (UIntPtr)tailUnused->curr;
newStartPage = PageTable.Page(newStartAddr);
UIntPtr regionSize = UnlinkUnusedPages(newStartPage);
VTable.Assert(newStartPage + regionSize == startPage);
}
PageType pageType =
fCleanPages ? PageType.UnusedClean : PageType.UnusedDirty;
PageTable.SetType(startPage, pageCount, pageType);
LinkUnusedPages(newStartPage, endPage - newStartPage, false);
} finally {
LeaveMutex(currentThread, iflag);
}
}
private static void LinkUnusedPages(UIntPtr startPage,
UIntPtr pageCount,
bool asVictim)
{
if (PageManager.SlowDebug)
{
for (UIntPtr i = startPage; i < startPage + pageCount; i++)
{
VTable.Assert(PageTable.IsUnusedPage(i) &&
PageTable.IsMyPage(i),
"Incorrect page to link into unused region");
}
}
Trace.Log(Trace.Area.Page,
"LinkUnusedPages start={0:x} count={1:x}",
__arglist(startPage, pageCount));
VTable.Deny(startPage > UIntPtr.Zero &&
PageTable.IsUnusedPage(startPage - 1) &&
PageTable.IsMyPage(startPage - 1));
VTable.Deny(startPage + pageCount > PageTable.pageTableCount);
VTable.Deny(startPage + pageCount < PageTable.pageTableCount &&
PageTable.IsUnusedPage(startPage + pageCount) &&
PageTable.IsMyPage(startPage + pageCount));
UnusedBlockHeader *header = (UnusedBlockHeader *)
PageTable.PageAddr(startPage);
UnusedBlockHeader.Initialize(header, pageCount);
int slot = SlotFromCount(pageCount);
// Unused blocks are linked into the free list either as the result of a collection
// or as a result of carving a big block into a smaller allocation and a remainder.
// When such a remainder is linked back into the free list, it is identified as a
// victim. We favor subsequent allocations from these victims, in an attempt to
// reduce fragmentation. This is achieved by keeping victims at the head of the
// free list.
//
// TODO: the long term solution is to perform best fit on the free list.
if (asVictim || unusedMemoryBlocks[slot].next == null)
{
fixed(UnusedBlockHeader *listHeader = &unusedMemoryBlocks[slot])
{
UnusedBlockHeader.InsertNext(listHeader, header);
}
}
else
{
UnusedBlockHeader *listHeader = unusedMemoryBlocks[slot].next;
UnusedBlockHeader.InsertNext(listHeader, header);
}
}
private static UIntPtr UnlinkUnusedPages(UIntPtr startPage)
{
VTable.Assert(PageTable.IsUnusedPage(startPage) &&
PageTable.IsMyPage(startPage));
VTable.Deny(startPage > UIntPtr.Zero &&
PageTable.IsUnusedPage(startPage - 1) &&
PageTable.IsMyPage(startPage - 1));
UnusedBlockHeader *header = (UnusedBlockHeader *)
PageTable.PageAddr(startPage);
UIntPtr pageCount = UnusedBlockHeader.Remove(header);
Trace.Log(Trace.Area.Page,
"UnlinkUnusedPages start={0:x} count={1:x}",
__arglist(startPage, pageCount));
return(pageCount);
}
internal override unsafe void Visit(UIntPtr *loc)
{
// <loc> is a traceable pointer; its referent
// either resides in the heap, the stack or in
// the static data area.
UIntPtr addr = *loc;
UIntPtr page = PageTable.Page(addr);
VTable.Assert(PageTable.IsMyPage(page),
"MemoryAccounting: !IsMyPage");
if (!PageTable.IsGcPage(page))
{
PageType pageType = PageTable.Type(page);
VTable.Assert(pageType == PageType.NonGC ||
pageType == PageType.Stack ||
pageType == PageType.Shared,
"unexpected page type");
if (pageType == PageType.NonGC)
{
// A managed pointer into the static data area.
managedPtrsToStaticData++;
}
else
{
// A managed pointer into the stack area.
managedPtrsToStack++;
}
return;
}
UIntPtr objAddr = GC.installedGC.FindObjectAddr(addr);
if (objAddr != addr)
{
// A "truly" interior pointer into a heap object.
interiorManagedHeapPtrs++;
}
else
{
exteriorManagedHeapPtrs++;
}
}
internal static uint TotalNumPages(PageType kind)
{
uint pageCount = 0;
for (UIntPtr i = UIntPtr.Zero; i < PageTable.pageTableCount;
i++)
{
if (PageTable.IsMyPage(i))
{
PageType pageType = PageTable.Type(i);
if (pageType == kind)
{
pageCount++;
}
}
}
return(pageCount);
}
internal unsafe override void Visit(UIntPtr *loc)
{
UIntPtr addr = *loc;
// Ignore pointers out of our memory area
if (PageTable.IsForeignAddr(addr))
{
return;
}
UIntPtr page = PageTable.Page(addr);
PageType pageType = PageTable.Type(page);
if (PageTable.IsGcPage(pageType))
{
Object obj = Magic.fromAddress(addr);
VTable.Assert(obj.GcMark() != UIntPtr.Zero);
VTable.Assert(PageTable.IsMyPage(page));
}
}
internal static bool MarkIfNecessary(UIntPtr value)
{
#if !SINGULARITY || CONCURRENT_MS_COLLECTOR
if (value == 0)
{
return(false);
}
UIntPtr marked = markedColor;
if (PageTable.IsGcPage(PageTable.Page(value)) &&
ThreadHeaderQueue.GcMark(Magic.fromAddress(value)) != marked)
{
VTable.Assert(PageTable.IsMyPage(PageTable.Page(value)));
Thread thread = Thread.CurrentThread;
UIntPtr unmarked = unmarkedColor;
ThreadHeaderQueue.Push(thread, value, marked, unmarked);
return(true);
}
#endif // CONCURRENT_MS_COLLECTOR
return(false);
}
internal unsafe override void Visit(UIntPtr *loc)
{
UIntPtr addr = *loc;
UIntPtr page = PageTable.Page(addr);
PageType pageType = PageTable.Type(page);
if (!PageTable.IsGcPage(pageType))
{
VTable.Assert(PageTable.IsNonGcPage(pageType) ||
PageTable.IsStackPage(pageType) ||
PageTable.IsSharedPage(pageType));
return;
}
VTable.Assert(PageTable.IsMyPage(page));
UIntPtr objectAddr = GC.installedGC.FindObjectAddr(addr);
VTable.Assert(objectAddr <= addr);
UIntPtr vtableAddr =
Magic.addressOf(Magic.fromAddress(objectAddr).vtable);
VTable.Assert(PageTable.IsNonGcPage(PageTable.Type(PageTable.Page(vtableAddr))));
VTable.Assert(Magic.fromAddress(vtableAddr) is VTable);
}
internal static bool TryReserveUnusedPages(Thread currentThread,
UIntPtr startPage,
UIntPtr pageCount,
PageType newType,
ref bool fCleanPages)
{
Trace.Log(Trace.Area.Page,
"TryReserveUnusedPages start={0:x} count={1:x}",
__arglist(startPage, pageCount));
VTable.Deny(PageTable.IsUnusedPageType(newType));
VTable.Assert(pageCount > UIntPtr.Zero);
VTable.Deny(startPage != UIntPtr.Zero &&
PageTable.IsUnusedPage(startPage - 1) &&
PageTable.IsMyPage(startPage - 1));
UIntPtr endPage = startPage + pageCount;
if (endPage > PageTable.pageTableCount)
{
return(false);
}
if (currentThread != null)
{
GC.CheckForNeededGCWork(currentThread);
}
bool iflag = EnterMutex(currentThread);
try {
// GC can occur and page can be collected.
if (startPage != UIntPtr.Zero &&
PageTable.IsUnusedPage(startPage - 1))
{
return(false);
}
if (!PageTable.IsUnusedPage(startPage) ||
!PageTable.IsMyPage(startPage))
{
return(false);
}
UnusedBlockHeader *header = (UnusedBlockHeader *)
PageTable.PageAddr(startPage);
if (header->count < pageCount)
{
return(false);
}
UIntPtr regionPages = UnlinkUnusedPages(startPage);
Trace.Log(Trace.Area.Page,
"TryReserveUnusedPages found={0:x}",
__arglist(regionPages));
SetPageTypeClean(startPage, pageCount, newType);
if (regionPages > pageCount)
{
UIntPtr suffixPages = regionPages - pageCount;
LinkUnusedPages(endPage, suffixPages, true);
}
} finally {
LeaveMutex(currentThread, iflag);
}
// Now that we are outside the Mutex, we should perform the
// real cleaning of the gotten pages
if (fCleanPages)
{
CleanFoundPages(startPage);
}
else
{
fCleanPages = FoundOnlyCleanPages(startPage);
}
return(true);
}
void VerifyPages(ObjectLayout.ObjectVisitor objectVisitor)
{
UIntPtr page = UIntPtr.Zero;
while (page < PageTable.pageTableCount)
{
UIntPtr startPage = page;
if (!PageTable.IsMyPage(startPage))
{
page++;
continue;
}
PageType pageType = PageTable.Type(page);
uint pageProcess = PageTable.Process(page);
do
{
page++;
} while (page < PageTable.pageTableCount &&
PageTable.Type(page) == pageType &&
PageTable.Process(page) == pageProcess);
UIntPtr endPage = page;
switch (pageType)
{
case PageType.Unallocated:
case PageType.Unknown:
case PageType.Shared: {
// The region does not belong to us, so there is
// nothing to check.
break;
}
case PageType.UnusedClean:
case PageType.UnusedDirty: {
PageManager.VerifyUnusedRegion(startPage, endPage);
break;
}
case PageType.System: {
// We have looked at the region, but it is off-limits
// for the verifier.
break;
}
case PageType.NonGC: {
// Since there may be non-objects in the static data
// pages, we cannot apply the heapVerifier to the
// region.
break;
}
case PageType.Stack: {
// The page contains (part of) the activation record
// stack for one or more threads.
break;
}
default: {
// We have found a data region
VTable.Assert(PageTable.IsGcPage(startPage));
UIntPtr startAddr = PageTable.PageAddr(startPage);
UIntPtr endAddr = PageTable.PageAddr(endPage);
GC.installedGC.VisitObjects(objectVisitor,
startAddr, endAddr);
break;
}
}
}
}
internal static bool IsMyGcPage(UIntPtr page)
{
return(PageTable.IsMyPage(page) && PageTable.IsGcPage(page));
}
internal static bool IsMyZombiePage(UIntPtr page)
{
return(PageTable.IsMyPage(page) &&
PageTable.IsZombiePage(PageTable.Type(page)));
}