internal static unsafe void SetStaticDataPages(UIntPtr startAddr,
UIntPtr size)
{
#if SINGULARITY
// It's perfectly fine to be given memory outside the page table region, so
// long as we intend to treat that memory as NonGC anyway.
if (startAddr < PageTable.baseAddr)
{
if (startAddr + size < PageTable.baseAddr)
{
// nothing to do. All pages are below the base covered by the page table
return;
}
// The range overlaps with the region covered by the page table
size -= (PageTable.baseAddr - startAddr);
startAddr = PageTable.baseAddr;
}
UIntPtr endAddr = startAddr + size;
if (endAddr > PageTable.limitAddr)
{
if (startAddr > PageTable.limitAddr)
{
// nothing to do. All pages are above the limit covered by the page table
return;
}
// The range overlaps with the region covered by the page table
size -= (PageTable.limitAddr - endAddr);
}
#endif
UIntPtr startIndex = PageTable.Page(startAddr);
UIntPtr pageCount = PageTable.PageCount(size);
PageTable.SetType(startIndex, pageCount, PageType.NonGC);
}
internal unsafe static void MakeZombiePage(UIntPtr page,
PageType destGeneration)
{
VTable.Assert(destGeneration >= nurseryGeneration &&
destGeneration <= MAX_GENERATION);
PageType pageType = PageTable.LiveToZombie(destGeneration);
PageTable.SetType(page, pageType);
}
internal void ProcessPinnedPages(ReferenceVisitor ptrVisitor)
{
if (pinnedPageList == null || pinnedPageList.Count == 0)
{
return;
}
pinnedPageList.Sort(comparer);
int limit = pinnedPageList.Count;
for (int i = 0; i < limit; i++)
{
UIntPtr page = (UIntPtr)pinnedPageList[i];
PageType fromSpaceType = PageTable.Type(page);
VTable.Assert(PageTable.IsZombiePage(fromSpaceType),
"Semispace:RegisterPinnedReference:2");
PageType toSpaceType =
PageTable.ZombieToLive(fromSpaceType);
PageTable.SetType(page, toSpaceType);
}
int pageIndex = 0;
while (pageIndex < limit)
{
UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex];
UIntPtr endPage = startPage + 1;
pageIndex++;
while (pageIndex < limit &&
(UIntPtr)pinnedPageList[pageIndex] == endPage)
{
pageIndex++;
endPage++;
}
UIntPtr objectAddr = FirstPinnedObjectAddr(startPage);
UIntPtr pastAddr = PostPinnedObjectAddr(endPage);
while (objectAddr < pastAddr)
{
if (Allocator.IsAlignment(objectAddr))
{
objectAddr += UIntPtr.Size;
}
else if (BumpAllocator.IsUnusedSpace(objectAddr))
{
objectAddr = (PageTable.PagePad(objectAddr) +
PreHeader.Size);
}
else
{
Object obj = Magic.fromAddress(objectAddr);
objectAddr += ptrVisitor.VisitReferenceFields(obj);
}
}
}
}
private static void MarkUnusedPages(Thread currentThread,
UIntPtr startPage,
UIntPtr pageCount,
bool fCleanPages)
{
Trace.Log(Trace.Area.Page,
"MarkUnusedPages start={0:x} count={1:x}",
__arglist(startPage, pageCount));
UIntPtr endPage = startPage + pageCount;
if (avoidDirtyPages && !fCleanPages)
{
UIntPtr dirtyStartAddr = PageTable.PageAddr(startPage);
UIntPtr dirtySize = PageTable.RegionSize(pageCount);
Util.MemClear(dirtyStartAddr, dirtySize);
fCleanPages = true;
}
bool iflag = EnterMutex(currentThread);
try {
if (endPage < PageTable.pageTableCount)
{
if (PageTable.IsUnusedPage(endPage) &&
PageTable.IsMyPage(endPage))
{
UIntPtr regionSize = UnlinkUnusedPages(endPage);
endPage += regionSize;
}
}
UIntPtr queryStartPage = startPage - 1;
UIntPtr newStartPage = startPage;
if (PageTable.IsUnusedPage(queryStartPage) &&
PageTable.IsMyPage(queryStartPage))
{
UnusedBlockHeader *tailUnused = (UnusedBlockHeader *)
PageTable.PageAddr(queryStartPage);
UIntPtr newStartAddr = (UIntPtr)tailUnused->curr;
newStartPage = PageTable.Page(newStartAddr);
UIntPtr regionSize = UnlinkUnusedPages(newStartPage);
VTable.Assert(newStartPage + regionSize == startPage);
}
PageType pageType =
fCleanPages ? PageType.UnusedClean : PageType.UnusedDirty;
PageTable.SetType(startPage, pageCount, pageType);
LinkUnusedPages(newStartPage, endPage - newStartPage, false);
} finally {
LeaveMutex(currentThread, iflag);
}
}
// Bartok uses the extra field in the PageTable to easily
// map from a stack address to the thread that owns that stack.
// This is used to implement GetCurrentThread.
// Singularity uses a different mechanism that does not involve
// the extra data at all.
private static unsafe void SetStackPages(UIntPtr startAddr,
UIntPtr endAddr,
Thread thread)
{
UIntPtr startPage = PageTable.Page(startAddr);
UIntPtr endPage = PageTable.Page(endAddr);
UIntPtr pageCount = endPage - startPage;
PageTable.VerifyType(startPage, pageCount, PageType.Unallocated);
PageTable.VerifyExtra(startPage, pageCount, 0);
PageTable.SetType(startPage, pageCount, PageType.Stack);
PageTable.SetExtra(startPage, pageCount,
(uint)thread.threadIndex);
}
// Tell the GC that a certain data area is no longer allowed to
// contain off-limits, non-moveable data.
private static unsafe void SetUnallocatedPages(UIntPtr startAddr,
UIntPtr size)
{
UIntPtr startPage = PageTable.Page(startAddr);
UIntPtr endAddr = startAddr + size;
UIntPtr endPage = PageTable.Page(endAddr);
UIntPtr pageCount = endPage - startPage;
if (pageCount == 1)
{
PageTable.SetExtra(startPage, 0);
PageTable.SetType(startPage, PageType.Unallocated);
PageTable.SetProcess(startPage, 0);
}
else
{
PageTable.SetExtra(startPage, pageCount, 0);
PageTable.SetType(startPage, pageCount, PageType.Unallocated);
PageTable.SetProcess(startPage, pageCount, 0);
}
}
private void CompactPhaseCleanup(Thread currentThread,
PageType generation,
UIntPtr newLimitPtr)
{
VTable.Assert(IsValidGeneration((int)generation));
registerThreadReferenceVisitor.Cleanup();
// Free up skipped pages
while (!this.skippedPageQueue.IsEmpty)
{
UIntPtr start = this.skippedPageQueue.Read();
UIntPtr finish = this.skippedPageQueue.Read();
InteriorPtrTable.ClearFirst(start, finish);
PageManager.FreePageRange(start, finish);
if (GC.remsetType == RemSetType.Cards)
{
OffsetTable.ClearLast(PageTable.PageAddr(start),
PageTable.PageAddr(finish) - 1);
}
}
this.skippedPageQueue.Cleanup(true);
// Release the queue standby pages
UnmanagedPageList.ReleaseStandbyPages();
// Update the ownership information for the copied data
PageType destGeneration =
(generation == MAX_GENERATION) ?
MAX_GENERATION :
(PageType)(generation + 1);
UIntPtr limitPage =
PageTable.Page(PageTable.PagePad(newLimitPtr));
for (UIntPtr i = UIntPtr.Zero; i < limitPage; i++)
{
if (IsMyZombiePage(i))
{
PageTable.SetType(i, (PageType)destGeneration);
}
}
}
//===============================
// Routines to mark special pages
private static unsafe void SetNonheapPages(UIntPtr startAddr,
UIntPtr size)
{
UIntPtr startPage = PageTable.Page(startAddr);
UIntPtr endAddr = startAddr + size;
UIntPtr endPage = PageTable.Page(endAddr);
if (!PageTable.PageAligned(endAddr))
{
endPage++;
}
UIntPtr pageCount = endPage - startPage;
if (pageCount == 1)
{
PageTable.SetType(startPage, PageType.System);
}
else
{
PageTable.SetType(startPage, pageCount, PageType.System);
}
}
private Object CopyLarge(Object obj, UIntPtr size)
{
// Don't copy large objects.
// Change the page type instead.
UIntPtr startAddr = Magic.addressOf(obj) - PreHeader.Size;
UIntPtr startPage = PageTable.Page(startAddr);
UIntPtr endPage = PageTable.Page(startAddr + size - 1) + 1;
VTable.Assert(this.pageType ==
PageTable.ZombieToLive(PageTable.Type(startPage)));
PageTable.SetType(startPage, (endPage - startPage),
this.pageType);
UIntPtr sizeOfPages = PageTable.PageSize * (endPage - startPage);
Trace.Log(Trace.Area.Pointer,
"FwdRef: large object: {0} gen: {1} pagesize: {2}",
__arglist(Magic.addressOf(obj), this.pageType,
sizeOfPages));
GenerationalGCData.gcPromotedTable[(int)this.pageType - 1] +=
sizeOfPages;
this.AddWork(startAddr, startAddr + size);
return(obj);
}
private static void SetPageTypeClean(UIntPtr startPage,
UIntPtr pageCount,
PageType newType)
{
UIntPtr *tableAddr = (UIntPtr *)PageTable.PageAddr(startPage);
UIntPtr *tableCursor = tableAddr + 1;
UIntPtr dirtyCount = UIntPtr.Zero;
UIntPtr endPage = startPage + pageCount;
for (UIntPtr i = startPage; i < endPage; i++)
{
PageType pageType = PageTable.Type(i);
if (pageType == PageType.UnusedDirty)
{
PageTable.SetType(i, newType);
UIntPtr j = i + 1;
while (j < endPage &&
PageTable.Type(j) == PageType.UnusedDirty)
{
PageTable.SetType(j, newType);
j++;
}
UIntPtr dirtyStartAddr = PageTable.PageAddr(i);
UIntPtr dirtyEndAddr = PageTable.PageAddr(j);
* tableCursor++ = dirtyStartAddr;
* tableCursor++ = dirtyEndAddr - dirtyStartAddr;
dirtyCount++;
i = j - 1;
}
else
{
PageTable.SetType(i, newType);
}
}
*tableAddr = dirtyCount;
PageTable.SetProcess(startPage, pageCount);
}
internal static UIntPtr EnsurePages(Thread currentThread,
UIntPtr pageCount,
PageType newType,
ref bool fCleanPages)
{
if (currentThread != null)
{
GC.CheckForNeededGCWork(currentThread);
}
VTable.Deny(PageTable.IsUnusedPageType(newType));
// Try to find already allocated but unused pages
UIntPtr foundPages =
FindUnusedPages(currentThread, pageCount, newType);
if (foundPages != UIntPtr.Zero)
{
if (fCleanPages)
{
CleanFoundPages(foundPages);
}
else
{
fCleanPages = FoundOnlyCleanPages(foundPages);
}
return(foundPages);
}
// We need to allocate new pages
bool iflag = EnterMutex(currentThread);
try {
UIntPtr bytesNeeded = PageTable.RegionSize(pageCount);
UIntPtr allocSize = Util.Pad(bytesNeeded, heap_commit_size);
UIntPtr startAddr = MemoryManager.AllocateMemory(allocSize);
if (startAddr == UIntPtr.Zero)
{
if (heap_commit_size > os_commit_size)
{
allocSize = Util.Pad(bytesNeeded, os_commit_size);
startAddr = MemoryManager.AllocateMemory(allocSize);
}
}
if (startAddr == UIntPtr.Zero)
{
// BUGBUG: if in CMS, should wait on one complete GC cycle and
// the retry. for STW, we may get here even if the collector
// hasn't triggered just prior.
PageTable.Dump("Out of memory");
throw outOfMemoryException;
}
UIntPtr startPage = PageTable.Page(startAddr);
PageTable.SetType(startPage, pageCount, newType);
PageTable.SetProcess(startPage, pageCount);
UIntPtr extraPages =
PageTable.PageCount(allocSize) - pageCount;
if (extraPages > 0)
{
// Mark the new memory pages as allocated-but-unused
MarkUnusedPages(/* avoid recursive locking */ null,
startPage + pageCount, extraPages,
true);
}
return(startPage);
} finally {
LeaveMutex(currentThread, iflag);
}
}
internal void CleanPinnedPages()
{
if (pinnedPageList == null || pinnedPageList.Count == 0)
{
return;
}
int pageIndex = 0;
int limit = pinnedPageList.Count;
UIntPtr lastPostPinnedAddr = UIntPtr.Zero;
while (pageIndex < limit)
{
UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex];
UIntPtr endPage = startPage + 1;
pageIndex++;
while (pageIndex < limit &&
(UIntPtr)pinnedPageList[pageIndex] == endPage)
{
pageIndex++;
endPage++;
}
// Zero out the area between the start of the page and
// the first object on the page
UIntPtr firstObjectAddr = FirstPinnedObjectAddr(startPage);
UIntPtr firstAddr = firstObjectAddr - PreHeader.Size;
UIntPtr trashAddr = PageTable.PageAddr(startPage);
if (firstAddr < trashAddr)
{
// The first object "spills" into the previous page,
// presumably by no more than HEADER_BYTES bytes
VTable.Assert(
PageTable.Page(firstAddr) == startPage - 1,
"Semispace:RegisterPinnedReference:3");
// Prepare to zero the preceding page unless it also
// had pinned data on it
trashAddr = PageTable.PageAddr(startPage - 1);
InteriorPtrTable.ClearFirst(startPage - 1);
if (trashAddr >= lastPostPinnedAddr)
{
// Need to mark the spilled-onto page live to
// keep the spilled data around
PageType fromSpaceType =
PageTable.Type(startPage - 1);
VTable.Assert(
PageTable.IsZombiePage(fromSpaceType),
"Semispace:RegisterPinnedReference:4");
PageType toSpaceType =
PageTable.ZombieToLive(fromSpaceType);
PageTable.SetType(startPage - 1, toSpaceType);
}
}
// If lastPostPinnedAddr is on the page that trashAddr
// starts, pinned data from the last run of pinned pages
// and pinned data from this run of pinned data are on the
// same page, so just write alignment tokens from
// lastPostPinnedAddr to the first pinned object.
// Otherwise, write an unused marker at lastPostPinnedAddr
// since the rest of its page must be copied or dead.
if (trashAddr < lastPostPinnedAddr)
{
trashAddr = lastPostPinnedAddr;
}
else
{
CleanPageTail(lastPostPinnedAddr);
}
if (GC.remsetType == RemSetType.Cards &&
trashAddr < firstAddr)
{
UIntPtr firstCard = CardTable.CardNo(trashAddr);
UIntPtr lastCard = CardTable.CardNo(firstAddr - 1);
if (!OffsetTable.NoObjectPtrToTheCard(firstCard))
{
UIntPtr offset = OffsetTable.GetOffset(firstCard);
UIntPtr objPtr =
CardTable.CardAddr(firstCard) + offset;
UIntPtr size = OffsetTable.ObjectSize(objPtr);
VTable.Assert
((objPtr + size - PreHeader.Size <= trashAddr) ||
(objPtr >= trashAddr),
"Object should be totally "
+ "above or below trashAddr");
if (objPtr >= trashAddr)
{
// The offset in this card needs to be updated
OffsetTable.ClearCards(firstCard, firstCard);
}
}
OffsetTable.ClearCards(firstCard + 1, lastCard - 1);
if (lastCard != CardTable.CardNo(firstObjectAddr))
{
OffsetTable.ClearCards(lastCard, lastCard);
}
else
{
VTable.Assert(OffsetTable.GetOffset(lastCard)
>= (firstObjectAddr
- CardTable.CardAddr(lastCard)),
"wrong offset");
}
}
{
// trashAddr should go back at most one page.
UIntPtr trashPage = PageTable.Page(trashAddr);
UIntPtr firstObjectAddrPage =
PageTable.Page(firstObjectAddr);
VTable.Assert((trashPage == firstObjectAddrPage - 1) ||
(trashPage == firstObjectAddrPage));
}
// If the InteriorPtrTable already had a value, then this is
// redundant, but if the call to First above has to compute
// the value, then (since it won't store it in the table) we
// should store it. Why? At this point the previous page
// would be "connected" to this one. After this collection
// the previous page will be unused or re-used and unrelated
// to this page and subsequent calls to First would then
// rely on it making the leap between unrelated pages.
InteriorPtrTable.SetFirst(firstObjectAddr);
while (trashAddr < firstAddr)
{
Allocator.WriteAlignment(trashAddr);
trashAddr += UIntPtr.Size;
}
// Zero out the area between the last whole object on
// the last page and the end of the last page
UIntPtr pastAddr = PostPinnedObjectAddr(endPage);
UIntPtr newEndPage =
PageTable.Page(PageTable.PagePad(pastAddr));
while (endPage < newEndPage)
{
// The last object spills into the next page(s), so
// mark those page(s) live
PageType fromPageType = PageTable.Type(endPage);
if (PageTable.IsZombiePage(fromPageType))
{
PageType toSpaceType =
PageTable.ZombieToLive(fromPageType);
PageTable.SetType(endPage, toSpaceType);
}
else
{
// final page might be live already because
// something else on it was pinned.
// pageIndex has already been incremented,
// so it points to the start of the next
// set of contiguous pages
VTable.Assert(
PageTable.IsLiveGcPage(fromPageType) &&
pageIndex < limit &&
endPage ==
(UIntPtr)pinnedPageList[pageIndex],
"Semispace:RegisterPinnedReference:5");
}
++endPage;
}
lastPostPinnedAddr = pastAddr;
}
CleanPageTail(lastPostPinnedAddr);
pinnedPageList = null;
comparer = null;
}