internal static void ClearLast(UIntPtr from, UIntPtr to)
{
UIntPtr first = CardTable.CardNo(from);
UIntPtr last = CardTable.CardNo(to);
ClearCards(first, last);
}
internal static void Initialize()
{
// The card table covers all the heap. Below is not accurate heap
// size. It includes all the memory space, starting from address 0.
// TODO: find more accurate heap size and start
UIntPtr heapSize = PageTable.pageTableCount * PageTable.PageSize;
VTable.Assert((heapSize >> CardBits) << CardBits == heapSize,
"Assumption: pageSize is expected to be multiple of CardSize");
UIntPtr heapStart = (UIntPtr)0;
totalCards = heapSize >> CardBits;
firstCardNo = heapStart >> CardBits;
UIntPtr tablePtr = PageManager.AllocateNonheapMemory(null, totalCards);
VTable.Assert(tablePtr != UIntPtr.Zero);
cardTablePtr = (byte *)tablePtr;
cardTableBase = (byte *)(tablePtr - firstCardNo);
CardTable.instance = (CardTable)
BootstrapMemory.Allocate(typeof(CardTable));
OffsetTable.Initialize(totalCards);
for (UIntPtr c = firstCardNo; c < firstCardNo + totalCards; c++)
{
VTable.Assert(!CardIsDirty(c), "Card table initialization error");
}
}
internal override UIntPtr Visit(Object obj)
{
#if DEBUG_OFFSETTABLE
VTable.DebugPrint("visit obj {0:x8}\n", __arglist(Magic.addressOf(obj)));
if (lastObjPtr != UIntPtr.Zero)
{
UIntPtr lastCard = CardTable.CardNo(lastObjPtr);
if (lastCard != CardTable.CardNo(Magic.addressOf(obj)))
{
if (!OffsetTable.NoObjectPtrToTheCard(lastCard))
{
UIntPtr realOffst = lastObjPtr - CardTable.CardAddr(lastCard);
UIntPtr currOffst = OffsetTable.GetOffset(lastCard);
if (realOffst != currOffst)
{
VTable.DebugPrint("Verifier: wrong offset. Card {0:x8} Offset {1:x8} Should be {2:x8}\n",
__arglist(lastCard, currOffst, realOffst));
VTable.Assert(false);
}
}
}
}
#endif
return(this.referenceVisitor.VisitReferenceFields(obj));
}
// OffsetTable records all objects promoted or directly allocated in mature
// generation.
internal static void SetLast(UIntPtr objPtr)
{
VTable.Assert(PageTable.IsGcPage(PageTable.Page(objPtr)), "Not GC page");
UIntPtr card = CardTable.CardNo(objPtr);
UIntPtr mask = (UIntPtr)(CardTable.CardSize - 1);
UIntPtr offset = objPtr & mask;
VTable.Assert(offset < CardTable.CardSize, "Overflow");
SetOffset(card, offset);
#if DEBUG_OFFSETTABLE
VTable.DebugPrint("SetLast objPtr {0:x8}, card {1:x8}, offset {2:x8}\n",
__arglist(objPtr, card, offset));
#endif
}
private static void InstallRemSet()
{
switch (GC.remsetType)
{
case RemSetType.SSB: {
SequentialStoreBuffer.Initialize();
installedRemSet = SequentialStoreBuffer.instance;
break;
}
case RemSetType.Cards: {
CardTable.Initialize();
installedRemSet = CardTable.instance;
break;
}
default: {
VTable.NotReached("Unsupported remembered set: " +
GC.remsetType);
break;
}
}
}
private static UIntPtr FirstPtrFromInteriorTable(UIntPtr c)
{
UIntPtr cardAddr = CardTable.CardAddr(c);
UIntPtr nextCardAddr = CardTable.NextCardAddr(c);
UIntPtr page = PageTable.Page(cardAddr);
UIntPtr pageAddr = PageTable.PageAddr(page);
UIntPtr currAddr;
if (page == 0)
{
currAddr = PtrToNextObject(pageAddr,
(UIntPtr)PreHeader.Size, nextCardAddr);
}
else
{
short offset = PageTable.Extra(page);
currAddr = UIntPtr.Zero;
if (offset != InteriorPtrTable.OFFSET_NO_DATA)
{
currAddr = pageAddr + (offset - InteriorPtrTable.OFFSET_SKEW);
}
// In general, we expect currAddr <= cardAddr. Or in the extreme
// case, when the object starts from the page boundary,
// currAddr - Object.HEADER_BYTES <= cardAddr. The contrary
// cases has to be handled by searching previous pages.
if (currAddr == UIntPtr.Zero ||
(currAddr > cardAddr &&
currAddr - PreHeader.Size > cardAddr))
{
// look from previous pages, in case that an object on
// them spans to the current page. In that case, we should
// should use that object's ptr.
currAddr = InteriorPtrTable.Last(page - 1);
// Usually, Last() returns a pointer before or at the page
// boundary. However, there is one exception: when an object
// exactly fits to the last byte of the previous page, and the next
// object starts right from the page boundary (the first byte of
// the next page), then the pointer to this next object is returned.
// Example found: objPtr =3d09fa8, size=60, pageboundary=
// 3d0a000, next objPtr=3d0a008. Then returned pointer is
// 3d0a008, which is beyond the page boundary.
VTable.Assert(currAddr <= pageAddr ||
currAddr - PreHeader.Size <= pageAddr,
"object is expected before page or right at the beginning of it");
}
}
VTable.Assert(currAddr < nextCardAddr, "object is expected before next card");
while (currAddr < nextCardAddr)
{
if (Allocator.IsAlignment(currAddr))
{
currAddr += UIntPtr.Size;
}
else if (BumpAllocator.IsUnusedSpace(currAddr))
{
currAddr = PageTable.PagePad(currAddr) + PreHeader.Size;
}
else
{
UIntPtr size = InteriorPtrTable.ObjectSize(currAddr);
if (currAddr + size - PreHeader.Size > cardAddr)
{
return(currAddr);
}
currAddr += size;
}
}
VTable.Assert(false, "No obj ptr found by looking at interior table");
return(UIntPtr.Zero);
}
internal static UIntPtr FirstObjPtrWithFieldInCard(UIntPtr c)
{
VTable.Assert(CardTable.IsValidCardNo(c), "Not valid card no");
VTable.Assert(CardTable.IsMyLiveGcCard(c), "Not my live GC card");
UIntPtr cardAddr = CardTable.CardAddr(c);
UIntPtr nextCardAddr = CardTable.NextCardAddr(c);
// Scan backward. May go to zombie areas
UIntPtr cardBefore = c - 1;
while (cardBefore >= CardTable.FirstCardNo &&
OffsetTable.NoObjectPtrToTheCard(cardBefore) &&
CardTable.IsMyGcCard(cardBefore))
{
cardBefore--;
}
UIntPtr ptr;
if (cardBefore < CardTable.FirstCardNo ||
!CardTable.IsMyGcCard(cardBefore))
{
// this case happens when c is the first live card that has an object.
VTable.Assert(CardTable.IsMyGcCard(cardBefore + 1),
"The next card must be a GC card");
VTable.Assert(CardTable.CardAddr(cardBefore + 1) ==
CardTable.PageAddrOfCard(cardBefore + 1),
"The next card must be at the boundary of a page");
UIntPtr pageAddr = CardTable.PageAddrOfCard(cardBefore + 1);
ptr = PtrToNextObject(pageAddr, (UIntPtr)PreHeader.Size, nextCardAddr);
}
else
{
VTable.Assert(!OffsetTable.NoObjectPtrToTheCard(cardBefore),
"A card with an object must have been found");
ptr = CardTable.CardAddr(cardBefore) +
OffsetTable.GetOffset(cardBefore);
}
VTable.Assert(ptr < nextCardAddr, "No objptr in this card");
// Scan forward. Update offset table in the course.
UIntPtr currentPtr = ptr;
while (currentPtr < cardAddr)
{
UIntPtr size = OffsetTable.ObjectSize(currentPtr);
if (currentPtr + size - PreHeader.Size > cardAddr)
{
VTable.Assert(CardTable.CardGeneration(CardTable.CardNo(currentPtr)) ==
CardTable.CardGeneration(c),
"The whole object must be in the same generation");
break;
}
else
{
ptr = currentPtr;
currentPtr = PtrToNextObject(currentPtr, size, nextCardAddr);
VTable.Assert(CardTable.CardNo(ptr) <= CardTable.CardNo(currentPtr),
"Previous ptr should be before current ptr");
if (CardTable.CardNo(ptr) < CardTable.CardNo(currentPtr))
{
UIntPtr ptrC = CardTable.CardNo(ptr);
UIntPtr offsetC = ptr - CardTable.CardAddr(ptrC);
if (offsetC > GetOffset(ptrC))
{
SetOffset(ptrC, offsetC);
}
}
}
}
VTable.Assert(currentPtr != UIntPtr.Zero, "current ptr is zero");
VTable.Assert(currentPtr < nextCardAddr, "No objptr found in this card");
#if DEBUG_OFFSETTABLE
UIntPtr temp = OffsetTable.FirstPtrFromInteriorTable(c);
if (temp != currentPtr)
{
VTable.DebugPrint("Found ptr ({0:x8}) inconsistent with first ptr from interior table({1:x8})\n",
__arglist(currentPtr, temp));
VTable.Assert(false);
}
#endif
return(currentPtr);
}
internal static bool NoObjectPtrToTheCard(UIntPtr c)
{
VTable.Assert(CardTable.IsValidCardNo(c), "NoObjectPtrToTheCard: invalid card no");
return(GetCompressedBiasedOffset(c) == NoObjPtrToCard);
}
private static void SetNoObjPtrToCard(UIntPtr c)
{
VTable.Assert(CardTable.IsValidCardNo(c), "SetNoObjPtrToCard: invalid card no");
SetCompressedBiasedOffset(c, NoObjPtrToCard);
}
private static void CleanPageTail(UIntPtr postPinnedAddr)
{
if (!PageTable.PageAligned(postPinnedAddr))
{
// If postPinnedAddr points to the first object on its page,
// then we are removing all objects (specifically the part
// of the object that the InteriorPtrTable tracks, the
// vtables) from the page, so we should clear the page's
// entry in the InteriorPtrTable.
UIntPtr page = PageTable.Page(postPinnedAddr);
UIntPtr firstObjPtr = InteriorPtrTable.First(page);
if (firstObjPtr > postPinnedAddr)
{
VTable.Assert
(firstObjPtr - PreHeader.Size >= postPinnedAddr,
"postPinnedAddr should not point to the "
+ "interior of an object (1)");
InteriorPtrTable.ClearFirst(page);
}
else if (!BumpAllocator.IsUnusedSpace(firstObjPtr))
{
UIntPtr firstObjSize =
InteriorPtrTable.ObjectSize(firstObjPtr);
VTable.Assert
(firstObjPtr + firstObjSize - PreHeader.Size
<= postPinnedAddr,
"postPinnedAddr should not point to the "
+ "interior of an object (2)");
}
UIntPtr byteCount = PageTable.PagePad(postPinnedAddr)
- postPinnedAddr;
Util.MemClear(postPinnedAddr, byteCount);
BumpAllocator.WriteUnusedMarker(postPinnedAddr);
if (GC.remsetType == RemSetType.Cards && byteCount > 0)
{
UIntPtr firstCard = CardTable.CardNo(postPinnedAddr);
UIntPtr lastCard =
CardTable.CardNo(postPinnedAddr + byteCount - 1);
if (!OffsetTable.NoObjectPtrToTheCard(firstCard))
{
UIntPtr offset = OffsetTable.GetOffset(firstCard);
UIntPtr objPtr =
CardTable.CardAddr(firstCard) + offset;
UIntPtr size = OffsetTable.ObjectSize(objPtr);
VTable.Assert
((objPtr + size - PreHeader.Size
<= postPinnedAddr) ||
(objPtr >= postPinnedAddr),
"Object should be totally "
+ "above or below postPinnedAddr");
if (objPtr >= postPinnedAddr)
{
OffsetTable.ClearCards(firstCard, firstCard);
}
}
OffsetTable.ClearCards(firstCard + 1, lastCard);
}
}
}
internal void CleanPinnedPages()
{
if (pinnedPageList == null || pinnedPageList.Count == 0)
{
return;
}
int pageIndex = 0;
int limit = pinnedPageList.Count;
UIntPtr lastPostPinnedAddr = UIntPtr.Zero;
while (pageIndex < limit)
{
UIntPtr startPage = (UIntPtr)pinnedPageList[pageIndex];
UIntPtr endPage = startPage + 1;
pageIndex++;
while (pageIndex < limit &&
(UIntPtr)pinnedPageList[pageIndex] == endPage)
{
pageIndex++;
endPage++;
}
// Zero out the area between the start of the page and
// the first object on the page
UIntPtr firstObjectAddr = FirstPinnedObjectAddr(startPage);
UIntPtr firstAddr = firstObjectAddr - PreHeader.Size;
UIntPtr trashAddr = PageTable.PageAddr(startPage);
if (firstAddr < trashAddr)
{
// The first object "spills" into the previous page,
// presumably by no more than HEADER_BYTES bytes
VTable.Assert(
PageTable.Page(firstAddr) == startPage - 1,
"Semispace:RegisterPinnedReference:3");
// Prepare to zero the preceding page unless it also
// had pinned data on it
trashAddr = PageTable.PageAddr(startPage - 1);
InteriorPtrTable.ClearFirst(startPage - 1);
if (trashAddr >= lastPostPinnedAddr)
{
// Need to mark the spilled-onto page live to
// keep the spilled data around
PageType fromSpaceType =
PageTable.Type(startPage - 1);
VTable.Assert(
PageTable.IsZombiePage(fromSpaceType),
"Semispace:RegisterPinnedReference:4");
PageType toSpaceType =
PageTable.ZombieToLive(fromSpaceType);
PageTable.SetType(startPage - 1, toSpaceType);
}
}
// If lastPostPinnedAddr is on the page that trashAddr
// starts, pinned data from the last run of pinned pages
// and pinned data from this run of pinned data are on the
// same page, so just write alignment tokens from
// lastPostPinnedAddr to the first pinned object.
// Otherwise, write an unused marker at lastPostPinnedAddr
// since the rest of its page must be copied or dead.
if (trashAddr < lastPostPinnedAddr)
{
trashAddr = lastPostPinnedAddr;
}
else
{
CleanPageTail(lastPostPinnedAddr);
}
if (GC.remsetType == RemSetType.Cards &&
trashAddr < firstAddr)
{
UIntPtr firstCard = CardTable.CardNo(trashAddr);
UIntPtr lastCard = CardTable.CardNo(firstAddr - 1);
if (!OffsetTable.NoObjectPtrToTheCard(firstCard))
{
UIntPtr offset = OffsetTable.GetOffset(firstCard);
UIntPtr objPtr =
CardTable.CardAddr(firstCard) + offset;
UIntPtr size = OffsetTable.ObjectSize(objPtr);
VTable.Assert
((objPtr + size - PreHeader.Size <= trashAddr) ||
(objPtr >= trashAddr),
"Object should be totally "
+ "above or below trashAddr");
if (objPtr >= trashAddr)
{
// The offset in this card needs to be updated
OffsetTable.ClearCards(firstCard, firstCard);
}
}
OffsetTable.ClearCards(firstCard + 1, lastCard - 1);
if (lastCard != CardTable.CardNo(firstObjectAddr))
{
OffsetTable.ClearCards(lastCard, lastCard);
}
else
{
VTable.Assert(OffsetTable.GetOffset(lastCard)
>= (firstObjectAddr
- CardTable.CardAddr(lastCard)),
"wrong offset");
}
}
{
// trashAddr should go back at most one page.
UIntPtr trashPage = PageTable.Page(trashAddr);
UIntPtr firstObjectAddrPage =
PageTable.Page(firstObjectAddr);
VTable.Assert((trashPage == firstObjectAddrPage - 1) ||
(trashPage == firstObjectAddrPage));
}
// If the InteriorPtrTable already had a value, then this is
// redundant, but if the call to First above has to compute
// the value, then (since it won't store it in the table) we
// should store it. Why? At this point the previous page
// would be "connected" to this one. After this collection
// the previous page will be unused or re-used and unrelated
// to this page and subsequent calls to First would then
// rely on it making the leap between unrelated pages.
InteriorPtrTable.SetFirst(firstObjectAddr);
while (trashAddr < firstAddr)
{
Allocator.WriteAlignment(trashAddr);
trashAddr += UIntPtr.Size;
}
// Zero out the area between the last whole object on
// the last page and the end of the last page
UIntPtr pastAddr = PostPinnedObjectAddr(endPage);
UIntPtr newEndPage =
PageTable.Page(PageTable.PagePad(pastAddr));
while (endPage < newEndPage)
{
// The last object spills into the next page(s), so
// mark those page(s) live
PageType fromPageType = PageTable.Type(endPage);
if (PageTable.IsZombiePage(fromPageType))
{
PageType toSpaceType =
PageTable.ZombieToLive(fromPageType);
PageTable.SetType(endPage, toSpaceType);
}
else
{
// final page might be live already because
// something else on it was pinned.
// pageIndex has already been incremented,
// so it points to the start of the next
// set of contiguous pages
VTable.Assert(
PageTable.IsLiveGcPage(fromPageType) &&
pageIndex < limit &&
endPage ==
(UIntPtr)pinnedPageList[pageIndex],
"Semispace:RegisterPinnedReference:5");
}
++endPage;
}
lastPostPinnedAddr = pastAddr;
}
CleanPageTail(lastPostPinnedAddr);
pinnedPageList = null;
comparer = null;
}