/// <summary>
/// Get next record in iterator
/// </summary>
/// <param name="key"></param>
/// <param name="value"></param>
/// <returns></returns>
public bool GetNext(out Key key, out Value value)
{
key = default(Key);
value = default(Value);
while (true)
{
// Check for boundary conditions
if (currentAddress >= endAddress)
{
return(false);
}
if (currentAddress < hlog.BeginAddress)
{
throw new Exception("Iterator address is less than log BeginAddress " + hlog.BeginAddress);
}
var currentPage = currentAddress >> hlog.LogPageSizeBits;
var currentFrame = currentPage % frameSize;
var offset = currentAddress & hlog.PageSizeMask;
if (currentAddress < hlog.HeadAddress)
{
BufferAndLoad(currentAddress, currentPage, currentFrame);
}
var recordSize = hlog.GetRecordSize(hlog.GetPhysicalAddress(currentAddress));
// Check if record fits on page, if not skip to next page
if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize)
{
currentAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits;
continue;
}
if (currentAddress >= hlog.HeadAddress)
{
// Read record from cached page memory
var _physicalAddress = hlog.GetPhysicalAddress(currentAddress);
key = hlog.GetKey(_physicalAddress);
value = hlog.GetValue(_physicalAddress);
currentAddress += hlog.GetRecordSize(_physicalAddress);
return(true);
}
var physicalAddress = frame.GetPhysicalAddress(currentFrame, offset);
key = hlog.GetKey(physicalAddress);
value = hlog.GetValue(physicalAddress);
currentAddress += hlog.GetRecordSize(physicalAddress);
return(true);
}
}
/// <summary>
/// Get next record in iterator
/// </summary>
/// <param name="recordInfo"></param>
/// <returns></returns>
public unsafe bool GetNext(out RecordInfo recordInfo)
{
recordInfo = default;
while (true)
{
currentAddress = nextAddress;
// Check for boundary conditions
if (currentAddress >= endAddress)
{
return(false);
}
epoch?.Resume();
var headAddress = hlog.HeadAddress;
if (currentAddress < hlog.BeginAddress && !forceInMemory)
{
epoch?.Suspend();
throw new FasterException("Iterator address is less than log BeginAddress " + hlog.BeginAddress);
}
if (frameSize == 0 && currentAddress < headAddress && !forceInMemory)
{
epoch?.Suspend();
throw new FasterException("Iterator address is less than log HeadAddress in memory-scan mode");
}
var currentPage = currentAddress >> hlog.LogPageSizeBits;
var offset = currentAddress & hlog.PageSizeMask;
if (currentAddress < headAddress && !forceInMemory)
{
BufferAndLoad(currentAddress, currentPage, currentPage % frameSize, headAddress, endAddress);
}
long physicalAddress;
if (currentAddress >= headAddress || forceInMemory)
{
physicalAddress = hlog.GetPhysicalAddress(currentAddress);
}
else
{
physicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset);
}
// Check if record fits on page, if not skip to next page
var recordSize = hlog.GetRecordSize(physicalAddress).Item2;
if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize)
{
nextAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits;
epoch?.Suspend();
continue;
}
nextAddress = currentAddress + recordSize;
ref var info = ref hlog.GetInfo(physicalAddress);
if (info.SkipOnScan || info.IsNull())
{
epoch?.Suspend();
continue;
}
currentPhysicalAddress = physicalAddress;
recordInfo = info;
if (currentAddress >= headAddress || forceInMemory)
{
memory?.Return();
memory = hlog.bufferPool.Get(recordSize);
Buffer.MemoryCopy((byte *)currentPhysicalAddress, memory.aligned_pointer, recordSize, recordSize);
currentPhysicalAddress = (long)memory.aligned_pointer;
}
epoch?.Suspend();
return(true);
}
/// <summary>
/// Get next record
/// </summary>
/// <param name="recordInfo"></param>
/// <returns>True if record found, false if end of scan</returns>
public bool GetNext(out RecordInfo recordInfo)
{
recordInfo = default;
while (true)
{
currentAddress = nextAddress;
// Check for boundary conditions
if (currentAddress >= endAddress)
{
return(false);
}
if (currentAddress < hlog.BeginAddress)
{
throw new FasterException("Iterator address is less than log BeginAddress " + hlog.BeginAddress);
}
if (frameSize == 0 && currentAddress < hlog.HeadAddress)
{
throw new FasterException("Iterator address is less than log HeadAddress in memory-scan mode");
}
var currentPage = currentAddress >> hlog.LogPageSizeBits;
var offset = currentAddress & hlog.PageSizeMask;
if (currentAddress < hlog.HeadAddress)
{
BufferAndLoad(currentAddress, currentPage, currentPage % frameSize);
}
long physicalAddress;
if (currentAddress >= hlog.HeadAddress)
{
physicalAddress = hlog.GetPhysicalAddress(currentAddress);
}
else
{
physicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset);
}
// Check if record fits on page, if not skip to next page
var recordSize = hlog.GetRecordSize(physicalAddress);
if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize)
{
nextAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits;
continue;
}
nextAddress = currentAddress + recordSize;
ref var info = ref hlog.GetInfo(physicalAddress);
if (info.Invalid || info.IsNull())
{
continue;
}
currentPhysicalAddress = physicalAddress;
recordInfo = info;
return(true);
}
/// <summary>
/// Get next entry
/// </summary>
/// <param name="physicalAddress"></param>
/// <param name="entryLength"></param>
/// <param name="type"></param>
/// <returns></returns>
public unsafe bool GetNext(out long physicalAddress, out int entryLength, out int type)
{
while (true)
{
physicalAddress = 0;
entryLength = 0;
currentAddress = nextAddress;
type = 0;
var _currentPage = currentAddress >> LogPageSizeBits;
var _currentFrame = _currentPage % frameSize;
var _currentOffset = currentAddress & PageSizeMask;
var _headAddress = long.MaxValue;
if (disposed)
{
return(false);
}
var _endAddress = endAddress;
if (tailAddress > _endAddress)
{
_endAddress = tailAddress;
}
if (currentAddress >= _endAddress)
{
return(false);
}
if (BufferAndLoad(currentAddress, _currentPage, _currentFrame, _headAddress, _endAddress))
{
continue;
}
physicalAddress = frame.GetPhysicalAddress(_currentFrame, _currentOffset);
// Get and check entry length
entryLength = GetHeader(physicalAddress).Length;
type = GetHeader(physicalAddress).Type;
if (entryLength == 0)
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
return(false);
}
else
{
continue;
}
}
int recordSize = (int)(Align(_currentOffset + HeaderSize + entryLength) - _currentOffset);
if (entryLength < 0 || (_currentOffset + recordSize > PageSize))
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
return(false);
}
else
{
continue;
}
}
// Verify checksum
if (!VerifyBlockChecksum((byte *)physicalAddress, entryLength))
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
return(false);
}
else
{
continue;
}
}
physicalAddress += HeaderSize;
if ((currentAddress & PageSizeMask) + recordSize == PageSize)
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
}
else
{
currentAddress += recordSize;
}
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out long oldCurrentAddress))
{
currentAddress = oldCurrentAddress;
return(true);
}
}
}
/// <summary>
/// Retrieve physical address of next iterator value
/// (under epoch protection if it is from main page buffer)
/// </summary>
/// <param name="physicalAddress"></param>
/// <param name="entryLength"></param>
/// <param name="currentAddress"></param>
/// <param name="nextAddress"></param>
/// <returns></returns>
private unsafe bool GetNextInternal(out long physicalAddress, out int entryLength, out long currentAddress, out long nextAddress)
{
while (true)
{
physicalAddress = 0;
entryLength = 0;
currentAddress = NextAddress;
nextAddress = NextAddress;
// Check for boundary conditions
if (currentAddress < allocator.BeginAddress)
{
currentAddress = allocator.BeginAddress;
}
var _currentPage = currentAddress >> allocator.LogPageSizeBits;
var _currentFrame = _currentPage % frameSize;
var _currentOffset = currentAddress & allocator.PageSizeMask;
var _headAddress = allocator.HeadAddress;
if (disposed)
{
return(false);
}
if ((currentAddress >= endAddress) || (currentAddress >= fasterLog.CommittedUntilAddress))
{
return(false);
}
if (currentAddress < _headAddress)
{
if (BufferAndLoad(currentAddress, _currentPage, _currentFrame, _headAddress))
{
continue;
}
physicalAddress = frame.GetPhysicalAddress(_currentFrame, _currentOffset);
}
else
{
physicalAddress = allocator.GetPhysicalAddress(currentAddress);
}
// Get and check entry length
entryLength = fasterLog.GetLength((byte *)physicalAddress);
if (entryLength == 0)
{
// We are likely at end of page, skip to next
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
Utility.MonotonicUpdate(ref NextAddress, currentAddress, out _);
if (0 != fasterLog.GetChecksum((byte *)physicalAddress))
{
epoch.Suspend();
var curPage = currentAddress >> allocator.LogPageSizeBits;
throw new FasterException("Invalid checksum found during scan, skipping page " + curPage);
}
else
{
continue;
}
}
int recordSize = headerSize + Align(entryLength);
if (entryLength < 0 || (_currentOffset + recordSize > allocator.PageSize))
{
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
if (Utility.MonotonicUpdate(ref NextAddress, currentAddress, out _))
{
epoch.Suspend();
throw new FasterException("Invalid length of record found: " + entryLength + " at address " + currentAddress + ", skipping page");
}
else
{
continue;
}
}
// Verify checksum if needed
if (currentAddress < _headAddress)
{
if (!fasterLog.VerifyChecksum((byte *)physicalAddress, entryLength))
{
var curPage = currentAddress >> allocator.LogPageSizeBits;
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
if (Utility.MonotonicUpdate(ref NextAddress, currentAddress, out _))
{
epoch.Suspend();
throw new FasterException("Invalid checksum found during scan, skipping page " + curPage);
}
else
{
continue;
}
}
}
if ((currentAddress & allocator.PageSizeMask) + recordSize == allocator.PageSize)
{
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
}
else
{
currentAddress += recordSize;
}
if (Utility.MonotonicUpdate(ref NextAddress, currentAddress, out long oldCurrentAddress))
{
nextAddress = currentAddress;
currentAddress = oldCurrentAddress;
return(true);
}
}
}
/// <summary>
/// Retrieve physical address of next iterator value
/// (under epoch protection if it is from main page buffer)
/// </summary>
/// <param name="physicalAddress"></param>
/// <param name="entryLength"></param>
/// <param name="epochTaken"></param>
/// <returns></returns>
private unsafe bool GetNextInternal(out long physicalAddress, out int entryLength, out bool epochTaken)
{
physicalAddress = 0;
entryLength = 0;
epochTaken = false;
currentAddress = nextAddress;
while (true)
{
// Check for boundary conditions
if (currentAddress < allocator.BeginAddress)
{
Debug.WriteLine("Iterator address is less than log BeginAddress " + allocator.BeginAddress + ", adjusting iterator address");
currentAddress = allocator.BeginAddress;
}
if ((currentAddress >= endAddress) || (currentAddress >= fasterLog.CommittedUntilAddress))
{
nextAddress = currentAddress;
return(false);
}
if (frameSize == 0 && currentAddress < allocator.HeadAddress)
{
throw new Exception("Iterator address is less than log HeadAddress in memory-scan mode");
}
var currentPage = currentAddress >> allocator.LogPageSizeBits;
var offset = currentAddress & allocator.PageSizeMask;
var headAddress = allocator.HeadAddress;
if (currentAddress < headAddress)
{
BufferAndLoad(currentAddress, currentPage, currentPage % frameSize);
physicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset);
}
else
{
epoch.Resume();
headAddress = allocator.HeadAddress;
if (currentAddress < headAddress) // rare case
{
epoch.Suspend();
continue;
}
physicalAddress = allocator.GetPhysicalAddress(currentAddress);
}
// Get and check entry length
entryLength = fasterLog.GetLength((byte *)physicalAddress);
if (entryLength == 0)
{
if (currentAddress >= headAddress)
{
epoch.Suspend();
}
nextAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
if (0 != fasterLog.GetChecksum((byte *)physicalAddress))
{
var curPage = currentAddress >> allocator.LogPageSizeBits;
throw new Exception("Invalid checksum found during scan, skipping page " + curPage);
}
else
{
// We are likely at end of page, skip to next
currentAddress = nextAddress;
continue;
}
}
int recordSize = headerSize + Align(entryLength);
if ((currentAddress & allocator.PageSizeMask) + recordSize > allocator.PageSize)
{
if (currentAddress >= headAddress)
{
epoch.Suspend();
}
nextAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
throw new Exception("Invalid length of record found: " + entryLength + ", skipping page");
}
// Verify checksum if needed
if (currentAddress < headAddress)
{
if (!fasterLog.VerifyChecksum((byte *)physicalAddress, entryLength))
{
var curPage = currentAddress >> allocator.LogPageSizeBits;
nextAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
throw new Exception("Invalid checksum found during scan, skipping page " + curPage);
}
}
if ((currentAddress & allocator.PageSizeMask) + recordSize == allocator.PageSize)
{
nextAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
}
else
{
nextAddress = currentAddress + recordSize;
}
epochTaken = currentAddress >= headAddress;
return(true);
}
}
/// <summary>
/// Retrieve physical address of next iterator value
/// (under epoch protection if it is from main page buffer)
/// </summary>
/// <param name="physicalAddress"></param>
/// <param name="entryLength"></param>
/// <param name="currentAddress"></param>
/// <param name="outNextAddress"></param>
/// <param name="commitRecord"></param>
/// <returns></returns>
private unsafe bool GetNextInternal(out long physicalAddress, out int entryLength, out long currentAddress, out long outNextAddress, out bool commitRecord)
{
while (true)
{
physicalAddress = 0;
entryLength = 0;
currentAddress = nextAddress;
outNextAddress = nextAddress;
commitRecord = false;
// Check for boundary conditions
if (currentAddress < allocator.BeginAddress)
{
currentAddress = allocator.BeginAddress;
}
var _currentPage = currentAddress >> allocator.LogPageSizeBits;
var _currentFrame = _currentPage % frameSize;
var _currentOffset = currentAddress & allocator.PageSizeMask;
var _headAddress = allocator.HeadAddress;
if (disposed)
{
return(false);
}
if ((currentAddress >= endAddress) || (currentAddress >= (scanUncommitted ? fasterLog.SafeTailAddress : fasterLog.CommittedUntilAddress)))
{
return(false);
}
if (currentAddress < _headAddress)
{
var _endAddress = endAddress;
if (fasterLog.readOnlyMode)
{
// Support partial page reads of committed data
var _flush = fasterLog.CommittedUntilAddress;
if (_flush < endAddress)
{
_endAddress = _flush;
}
}
if (BufferAndLoad(currentAddress, _currentPage, _currentFrame, _headAddress, _endAddress))
{
continue;
}
physicalAddress = frame.GetPhysicalAddress(_currentFrame, _currentOffset);
}
else
{
physicalAddress = allocator.GetPhysicalAddress(currentAddress);
}
// Get and check entry length
entryLength = fasterLog.GetLength((byte *)physicalAddress);
// We may encounter zeroed out bits at the end of page in a normal log, therefore, we need to check
// whether that is the case
if (entryLength == 0)
{
// Zero-ed out bytes could be padding at the end of page, first jump to the start of next page.
var nextStart = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, nextStart, out _))
{
var pageOffset = currentAddress & ((1 << allocator.LogPageSizeBits) - 1);
// If zeroed out field is at page start, we encountered an uninitialized page and should signal up
if (pageOffset == 0)
{
throw new FasterException("Uninitialized page found during scan at page " + (currentAddress >> allocator.LogPageSizeBits));
}
}
continue;
}
// commit records have negative length fields
if (entryLength < 0)
{
commitRecord = true;
entryLength = -entryLength;
}
int recordSize = headerSize + Align(entryLength);
if (_currentOffset + recordSize > allocator.PageSize)
{
currentAddress += headerSize;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
throw new FasterException("Invalid length of record found: " + entryLength + " at address " + currentAddress);
}
continue;
}
// Verify checksum if needed
if (currentAddress < _headAddress)
{
if (!fasterLog.VerifyChecksum((byte *)physicalAddress, entryLength))
{
currentAddress += headerSize;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
throw new FasterException("Invalid checksum found during scan, skipping");
}
continue;
}
}
if ((currentAddress & allocator.PageSizeMask) + recordSize == allocator.PageSize)
{
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
}
else
{
currentAddress += recordSize;
}
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out long oldCurrentAddress))
{
outNextAddress = currentAddress;
currentAddress = oldCurrentAddress;
return(true);
}
}
}