private bool WaitForFrameLoad(long currentAddress, long currentFrame)
{
if (loaded[currentFrame].IsSet)
{
return(false);
}
try
{
epoch?.Suspend();
loaded[currentFrame].Wait(loadedCancel[currentFrame].Token); // Ensure we have completed ongoing load
}
catch (Exception e)
{
loadedPage[currentFrame] = -1;
loadedCancel[currentFrame] = new CancellationTokenSource();
Utility.MonotonicUpdate(ref nextAddress, (1 + (currentAddress >> logPageSizeBits)) << logPageSizeBits, out _);
throw new FasterException("Page read from storage failed, skipping page. Inner exception: " + e.ToString());
}
finally
{
epoch?.Resume();
}
return(true);
}
/// <summary>
/// Handle space utilization of limited capacity devices by invoking segment truncation if necessary
/// </summary>
/// <param name="segment">Segment being written to</param>
protected void HandleCapacity(int segment)
{
// If the device has bounded space, and we are writing a new segment, need to check whether an existing segment needs to be evicted.
if (Capacity != Devices.CAPACITY_UNSPECIFIED && Utility.MonotonicUpdate(ref endSegment, segment, out int oldEnd))
{
// Attempt to update the stored range until there is enough space on the tier to accomodate the current segment
int newStartSegment = endSegment - (int)(Capacity >> segmentSizeBits);
// Assuming that we still have enough physical capacity to write another segment, even if delete does not immediately free up space.
TruncateUntilSegmentAsync(newStartSegment, r => { }, null);
}
}
/// <summary>
///
/// </summary>
/// <param name="alignedSourceAddress"></param>
/// <param name="alignedDestinationAddress"></param>
/// <param name="numBytesToWrite"></param>
/// <param name="callback"></param>
/// <param name="asyncResult"></param>
public void WriteAsync(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite, IOCompletionCallback callback, IAsyncResult asyncResult)
{
int segment = (int)(segmentSizeBits < 64 ? alignedDestinationAddress >> segmentSizeBits : 0);
// If the device has bounded space, and we are writing a new segment, need to check whether an existing segment needs to be evicted.
if (Capacity != Devices.CAPACITY_UNSPECIFIED && Utility.MonotonicUpdate(ref endSegment, segment, out int oldEnd))
{
// Attempt to update the stored range until there are enough space on the tier to accomodate the current logTail
int newStartSegment = endSegment - (int)(Capacity >> segmentSizeBits);
// Assuming that we still have enough physical capacity to write another segment, even if delete does not immediately free up space.
TruncateUntilSegmentAsync(newStartSegment, r => { }, null);
}
WriteAsync(
alignedSourceAddress,
segment,
alignedDestinationAddress & segmentSizeMask,
numBytesToWrite, callback, asyncResult);
}
/// <summary>
/// Get next entry
/// </summary>
/// <param name="physicalAddress"></param>
/// <param name="entryLength"></param>
/// <param name="type"></param>
/// <returns></returns>
public unsafe bool GetNext(out long physicalAddress, out int entryLength, out int type)
{
while (true)
{
physicalAddress = 0;
entryLength = 0;
currentAddress = nextAddress;
type = 0;
var _currentPage = currentAddress >> LogPageSizeBits;
var _currentFrame = _currentPage % frameSize;
var _currentOffset = currentAddress & PageSizeMask;
var _headAddress = long.MaxValue;
if (disposed)
{
return(false);
}
var _endAddress = endAddress;
if (tailAddress > _endAddress)
{
_endAddress = tailAddress;
}
if (currentAddress >= _endAddress)
{
return(false);
}
if (BufferAndLoad(currentAddress, _currentPage, _currentFrame, _headAddress, _endAddress))
{
continue;
}
physicalAddress = frame.GetPhysicalAddress(_currentFrame, _currentOffset);
// Get and check entry length
entryLength = GetHeader(physicalAddress).Length;
type = GetHeader(physicalAddress).Type;
if (entryLength == 0)
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
return(false);
}
else
{
continue;
}
}
int recordSize = (int)(Align(_currentOffset + HeaderSize + entryLength) - _currentOffset);
if (entryLength < 0 || (_currentOffset + recordSize > PageSize))
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
return(false);
}
else
{
continue;
}
}
// Verify checksum
if (!VerifyBlockChecksum((byte *)physicalAddress, entryLength))
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
return(false);
}
else
{
continue;
}
}
physicalAddress += HeaderSize;
if ((currentAddress & PageSizeMask) + recordSize == PageSize)
{
currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits;
}
else
{
currentAddress += recordSize;
}
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out long oldCurrentAddress))
{
currentAddress = oldCurrentAddress;
return(true);
}
}
}
/// <summary>
/// Retrieve physical address of next iterator value
/// (under epoch protection if it is from main page buffer)
/// </summary>
/// <param name="physicalAddress"></param>
/// <param name="entryLength"></param>
/// <param name="currentAddress"></param>
/// <param name="nextAddress"></param>
/// <returns></returns>
private unsafe bool GetNextInternal(out long physicalAddress, out int entryLength, out long currentAddress, out long nextAddress)
{
while (true)
{
physicalAddress = 0;
entryLength = 0;
currentAddress = NextAddress;
nextAddress = NextAddress;
// Check for boundary conditions
if (currentAddress < allocator.BeginAddress)
{
currentAddress = allocator.BeginAddress;
}
var _currentPage = currentAddress >> allocator.LogPageSizeBits;
var _currentFrame = _currentPage % frameSize;
var _currentOffset = currentAddress & allocator.PageSizeMask;
var _headAddress = allocator.HeadAddress;
if (disposed)
{
return(false);
}
if ((currentAddress >= endAddress) || (currentAddress >= fasterLog.CommittedUntilAddress))
{
return(false);
}
if (currentAddress < _headAddress)
{
if (BufferAndLoad(currentAddress, _currentPage, _currentFrame, _headAddress))
{
continue;
}
physicalAddress = frame.GetPhysicalAddress(_currentFrame, _currentOffset);
}
else
{
physicalAddress = allocator.GetPhysicalAddress(currentAddress);
}
// Get and check entry length
entryLength = fasterLog.GetLength((byte *)physicalAddress);
if (entryLength == 0)
{
// We are likely at end of page, skip to next
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
Utility.MonotonicUpdate(ref NextAddress, currentAddress, out _);
if (0 != fasterLog.GetChecksum((byte *)physicalAddress))
{
epoch.Suspend();
var curPage = currentAddress >> allocator.LogPageSizeBits;
throw new FasterException("Invalid checksum found during scan, skipping page " + curPage);
}
else
{
continue;
}
}
int recordSize = headerSize + Align(entryLength);
if (entryLength < 0 || (_currentOffset + recordSize > allocator.PageSize))
{
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
if (Utility.MonotonicUpdate(ref NextAddress, currentAddress, out _))
{
epoch.Suspend();
throw new FasterException("Invalid length of record found: " + entryLength + " at address " + currentAddress + ", skipping page");
}
else
{
continue;
}
}
// Verify checksum if needed
if (currentAddress < _headAddress)
{
if (!fasterLog.VerifyChecksum((byte *)physicalAddress, entryLength))
{
var curPage = currentAddress >> allocator.LogPageSizeBits;
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
if (Utility.MonotonicUpdate(ref NextAddress, currentAddress, out _))
{
epoch.Suspend();
throw new FasterException("Invalid checksum found during scan, skipping page " + curPage);
}
else
{
continue;
}
}
}
if ((currentAddress & allocator.PageSizeMask) + recordSize == allocator.PageSize)
{
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
}
else
{
currentAddress += recordSize;
}
if (Utility.MonotonicUpdate(ref NextAddress, currentAddress, out long oldCurrentAddress))
{
nextAddress = currentAddress;
currentAddress = oldCurrentAddress;
return(true);
}
}
}
internal void UpdateCompletedUntilAddress(long address)
{
Utility.MonotonicUpdate(ref CompletedUntilAddress, address, out _);
}
/// <summary>
/// Mark iterator complete until specified address. Info is not
/// persisted until a subsequent commit operation on the log.
/// </summary>
/// <param name="address"></param>
public void CompleteUntil(long address)
{
Utility.MonotonicUpdate(ref requestedCompletedUntilAddress, address, out _);
}
/// <summary>
/// Retrieve physical address of next iterator value
/// (under epoch protection if it is from main page buffer)
/// </summary>
/// <param name="physicalAddress"></param>
/// <param name="entryLength"></param>
/// <param name="currentAddress"></param>
/// <param name="outNextAddress"></param>
/// <param name="commitRecord"></param>
/// <returns></returns>
private unsafe bool GetNextInternal(out long physicalAddress, out int entryLength, out long currentAddress, out long outNextAddress, out bool commitRecord)
{
while (true)
{
physicalAddress = 0;
entryLength = 0;
currentAddress = nextAddress;
outNextAddress = nextAddress;
commitRecord = false;
// Check for boundary conditions
if (currentAddress < allocator.BeginAddress)
{
currentAddress = allocator.BeginAddress;
}
var _currentPage = currentAddress >> allocator.LogPageSizeBits;
var _currentFrame = _currentPage % frameSize;
var _currentOffset = currentAddress & allocator.PageSizeMask;
var _headAddress = allocator.HeadAddress;
if (disposed)
{
return(false);
}
if ((currentAddress >= endAddress) || (currentAddress >= (scanUncommitted ? fasterLog.SafeTailAddress : fasterLog.CommittedUntilAddress)))
{
return(false);
}
if (currentAddress < _headAddress)
{
var _endAddress = endAddress;
if (fasterLog.readOnlyMode)
{
// Support partial page reads of committed data
var _flush = fasterLog.CommittedUntilAddress;
if (_flush < endAddress)
{
_endAddress = _flush;
}
}
if (BufferAndLoad(currentAddress, _currentPage, _currentFrame, _headAddress, _endAddress))
{
continue;
}
physicalAddress = frame.GetPhysicalAddress(_currentFrame, _currentOffset);
}
else
{
physicalAddress = allocator.GetPhysicalAddress(currentAddress);
}
// Get and check entry length
entryLength = fasterLog.GetLength((byte *)physicalAddress);
// We may encounter zeroed out bits at the end of page in a normal log, therefore, we need to check
// whether that is the case
if (entryLength == 0)
{
// Zero-ed out bytes could be padding at the end of page, first jump to the start of next page.
var nextStart = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
if (Utility.MonotonicUpdate(ref nextAddress, nextStart, out _))
{
var pageOffset = currentAddress & ((1 << allocator.LogPageSizeBits) - 1);
// If zeroed out field is at page start, we encountered an uninitialized page and should signal up
if (pageOffset == 0)
{
throw new FasterException("Uninitialized page found during scan at page " + (currentAddress >> allocator.LogPageSizeBits));
}
}
continue;
}
// commit records have negative length fields
if (entryLength < 0)
{
commitRecord = true;
entryLength = -entryLength;
}
int recordSize = headerSize + Align(entryLength);
if (_currentOffset + recordSize > allocator.PageSize)
{
currentAddress += headerSize;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
throw new FasterException("Invalid length of record found: " + entryLength + " at address " + currentAddress);
}
continue;
}
// Verify checksum if needed
if (currentAddress < _headAddress)
{
if (!fasterLog.VerifyChecksum((byte *)physicalAddress, entryLength))
{
currentAddress += headerSize;
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _))
{
throw new FasterException("Invalid checksum found during scan, skipping");
}
continue;
}
}
if ((currentAddress & allocator.PageSizeMask) + recordSize == allocator.PageSize)
{
currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits;
}
else
{
currentAddress += recordSize;
}
if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out long oldCurrentAddress))
{
outNextAddress = currentAddress;
currentAddress = oldCurrentAddress;
return(true);
}
}
}