private static async ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > SlowReadAsync <Input, Output, Context, Functions>(
FasterKV <Key, Value> @this,
ClientSession <Key, Value, Input, Output, Context, Functions> clientSession,
PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, CancellationToken token = default)
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
clientSession.ctx.asyncPendingCount++;
clientSession.ctx.pendingReads.Add();
ExceptionDispatchInfo exceptionDispatchInfo = default;
try
{
token.ThrowIfCancellationRequested();
if (@this.epoch.ThisInstanceProtected())
{
throw new NotSupportedException("Async operations not supported over protected epoch");
}
using (token.Register(() => diskRequest.asyncOperation.TrySetCanceled()))
diskRequest = await diskRequest.asyncOperation.Task;
}
catch (Exception e)
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(e);
}
return(new ReadAsyncResult <Input, Output, Context, Functions>(@this, clientSession, pendingContext, diskRequest, exceptionDispatchInfo));
}
private static async ValueTask <ReadAsyncResult <Input, Output, Context> > SlowReadAsync <Input, Output, Context>(
FasterKV <Key, Value> @this,
IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx,
PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, CancellationToken token = default)
{
currentCtx.asyncPendingCount++;
currentCtx.pendingReads.Add();
ExceptionDispatchInfo exceptionDispatchInfo = default;
try
{
token.ThrowIfCancellationRequested();
if (@this.epoch.ThisInstanceProtected())
{
throw new NotSupportedException("Async operations not supported over protected epoch");
}
using (token.Register(() => diskRequest.asyncOperation.TrySetCanceled()))
diskRequest = await diskRequest.asyncOperation.Task.WithCancellationAsync(token).ConfigureAwait(false);
}
catch (Exception e)
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(e);
}
return(new ReadAsyncResult <Input, Output, Context>(@this, fasterSession, currentCtx, pendingContext, diskRequest, exceptionDispatchInfo));
}
private static async ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > SlowReadAsync <Input, Output, Context, Functions>(
FasterKV <Key, Value> @this,
ClientSession <Key, Value, Input, Output, Context, Functions> clientSession,
PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, CancellationToken token = default)
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
clientSession.ctx.asyncPendingCount++;
clientSession.ctx.pendingReads.Add();
try
{
token.ThrowIfCancellationRequested();
if (@this.epoch.ThisInstanceProtected())
{
throw new NotSupportedException("Async operations not supported over protected epoch");
}
diskRequest = await diskRequest.asyncOperation.Task;
}
catch
{
clientSession.ctx.ioPendingRequests.Remove(pendingContext.id);
clientSession.ctx.asyncPendingCount--;
throw;
}
finally
{
clientSession.ctx.pendingReads.Remove();
}
return(new ReadAsyncResult <Input, Output, Context, Functions>(@this, clientSession, pendingContext, diskRequest));
}
internal async ValueTask <RmwAsyncResult <Input, Output, Context> > CompleteAsync(CancellationToken token = default)
{
Debug.Assert(_fasterKV.RelaxedCPR);
AsyncIOContext <Key, Value> newDiskRequest = default;
if ((_diskRequest.asyncOperation != null || _flushTask is { }) &&
internal ReadAsyncResult(Status status, Output output)
{
_exception = default;
_result = (status, output);
_fasterKV = default;
_clientSession = default;
_pendingContext = default;
_diskRequest = default;
}
internal ReadAsyncResult(
FasterKV <Key, Value, Input, Output, Context, Functions> fasterKV,
ClientSession <Key, Value, Input, Output, Context, Functions> clientSession,
PendingContext pendingContext, AsyncIOContext <Key, Value> diskRequest)
{
status = Status.PENDING;
output = default;
readAsyncInternal = new ReadAsyncInternal(fasterKV, clientSession, pendingContext, diskRequest);
}
internal RmwAsyncResult(FasterKV <Key, Value> fasterKV, IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, PendingContext <Input, Output, Context> pendingContext,
AsyncIOContext <Key, Value> diskRequest, ExceptionDispatchInfo exceptionDispatchInfo)
{
Status = Status.PENDING;
output = default;
updateAsyncInternal = new UpdateAsyncInternal <Input, Output, Context, RmwAsyncOperation <Input, Output, Context>, RmwAsyncResult <Input, Output, Context> >(
fasterKV, fasterSession, currentCtx, pendingContext, exceptionDispatchInfo, new RmwAsyncOperation <Input, Output, Context>(diskRequest));
}
internal ReadAsyncInternal(FasterKV <Key, Value, Input, Output, Context, Functions> fasterKV, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, PendingContext pendingContext, AsyncIOContext <Key, Value> diskRequest)
{
_exception = default;
_fasterKV = fasterKV;
_clientSession = clientSession;
_pendingContext = pendingContext;
_diskRequest = diskRequest;
CompletionComputeStatus = Pending;
}
internal RmwAsyncResult(
FasterKV <Key, Value> fasterKV,
ClientSession <Key, Value, Input, Output, Context, Functions> clientSession,
PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, ExceptionDispatchInfo exceptionDispatchInfo)
{
status = Status.PENDING;
output = default;
rmwAsyncInternal = new RmwAsyncInternal <Input, Output, Context, Functions>(fasterKV, clientSession, pendingContext, diskRequest, exceptionDispatchInfo);
}
internal RmwAsyncInternal(FasterKV <Key, Value> fasterKV, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, ExceptionDispatchInfo exceptionDispatchInfo)
{
_exception = exceptionDispatchInfo;
_fasterKV = fasterKV;
_clientSession = clientSession;
_pendingContext = pendingContext;
_diskRequest = diskRequest;
CompletionComputeStatus = Pending;
}
internal ValueTask <RmwAsyncResult <Input, Output, Context, Functions> > CompleteAsync(CancellationToken token = default)
{
Debug.Assert(_fasterKV.RelaxedCPR);
AsyncIOContext <Key, Value> newDiskRequest = default;
if (_diskRequest.asyncOperation != null &&
CompletionComputeStatus != Completed &&
Interlocked.CompareExchange(ref CompletionComputeStatus, Completed, Pending) == Pending)
{
try
{
if (_exception == default)
{
if (_clientSession.SupportAsync)
{
_clientSession.UnsafeResumeThread();
}
try
{
var status = _fasterKV.InternalCompletePendingRequestFromContext(_clientSession.ctx, _clientSession.ctx, _clientSession.FasterSession, _diskRequest, ref _pendingContext, true, out newDiskRequest);
_pendingContext.Dispose();
if (status != Status.PENDING)
{
return(new ValueTask <RmwAsyncResult <Input, Output, Context, Functions> >(new RmwAsyncResult <Input, Output, Context, Functions>(status, default)));
}
}
finally
{
if (_clientSession.SupportAsync)
{
_clientSession.UnsafeSuspendThread();
}
}
}
}
catch (Exception e)
{
_exception = ExceptionDispatchInfo.Capture(e);
}
finally
{
_clientSession.ctx.ioPendingRequests.Remove(_pendingContext.id);
_clientSession.ctx.asyncPendingCount--;
_clientSession.ctx.pendingReads.Remove();
}
}
if (_exception != default)
{
_exception.Throw();
}
return(SlowRmwAsync(_fasterKV, _clientSession, _pendingContext, newDiskRequest, token));
}
internal ReadAsyncResult(
FasterKV <Key, Value> fasterKV,
IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx,
PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, ExceptionDispatchInfo exceptionDispatchInfo)
{
status = Status.PENDING;
output = default;
this.recordInfo = default;
readAsyncInternal = new ReadAsyncInternal <Input, Output, Context>(fasterKV, fasterSession, currentCtx, pendingContext, diskRequest, exceptionDispatchInfo);
}
internal ReadAsyncInternal(FasterKV <Key, Value> fasterKV, IFasterSession <Key, Value, Input, Output, Context> fasterSession, FasterExecutionContext <Input, Output, Context> currentCtx,
PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, ExceptionDispatchInfo exceptionDispatchInfo)
{
_exception = exceptionDispatchInfo;
_fasterKV = fasterKV;
_fasterSession = fasterSession;
_currentCtx = currentCtx;
_pendingContext = pendingContext;
_diskRequest = diskRequest;
CompletionComputeStatus = Pending;
_recordInfo = default;
}
private bool RetrievedObjects(byte *record, AsyncIOContext ctx)
{
if (!Key.HasObjectsToSerialize() && !Value.HasObjectsToSerialize())
{
return(true);
}
if (ctx.objBuffer.buffer == null)
{
// Issue IO for objects
long startAddress = -1;
long numBytes = 0;
if (Key.HasObjectsToSerialize())
{
var x = (AddressInfo *)Layout.GetKey((long)record);
numBytes += x->Size;
startAddress = x->Address;
}
if (Value.HasObjectsToSerialize())
{
var x = (AddressInfo *)Layout.GetValue((long)record);
numBytes += x->Size;
if (startAddress == -1)
{
startAddress = x->Address;
}
}
// We are limited to a 2GB size per key-value
if (numBytes > int.MaxValue)
{
throw new Exception("Size of key-value exceeds max of 2GB: " + numBytes);
}
AsyncGetFromDisk(startAddress, (int)numBytes,
AsyncGetFromDiskCallback, ctx, ctx.record);
return(false);
}
// Parse the key and value objects
MemoryStream ms = new MemoryStream(ctx.objBuffer.buffer);
ms.Seek(ctx.objBuffer.offset + ctx.objBuffer.valid_offset, SeekOrigin.Begin);
Key.Deserialize(Layout.GetKey((long)record), ms);
Value.Deserialize(Layout.GetValue((long)record), ms);
ctx.objBuffer.Return();
return(true);
}
private bool RetrievedObjects(byte *record, AsyncIOContext ctx)
{
if (!Key.HasObjectsToSerialize() && !Value.HasObjectsToSerialize())
{
return(true);
}
if (ctx.objBuffer.buffer == null)
{
// Issue IO for objects
long startAddress = -1;
int numBytes = 0;
if (Key.HasObjectsToSerialize())
{
var x = (AddressInfo *)Layout.GetKey((long)record);
if (x->IsDiskAddress)
{
numBytes += x->Size;
startAddress = x->Address;
}
}
if (Value.HasObjectsToSerialize())
{
var x = (AddressInfo *)Layout.GetValue((long)record);
if (x->IsDiskAddress)
{
numBytes += x->Size;
if (startAddress == -1)
{
startAddress = x->Address;
}
}
}
AsyncGetFromDisk(startAddress, numBytes,
AsyncGetFromDiskCallback, ctx, ctx.record);
return(false);
}
// Parse the key and value objects
MemoryStream ms = new MemoryStream(ctx.objBuffer.buffer);
ms.Seek(ctx.objBuffer.offset + ctx.objBuffer.valid_offset, SeekOrigin.Begin);
Key.Deserialize(Layout.GetKey((long)record), ms);
Value.Deserialize(Layout.GetValue((long)record), ms);
ctx.objBuffer.Return();
return(true);
}
private void AsyncGetFromDisk(long fromLogical,
int numRecords,
IOCompletionCallback callback,
AsyncIOContext context,
SectorAlignedMemory result = default(SectorAlignedMemory))
{
while (numPendingReads > 120)
{
Thread.SpinWait(100);
// Do not protect if we are not already protected
// E.g., we are in an IO thread
if (epoch.IsProtected())
epoch.ProtectAndDrain();
}
Interlocked.Increment(ref numPendingReads);
hlog.AsyncReadRecordToMemory(fromLogical, numRecords, callback, context, result);
}
protected void InternalContinuePendingRequestAndCallback(
ExecutionContext ctx,
AsyncIOContext request)
{
var handleLatches = false;
if ((ctx.version < threadCtx.version) // Thread has already shifted to (v+1)
||
(threadCtx.phase == Phase.PREPARE)) // Thread still in version v, but acquired shared-latch
{
handleLatches = true;
}
if (ctx.ioPendingRequests.TryGetValue(request.id, out PendingContext pendingContext))
{
var status = default(Status);
var internalStatus = default(OperationStatus);
// Remove from pending dictionary
ctx.ioPendingRequests.Remove(request.id);
// Issue the continue command
if (pendingContext.type == OperationType.READ)
{
internalStatus = InternalContinuePendingRead(ctx, request, ref pendingContext);
}
else
{
internalStatus = InternalContinuePendingRMW(ctx, request, ref pendingContext);;
}
// Delete key, value, record
if (Key.HasObjectsToSerialize())
{
var physicalAddress = (long)request.record.GetValidPointer();
Key.Free(Layout.GetKey(physicalAddress));
}
if (Value.HasObjectsToSerialize())
{
var physicalAddress = (long)request.record.GetValidPointer();
Value.Free(Layout.GetValue(physicalAddress));
}
request.record.Return();
// Handle operation status
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(ctx, pendingContext, internalStatus);
}
// If done, callback user code
if (status == Status.OK || status == Status.NOTFOUND)
{
if (handleLatches)
{
ReleaseSharedLatch(pendingContext.key);
}
if (pendingContext.type == OperationType.READ)
{
Functions.ReadCompletionCallback(pendingContext.key,
pendingContext.input,
pendingContext.output,
pendingContext.userContext,
status);
}
else
{
Functions.RMWCompletionCallback(pendingContext.key,
pendingContext.input,
pendingContext.userContext,
status);
}
}
}
}
internal void InternalContinuePendingRequestAndCallback(
FasterExecutionContext ctx,
AsyncIOContext <Key, Value> request)
{
var handleLatches = false;
if ((ctx.version < threadCtx.version) // Thread has already shifted to (v+1)
||
(threadCtx.phase == Phase.PREPARE)) // Thread still in version v, but acquired shared-latch
{
handleLatches = true;
}
if (ctx.ioPendingRequests.TryGetValue(request.id, out PendingContext pendingContext))
{
var status = default(Status);
var internalStatus = default(OperationStatus);
// Remove from pending dictionary
ctx.ioPendingRequests.Remove(request.id);
// Issue the continue command
if (pendingContext.type == OperationType.READ)
{
internalStatus = InternalContinuePendingRead(ctx, request, ref pendingContext);
}
else
{
internalStatus = InternalContinuePendingRMW(ctx, request, ref pendingContext);;
}
request.record.Return();
// Handle operation status
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(ctx, pendingContext, internalStatus);
}
// If done, callback user code
if (status == Status.OK || status == Status.NOTFOUND)
{
if (handleLatches)
{
ReleaseSharedLatch(pendingContext.key);
}
if (pendingContext.type == OperationType.READ)
{
functions.ReadCompletionCallback(ref pendingContext.key,
ref pendingContext.input,
ref pendingContext.output,
pendingContext.userContext,
status);
}
else
{
functions.RMWCompletionCallback(ref pendingContext.key,
ref pendingContext.input,
pendingContext.userContext,
status);
}
}
}
}
/// <summary>
/// Retrieve objects from object log
/// </summary>
/// <param name="record"></param>
/// <param name="ctx"></param>
/// <returns></returns>
protected override bool RetrievedFullRecord(byte *record, ref AsyncIOContext <Key, Value> ctx)
{
ShallowCopy(ref GetKey((long)record), ref ctx.key);
ShallowCopy(ref GetValue((long)record), ref ctx.value);
return(true);
}
/// <summary>
/// Invoked by users to obtain a record from disk. It uses sector aligned memory to read
/// the record efficiently into memory.
/// </summary>
/// <param name="fromLogical"></param>
/// <param name="numBytes"></param>
/// <param name="callback"></param>
/// <param name="context"></param>
/// <param name="result"></param>
protected override void AsyncReadRecordObjectsToMemory(long fromLogical, int numBytes, IOCompletionCallback callback, AsyncIOContext <Key, Value> context, SectorAlignedMemory result = default(SectorAlignedMemory))
{
throw new InvalidOperationException("AsyncReadRecordObjectsToMemory invalid for BlittableAllocator");
}
/// <summary>
/// Retrieve objects from object log
/// </summary>
/// <param name="record"></param>
/// <param name="ctx"></param>
/// <returns></returns>
protected override bool RetrievedFullRecord(byte *record, ref AsyncIOContext <Key, Value> ctx)
{
return(true);
}
internal RmwAsyncOperation(AsyncIOContext <Key, Value> diskRequest) => this.diskRequest = diskRequest;
public void AsyncReadRecordToMemory(long fromLogical, int numRecords, IOCompletionCallback callback, AsyncIOContext context, SectorAlignedMemory result = default(SectorAlignedMemory))
{
ulong fileOffset = (ulong)(AlignedPageSizeBytes * (fromLogical >> LogPageSizeBits) + (fromLogical & PageSizeMask));
ulong alignedFileOffset = (ulong)(((long)fileOffset / sectorSize) * sectorSize);
uint alignedReadLength = (uint)((long)fileOffset + numRecords - (long)alignedFileOffset);
alignedReadLength = (uint)((alignedReadLength + (sectorSize - 1)) & ~(sectorSize - 1));
var record = readBufferPool.Get((int)alignedReadLength);
record.valid_offset = (int)(fileOffset - alignedFileOffset);
record.available_bytes = (int)(alignedReadLength - (fileOffset - alignedFileOffset));
record.required_bytes = numRecords;
var asyncResult = default(AsyncGetFromDiskResult <AsyncIOContext>);
asyncResult.context = context;
if (result.buffer == null)
{
asyncResult.context.record = record;
device.ReadAsync(alignedFileOffset,
(IntPtr)asyncResult.context.record.aligned_pointer,
alignedReadLength,
callback,
asyncResult);
}
else
{
asyncResult.context.record = result;
asyncResult.context.objBuffer = record;
objlogDevice.ReadAsync(
(int)(context.logicalAddress >> LogSegmentSizeBits),
alignedFileOffset,
(IntPtr)asyncResult.context.objBuffer.aligned_pointer,
alignedReadLength,
callback,
asyncResult);
}
}
public override ref Value GetContextRecordValue(ref AsyncIOContext <Key, Value> ctx)
{
return(ref GetValue((long)ctx.record.GetValidPointer()));
}