private static async ValueTask <ReadAsyncResult> SlowReadAsync(FasterKV <Key, Value, Input, Output, Context, Functions> @this, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, PendingContext pendingContext, CancellationToken token = default(CancellationToken))
{
var diskRequest = @this.ScheduleGetFromDisk(clientSession.ctx, ref pendingContext);
clientSession.ctx.ioPendingRequests.Add(pendingContext.id, pendingContext);
clientSession.ctx.asyncPendingCount++;
clientSession.ctx.pendingReads.Add();
try
{
token.ThrowIfCancellationRequested();
if (@this.epoch.ThisInstanceProtected())
{
throw new NotSupportedException("Async operations not supported over protected epoch");
}
diskRequest = await diskRequest.asyncOperation.ValueTaskOfT;
}
catch
{
clientSession.ctx.ioPendingRequests.Remove(pendingContext.id);
clientSession.ctx.asyncPendingCount--;
throw;
}
finally
{
clientSession.ctx.pendingReads.Remove();
}
return(new ReadAsyncResult(@this, clientSession, pendingContext, diskRequest));
}
internal async ValueTask <(Status, Output)> CompleteIOPendingReadRequestsAsync(long serialNo, FasterExecutionContext opCtx, FasterExecutionContext currentCtx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default)
{
(Status, Output)s = default;
while (opCtx.ioPendingRequests.Count > 0)
{
AsyncIOContext <Key, Value> request;
if (opCtx.readyResponses.Count > 0)
{
clientSession.UnsafeResumeThread();
while (opCtx.readyResponses.Count > 0)
{
opCtx.readyResponses.TryDequeue(out request);
s = InternalContinuePendingRequestAndCallback(serialNo, opCtx, currentCtx, request);
}
clientSession.UnsafeSuspendThread();
}
else
{
request = await opCtx.readyResponses.DequeueAsync(token);
clientSession.UnsafeResumeThread();
s = InternalContinuePendingRequestAndCallback(serialNo, opCtx, currentCtx, request);
clientSession.UnsafeSuspendThread();
}
}
return(s);
}
internal ReadAsyncInternal(FasterKV <Key, Value, Input, Output, Context, Functions> fasterKV, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, PendingContext pendingContext, AsyncIOContext <Key, Value> diskRequest)
{
_exception = default;
_fasterKV = fasterKV;
_clientSession = clientSession;
_pendingContext = pendingContext;
_diskRequest = diskRequest;
CompletionComputeStatus = Pending;
}
/// <summary>
/// Complete outstanding pending operations
/// </summary>
/// <returns></returns>
internal async ValueTask <(Status, Output)> CompletePendingReadAsync(long serialNo, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default)
{
bool done = true;
#region Previous pending requests
if (!RelaxedCPR)
{
if (clientSession.ctx.phase == Phase.IN_PROGRESS
||
clientSession.ctx.phase == Phase.WAIT_PENDING)
{
await CompleteIOPendingRequestsAsync(clientSession.ctx.prevCtx, clientSession.ctx, clientSession, token);
Debug.Assert(clientSession.ctx.prevCtx.ioPendingRequests.Count == 0);
if (clientSession.ctx.prevCtx.retryRequests.Count > 0)
{
CompleteRetryRequests(clientSession.ctx.prevCtx, clientSession.ctx, clientSession);
}
done &= (clientSession.ctx.prevCtx.ioPendingRequests.Count == 0);
done &= (clientSession.ctx.prevCtx.retryRequests.Count == 0);
}
}
#endregion
var s = await CompleteIOPendingReadRequestsAsync(serialNo, clientSession.ctx, clientSession.ctx, clientSession, token);
CompleteRetryRequests(clientSession.ctx, clientSession.ctx, clientSession);
Debug.Assert(clientSession.ctx.ioPendingRequests.Count == 0);
done &= (clientSession.ctx.ioPendingRequests.Count == 0);
done &= (clientSession.ctx.retryRequests.Count == 0);
if (!done)
{
throw new Exception("CompletePendingAsync did not complete");
}
return(s);
}
internal void CompleteRetryRequests(FasterExecutionContext opCtx, FasterExecutionContext currentCtx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession)
{
int count = opCtx.retryRequests.Count;
if (count == 0)
{
return;
}
clientSession.UnsafeResumeThread();
for (int i = 0; i < count; i++)
{
var pendingContext = opCtx.retryRequests.Dequeue();
InternalRetryRequestAndCallback(opCtx, currentCtx, pendingContext);
}
clientSession.UnsafeSuspendThread();
}
/// <summary>
/// Complete outstanding pending operations that were issued synchronously
/// Async operations (e.g., ReadAsync) need to be completed individually
/// </summary>
/// <returns></returns>
internal async ValueTask CompletePendingAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default)
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
bool done = true;
#region Previous pending requests
if (!RelaxedCPR)
{
if (clientSession.ctx.phase == Phase.IN_PROGRESS
||
clientSession.ctx.phase == Phase.WAIT_PENDING)
{
await clientSession.ctx.prevCtx.pendingReads.WaitEmptyAsync();
await InternalCompletePendingRequestsAsync(clientSession.ctx.prevCtx, clientSession.ctx, clientSession.FasterSession, token);
Debug.Assert(clientSession.ctx.prevCtx.SyncIoPendingCount == 0);
if (clientSession.ctx.prevCtx.retryRequests.Count > 0)
{
clientSession.FasterSession.UnsafeResumeThread();
InternalCompleteRetryRequests(clientSession.ctx.prevCtx, clientSession.ctx, clientSession.FasterSession);
clientSession.FasterSession.UnsafeSuspendThread();
}
done &= (clientSession.ctx.prevCtx.HasNoPendingRequests);
}
}
#endregion
await InternalCompletePendingRequestsAsync(clientSession.ctx, clientSession.ctx, clientSession.FasterSession, token);
clientSession.FasterSession.UnsafeResumeThread();
InternalCompleteRetryRequests(clientSession.ctx, clientSession.ctx, clientSession.FasterSession);
clientSession.FasterSession.UnsafeSuspendThread();
Debug.Assert(clientSession.ctx.HasNoPendingRequests);
done &= (clientSession.ctx.HasNoPendingRequests);
if (!done)
{
throw new Exception("CompletePendingAsync did not complete");
}
}
private async ValueTask HandleCheckpointingPhasesAsync(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default)
{
if (async)
{
clientSession?.UnsafeResumeThread();
}
var finalState = SystemState.Copy(ref _systemState);
while (finalState.phase == Phase.INTERMEDIATE)
{
finalState = SystemState.Copy(ref _systemState);
}
var previousState = ctx != null?SystemState.Make(ctx.phase, ctx.version) : finalState;
// We need to move from previousState to finalState one step at a time
var currentState = previousState;
SystemState startState = GetStartState(finalState);
if ((currentState.version < startState.version) ||
(currentState.version == startState.version && currentState.phase < startState.phase))
{
// Fast-forward to beginning of current checkpoint cycle
currentState = startState;
}
do
{
switch (currentState.phase)
{
case Phase.PREP_INDEX_CHECKPOINT:
{
if (ctx != null)
{
if (!ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt])
{
ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = true;
}
epoch.Mark(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.INDEX_CHECKPOINT:
{
if (_checkpointType == CheckpointType.INDEX_ONLY && ctx != null)
{
// Reseting the marker for a potential FULL or INDEX_ONLY checkpoint in the future
ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = false;
}
if (async && !IsIndexFuzzyCheckpointCompleted())
{
clientSession?.UnsafeSuspendThread();
await IsIndexFuzzyCheckpointCompletedAsync(token);
clientSession?.UnsafeResumeThread();
}
GlobalMoveToNextCheckpointState(currentState);
break;
}
case Phase.PREPARE:
{
if (ctx != null)
{
if (!ctx.markers[EpochPhaseIdx.Prepare])
{
if (!RelaxedCPR)
{
AcquireSharedLatchesForAllPendingRequests(ctx);
}
ctx.markers[EpochPhaseIdx.Prepare] = true;
}
epoch.Mark(EpochPhaseIdx.Prepare, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.Prepare, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.IN_PROGRESS:
{
if (ctx != null)
{
// Need to be very careful here as threadCtx is changing
FasterExecutionContext _ctx;
if (previousState.phase == Phase.IN_PROGRESS)
{
_ctx = ctx.prevCtx;
}
else
{
_ctx = ctx;
}
if (!_ctx.markers[EpochPhaseIdx.InProgress])
{
AtomicSwitch(ctx, ctx.prevCtx, _ctx.version);
InitContext(ctx, ctx.prevCtx.guid, ctx.prevCtx.serialNum);
// Has to be prevCtx, not ctx
ctx.prevCtx.markers[EpochPhaseIdx.InProgress] = true;
}
epoch.Mark(EpochPhaseIdx.InProgress, currentState.version);
}
// Has to be prevCtx, not ctx
if (epoch.CheckIsComplete(EpochPhaseIdx.InProgress, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.WAIT_PENDING:
{
if (ctx != null)
{
if (!ctx.prevCtx.markers[EpochPhaseIdx.WaitPending])
{
var notify = (ctx.prevCtx.ioPendingRequests.Count == 0);
notify = notify && (ctx.prevCtx.retryRequests.Count == 0);
if (notify)
{
ctx.prevCtx.markers[EpochPhaseIdx.WaitPending] = true;
}
else
{
break;
}
}
epoch.Mark(EpochPhaseIdx.WaitPending, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.WaitPending, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.WAIT_FLUSH:
{
if (ctx == null || !ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush])
{
bool notify;
if (FoldOverSnapshot)
{
notify = (hlog.FlushedUntilAddress >= _hybridLogCheckpoint.info.finalLogicalAddress);
}
else
{
notify = (_hybridLogCheckpoint.flushedSemaphore != null) && _hybridLogCheckpoint.flushedSemaphore.CurrentCount > 0;
}
if (async && !notify)
{
Debug.Assert(_hybridLogCheckpoint.flushedSemaphore != null);
clientSession?.UnsafeSuspendThread();
await _hybridLogCheckpoint.flushedSemaphore.WaitAsync(token);
clientSession?.UnsafeResumeThread();
_hybridLogCheckpoint.flushedSemaphore.Release();
notify = true;
}
if (_checkpointType == CheckpointType.FULL)
{
notify = notify && IsIndexFuzzyCheckpointCompleted();
if (async && !notify)
{
clientSession?.UnsafeSuspendThread();
await IsIndexFuzzyCheckpointCompletedAsync(token);
clientSession?.UnsafeResumeThread();
notify = true;
}
}
if (notify)
{
if (ctx != null)
{
ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush] = true;
}
}
else
{
break;
}
}
if (ctx != null)
{
epoch.Mark(EpochPhaseIdx.WaitFlush, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.WaitFlush, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.PERSISTENCE_CALLBACK:
{
if (ctx != null)
{
if (!ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback])
{
if (ctx.prevCtx.serialNum != -1)
{
var commitPoint = new CommitPoint
{
UntilSerialNo = ctx.prevCtx.serialNum,
ExcludedSerialNos = ctx.prevCtx.excludedSerialNos
};
// Thread local action
functions.CheckpointCompletionCallback(ctx.guid, commitPoint);
if (clientSession != null)
{
clientSession.LatestCommitPoint = commitPoint;
}
}
ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback] = true;
}
epoch.Mark(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.REST:
{
break;
}
default:
throw new FasterException("Invalid state found during checkpointing");
}
if (ctx != null)
{
// update thread local variables
ctx.phase = currentState.phase;
ctx.version = currentState.version;
}
previousState.word = currentState.word;
currentState = GetNextState(currentState, _checkpointType);
} while (previousState.word != finalState.word);
if (async)
{
clientSession?.UnsafeSuspendThread();
}
}
internal ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > ReadAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession,
ref Key key, ref Input input, Context context = default, CancellationToken token = default)
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
var pcontext = default(PendingContext <Input, Output, Context>);
Output output = default;
OperationStatus internalStatus;
var nextSerialNum = clientSession.ctx.serialNum + 1;
if (clientSession.SupportAsync)
{
clientSession.UnsafeResumeThread();
}
try
{
TryReadAgain:
internalStatus = InternalRead(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, nextSerialNum);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, output)));
}
if (internalStatus == OperationStatus.CPR_SHIFT_DETECTED)
{
SynchronizeEpoch(clientSession.ctx, clientSession.ctx, ref pcontext, clientSession.FasterSession);
goto TryReadAgain;
}
}
finally
{
clientSession.ctx.serialNum = nextSerialNum;
if (clientSession.SupportAsync)
{
clientSession.UnsafeSuspendThread();
}
}
return(SlowReadAsync(this, clientSession, pcontext, token));
}
/// <summary>
/// Check if at least one (sync) request is ready for CompletePending to operate on
/// </summary>
/// <param name="clientSession"></param>
/// <param name="token"></param>
/// <returns></returns>
internal async ValueTask ReadyToCompletePendingAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default)
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
#region Previous pending requests
if (!RelaxedCPR)
{
if (clientSession.ctx.phase == Phase.IN_PROGRESS || clientSession.ctx.phase == Phase.WAIT_PENDING)
{
if (clientSession.ctx.prevCtx.SyncIoPendingCount != 0)
{
await clientSession.ctx.prevCtx.readyResponses.WaitForEntryAsync(token);
}
}
}
#endregion
if (clientSession.ctx.SyncIoPendingCount != 0)
{
await clientSession.ctx.readyResponses.WaitForEntryAsync(token);
}
}
internal ValueTask <RmwAsyncResult <Input, Output, Context, Functions> > RmwAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession,
ref Key key, ref Input input, Context context, long serialNo, CancellationToken token = default)
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
var pcontext = default(PendingContext <Input, Output, Context>);
var diskRequest = default(AsyncIOContext <Key, Value>);
if (clientSession.SupportAsync)
{
clientSession.UnsafeResumeThread();
}
try
{
OperationStatus internalStatus;
do
{
internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, serialNo);
}while (internalStatus == OperationStatus.RETRY_NOW || internalStatus == OperationStatus.RETRY_LATER);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <RmwAsyncResult <Input, Output, Context, Functions> >(new RmwAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, default)));
internal ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > ReadAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession,
ref Key key, ref Input input, Context context, long serialNo, CancellationToken token)
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
var pcontext = default(PendingContext <Input, Output, Context>);
var diskRequest = default(AsyncIOContext <Key, Value>);
Output output = default;
if (clientSession.SupportAsync)
{
clientSession.UnsafeResumeThread();
}
try
{
OperationStatus internalStatus = InternalRead(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, serialNo);
Debug.Assert(internalStatus != OperationStatus.RETRY_NOW);
Debug.Assert(internalStatus != OperationStatus.RETRY_LATER);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, output)));
}
else
{
var status = HandleOperationStatus(clientSession.ctx, clientSession.ctx, ref pcontext, clientSession.FasterSession, internalStatus, true, out diskRequest);
if (status != Status.PENDING)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>(status, output)));
}
}
}
finally
{
Debug.Assert(serialNo >= clientSession.ctx.serialNum, "Operation serial numbers must be non-decreasing");
clientSession.ctx.serialNum = serialNo;
if (clientSession.SupportAsync)
{
clientSession.UnsafeSuspendThread();
}
}
return(SlowReadAsync(this, clientSession, pcontext, diskRequest, token));
}
private void HandleCheckpointingPhases(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession)
{
var _ = HandleCheckpointingPhasesAsync(ctx, clientSession, false);
return;
}
public AsyncFasterSession(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession)
{
_clientSession = clientSession;
}
internal UnsafeContext(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession)
{
this.clientSession = clientSession;
FasterSession = new InternalFasterSession(clientSession);
}
