internal Status ContextDelete <Input, Output, Context, FasterSession>(
ref Key key,
Context context,
FasterSession fasterSession,
long serialNo,
FasterExecutionContext <Input, Output, Context> sessionCtx)
where FasterSession : IFasterSession <Key, Value, Input, Output, Context>
{
var pcontext = default(PendingContext <Input, Output, Context>);
var internalStatus = InternalDelete(ref key, ref context, ref pcontext, fasterSession, sessionCtx, serialNo);
Status status;
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus);
}
sessionCtx.serialNum = serialNo;
return(status);
}
internal Status ContextRead <Input, Output, Context, FasterSession>(ref Key key, ref Input input, ref Output output, Context context, FasterSession fasterSession, long serialNo,
FasterExecutionContext <Input, Output, Context> sessionCtx)
where FasterSession : IFasterSession <Key, Value, Input, Output, Context>
{
var pcontext = default(PendingContext <Input, Output, Context>);
var internalStatus = InternalRead(ref key, ref input, ref output, ref context, ref pcontext, fasterSession, sessionCtx, serialNo);
Debug.Assert(internalStatus != OperationStatus.RETRY_NOW);
Status status;
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(sessionCtx, sessionCtx, ref pcontext, fasterSession, internalStatus, false, out _);
}
Debug.Assert(serialNo >= sessionCtx.serialNum, "Operation serial numbers must be non-decreasing");
sessionCtx.serialNum = serialNo;
return(status);
}
internal void AcquireSharedLatchesForAllPendingRequests <Input, Output, Context>(FasterExecutionContext <Input, Output, Context> ctx)
{
foreach (var _ctx in ctx.retryRequests)
{
AcquireSharedLatch(_ctx.key.Get());
}
foreach (var _ctx in ctx.ioPendingRequests.Values)
{
AcquireSharedLatch(_ctx.key.Get());
}
}
internal ValueTask <RmwAsyncResult <Input, Output, Context> > RmwAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref Key key, ref Input input, Context context, long serialNo, CancellationToken token = default)
{
var pcontext = default(PendingContext <Input, Output, Context>);
var diskRequest = default(AsyncIOContext <Key, Value>);
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, fasterSession, currentCtx, serialNo);
}while (internalStatus == OperationStatus.RETRY_NOW || internalStatus == OperationStatus.RETRY_LATER);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <RmwAsyncResult <Input, Output, Context> >(new RmwAsyncResult <Input, Output, Context>((Status)internalStatus, default)));
private static async ValueTask <ExceptionDispatchInfo> WaitForFlushCompletionAsync <Input, Output, Context>(FasterKV <Key, Value> @this, FasterExecutionContext <Input, Output, Context> currentCtx, CompletionEvent flushEvent, CancellationToken token)
{
ExceptionDispatchInfo exceptionDispatchInfo = default;
try
{
token.ThrowIfCancellationRequested();
if (@this.epoch.ThisInstanceProtected())
{
throw new NotSupportedException("Async operations not supported over protected epoch");
}
await flushEvent.WaitAsync(token).ConfigureAwait(false);
}
catch (Exception e)
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(e);
}
return(exceptionDispatchInfo);
}
private void HandleCheckpointingPhases(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession)
{
var _ = HandleCheckpointingPhasesAsync(ctx, clientSession, false);
return;
}
internal async ValueTask CompleteIOPendingRequestsAsync(FasterExecutionContext opCtx, FasterExecutionContext currentCtx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default)
{
while (opCtx.ioPendingRequests.Count > 0)
{
AsyncIOContext <Key, Value> request;
if (opCtx.readyResponses.Count > 0)
{
clientSession.UnsafeResumeThread();
while (opCtx.readyResponses.Count > 0)
{
opCtx.readyResponses.TryDequeue(out request);
InternalContinuePendingRequestAndCallback(opCtx, currentCtx, request);
}
clientSession.UnsafeSuspendThread();
}
else
{
request = await opCtx.readyResponses.DequeueAsync(token);
clientSession.UnsafeResumeThread();
InternalContinuePendingRequestAndCallback(opCtx, currentCtx, request);
clientSession.UnsafeSuspendThread();
}
}
}
private async ValueTask HandleCheckpointingPhasesAsync(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default)
{
if (async)
{
clientSession?.UnsafeResumeThread();
}
var finalState = SystemState.Copy(ref _systemState);
while (finalState.phase == Phase.INTERMEDIATE)
{
finalState = SystemState.Copy(ref _systemState);
}
var previousState = ctx != null?SystemState.Make(ctx.phase, ctx.version) : finalState;
// We need to move from previousState to finalState one step at a time
var currentState = previousState;
SystemState startState = GetStartState(finalState);
if ((currentState.version < startState.version) ||
(currentState.version == startState.version && currentState.phase < startState.phase))
{
// Fast-forward to beginning of current checkpoint cycle
currentState = startState;
}
do
{
switch (currentState.phase)
{
case Phase.PREP_INDEX_CHECKPOINT:
{
if (ctx != null)
{
if (!ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt])
{
ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = true;
}
epoch.Mark(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.INDEX_CHECKPOINT:
{
if (_checkpointType == CheckpointType.INDEX_ONLY && ctx != null)
{
// Reseting the marker for a potential FULL or INDEX_ONLY checkpoint in the future
ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = false;
}
if (async && !IsIndexFuzzyCheckpointCompleted())
{
clientSession?.UnsafeSuspendThread();
await IsIndexFuzzyCheckpointCompletedAsync(token);
clientSession?.UnsafeResumeThread();
}
GlobalMoveToNextCheckpointState(currentState);
break;
}
case Phase.PREPARE:
{
if (ctx != null)
{
if (!ctx.markers[EpochPhaseIdx.Prepare])
{
if (!RelaxedCPR)
{
AcquireSharedLatchesForAllPendingRequests(ctx);
}
ctx.markers[EpochPhaseIdx.Prepare] = true;
}
epoch.Mark(EpochPhaseIdx.Prepare, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.Prepare, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.IN_PROGRESS:
{
if (ctx != null)
{
// Need to be very careful here as threadCtx is changing
FasterExecutionContext _ctx;
if (previousState.phase == Phase.IN_PROGRESS)
{
_ctx = ctx.prevCtx;
}
else
{
_ctx = ctx;
}
if (!_ctx.markers[EpochPhaseIdx.InProgress])
{
AtomicSwitch(ctx, ctx.prevCtx, _ctx.version);
InitContext(ctx, ctx.prevCtx.guid, ctx.prevCtx.serialNum);
// Has to be prevCtx, not ctx
ctx.prevCtx.markers[EpochPhaseIdx.InProgress] = true;
}
epoch.Mark(EpochPhaseIdx.InProgress, currentState.version);
}
// Has to be prevCtx, not ctx
if (epoch.CheckIsComplete(EpochPhaseIdx.InProgress, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.WAIT_PENDING:
{
if (ctx != null)
{
if (!ctx.prevCtx.markers[EpochPhaseIdx.WaitPending])
{
var notify = (ctx.prevCtx.ioPendingRequests.Count == 0);
notify = notify && (ctx.prevCtx.retryRequests.Count == 0);
if (notify)
{
ctx.prevCtx.markers[EpochPhaseIdx.WaitPending] = true;
}
else
{
break;
}
}
epoch.Mark(EpochPhaseIdx.WaitPending, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.WaitPending, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.WAIT_FLUSH:
{
if (ctx == null || !ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush])
{
bool notify;
if (FoldOverSnapshot)
{
notify = (hlog.FlushedUntilAddress >= _hybridLogCheckpoint.info.finalLogicalAddress);
}
else
{
notify = (_hybridLogCheckpoint.flushedSemaphore != null) && _hybridLogCheckpoint.flushedSemaphore.CurrentCount > 0;
}
if (async && !notify)
{
Debug.Assert(_hybridLogCheckpoint.flushedSemaphore != null);
clientSession?.UnsafeSuspendThread();
await _hybridLogCheckpoint.flushedSemaphore.WaitAsync(token);
clientSession?.UnsafeResumeThread();
_hybridLogCheckpoint.flushedSemaphore.Release();
notify = true;
}
if (_checkpointType == CheckpointType.FULL)
{
notify = notify && IsIndexFuzzyCheckpointCompleted();
if (async && !notify)
{
clientSession?.UnsafeSuspendThread();
await IsIndexFuzzyCheckpointCompletedAsync(token);
clientSession?.UnsafeResumeThread();
notify = true;
}
}
if (notify)
{
if (ctx != null)
{
ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush] = true;
}
}
else
{
break;
}
}
if (ctx != null)
{
epoch.Mark(EpochPhaseIdx.WaitFlush, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.WaitFlush, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.PERSISTENCE_CALLBACK:
{
if (ctx != null)
{
if (!ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback])
{
if (ctx.prevCtx.serialNum != -1)
{
var commitPoint = new CommitPoint
{
UntilSerialNo = ctx.prevCtx.serialNum,
ExcludedSerialNos = ctx.prevCtx.excludedSerialNos
};
// Thread local action
functions.CheckpointCompletionCallback(ctx.guid, commitPoint);
if (clientSession != null)
{
clientSession.LatestCommitPoint = commitPoint;
}
}
ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback] = true;
}
epoch.Mark(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.REST:
{
break;
}
default:
throw new FasterException("Invalid state found during checkpointing");
}
if (ctx != null)
{
// update thread local variables
ctx.phase = currentState.phase;
ctx.version = currentState.version;
}
previousState.word = currentState.word;
currentState = GetNextState(currentState, _checkpointType);
} while (previousState.word != finalState.word);
if (async)
{
clientSession?.UnsafeSuspendThread();
}
}
/// <summary>
/// Steps the thread's local state machine. Threads catch up to the current global state and performs
/// necessary actions associated with the state as defined by the current state machine
/// </summary>
/// <param name="ctx">null if calling without a context (e.g. waiting on a checkpoint)</param>
/// <param name="fasterSession">Faster session.</param>
/// <param name="valueTasks">Return list of tasks that caller needs to await, to continue checkpointing</param>
/// <param name="token">Cancellation token</param>
/// <returns></returns>
private void ThreadStateMachineStep <Input, Output, Context, FasterSession>(
FasterExecutionContext <Input, Output, Context> ctx,
FasterSession fasterSession,
List <ValueTask> valueTasks,
CancellationToken token = default)
where FasterSession : IFasterSession
{
#region Capture current (non-intermediate) system state
var currentTask = currentSyncStateMachine;
var targetState = SystemState.Copy(ref systemState);
SystemState.RemoveIntermediate(ref targetState);
while (currentSyncStateMachine != currentTask)
{
currentTask = currentSyncStateMachine;
targetState = SystemState.Copy(ref systemState);
SystemState.RemoveIntermediate(ref targetState);
}
#endregion
var currentState = ctx == null ? targetState : SystemState.Make(ctx.phase, ctx.version);
var targetStartState = StartOfCurrentCycle(targetState);
#region Get returning thread to start of current cycle, issuing completion callbacks if needed
if (ctx != null)
{
if (ctx.version < targetStartState.version)
{
// Issue CPR callback for full session
if (ctx.serialNum != -1)
{
List <long> excludedSerialNos = new List <long>();
foreach (var v in ctx.ioPendingRequests.Values)
{
excludedSerialNos.Add(v.serialNum);
}
foreach (var v in ctx.retryRequests)
{
excludedSerialNos.Add(v.serialNum);
}
var commitPoint = new CommitPoint
{
UntilSerialNo = ctx.serialNum,
ExcludedSerialNos = excludedSerialNos
};
// Thread local action
fasterSession?.CheckpointCompletionCallback(ctx.guid, commitPoint);
}
}
if ((ctx.version == targetStartState.version) && (ctx.phase < Phase.REST))
{
IssueCompletionCallback(ctx, fasterSession);
}
}
#endregion
// No state machine associated with target, or target is in REST phase:
// we can directly fast forward session to target state
if (currentTask == null || targetState.phase == Phase.REST)
{
if (ctx != null)
{
ctx.phase = targetState.phase;
ctx.version = targetState.version;
}
return;
}
#region Jump on and execute current state machine
// We start at either the start point or our previous position in the state machine.
// If we are calling from somewhere other than an execution thread (e.g. waiting on
// a checkpoint to complete on a client app thread), we start at current system state
var threadState =
ctx == null ? targetState :
FastForwardToCurrentCycle(currentState, targetStartState);
var previousState = threadState;
do
{
Debug.Assert(
(threadState.version < targetState.version) ||
(threadState.version == targetState.version &&
(threadState.phase <= targetState.phase || currentTask is IndexSnapshotStateMachine)
));
currentTask.OnThreadEnteringState(threadState, previousState, this, ctx, fasterSession, valueTasks, token);
if (ctx != null)
{
ctx.phase = threadState.phase;
ctx.version = threadState.version;
}
previousState.word = threadState.word;
threadState = currentTask.NextState(threadState);
if (systemState.word != targetState.word)
{
var tmp = SystemState.Copy(ref systemState);
if (currentSyncStateMachine == currentTask)
{
targetState = tmp;
SystemState.RemoveIntermediate(ref targetState);
}
}
} while (previousState.word != targetState.word);
#endregion
return;
}
private ValueTask <UpsertAsyncResult <Input, Output, Context> > UpsertAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref PendingContext <Input, Output, Context> pcontext, ref Key key, ref Value value, Context userContext, long serialNo, CancellationToken token)
{
CompletionEvent flushEvent;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
flushEvent = hlog.FlushEvent;
internalStatus = InternalUpsert(ref key, ref value, ref userContext, ref pcontext, fasterSession, currentCtx, serialNo);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <UpsertAsyncResult <Input, Output, Context> >(new UpsertAsyncResult <Input, Output, Context>((Status)internalStatus)));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowUpsertAsync(this, fasterSession, currentCtx, pcontext, flushEvent, token));
}
/// <summary>
/// Check if at least one (sync) request is ready for CompletePending to operate on
/// </summary>
/// <param name="currentCtx"></param>
/// <param name="token"></param>
/// <returns></returns>
internal async ValueTask ReadyToCompletePendingAsync <Input, Output, Context>(FasterExecutionContext <Input, Output, Context> currentCtx, CancellationToken token = default)
{
#region Previous pending requests
if (!RelaxedCPR)
{
if (currentCtx.phase == Phase.IN_PROGRESS || currentCtx.phase == Phase.WAIT_PENDING)
{
if (currentCtx.prevCtx.SyncIoPendingCount != 0)
{
await currentCtx.prevCtx.readyResponses.WaitForEntryAsync(token).ConfigureAwait(false);
}
}
}
#endregion
if (currentCtx.SyncIoPendingCount != 0)
{
await currentCtx.readyResponses.WaitForEntryAsync(token).ConfigureAwait(false);
}
}
private static ExceptionDispatchInfo GetSlowUpdelAsyncExceptionDispatchInfo <Input, Output, Context>(FasterKV <Key, Value> @this, FasterExecutionContext <Input, Output, Context> currentCtx, CancellationToken token)
{
currentCtx.asyncPendingCount++;
ExceptionDispatchInfo exceptionDispatchInfo = default;
try
{
token.ThrowIfCancellationRequested();
if (@this.epoch.ThisInstanceProtected())
{
throw new NotSupportedException("Async operations not supported over protected epoch");
}
}
catch (Exception e)
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(e);
}
return(exceptionDispatchInfo);
}
internal ValueTask <DeleteAsyncResult <Input, Output, Context> > DeleteAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref Key key, Context userContext, long serialNo, CancellationToken token = default)
{
var pcontext = default(PendingContext <Input, Output, Context>);
pcontext.IsAsync = true;
Task flushTask;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
flushTask = hlog.FlushTask;
internalStatus = InternalDelete(ref key, ref userContext, ref pcontext, fasterSession, currentCtx, serialNo);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <DeleteAsyncResult <Input, Output, Context> >(new DeleteAsyncResult <Input, Output, Context>(internalStatus)));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowDeleteAsync(this, fasterSession, currentCtx, pcontext, flushTask, token));
}
public OperationStatus DoFastOperation(FasterKV <Key, Value> fasterKV, PendingContext <Input, Output, Context> pendingContext, IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx)
=> fasterKV.InternalDelete(ref pendingContext.key.Get(), ref pendingContext.userContext, ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum);
public ValueTask <UpsertAsyncResult <Input, Output, Context> > DoSlowOperation(FasterKV <Key, Value> fasterKV, IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, PendingContext <Input, Output, Context> pendingContext, Task flushTask, CancellationToken token)
=> SlowUpsertAsync(fasterKV, fasterSession, currentCtx, pendingContext, flushTask, token);
internal Status ContextRMW(ref Key key, ref Input input, Context context, long serialNo, FasterExecutionContext sessionCtx)
{
var pcontext = default(PendingContext);
var internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, sessionCtx, serialNo);
Status status;
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(sessionCtx, sessionCtx, pcontext, internalStatus);
}
sessionCtx.serialNum = serialNo;
return(status);
}
internal ValueTask <ReadAsyncResult <Input, Output, Context> > ReadAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx,
ref Key key, ref Input input, long startAddress, Context context, long serialNo, CancellationToken token, byte operationFlags = 0)
{
var pcontext = default(PendingContext <Input, Output, Context>);
pcontext.SetOperationFlags(operationFlags, startAddress);
var diskRequest = default(AsyncIOContext <Key, Value>);
Output output = default;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus = InternalRead(ref key, ref input, ref output, startAddress, ref context, ref pcontext, fasterSession, currentCtx, serialNo);
Debug.Assert(internalStatus != OperationStatus.RETRY_NOW);
Debug.Assert(internalStatus != OperationStatus.RETRY_LATER);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context> >(new ReadAsyncResult <Input, Output, Context>((Status)internalStatus, output, pcontext.recordInfo)));
}
else
{
var status = HandleOperationStatus(currentCtx, currentCtx, ref pcontext, fasterSession, internalStatus, true, out diskRequest);
if (status != Status.PENDING)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context> >(new ReadAsyncResult <Input, Output, Context>(status, output, pcontext.recordInfo)));
}
}
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowReadAsync(this, fasterSession, currentCtx, pcontext, diskRequest, token));
}
internal void InternalRetryRequestAndCallback(
FasterExecutionContext ctx,
PendingContext pendingContext)
{
var status = default(Status);
var internalStatus = default(OperationStatus);
#region Entry latch operation
var handleLatches = false;
if ((ctx.version < threadCtx.version) // Thread has already shifted to (v+1)
||
(threadCtx.phase == Phase.PREPARE)) // Thread still in version v, but acquired shared-latch
{
handleLatches = true;
}
#endregion
// Issue retry command
switch (pendingContext.type)
{
case OperationType.RMW:
internalStatus = InternalRetryPendingRMW(ctx, ref pendingContext);
break;
case OperationType.UPSERT:
internalStatus = InternalUpsert(ref pendingContext.key,
ref pendingContext.value,
ref pendingContext.userContext,
ref pendingContext);
break;
case OperationType.READ:
throw new Exception("Cannot happen!");
}
// Handle operation status
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(ctx, pendingContext, internalStatus);
}
// If done, callback user code.
if (status == Status.OK || status == Status.NOTFOUND)
{
if (handleLatches)
{
ReleaseSharedLatch(pendingContext.key);
}
switch (pendingContext.type)
{
case OperationType.RMW:
functions.RMWCompletionCallback(ref pendingContext.key,
ref pendingContext.input,
pendingContext.userContext, status);
break;
case OperationType.UPSERT:
functions.UpsertCompletionCallback(ref pendingContext.key,
ref pendingContext.value,
pendingContext.userContext);
break;
default:
throw new Exception("Operation type not allowed for retry");
}
}
}
internal ReadAsyncInternal(FasterKV <Key, Value> fasterKV, IFasterSession <Key, Value, Input, Output, Context> fasterSession, FasterExecutionContext <Input, Output, Context> currentCtx,
PendingContext <Input, Output, Context> pendingContext, AsyncIOContext <Key, Value> diskRequest, ExceptionDispatchInfo exceptionDispatchInfo)
{
_exception = exceptionDispatchInfo;
_fasterKV = fasterKV;
_fasterSession = fasterSession;
_currentCtx = currentCtx;
_pendingContext = pendingContext;
_diskRequest = diskRequest;
CompletionComputeStatus = Pending;
_recordInfo = default;
}
internal bool AtomicSwitch <Input, Output, Context>(FasterExecutionContext <Input, Output, Context> fromCtx, FasterExecutionContext <Input, Output, Context> toCtx, int version, ConcurrentDictionary <string, CommitPoint> tokens)
{
lock (toCtx)
{
if (toCtx.version < version)
{
CopyContext(fromCtx, toCtx);
if (toCtx.serialNum != -1)
{
tokens.TryAdd(toCtx.guid,
new CommitPoint
{
UntilSerialNo = toCtx.serialNum,
ExcludedSerialNos = toCtx.excludedSerialNos
});
}
return(true);
}
}
return(false);
}
internal bool AtomicSwitch <Input, Output, Context>(FasterExecutionContext <Input, Output, Context> fromCtx, FasterExecutionContext <Input, Output, Context> toCtx, int version)
{
lock (toCtx)
{
if (toCtx.version < version)
{
CopyContext(fromCtx, toCtx);
if (toCtx.serialNum != -1)
{
_hybridLogCheckpoint.info.checkpointTokens.TryAdd(toCtx.guid,
new CommitPoint
{
UntilSerialNo = toCtx.serialNum,
ExcludedSerialNos = toCtx.excludedSerialNos
});
}
return(true);
}
}
return(false);
}
/// <inheritdoc/>
public ValueTask <DeleteAsyncResult <Input, Output, Context> > DoSlowOperation(FasterKV <Key, Value> fasterKV, IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, PendingContext <Input, Output, Context> pendingContext, CompletionEvent flushEvent, CancellationToken token)
=> SlowDeleteAsync(fasterKV, fasterSession, currentCtx, pendingContext, flushEvent, token);
internal void InternalContinuePendingRequestAndCallback(
FasterExecutionContext ctx,
AsyncIOContext <Key, Value> request)
{
var handleLatches = false;
if ((ctx.version < threadCtx.version) // Thread has already shifted to (v+1)
||
(threadCtx.phase == Phase.PREPARE)) // Thread still in version v, but acquired shared-latch
{
handleLatches = true;
}
if (ctx.ioPendingRequests.TryGetValue(request.id, out PendingContext pendingContext))
{
var status = default(Status);
var internalStatus = default(OperationStatus);
// Remove from pending dictionary
ctx.ioPendingRequests.Remove(request.id);
// Issue the continue command
if (pendingContext.type == OperationType.READ)
{
internalStatus = InternalContinuePendingRead(ctx, request, ref pendingContext);
}
else
{
internalStatus = InternalContinuePendingRMW(ctx, request, ref pendingContext);;
}
request.record.Return();
// Handle operation status
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
status = (Status)internalStatus;
}
else
{
status = HandleOperationStatus(ctx, pendingContext, internalStatus);
}
// If done, callback user code
if (status == Status.OK || status == Status.NOTFOUND)
{
if (handleLatches)
{
ReleaseSharedLatch(pendingContext.key);
}
if (pendingContext.type == OperationType.READ)
{
functions.ReadCompletionCallback(ref pendingContext.key,
ref pendingContext.input,
ref pendingContext.output,
pendingContext.userContext,
status);
}
else
{
functions.RMWCompletionCallback(ref pendingContext.key,
ref pendingContext.input,
pendingContext.userContext,
status);
}
}
}
}
/// <inheritdoc/>
public void DecrementPending(FasterExecutionContext <Input, Output, Context> currentCtx, ref PendingContext <Input, Output, Context> pendingContext)
{
}
/// <summary>
/// Issue completion callback if needed, for the given context's prevCtx
/// </summary>
internal void IssueCompletionCallback <Input, Output, Context, FasterSession>(FasterExecutionContext <Input, Output, Context> ctx, FasterSession fasterSession)
where FasterSession : IFasterSession
{
CommitPoint commitPoint = default;
if (ctx.prevCtx.excludedSerialNos != null)
{
lock (ctx.prevCtx)
{
if (ctx.prevCtx.serialNum != -1)
{
commitPoint = new CommitPoint
{
UntilSerialNo = ctx.prevCtx.serialNum,
ExcludedSerialNos = ctx.prevCtx.excludedSerialNos
};
ctx.prevCtx.excludedSerialNos = null;
}
}
if (commitPoint.ExcludedSerialNos != null)
{
fasterSession?.CheckpointCompletionCallback(ctx.guid, commitPoint);
}
}
}
internal ValueTask <DeleteAsyncResult <Input, Output, Context> > DeleteAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref Key key, Context userContext, long serialNo, CancellationToken token = default)
{
var pcontext = new PendingContext <Input, Output, Context> {
IsAsync = true
};
return(DeleteAsync(fasterSession, currentCtx, ref pcontext, ref key, userContext, serialNo, token));
}
internal void InternalCompleteRetryRequest(FasterExecutionContext opCtx, FasterExecutionContext currentCtx, PendingContext pendingContext)
{
var internalStatus = default(OperationStatus);
ref Key key = ref pendingContext.key.Get();
internal ValueTask <DeleteAsyncResult <Input, Output, Context> > DeleteAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref PendingContext <Input, Output, Context> pcontext, ref Key key, Context userContext, long serialNo, CancellationToken token)
{
CompletionEvent flushEvent;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
flushEvent = hlog.FlushEvent;
internalStatus = InternalDelete(ref key, ref userContext, ref pcontext, fasterSession, currentCtx, serialNo);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (OperationStatusUtils.TryConvertToStatusCode(internalStatus, out Status status))
{
return(new ValueTask <DeleteAsyncResult <Input, Output, Context> >(new DeleteAsyncResult <Input, Output, Context>(new(internalStatus))));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowDeleteAsync(this, fasterSession, currentCtx, pcontext, flushEvent, token));
}
internal void CopyContext <Input, Output, Context>(FasterExecutionContext <Input, Output, Context> src, FasterExecutionContext <Input, Output, Context> dst)
{
dst.phase = src.phase;
dst.version = src.version;
dst.threadStateMachine = src.threadStateMachine;
dst.markers = src.markers;
dst.serialNum = src.serialNum;
dst.guid = src.guid;
dst.excludedSerialNos = new List <long>();
if (!RelaxedCPR)
{
dst.totalPending = src.totalPending;
dst.retryRequests = src.retryRequests;
dst.readyResponses = src.readyResponses;
dst.ioPendingRequests = src.ioPendingRequests;
dst.pendingReads = src.pendingReads;
}
else
{
foreach (var v in src.ioPendingRequests.Values)
{
dst.excludedSerialNos.Add(v.serialNum);
}
foreach (var v in src.retryRequests)
{
dst.excludedSerialNos.Add(v.serialNum);
}
}
}
/// <summary>
/// Check if at least one (sync) request is ready for CompletePending to operate on
/// </summary>
/// <param name="currentCtx"></param>
/// <param name="token"></param>
/// <returns></returns>
internal async ValueTask ReadyToCompletePendingAsync <Input, Output, Context>(FasterExecutionContext <Input, Output, Context> currentCtx, CancellationToken token = default)
{
if (currentCtx.SyncIoPendingCount != 0)
{
await currentCtx.readyResponses.WaitForEntryAsync(token).ConfigureAwait(false);
}
}
