internal async ValueTask CompleteIOPendingRequestsAsync(FasterExecutionContext opCtx, FasterExecutionContext currentCtx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default) { while (opCtx.ioPendingRequests.Count > 0) { AsyncIOContext <Key, Value> request; if (opCtx.readyResponses.Count > 0) { clientSession.UnsafeResumeThread(); while (opCtx.readyResponses.Count > 0) { opCtx.readyResponses.TryDequeue(out request); InternalContinuePendingRequestAndCallback(opCtx, currentCtx, request); } clientSession.UnsafeSuspendThread(); } else { request = await opCtx.readyResponses.DequeueAsync(token); clientSession.UnsafeResumeThread(); InternalContinuePendingRequestAndCallback(opCtx, currentCtx, request); clientSession.UnsafeSuspendThread(); } } }
internal ValueTask <RmwAsyncResult <Input, Output, Context, Functions> > CompleteAsync(CancellationToken token = default) { Debug.Assert(_fasterKV.RelaxedCPR); AsyncIOContext <Key, Value> newDiskRequest = default; if (_diskRequest.asyncOperation != null && CompletionComputeStatus != Completed && Interlocked.CompareExchange(ref CompletionComputeStatus, Completed, Pending) == Pending) { try { if (_exception == default) { if (_clientSession.SupportAsync) { _clientSession.UnsafeResumeThread(); } try { var status = _fasterKV.InternalCompletePendingRequestFromContext(_clientSession.ctx, _clientSession.ctx, _clientSession.FasterSession, _diskRequest, ref _pendingContext, true, out newDiskRequest); _pendingContext.Dispose(); if (status != Status.PENDING) { return(new ValueTask <RmwAsyncResult <Input, Output, Context, Functions> >(new RmwAsyncResult <Input, Output, Context, Functions>(status, default))); } } finally { if (_clientSession.SupportAsync) { _clientSession.UnsafeSuspendThread(); } } } } catch (Exception e) { _exception = ExceptionDispatchInfo.Capture(e); } finally { _clientSession.ctx.ioPendingRequests.Remove(_pendingContext.id); _clientSession.ctx.asyncPendingCount--; _clientSession.ctx.pendingReads.Remove(); } } if (_exception != default) { _exception.Throw(); } return(SlowRmwAsync(_fasterKV, _clientSession, _pendingContext, newDiskRequest, token)); }
internal (Status, Output) Complete() { (Status, Output)_result = default; if (_diskRequest.asyncOperation != null && CompletionComputeStatus != Completed && Interlocked.CompareExchange(ref CompletionComputeStatus, Completed, Pending) == Pending) { try { if (_exception == default) { if (_clientSession.SupportAsync) { _clientSession.UnsafeResumeThread(); } try { Debug.Assert(_fasterKV.RelaxedCPR); var status = _fasterKV.InternalCompletePendingRequestFromContext(_clientSession.ctx, _clientSession.ctx, _clientSession.FasterSession, _diskRequest, ref _pendingContext, true, out _); Debug.Assert(status != Status.PENDING); _result = (status, _pendingContext.output); _pendingContext.Dispose(); } finally { if (_clientSession.SupportAsync) { _clientSession.UnsafeSuspendThread(); } } } } catch (Exception e) { _exception = ExceptionDispatchInfo.Capture(e); } finally { _clientSession.ctx.ioPendingRequests.Remove(_pendingContext.id); _clientSession.ctx.asyncPendingCount--; _clientSession.ctx.pendingReads.Remove(); } } if (_exception != default) { _exception.Throw(); } return(_result); }
/// <summary> /// Steps the thread's local state machine. Threads catch up to the current global state and performs /// necessary actions associated with the state as defined by the current state machine /// </summary> /// <param name="ctx">null if calling without a context (e.g. waiting on a checkpoint)</param> /// <param name="clientSession">null if calling without a session (e.g. waiting on a checkpoint)</param> /// <param name="async"></param> /// <param name="token"></param> /// <returns></returns> private async ValueTask ThreadStateMachineStep(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default) { if (async) { clientSession?.UnsafeResumeThread(); } // Target state is the current (non-intermediate state) system state thread needs to catch up to var(currentTask, targetState) = CaptureTaskAndTargetState(); // No state machine associated with target, we can // directly fast forward session to target state if (currentTask == null) { if (ctx != null) { ctx.phase = targetState.phase; ctx.version = targetState.version; } return; } // the current thread state is what the thread remembers, or simply what the current system // is if we are calling from somewhere other than an execution thread (e.g. waiting on // a checkpoint to complete on a client app thread) var threadState = ctx == null ? targetState : SystemState.Make(ctx.phase, ctx.version); // If the thread was in the middle of handling some older, unrelated task, fast-forward to the current task // as the old one is no longer relevant threadState = FastForwardToCurrentCycle(threadState, targetState); var previousState = threadState; do { await currentTask.OnThreadEnteringState(threadState, previousState, this, ctx, clientSession, async, token); if (ctx != null) { ctx.phase = threadState.phase; ctx.version = threadState.version; } previousState.word = threadState.word; threadState = currentTask.NextState(threadState); } while (previousState.word != targetState.word); }
/// <inheritdoc /> public async ValueTask OnThreadState <Key, Value, Input, Output, Context, Functions>( SystemState current, SystemState prev, FasterKV <Key, Value, Input, Output, Context, Functions> faster, FasterKV <Key, Value, Input, Output, Context, Functions> .FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default) where Key : new() where Value : new() where Functions : IFunctions <Key, Value, Input, Output, Context> { switch (current.phase) { case Phase.PREP_INDEX_CHECKPOINT: if (ctx != null) { if (!ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt]) { ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = true; } faster.epoch.Mark(EpochPhaseIdx.PrepareForIndexCheckpt, current.version); } if (faster.epoch.CheckIsComplete(EpochPhaseIdx.PrepareForIndexCheckpt, current.version)) { faster.GlobalStateMachineStep(current); } break; case Phase.INDEX_CHECKPOINT: if (ctx != null) { // Resetting the marker for a potential FULL or INDEX_ONLY checkpoint in the future ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = false; } if (async && !faster.IsIndexFuzzyCheckpointCompleted()) { clientSession?.UnsafeSuspendThread(); await faster.IsIndexFuzzyCheckpointCompletedAsync(token); clientSession?.UnsafeResumeThread(); } faster.GlobalStateMachineStep(current); break; } }
internal void CompleteRetryRequests(FasterExecutionContext opCtx, FasterExecutionContext currentCtx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession) { int count = opCtx.retryRequests.Count; if (count == 0) { return; } clientSession.UnsafeResumeThread(); for (int i = 0; i < count; i++) { var pendingContext = opCtx.retryRequests.Dequeue(); InternalRetryRequestAndCallback(opCtx, currentCtx, pendingContext); } clientSession.UnsafeSuspendThread(); }
internal ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > ReadAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, ref Key key, ref Input input, Context context, long serialNo, CancellationToken token) where Functions : IFunctions <Key, Value, Input, Output, Context> { var pcontext = default(PendingContext <Input, Output, Context>); var diskRequest = default(AsyncIOContext <Key, Value>); Output output = default; if (clientSession.SupportAsync) { clientSession.UnsafeResumeThread(); } try { OperationStatus internalStatus = InternalRead(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, serialNo); Debug.Assert(internalStatus != OperationStatus.RETRY_NOW); Debug.Assert(internalStatus != OperationStatus.RETRY_LATER); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, output))); } else { var status = HandleOperationStatus(clientSession.ctx, clientSession.ctx, ref pcontext, clientSession.FasterSession, internalStatus, true, out diskRequest); if (status != Status.PENDING) { return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>(status, output))); } } } finally { Debug.Assert(serialNo >= clientSession.ctx.serialNum, "Operation serial numbers must be non-decreasing"); clientSession.ctx.serialNum = serialNo; if (clientSession.SupportAsync) { clientSession.UnsafeSuspendThread(); } } return(SlowReadAsync(this, clientSession, pcontext, diskRequest, token)); }
internal (Status, Output) CompleteRead() { (Status, Output)_result = default; if (_diskRequest.asyncOperation != null && CompletionComputeStatus != Completed && Interlocked.CompareExchange(ref CompletionComputeStatus, Completed, Pending) == Pending) { try { if (_clientSession.SupportAsync) { _clientSession.UnsafeResumeThread(); } try { Debug.Assert(_fasterKV.RelaxedCPR); _result = _fasterKV.InternalCompleteIOPendingReadRequestsAsync( _clientSession.ctx, _clientSession.ctx, _diskRequest, _pendingContext); } finally { if (_clientSession.SupportAsync) { _clientSession.UnsafeSuspendThread(); } } } catch (Exception e) { _exception = ExceptionDispatchInfo.Capture(e); } finally { _clientSession.ctx.ioPendingRequests.Remove(_pendingContext.id); } } if (_exception != default) { _exception.Throw(); } return(_result); }
internal ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > ReadAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, ref Key key, ref Input input, Context context = default, CancellationToken token = default) where Functions : IFunctions <Key, Value, Input, Output, Context> { var pcontext = default(PendingContext <Input, Output, Context>); Output output = default; OperationStatus internalStatus; var nextSerialNum = clientSession.ctx.serialNum + 1; if (clientSession.SupportAsync) { clientSession.UnsafeResumeThread(); } try { TryReadAgain: internalStatus = InternalRead(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, nextSerialNum); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, output))); } if (internalStatus == OperationStatus.CPR_SHIFT_DETECTED) { SynchronizeEpoch(clientSession.ctx, clientSession.ctx, ref pcontext, clientSession.FasterSession); goto TryReadAgain; } } finally { clientSession.ctx.serialNum = nextSerialNum; if (clientSession.SupportAsync) { clientSession.UnsafeSuspendThread(); } } return(SlowReadAsync(this, clientSession, pcontext, token)); }
/// <inheritdoc /> public async ValueTask OnThreadState <Key, Value, Input, Output, Context, Functions>(SystemState current, SystemState prev, FasterKV <Key, Value, Input, Output, Context, Functions> faster, FasterKV <Key, Value, Input, Output, Context, Functions> .FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default) where Key : new() where Value : new() where Functions : IFunctions <Key, Value, Input, Output, Context> { if (current.phase != Phase.WAIT_INDEX_CHECKPOINT) { return; } if (async && !faster.IsIndexFuzzyCheckpointCompleted()) { clientSession?.UnsafeSuspendThread(); await faster.IsIndexFuzzyCheckpointCompletedAsync(token); clientSession?.UnsafeResumeThread(); } faster.GlobalStateMachineStep(current); }
internal ValueTask <RmwAsyncResult <Input, Output, Context, Functions> > RmwAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, ref Key key, ref Input input, Context context, long serialNo, CancellationToken token = default) where Functions : IFunctions <Key, Value, Input, Output, Context> { var pcontext = default(PendingContext <Input, Output, Context>); var diskRequest = default(AsyncIOContext <Key, Value>); if (clientSession.SupportAsync) { clientSession.UnsafeResumeThread(); } try { OperationStatus internalStatus; do { internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, serialNo); }while (internalStatus == OperationStatus.RETRY_NOW || internalStatus == OperationStatus.RETRY_LATER); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { return(new ValueTask <RmwAsyncResult <Input, Output, Context, Functions> >(new RmwAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, default)));
public void UnsafeResumeThread() { _clientSession.UnsafeResumeThread(); }
private async ValueTask HandleCheckpointingPhasesAsync(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default) { if (async) { clientSession?.UnsafeResumeThread(); } var finalState = SystemState.Copy(ref _systemState); while (finalState.phase == Phase.INTERMEDIATE) { finalState = SystemState.Copy(ref _systemState); } var previousState = ctx != null?SystemState.Make(ctx.phase, ctx.version) : finalState; // We need to move from previousState to finalState one step at a time var currentState = previousState; SystemState startState = GetStartState(finalState); if ((currentState.version < startState.version) || (currentState.version == startState.version && currentState.phase < startState.phase)) { // Fast-forward to beginning of current checkpoint cycle currentState = startState; } do { switch (currentState.phase) { case Phase.PREP_INDEX_CHECKPOINT: { if (ctx != null) { if (!ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt]) { ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = true; } epoch.Mark(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.INDEX_CHECKPOINT: { if (_checkpointType == CheckpointType.INDEX_ONLY && ctx != null) { // Reseting the marker for a potential FULL or INDEX_ONLY checkpoint in the future ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = false; } if (async && !IsIndexFuzzyCheckpointCompleted()) { clientSession?.UnsafeSuspendThread(); await IsIndexFuzzyCheckpointCompletedAsync(token); clientSession?.UnsafeResumeThread(); } GlobalMoveToNextCheckpointState(currentState); break; } case Phase.PREPARE: { if (ctx != null) { if (!ctx.markers[EpochPhaseIdx.Prepare]) { if (!RelaxedCPR) { AcquireSharedLatchesForAllPendingRequests(ctx); } ctx.markers[EpochPhaseIdx.Prepare] = true; } epoch.Mark(EpochPhaseIdx.Prepare, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.Prepare, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.IN_PROGRESS: { if (ctx != null) { // Need to be very careful here as threadCtx is changing FasterExecutionContext _ctx; if (previousState.phase == Phase.IN_PROGRESS) { _ctx = ctx.prevCtx; } else { _ctx = ctx; } if (!_ctx.markers[EpochPhaseIdx.InProgress]) { AtomicSwitch(ctx, ctx.prevCtx, _ctx.version); InitContext(ctx, ctx.prevCtx.guid, ctx.prevCtx.serialNum); // Has to be prevCtx, not ctx ctx.prevCtx.markers[EpochPhaseIdx.InProgress] = true; } epoch.Mark(EpochPhaseIdx.InProgress, currentState.version); } // Has to be prevCtx, not ctx if (epoch.CheckIsComplete(EpochPhaseIdx.InProgress, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.WAIT_PENDING: { if (ctx != null) { if (!ctx.prevCtx.markers[EpochPhaseIdx.WaitPending]) { var notify = (ctx.prevCtx.ioPendingRequests.Count == 0); notify = notify && (ctx.prevCtx.retryRequests.Count == 0); if (notify) { ctx.prevCtx.markers[EpochPhaseIdx.WaitPending] = true; } else { break; } } epoch.Mark(EpochPhaseIdx.WaitPending, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.WaitPending, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.WAIT_FLUSH: { if (ctx == null || !ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush]) { bool notify; if (FoldOverSnapshot) { notify = (hlog.FlushedUntilAddress >= _hybridLogCheckpoint.info.finalLogicalAddress); } else { notify = (_hybridLogCheckpoint.flushedSemaphore != null) && _hybridLogCheckpoint.flushedSemaphore.CurrentCount > 0; } if (async && !notify) { Debug.Assert(_hybridLogCheckpoint.flushedSemaphore != null); clientSession?.UnsafeSuspendThread(); await _hybridLogCheckpoint.flushedSemaphore.WaitAsync(token); clientSession?.UnsafeResumeThread(); _hybridLogCheckpoint.flushedSemaphore.Release(); notify = true; } if (_checkpointType == CheckpointType.FULL) { notify = notify && IsIndexFuzzyCheckpointCompleted(); if (async && !notify) { clientSession?.UnsafeSuspendThread(); await IsIndexFuzzyCheckpointCompletedAsync(token); clientSession?.UnsafeResumeThread(); notify = true; } } if (notify) { if (ctx != null) { ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush] = true; } } else { break; } } if (ctx != null) { epoch.Mark(EpochPhaseIdx.WaitFlush, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.WaitFlush, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.PERSISTENCE_CALLBACK: { if (ctx != null) { if (!ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback]) { if (ctx.prevCtx.serialNum != -1) { var commitPoint = new CommitPoint { UntilSerialNo = ctx.prevCtx.serialNum, ExcludedSerialNos = ctx.prevCtx.excludedSerialNos }; // Thread local action functions.CheckpointCompletionCallback(ctx.guid, commitPoint); if (clientSession != null) { clientSession.LatestCommitPoint = commitPoint; } } ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback] = true; } epoch.Mark(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.REST: { break; } default: throw new FasterException("Invalid state found during checkpointing"); } if (ctx != null) { // update thread local variables ctx.phase = currentState.phase; ctx.version = currentState.version; } previousState.word = currentState.word; currentState = GetNextState(currentState, _checkpointType); } while (previousState.word != finalState.word); if (async) { clientSession?.UnsafeSuspendThread(); } }