private static async ValueTask <ReadAsyncResult> SlowReadAsync(FasterKV <Key, Value, Input, Output, Context, Functions> @this, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, PendingContext pendingContext, CancellationToken token = default(CancellationToken)) { var diskRequest = @this.ScheduleGetFromDisk(clientSession.ctx, ref pendingContext); clientSession.ctx.ioPendingRequests.Add(pendingContext.id, pendingContext); clientSession.ctx.asyncPendingCount++; clientSession.ctx.pendingReads.Add(); try { token.ThrowIfCancellationRequested(); if (@this.epoch.ThisInstanceProtected()) { throw new NotSupportedException("Async operations not supported over protected epoch"); } diskRequest = await diskRequest.asyncOperation.ValueTaskOfT; } catch { clientSession.ctx.ioPendingRequests.Remove(pendingContext.id); clientSession.ctx.asyncPendingCount--; throw; } finally { clientSession.ctx.pendingReads.Remove(); } return(new ReadAsyncResult(@this, clientSession, pendingContext, diskRequest)); }
internal async ValueTask <(Status, Output)> CompleteIOPendingReadRequestsAsync(long serialNo, FasterExecutionContext opCtx, FasterExecutionContext currentCtx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default) { (Status, Output)s = default; while (opCtx.ioPendingRequests.Count > 0) { AsyncIOContext <Key, Value> request; if (opCtx.readyResponses.Count > 0) { clientSession.UnsafeResumeThread(); while (opCtx.readyResponses.Count > 0) { opCtx.readyResponses.TryDequeue(out request); s = InternalContinuePendingRequestAndCallback(serialNo, opCtx, currentCtx, request); } clientSession.UnsafeSuspendThread(); } else { request = await opCtx.readyResponses.DequeueAsync(token); clientSession.UnsafeResumeThread(); s = InternalContinuePendingRequestAndCallback(serialNo, opCtx, currentCtx, request); clientSession.UnsafeSuspendThread(); } } return(s); }
internal ReadAsyncInternal(FasterKV <Key, Value, Input, Output, Context, Functions> fasterKV, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, PendingContext pendingContext, AsyncIOContext <Key, Value> diskRequest) { _exception = default; _fasterKV = fasterKV; _clientSession = clientSession; _pendingContext = pendingContext; _diskRequest = diskRequest; CompletionComputeStatus = Pending; }
/// <summary> /// Complete outstanding pending operations /// </summary> /// <returns></returns> internal async ValueTask <(Status, Output)> CompletePendingReadAsync(long serialNo, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default) { bool done = true; #region Previous pending requests if (!RelaxedCPR) { if (clientSession.ctx.phase == Phase.IN_PROGRESS || clientSession.ctx.phase == Phase.WAIT_PENDING) { await CompleteIOPendingRequestsAsync(clientSession.ctx.prevCtx, clientSession.ctx, clientSession, token); Debug.Assert(clientSession.ctx.prevCtx.ioPendingRequests.Count == 0); if (clientSession.ctx.prevCtx.retryRequests.Count > 0) { CompleteRetryRequests(clientSession.ctx.prevCtx, clientSession.ctx, clientSession); } done &= (clientSession.ctx.prevCtx.ioPendingRequests.Count == 0); done &= (clientSession.ctx.prevCtx.retryRequests.Count == 0); } } #endregion var s = await CompleteIOPendingReadRequestsAsync(serialNo, clientSession.ctx, clientSession.ctx, clientSession, token); CompleteRetryRequests(clientSession.ctx, clientSession.ctx, clientSession); Debug.Assert(clientSession.ctx.ioPendingRequests.Count == 0); done &= (clientSession.ctx.ioPendingRequests.Count == 0); done &= (clientSession.ctx.retryRequests.Count == 0); if (!done) { throw new Exception("CompletePendingAsync did not complete"); } return(s); }
internal void CompleteRetryRequests(FasterExecutionContext opCtx, FasterExecutionContext currentCtx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession) { int count = opCtx.retryRequests.Count; if (count == 0) { return; } clientSession.UnsafeResumeThread(); for (int i = 0; i < count; i++) { var pendingContext = opCtx.retryRequests.Dequeue(); InternalRetryRequestAndCallback(opCtx, currentCtx, pendingContext); } clientSession.UnsafeSuspendThread(); }
/// <summary> /// Complete outstanding pending operations that were issued synchronously /// Async operations (e.g., ReadAsync) need to be completed individually /// </summary> /// <returns></returns> internal async ValueTask CompletePendingAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default) where Functions : IFunctions <Key, Value, Input, Output, Context> { bool done = true; #region Previous pending requests if (!RelaxedCPR) { if (clientSession.ctx.phase == Phase.IN_PROGRESS || clientSession.ctx.phase == Phase.WAIT_PENDING) { await clientSession.ctx.prevCtx.pendingReads.WaitEmptyAsync(); await InternalCompletePendingRequestsAsync(clientSession.ctx.prevCtx, clientSession.ctx, clientSession.FasterSession, token); Debug.Assert(clientSession.ctx.prevCtx.SyncIoPendingCount == 0); if (clientSession.ctx.prevCtx.retryRequests.Count > 0) { clientSession.FasterSession.UnsafeResumeThread(); InternalCompleteRetryRequests(clientSession.ctx.prevCtx, clientSession.ctx, clientSession.FasterSession); clientSession.FasterSession.UnsafeSuspendThread(); } done &= (clientSession.ctx.prevCtx.HasNoPendingRequests); } } #endregion await InternalCompletePendingRequestsAsync(clientSession.ctx, clientSession.ctx, clientSession.FasterSession, token); clientSession.FasterSession.UnsafeResumeThread(); InternalCompleteRetryRequests(clientSession.ctx, clientSession.ctx, clientSession.FasterSession); clientSession.FasterSession.UnsafeSuspendThread(); Debug.Assert(clientSession.ctx.HasNoPendingRequests); done &= (clientSession.ctx.HasNoPendingRequests); if (!done) { throw new Exception("CompletePendingAsync did not complete"); } }
private async ValueTask HandleCheckpointingPhasesAsync(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default) { if (async) { clientSession?.UnsafeResumeThread(); } var finalState = SystemState.Copy(ref _systemState); while (finalState.phase == Phase.INTERMEDIATE) { finalState = SystemState.Copy(ref _systemState); } var previousState = ctx != null?SystemState.Make(ctx.phase, ctx.version) : finalState; // We need to move from previousState to finalState one step at a time var currentState = previousState; SystemState startState = GetStartState(finalState); if ((currentState.version < startState.version) || (currentState.version == startState.version && currentState.phase < startState.phase)) { // Fast-forward to beginning of current checkpoint cycle currentState = startState; } do { switch (currentState.phase) { case Phase.PREP_INDEX_CHECKPOINT: { if (ctx != null) { if (!ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt]) { ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = true; } epoch.Mark(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.INDEX_CHECKPOINT: { if (_checkpointType == CheckpointType.INDEX_ONLY && ctx != null) { // Reseting the marker for a potential FULL or INDEX_ONLY checkpoint in the future ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = false; } if (async && !IsIndexFuzzyCheckpointCompleted()) { clientSession?.UnsafeSuspendThread(); await IsIndexFuzzyCheckpointCompletedAsync(token); clientSession?.UnsafeResumeThread(); } GlobalMoveToNextCheckpointState(currentState); break; } case Phase.PREPARE: { if (ctx != null) { if (!ctx.markers[EpochPhaseIdx.Prepare]) { if (!RelaxedCPR) { AcquireSharedLatchesForAllPendingRequests(ctx); } ctx.markers[EpochPhaseIdx.Prepare] = true; } epoch.Mark(EpochPhaseIdx.Prepare, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.Prepare, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.IN_PROGRESS: { if (ctx != null) { // Need to be very careful here as threadCtx is changing FasterExecutionContext _ctx; if (previousState.phase == Phase.IN_PROGRESS) { _ctx = ctx.prevCtx; } else { _ctx = ctx; } if (!_ctx.markers[EpochPhaseIdx.InProgress]) { AtomicSwitch(ctx, ctx.prevCtx, _ctx.version); InitContext(ctx, ctx.prevCtx.guid, ctx.prevCtx.serialNum); // Has to be prevCtx, not ctx ctx.prevCtx.markers[EpochPhaseIdx.InProgress] = true; } epoch.Mark(EpochPhaseIdx.InProgress, currentState.version); } // Has to be prevCtx, not ctx if (epoch.CheckIsComplete(EpochPhaseIdx.InProgress, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.WAIT_PENDING: { if (ctx != null) { if (!ctx.prevCtx.markers[EpochPhaseIdx.WaitPending]) { var notify = (ctx.prevCtx.ioPendingRequests.Count == 0); notify = notify && (ctx.prevCtx.retryRequests.Count == 0); if (notify) { ctx.prevCtx.markers[EpochPhaseIdx.WaitPending] = true; } else { break; } } epoch.Mark(EpochPhaseIdx.WaitPending, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.WaitPending, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.WAIT_FLUSH: { if (ctx == null || !ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush]) { bool notify; if (FoldOverSnapshot) { notify = (hlog.FlushedUntilAddress >= _hybridLogCheckpoint.info.finalLogicalAddress); } else { notify = (_hybridLogCheckpoint.flushedSemaphore != null) && _hybridLogCheckpoint.flushedSemaphore.CurrentCount > 0; } if (async && !notify) { Debug.Assert(_hybridLogCheckpoint.flushedSemaphore != null); clientSession?.UnsafeSuspendThread(); await _hybridLogCheckpoint.flushedSemaphore.WaitAsync(token); clientSession?.UnsafeResumeThread(); _hybridLogCheckpoint.flushedSemaphore.Release(); notify = true; } if (_checkpointType == CheckpointType.FULL) { notify = notify && IsIndexFuzzyCheckpointCompleted(); if (async && !notify) { clientSession?.UnsafeSuspendThread(); await IsIndexFuzzyCheckpointCompletedAsync(token); clientSession?.UnsafeResumeThread(); notify = true; } } if (notify) { if (ctx != null) { ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush] = true; } } else { break; } } if (ctx != null) { epoch.Mark(EpochPhaseIdx.WaitFlush, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.WaitFlush, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.PERSISTENCE_CALLBACK: { if (ctx != null) { if (!ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback]) { if (ctx.prevCtx.serialNum != -1) { var commitPoint = new CommitPoint { UntilSerialNo = ctx.prevCtx.serialNum, ExcludedSerialNos = ctx.prevCtx.excludedSerialNos }; // Thread local action functions.CheckpointCompletionCallback(ctx.guid, commitPoint); if (clientSession != null) { clientSession.LatestCommitPoint = commitPoint; } } ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback] = true; } epoch.Mark(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version); } if (epoch.CheckIsComplete(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version)) { GlobalMoveToNextCheckpointState(currentState); } break; } case Phase.REST: { break; } default: throw new FasterException("Invalid state found during checkpointing"); } if (ctx != null) { // update thread local variables ctx.phase = currentState.phase; ctx.version = currentState.version; } previousState.word = currentState.word; currentState = GetNextState(currentState, _checkpointType); } while (previousState.word != finalState.word); if (async) { clientSession?.UnsafeSuspendThread(); } }
internal ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > ReadAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, ref Key key, ref Input input, Context context = default, CancellationToken token = default) where Functions : IFunctions <Key, Value, Input, Output, Context> { var pcontext = default(PendingContext <Input, Output, Context>); Output output = default; OperationStatus internalStatus; var nextSerialNum = clientSession.ctx.serialNum + 1; if (clientSession.SupportAsync) { clientSession.UnsafeResumeThread(); } try { TryReadAgain: internalStatus = InternalRead(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, nextSerialNum); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, output))); } if (internalStatus == OperationStatus.CPR_SHIFT_DETECTED) { SynchronizeEpoch(clientSession.ctx, clientSession.ctx, ref pcontext, clientSession.FasterSession); goto TryReadAgain; } } finally { clientSession.ctx.serialNum = nextSerialNum; if (clientSession.SupportAsync) { clientSession.UnsafeSuspendThread(); } } return(SlowReadAsync(this, clientSession, pcontext, token)); }
/// <summary> /// Check if at least one (sync) request is ready for CompletePending to operate on /// </summary> /// <param name="clientSession"></param> /// <param name="token"></param> /// <returns></returns> internal async ValueTask ReadyToCompletePendingAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, CancellationToken token = default) where Functions : IFunctions <Key, Value, Input, Output, Context> { #region Previous pending requests if (!RelaxedCPR) { if (clientSession.ctx.phase == Phase.IN_PROGRESS || clientSession.ctx.phase == Phase.WAIT_PENDING) { if (clientSession.ctx.prevCtx.SyncIoPendingCount != 0) { await clientSession.ctx.prevCtx.readyResponses.WaitForEntryAsync(token); } } } #endregion if (clientSession.ctx.SyncIoPendingCount != 0) { await clientSession.ctx.readyResponses.WaitForEntryAsync(token); } }
internal ValueTask <RmwAsyncResult <Input, Output, Context, Functions> > RmwAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, ref Key key, ref Input input, Context context, long serialNo, CancellationToken token = default) where Functions : IFunctions <Key, Value, Input, Output, Context> { var pcontext = default(PendingContext <Input, Output, Context>); var diskRequest = default(AsyncIOContext <Key, Value>); if (clientSession.SupportAsync) { clientSession.UnsafeResumeThread(); } try { OperationStatus internalStatus; do { internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, serialNo); }while (internalStatus == OperationStatus.RETRY_NOW || internalStatus == OperationStatus.RETRY_LATER); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { return(new ValueTask <RmwAsyncResult <Input, Output, Context, Functions> >(new RmwAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, default)));
internal ValueTask <ReadAsyncResult <Input, Output, Context, Functions> > ReadAsync <Input, Output, Context, Functions>(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, ref Key key, ref Input input, Context context, long serialNo, CancellationToken token) where Functions : IFunctions <Key, Value, Input, Output, Context> { var pcontext = default(PendingContext <Input, Output, Context>); var diskRequest = default(AsyncIOContext <Key, Value>); Output output = default; if (clientSession.SupportAsync) { clientSession.UnsafeResumeThread(); } try { OperationStatus internalStatus = InternalRead(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, clientSession.ctx, serialNo); Debug.Assert(internalStatus != OperationStatus.RETRY_NOW); Debug.Assert(internalStatus != OperationStatus.RETRY_LATER); if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>((Status)internalStatus, output))); } else { var status = HandleOperationStatus(clientSession.ctx, clientSession.ctx, ref pcontext, clientSession.FasterSession, internalStatus, true, out diskRequest); if (status != Status.PENDING) { return(new ValueTask <ReadAsyncResult <Input, Output, Context, Functions> >(new ReadAsyncResult <Input, Output, Context, Functions>(status, output))); } } } finally { Debug.Assert(serialNo >= clientSession.ctx.serialNum, "Operation serial numbers must be non-decreasing"); clientSession.ctx.serialNum = serialNo; if (clientSession.SupportAsync) { clientSession.UnsafeSuspendThread(); } } return(SlowReadAsync(this, clientSession, pcontext, diskRequest, token)); }
private void HandleCheckpointingPhases(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession) { var _ = HandleCheckpointingPhasesAsync(ctx, clientSession, false); return; }
public AsyncFasterSession(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession) { _clientSession = clientSession; }
internal UnsafeContext(ClientSession <Key, Value, Input, Output, Context, Functions> clientSession) { this.clientSession = clientSession; FasterSession = new InternalFasterSession(clientSession); }