/// <summary>
/// Complete outstanding pending operations that were issued synchronously
/// Async operations (e.g., ReadAsync) need to be completed individually
/// </summary>
/// <returns></returns>
internal async ValueTask CompletePendingAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, CancellationToken token,
CompletedOutputIterator <Key, Value, Input, Output, Context> completedOutputs)
{
while (true)
{
bool done = true;
#region Previous pending requests
if (!RelaxedCPR)
{
if (currentCtx.phase == Phase.IN_PROGRESS || currentCtx.phase == Phase.WAIT_PENDING)
{
fasterSession.UnsafeResumeThread();
try
{
InternalCompletePendingRequests(currentCtx.prevCtx, currentCtx, fasterSession, completedOutputs);
InternalCompleteRetryRequests(currentCtx.prevCtx, currentCtx, fasterSession);
}
finally
{
fasterSession.UnsafeSuspendThread();
}
await currentCtx.prevCtx.WaitPendingAsync(token).ConfigureAwait(false);
done &= currentCtx.prevCtx.HasNoPendingRequests;
}
}
#endregion
fasterSession.UnsafeResumeThread();
try
{
InternalCompletePendingRequests(currentCtx, currentCtx, fasterSession, completedOutputs);
InternalCompleteRetryRequests(currentCtx, currentCtx, fasterSession);
}
finally
{
fasterSession.UnsafeSuspendThread();
}
await currentCtx.WaitPendingAsync(token).ConfigureAwait(false);
done &= currentCtx.HasNoPendingRequests;
if (done)
{
return;
}
InternalRefresh(currentCtx, fasterSession);
Thread.Yield();
}
}
internal ValueTask <DeleteAsyncResult <Input, Output, Context> > DeleteAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref PendingContext <Input, Output, Context> pcontext, ref Key key, Context userContext, long serialNo, CancellationToken token)
{
CompletionEvent flushEvent;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
flushEvent = hlog.FlushEvent;
internalStatus = InternalDelete(ref key, ref userContext, ref pcontext, fasterSession, currentCtx, serialNo);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (OperationStatusUtils.TryConvertToStatusCode(internalStatus, out Status status))
{
return(new ValueTask <DeleteAsyncResult <Input, Output, Context> >(new DeleteAsyncResult <Input, Output, Context>(new(internalStatus))));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowDeleteAsync(this, fasterSession, currentCtx, pcontext, flushEvent, token));
}
private bool TryCompleteSync(bool asyncOp, out CompletionEvent flushEvent, out TAsyncResult asyncResult)
{
_fasterSession.UnsafeResumeThread();
try
{
Status status = _asyncOperation.DoFastOperation(_fasterKV, ref _pendingContext, _fasterSession, _currentCtx, asyncOp, out flushEvent, out Output output);
if (!status.IsPending)
{
_pendingContext.Dispose();
asyncResult = _asyncOperation.CreateResult(status, output, new RecordMetadata(_pendingContext.recordInfo, _pendingContext.logicalAddress));
return(true);
}
}
catch (Exception e)
{
_exception = ExceptionDispatchInfo.Capture(e);
flushEvent = default;
}
finally
{
_fasterSession.UnsafeSuspendThread();
}
asyncResult = default;
return(false);
}
internal ValueTask <DeleteAsyncResult <Input, Output, Context> > DeleteAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref Key key, Context userContext, long serialNo, CancellationToken token = default)
{
var pcontext = default(PendingContext <Input, Output, Context>);
pcontext.IsAsync = true;
Task flushTask;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
flushTask = hlog.FlushTask;
internalStatus = InternalDelete(ref key, ref userContext, ref pcontext, fasterSession, currentCtx, serialNo);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <DeleteAsyncResult <Input, Output, Context> >(new DeleteAsyncResult <Input, Output, Context>(internalStatus)));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowDeleteAsync(this, fasterSession, currentCtx, pcontext, flushTask, token));
}
private ValueTask <UpsertAsyncResult <Input, Output, Context> > UpsertAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref PendingContext <Input, Output, Context> pcontext, ref Key key, ref Value value, Context userContext, long serialNo, CancellationToken token)
{
CompletionEvent flushEvent;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
flushEvent = hlog.FlushEvent;
internalStatus = InternalUpsert(ref key, ref value, ref userContext, ref pcontext, fasterSession, currentCtx, serialNo);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <UpsertAsyncResult <Input, Output, Context> >(new UpsertAsyncResult <Input, Output, Context>((Status)internalStatus)));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowUpsertAsync(this, fasterSession, currentCtx, pcontext, flushEvent, token));
}
/// <summary>
/// Complete outstanding pending operations that were issued synchronously
/// Async operations (e.g., ReadAsync) need to be completed individually
/// </summary>
/// <returns></returns>
internal async ValueTask CompletePendingAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, CancellationToken token,
CompletedOutputIterator <Key, Value, Input, Output, Context> completedOutputs)
{
while (true)
{
fasterSession.UnsafeResumeThread();
try
{
InternalCompletePendingRequests(currentCtx, currentCtx, fasterSession, completedOutputs);
InternalCompleteRetryRequests(currentCtx, currentCtx, fasterSession);
}
finally
{
fasterSession.UnsafeSuspendThread();
}
await currentCtx.WaitPendingAsync(token).ConfigureAwait(false);
if (currentCtx.HasNoPendingRequests)
{
return;
}
InternalRefresh(currentCtx, fasterSession);
Thread.Yield();
}
}
private ValueTask <UpsertAsyncResult <Input, Output, Context> > UpsertAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, ref PendingContext <Input, Output, Context> pcontext, ref Key key, ref Input input, ref Value value, Context userContext, long serialNo, CancellationToken token)
{
CompletionEvent flushEvent;
Output output = default;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
flushEvent = hlog.FlushEvent;
internalStatus = InternalUpsert(ref key, ref input, ref value, ref output, ref userContext, ref pcontext, fasterSession, currentCtx, serialNo);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (OperationStatusUtils.TryConvertToStatusCode(internalStatus, out Status status))
{
return(new ValueTask <UpsertAsyncResult <Input, Output, Context> >(new UpsertAsyncResult <Input, Output, Context>(status, output, new RecordMetadata(pcontext.recordInfo, pcontext.logicalAddress))));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowUpsertAsync(this, fasterSession, currentCtx, pcontext, flushEvent, token));
}
/// <summary>
/// Complete outstanding pending operations that were issued synchronously
/// Async operations (e.g., ReadAsync) need to be completed individually
/// </summary>
/// <returns></returns>
internal async ValueTask CompletePendingAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx, CancellationToken token = default)
{
bool done = true;
#region Previous pending requests
if (!RelaxedCPR)
{
if (currentCtx.phase == Phase.IN_PROGRESS
||
currentCtx.phase == Phase.WAIT_PENDING)
{
await currentCtx.prevCtx.pendingReads.WaitUntilEmptyAsync(token);
await InternalCompletePendingRequestsAsync(currentCtx.prevCtx, currentCtx, fasterSession, token);
Debug.Assert(currentCtx.prevCtx.SyncIoPendingCount == 0);
if (currentCtx.prevCtx.retryRequests.Count > 0)
{
fasterSession.UnsafeResumeThread();
InternalCompleteRetryRequests(currentCtx.prevCtx, currentCtx, fasterSession);
fasterSession.UnsafeSuspendThread();
}
done &= (currentCtx.prevCtx.HasNoPendingRequests);
}
}
#endregion
await InternalCompletePendingRequestsAsync(currentCtx, currentCtx, fasterSession, token);
fasterSession.UnsafeResumeThread();
InternalCompleteRetryRequests(currentCtx, currentCtx, fasterSession);
fasterSession.UnsafeSuspendThread();
Debug.Assert(currentCtx.HasNoPendingRequests);
done &= (currentCtx.HasNoPendingRequests);
if (!done)
{
throw new Exception("CompletePendingAsync did not complete");
}
}
internal async ValueTask <TAsyncResult> CompleteAsync(CancellationToken token = default)
{
Debug.Assert(_fasterKV.RelaxedCPR);
if (CompletionComputeStatus != Completed &&
Interlocked.CompareExchange(ref CompletionComputeStatus, Completed, Pending) == Pending)
{
try
{
if (_exception == default)
{
await _flushTask.WithCancellationAsync(token);
_fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
_flushTask = _fasterKV.hlog.FlushTask;
internalStatus = asyncOperation.DoFastOperation(_fasterKV, ref _pendingContext, _fasterSession, _currentCtx);
} while (internalStatus == OperationStatus.RETRY_NOW);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
_pendingContext.Dispose();
return(asyncOperation.CreateResult(internalStatus));
}
Debug.Assert(internalStatus == OperationStatus.ALLOCATE_FAILED);
}
finally
{
_fasterSession.UnsafeSuspendThread();
}
}
}
catch (Exception e)
{
_exception = ExceptionDispatchInfo.Capture(e);
}
finally
{
_currentCtx.asyncPendingCount--;
}
}
if (_exception != default)
{
_exception.Throw();
}
return(await asyncOperation.DoSlowOperation(_fasterKV, _fasterSession, _currentCtx, _pendingContext, _flushTask, token));
}
internal ValueTask <RmwAsyncResult <Input, Output, Context> > CompleteAsync(CancellationToken token = default)
{
Debug.Assert(_fasterKV.RelaxedCPR);
AsyncIOContext <Key, Value> newDiskRequest = default;
if (_diskRequest.asyncOperation != null &&
CompletionComputeStatus != Completed &&
Interlocked.CompareExchange(ref CompletionComputeStatus, Completed, Pending) == Pending)
{
try
{
if (_exception == default)
{
_fasterSession.UnsafeResumeThread();
try
{
var status = _fasterKV.InternalCompletePendingRequestFromContext(_currentCtx, _currentCtx, _fasterSession, _diskRequest, ref _pendingContext, true, out newDiskRequest);
_pendingContext.Dispose();
if (status != Status.PENDING)
{
return(new ValueTask <RmwAsyncResult <Input, Output, Context> >(new RmwAsyncResult <Input, Output, Context>(status, default)));
}
}
finally
{
_fasterSession.UnsafeSuspendThread();
}
}
}
catch (Exception e)
{
_exception = ExceptionDispatchInfo.Capture(e);
}
finally
{
_currentCtx.ioPendingRequests.Remove(_pendingContext.id);
_currentCtx.asyncPendingCount--;
_currentCtx.pendingReads.Remove();
}
}
if (_exception != default)
{
_exception.Throw();
}
return(SlowRmwAsync(_fasterKV, _fasterSession, _currentCtx, _pendingContext, newDiskRequest, token));
}
internal (Status, Output) Complete()
{
(Status, Output)_result = default;
if (_diskRequest.asyncOperation != null &&
CompletionComputeStatus != Completed &&
Interlocked.CompareExchange(ref CompletionComputeStatus, Completed, Pending) == Pending)
{
try
{
if (_exception == default)
{
_fasterSession.UnsafeResumeThread();
try
{
Debug.Assert(_fasterKV.RelaxedCPR);
var status = _fasterKV.InternalCompletePendingRequestFromContext(_currentCtx, _currentCtx, _fasterSession, _diskRequest, ref _pendingContext, true, out _);
Debug.Assert(status != Status.PENDING);
_result = (status, _pendingContext.output);
_recordInfo = _pendingContext.recordInfo;
_pendingContext.Dispose();
}
finally
{
_fasterSession.UnsafeSuspendThread();
}
}
}
catch (Exception e)
{
_exception = ExceptionDispatchInfo.Capture(e);
}
finally
{
_currentCtx.ioPendingRequests.Remove(_pendingContext.id);
_currentCtx.asyncPendingCount--;
_currentCtx.pendingReads.Remove();
}
}
if (_exception != default)
{
_exception.Throw();
}
return(_result);
}
internal ValueTask <ReadAsyncResult <Input, Output, Context> > ReadAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx,
ref Key key, ref Input input, ref ReadOptions readOptions, Context context, long serialNo, CancellationToken token, bool noKey = false)
{
var pcontext = default(PendingContext <Input, Output, Context>);
var operationFlags = PendingContext <Input, Output, Context> .GetOperationFlags(MergeReadFlags(currentCtx.ReadFlags, readOptions.ReadFlags), noKey);
pcontext.SetOperationFlags(operationFlags, readOptions.StopAddress);
var diskRequest = default(AsyncIOContext <Key, Value>);
Output output = default;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus;
do
{
internalStatus = InternalRead(ref key, ref input, ref output, readOptions.StartAddress, ref context, ref pcontext, fasterSession, currentCtx, serialNo);
}while (internalStatus == OperationStatus.RETRY_NOW);
Debug.Assert(internalStatus != OperationStatus.RETRY_LATER);
if (OperationStatusUtils.TryConvertToStatusCode(internalStatus, out Status status))
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context> >(new ReadAsyncResult <Input, Output, Context>(new(internalStatus), output, new RecordMetadata(pcontext.recordInfo, pcontext.logicalAddress))));
}
status = HandleOperationStatus(currentCtx, currentCtx, ref pcontext, fasterSession, internalStatus, true, out diskRequest);
if (!status.IsPending)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context> >(new ReadAsyncResult <Input, Output, Context>(status, output, new RecordMetadata(pcontext.recordInfo, pcontext.logicalAddress))));
}
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowReadAsync(this, fasterSession, currentCtx, pcontext, diskRequest, token));
}
internal ValueTask <ReadAsyncResult <Input, Output, Context> > ReadAsync <Input, Output, Context>(IFasterSession <Key, Value, Input, Output, Context> fasterSession,
FasterExecutionContext <Input, Output, Context> currentCtx,
ref Key key, ref Input input, long startAddress, Context context, long serialNo, CancellationToken token, byte operationFlags = 0)
{
var pcontext = default(PendingContext <Input, Output, Context>);
pcontext.SetOperationFlags(operationFlags, startAddress);
var diskRequest = default(AsyncIOContext <Key, Value>);
Output output = default;
fasterSession.UnsafeResumeThread();
try
{
OperationStatus internalStatus = InternalRead(ref key, ref input, ref output, startAddress, ref context, ref pcontext, fasterSession, currentCtx, serialNo);
Debug.Assert(internalStatus != OperationStatus.RETRY_NOW);
Debug.Assert(internalStatus != OperationStatus.RETRY_LATER);
if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context> >(new ReadAsyncResult <Input, Output, Context>((Status)internalStatus, output, pcontext.recordInfo)));
}
else
{
var status = HandleOperationStatus(currentCtx, currentCtx, ref pcontext, fasterSession, internalStatus, true, out diskRequest);
if (status != Status.PENDING)
{
return(new ValueTask <ReadAsyncResult <Input, Output, Context> >(new ReadAsyncResult <Input, Output, Context>(status, output, pcontext.recordInfo)));
}
}
}
finally
{
Debug.Assert(serialNo >= currentCtx.serialNum, "Operation serial numbers must be non-decreasing");
currentCtx.serialNum = serialNo;
fasterSession.UnsafeSuspendThread();
}
return(SlowReadAsync(this, fasterSession, currentCtx, pcontext, diskRequest, token));
}
private bool TryCompleteSync(bool asyncOp, out CompletionEvent flushEvent, out TAsyncResult asyncResult)
{
_fasterSession.UnsafeResumeThread();
try
{
Status status = _asyncOperation.DoFastOperation(_fasterKV, ref _pendingContext, _fasterSession, _currentCtx, asyncOp, out flushEvent, out Output output);
if (status != Status.PENDING)
{
_pendingContext.Dispose();
asyncResult = _asyncOperation.CreateResult(status, output);
return(true);
}
}
finally
{
_fasterSession.UnsafeSuspendThread();
}
asyncResult = default;
return(false);
}