/// <summary>
/// Create FASTER instance
/// </summary>
/// <param name="size">Size of core index (#cache lines)</param>
/// <param name="comparer">FASTER equality comparer for key</param>
/// <param name="variableLengthStructSettings"></param>
/// <param name="functions">Callback functions</param>
/// <param name="logSettings">Log settings</param>
/// <param name="checkpointSettings">Checkpoint settings</param>
/// <param name="serializerSettings">Serializer settings</param>
public FasterKV(long size, Functions functions, LogSettings logSettings, CheckpointSettings checkpointSettings = null, SerializerSettings <Key, Value> serializerSettings = null, IFasterEqualityComparer <Key> comparer = null, VariableLengthStructSettings <Key, Value> variableLengthStructSettings = null)
{
if (comparer != null)
{
this.comparer = comparer;
}
else
{
if (typeof(IFasterEqualityComparer <Key>).IsAssignableFrom(typeof(Key)))
{
this.comparer = new Key() as IFasterEqualityComparer <Key>;
}
else
{
Console.WriteLine("***WARNING*** Creating default FASTER key equality comparer based on potentially slow EqualityComparer<Key>.Default. To avoid this, provide a comparer (IFasterEqualityComparer<Key>) as an argument to FASTER's constructor, or make Key implement the interface IFasterEqualityComparer<Key>");
this.comparer = FasterEqualityComparer <Key> .Default;
}
}
if (checkpointSettings == null)
{
checkpointSettings = new CheckpointSettings();
}
if (checkpointSettings.CheckpointDir != null && checkpointSettings.CheckpointManager != null)
{
throw new FasterException("Specify either CheckpointManager or CheckpointDir for CheckpointSettings, not both");
}
checkpointManager = checkpointSettings.CheckpointManager ?? new LocalCheckpointManager(checkpointSettings.CheckpointDir ?? "");
FoldOverSnapshot = checkpointSettings.CheckPointType == core.CheckpointType.FoldOver;
CopyReadsToTail = logSettings.CopyReadsToTail;
this.functions = functions;
if (logSettings.ReadCacheSettings != null)
{
CopyReadsToTail = false;
UseReadCache = true;
}
if (Utility.IsBlittable <Key>() && Utility.IsBlittable <Value>())
{
if (variableLengthStructSettings != null)
{
hlog = new VariableLengthBlittableAllocator <Key, Value>(logSettings, variableLengthStructSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context, Functions>(this, hlog);
if (UseReadCache)
{
readcache = new VariableLengthBlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, variableLengthStructSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context, Functions>(this, readcache);
}
}
else
{
hlog = new BlittableAllocator <Key, Value>(logSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context, Functions>(this, hlog);
if (UseReadCache)
{
readcache = new BlittableAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context, Functions>(this, readcache);
}
}
}
else
{
WriteDefaultOnDelete = true;
hlog = new GenericAllocator <Key, Value>(logSettings, serializerSettings, this.comparer, null, epoch);
Log = new LogAccessor <Key, Value, Input, Output, Context, Functions>(this, hlog);
if (UseReadCache)
{
readcache = new GenericAllocator <Key, Value>(
new LogSettings
{
PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits,
MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits,
MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction
}, serializerSettings, this.comparer, ReadCacheEvict, epoch);
readcache.Initialize();
ReadCache = new LogAccessor <Key, Value, Input, Output, Context, Functions>(this, readcache);
}
}
hlog.Initialize();
sectorSize = (int)logSettings.LogDevice.SectorSize;
Initialize(size, sectorSize);
_systemState = default;
_systemState.phase = Phase.REST;
_systemState.version = 1;
_checkpointType = CheckpointType.HYBRID_LOG_ONLY;
}
/// <summary>
/// Compare two <see cref="SystemState"/>s for equality
/// </summary>
internal static bool Equal(SystemState s1, SystemState s2)
{
return(s1.Word == s2.Word);
}
/// <summary>
/// Compare the current <see cref="SystemState"/> to <paramref name="other"/> for equality
/// </summary>
private bool Equals(SystemState other)
{
return(Word == other.Word);
}
public static bool Equal(SystemState s1, SystemState s2)
{
return(s1.word == s2.word);
}
internal static void RemoveIntermediate(ref SystemState state)
{
state.Phase &= ~Phase.INTERMEDIATE;
}
protected bool GlobalMoveToNextState(SystemState currentState, SystemState nextState, ref long context)
{
var intermediateState = SystemState.Make(Phase.INTERMEDIATE, currentState.version);
// Move from S1 to I
if (MakeTransition(currentState, intermediateState))
{
// Acquired ownership to make the transition from S1 to S2
switch (nextState.phase)
{
case Phase.PREP_INDEX_CHECKPOINT:
{
_checkpointType = (CheckpointType)context;
switch (_checkpointType)
{
case CheckpointType.INDEX_ONLY:
{
_indexCheckpointToken = Guid.NewGuid();
InitializeIndexCheckpoint(_indexCheckpointToken);
break;
}
case CheckpointType.FULL:
{
var fullCheckpointToken = Guid.NewGuid();
_indexCheckpointToken = fullCheckpointToken;
_hybridLogCheckpointToken = fullCheckpointToken;
InitializeIndexCheckpoint(_indexCheckpointToken);
InitializeHybridLogCheckpoint(_hybridLogCheckpointToken, currentState.version);
break;
}
default:
throw new Exception();
}
ObtainCurrentTailAddress(ref _indexCheckpoint.info.startLogicalAddress);
MakeTransition(intermediateState, nextState);
break;
}
case Phase.INDEX_CHECKPOINT:
{
TakeIndexFuzzyCheckpoint();
MakeTransition(intermediateState, nextState);
break;
}
case Phase.PREPARE:
{
switch (currentState.phase)
{
case Phase.REST:
{
_checkpointType = (CheckpointType)context;
Debug.Assert(_checkpointType == CheckpointType.HYBRID_LOG_ONLY);
_hybridLogCheckpointToken = Guid.NewGuid();
InitializeHybridLogCheckpoint(_hybridLogCheckpointToken, currentState.version);
break;
}
case Phase.PREP_INDEX_CHECKPOINT:
{
TakeIndexFuzzyCheckpoint();
break;
}
default:
throw new Exception();
}
ObtainCurrentTailAddress(ref _hybridLogCheckpoint.info.startLogicalAddress);
if (!Constants.kFoldOverSnapshot)
{
_hybridLogCheckpoint.info.flushedLogicalAddress = hlog.FlushedUntilAddress;
_hybridLogCheckpoint.info.useSnapshotFile = 1;
}
MakeTransition(intermediateState, nextState);
break;
}
case Phase.IN_PROGRESS:
{
MakeTransition(intermediateState, nextState);
break;
}
case Phase.WAIT_PENDING:
{
MakeTransition(intermediateState, nextState);
break;
}
case Phase.WAIT_FLUSH:
{
if (_checkpointType == CheckpointType.FULL)
{
_indexCheckpoint.info.num_buckets = overflowBucketsAllocator.GetMaxValidAddress();
ObtainCurrentTailAddress(ref _indexCheckpoint.info.finalLogicalAddress);
WriteIndexMetaFile();
}
if (Constants.kFoldOverSnapshot)
{
hlog.ShiftReadOnlyToTail(out long tailAddress);
_hybridLogCheckpoint.info.finalLogicalAddress = tailAddress;
}
else
{
ObtainCurrentTailAddress(ref _hybridLogCheckpoint.info.finalLogicalAddress);
_hybridLogCheckpoint.snapshotFileDevice = FasterFactory.CreateLogDevice
(DirectoryConfiguration.GetHybridLogCheckpointFileName(_hybridLogCheckpointToken));
_hybridLogCheckpoint.snapshotFileObjectLogDevice = FasterFactory.CreateObjectLogDevice
(DirectoryConfiguration.GetHybridLogCheckpointFileName(_hybridLogCheckpointToken));
long startPage = hlog.GetPage(_hybridLogCheckpoint.info.flushedLogicalAddress);
long endPage = hlog.GetPage(_hybridLogCheckpoint.info.finalLogicalAddress);
if (_hybridLogCheckpoint.info.finalLogicalAddress > hlog.GetStartLogicalAddress(endPage))
{
endPage++;
}
new Thread(() =>
hlog.AsyncFlushPagesToDevice(startPage,
endPage,
_hybridLogCheckpoint.snapshotFileDevice,
_hybridLogCheckpoint.snapshotFileObjectLogDevice,
out _hybridLogCheckpoint.flushed)).Start();
}
WriteHybridLogMetaInfo();
MakeTransition(intermediateState, nextState);
break;
}
case Phase.PERSISTENCE_CALLBACK:
{
MakeTransition(intermediateState, nextState);
break;
}
case Phase.GC:
{
var tmp = hlog.BeginAddress;
hlog.BeginAddress = context;
context = tmp;
int numChunks = (int)(state[resizeInfo.version].size / Constants.kSizeofChunk);
if (numChunks == 0)
{
numChunks = 1; // at least one chunk
}
numPendingChunksToBeGCed = numChunks;
gcStatus = new long[numChunks];
MakeTransition(intermediateState, nextState);
break;
}
case Phase.PREPARE_GROW:
{
// Note that the transition must be done before bumping epoch here!
MakeTransition(intermediateState, nextState);
epoch.BumpCurrentEpoch(() =>
{
long _context = 0;
GlobalMoveToNextState(nextState, SystemState.Make(Phase.IN_PROGRESS_GROW, nextState.version), ref _context);
});
break;
}
case Phase.IN_PROGRESS_GROW:
{
// Set up the transition to new version of HT
int numChunks = (int)(state[resizeInfo.version].size / Constants.kSizeofChunk);
if (numChunks == 0)
{
numChunks = 1; // at least one chunk
}
numPendingChunksToBeSplit = numChunks;
splitStatus = new long[numChunks];
Initialize(1 - resizeInfo.version, state[resizeInfo.version].size * 2, hlog.GetSectorSize());
resizeInfo.version = 1 - resizeInfo.version;
MakeTransition(intermediateState, nextState);
break;
}
case Phase.REST:
{
switch (_checkpointType)
{
case CheckpointType.INDEX_ONLY:
{
_indexCheckpoint.info.num_buckets = overflowBucketsAllocator.GetMaxValidAddress();
ObtainCurrentTailAddress(ref _indexCheckpoint.info.finalLogicalAddress);
WriteIndexMetaFile();
_indexCheckpoint.Reset();
break;
}
case CheckpointType.FULL:
{
_indexCheckpoint.Reset();
_hybridLogCheckpoint.Reset();
break;
}
case CheckpointType.HYBRID_LOG_ONLY:
{
_hybridLogCheckpoint.Reset();
break;
}
case CheckpointType.NONE:
break;
default:
throw new Exception();
}
_checkpointType = CheckpointType.NONE;
MakeTransition(intermediateState, nextState);
break;
}
}
return(true);
}
else
{
return(false);
}
}
protected bool GlobalMoveToNextCheckpointState(SystemState currentState)
{
long context = 0;
return(GlobalMoveToNextState(currentState, GetNextState(currentState, _checkpointType), ref context));
}
/// <inheritdoc />
public virtual void GlobalBeforeEnteringState <Key, Value, Input, Output, Context, Functions>(SystemState next,
FasterKV <Key, Value, Input, Output, Context, Functions> faster)
where Key : new()
where Value : new()
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
switch (next.phase)
{
case Phase.PREPARE:
if (faster._hybridLogCheckpointToken == default)
{
faster._hybridLogCheckpointToken = Guid.NewGuid();
faster.InitializeHybridLogCheckpoint(faster._hybridLogCheckpointToken, next.version);
}
faster.ObtainCurrentTailAddress(ref faster._hybridLogCheckpoint.info.startLogicalAddress);
break;
case Phase.WAIT_FLUSH:
faster._hybridLogCheckpoint.info.headAddress = faster.hlog.HeadAddress;
faster._hybridLogCheckpoint.info.beginAddress = faster.hlog.BeginAddress;
break;
case Phase.PERSISTENCE_CALLBACK:
// Collect object log offsets only after flushes
// are completed
var seg = faster.hlog.GetSegmentOffsets();
if (seg != null)
{
faster._hybridLogCheckpoint.info.objectLogSegmentOffsets = new long[seg.Length];
Array.Copy(seg, faster._hybridLogCheckpoint.info.objectLogSegmentOffsets, seg.Length);
}
if (faster._activeSessions != null)
{
// write dormant sessions to checkpoint
foreach (var kvp in faster._activeSessions)
{
faster.AtomicSwitch(kvp.Value.ctx, kvp.Value.ctx.prevCtx, next.version - 1);
}
}
faster.WriteHybridLogMetaInfo();
break;
case Phase.REST:
faster._hybridLogCheckpointToken = default;
faster._hybridLogCheckpoint.Reset();
var nextTcs = new TaskCompletionSource <LinkedCheckpointInfo>(TaskCreationOptions.RunContinuationsAsynchronously);
faster.checkpointTcs.SetResult(new LinkedCheckpointInfo {
NextTask = nextTcs.Task
});
faster.checkpointTcs = nextTcs;
break;
}
}
/// <inheritdoc />
public virtual void GlobalAfterEnteringState <Key, Value, Input, Output, Context, Functions>(SystemState next,
FasterKV <Key, Value, Input, Output, Context, Functions> faster)
where Key : new()
where Value : new()
where Functions : IFunctions <Key, Value, Input, Output, Context>
{
}
private SystemState FastForwardToCurrentCycle(SystemState currentThreadState, SystemState currentGlobalState)
{
var startState = StartOfCurrentCycle(currentGlobalState);
if (currentThreadState.version < startState.version ||
currentThreadState.version == startState.version && currentThreadState.phase < startState.phase)
{
return(startState);
}
return(currentThreadState);
}
private SystemState StartOfCurrentCycle(SystemState currentGlobalState)
{
return(currentGlobalState.phase <= Phase.REST
? SystemState.Make(Phase.REST, currentGlobalState.version - 1)
: SystemState.Make(Phase.REST, currentGlobalState.version));
}
private bool GlobalMoveToNextState(SystemState currentState, SystemState nextState, ref long context)
{
var intermediateState = SystemState.Make(Phase.INTERMEDIATE, currentState.version);
// Move from S1 to I
if (MakeTransition(currentState, intermediateState))
{
// Acquired ownership to make the transition from S1 to S2
switch (nextState.phase)
{
case Phase.PREP_INDEX_CHECKPOINT:
{
_checkpointType = (CheckpointType)context;
switch (_checkpointType)
{
case CheckpointType.INDEX_ONLY:
{
_indexCheckpointToken = Guid.NewGuid();
InitializeIndexCheckpoint(_indexCheckpointToken);
break;
}
case CheckpointType.FULL:
{
var fullCheckpointToken = Guid.NewGuid();
_indexCheckpointToken = fullCheckpointToken;
_hybridLogCheckpointToken = fullCheckpointToken;
InitializeIndexCheckpoint(_indexCheckpointToken);
InitializeHybridLogCheckpoint(_hybridLogCheckpointToken, currentState.version);
break;
}
default:
throw new FasterException();
}
ObtainCurrentTailAddress(ref _indexCheckpoint.info.startLogicalAddress);
MakeTransition(intermediateState, nextState);
break;
}
case Phase.INDEX_CHECKPOINT:
{
if (UseReadCache && this.ReadCache.BeginAddress != this.ReadCache.TailAddress)
{
throw new FasterException("Index checkpoint with read cache is not supported");
}
TakeIndexFuzzyCheckpoint();
MakeTransition(intermediateState, nextState);
break;
}
case Phase.PREPARE:
{
switch (currentState.phase)
{
case Phase.REST:
{
_checkpointType = (CheckpointType)context;
Debug.Assert(_checkpointType == CheckpointType.HYBRID_LOG_ONLY);
_hybridLogCheckpointToken = Guid.NewGuid();
InitializeHybridLogCheckpoint(_hybridLogCheckpointToken, currentState.version);
break;
}
case Phase.PREP_INDEX_CHECKPOINT:
{
if (UseReadCache && this.ReadCache.BeginAddress != this.ReadCache.TailAddress)
{
throw new FasterException("Index checkpoint with read cache is not supported");
}
TakeIndexFuzzyCheckpoint();
break;
}
default:
throw new FasterException();
}
ObtainCurrentTailAddress(ref _hybridLogCheckpoint.info.startLogicalAddress);
if (!FoldOverSnapshot)
{
_hybridLogCheckpoint.info.flushedLogicalAddress = hlog.FlushedUntilAddress;
_hybridLogCheckpoint.info.useSnapshotFile = 1;
}
MakeTransition(intermediateState, nextState);
break;
}
case Phase.IN_PROGRESS:
{
MakeTransition(intermediateState, nextState);
break;
}
case Phase.WAIT_PENDING:
{
MakeTransition(intermediateState, nextState);
break;
}
case Phase.WAIT_FLUSH:
{
if (_checkpointType == CheckpointType.FULL)
{
_indexCheckpoint.info.num_buckets = overflowBucketsAllocator.GetMaxValidAddress();
ObtainCurrentTailAddress(ref _indexCheckpoint.info.finalLogicalAddress);
}
_hybridLogCheckpoint.info.headAddress = hlog.HeadAddress;
_hybridLogCheckpoint.info.beginAddress = hlog.BeginAddress;
var seg = hlog.GetSegmentOffsets();
if (seg != null)
{
_hybridLogCheckpoint.info.objectLogSegmentOffsets = new long[seg.Length];
Array.Copy(seg, _hybridLogCheckpoint.info.objectLogSegmentOffsets, seg.Length);
}
if (FoldOverSnapshot)
{
hlog.ShiftReadOnlyToTail(out long tailAddress, out _hybridLogCheckpoint.flushedSemaphore);
_hybridLogCheckpoint.info.finalLogicalAddress = tailAddress;
}
else
{
ObtainCurrentTailAddress(ref _hybridLogCheckpoint.info.finalLogicalAddress);
_hybridLogCheckpoint.snapshotFileDevice = checkpointManager.GetSnapshotLogDevice(_hybridLogCheckpointToken);
_hybridLogCheckpoint.snapshotFileObjectLogDevice = checkpointManager.GetSnapshotObjectLogDevice(_hybridLogCheckpointToken);
_hybridLogCheckpoint.snapshotFileDevice.Initialize(hlog.GetSegmentSize());
_hybridLogCheckpoint.snapshotFileObjectLogDevice.Initialize(-1);
long startPage = hlog.GetPage(_hybridLogCheckpoint.info.flushedLogicalAddress);
long endPage = hlog.GetPage(_hybridLogCheckpoint.info.finalLogicalAddress);
if (_hybridLogCheckpoint.info.finalLogicalAddress > hlog.GetStartLogicalAddress(endPage))
{
endPage++;
}
// This can be run on a new thread if we want to immediately parallelize
// the rest of the log flush
hlog.AsyncFlushPagesToDevice(
startPage,
endPage,
_hybridLogCheckpoint.info.finalLogicalAddress,
_hybridLogCheckpoint.snapshotFileDevice,
_hybridLogCheckpoint.snapshotFileObjectLogDevice,
out _hybridLogCheckpoint.flushedSemaphore);
}
MakeTransition(intermediateState, nextState);
break;
}
case Phase.PERSISTENCE_CALLBACK:
{
if (_activeSessions != null)
{
// write dormant sessions to checkpoint
foreach (var kvp in _activeSessions)
{
AtomicSwitch(kvp.Value.ctx, kvp.Value.ctx.prevCtx, currentState.version - 1);
}
}
WriteHybridLogMetaInfo();
if (_checkpointType == CheckpointType.FULL)
{
WriteIndexMetaInfo();
}
MakeTransition(intermediateState, nextState);
break;
}
case Phase.PREPARE_GROW:
{
// Note that the transition must be done before bumping epoch here!
MakeTransition(intermediateState, nextState);
epoch.BumpCurrentEpoch(() =>
{
long _context = 0;
GlobalMoveToNextState(nextState, SystemState.Make(Phase.IN_PROGRESS_GROW, nextState.version), ref _context);
});
break;
}
case Phase.IN_PROGRESS_GROW:
{
// Set up the transition to new version of HT
int numChunks = (int)(state[resizeInfo.version].size / Constants.kSizeofChunk);
if (numChunks == 0)
{
numChunks = 1; // at least one chunk
}
numPendingChunksToBeSplit = numChunks;
splitStatus = new long[numChunks];
Initialize(1 - resizeInfo.version, state[resizeInfo.version].size * 2, sectorSize);
resizeInfo.version = 1 - resizeInfo.version;
MakeTransition(intermediateState, nextState);
break;
}
case Phase.REST:
{
switch (_checkpointType)
{
case CheckpointType.INDEX_ONLY:
{
_indexCheckpoint.info.num_buckets = overflowBucketsAllocator.GetMaxValidAddress();
ObtainCurrentTailAddress(ref _indexCheckpoint.info.finalLogicalAddress);
WriteIndexMetaInfo();
_indexCheckpoint.Reset();
break;
}
case CheckpointType.FULL:
{
_indexCheckpoint.Reset();
_hybridLogCheckpoint.Reset();
break;
}
case CheckpointType.HYBRID_LOG_ONLY:
{
_hybridLogCheckpoint.Reset();
break;
}
default:
throw new FasterException();
}
var nextTcs = new TaskCompletionSource <LinkedCheckpointInfo>(TaskCreationOptions.RunContinuationsAsynchronously);
checkpointTcs.SetResult(new LinkedCheckpointInfo {
NextTask = nextTcs.Task
});
checkpointTcs = nextTcs;
MakeTransition(intermediateState, nextState);
break;
}
}
return(true);
}
else
{
return(false);
}
}
/// <inheritdoc /> public abstract SystemState NextState(SystemState start);
/// <summary>
/// Check whether thread is in same cycle compared to current systemState
/// </summary>
/// <param name="ctx"></param>
/// <param name="threadState"></param>
/// <returns></returns>
internal bool SameCycle <Input, Output, Context>(FasterExecutionContext <Input, Output, Context> ctx, SystemState threadState)
{
if (ctx == null)
{
var _systemState = SystemState.Copy(ref systemState);
SystemState.RemoveIntermediate(ref _systemState);
return(StartOfCurrentCycle(threadState).Version == StartOfCurrentCycle(_systemState).Version);
}
return(ctx.threadStateMachine == currentSyncStateMachine);
}
/// <inheritdoc />
public virtual void GlobalAfterEnteringState <Key, Value>(SystemState next,
FasterKV <Key, Value> faster)
where Key : new()
where Value : new()
{
}
private async ValueTask HandleCheckpointingPhasesAsync(FasterExecutionContext ctx, ClientSession <Key, Value, Input, Output, Context, Functions> clientSession, bool async = true, CancellationToken token = default)
{
if (async)
{
clientSession?.UnsafeResumeThread();
}
var finalState = SystemState.Copy(ref _systemState);
while (finalState.phase == Phase.INTERMEDIATE)
{
finalState = SystemState.Copy(ref _systemState);
}
var previousState = ctx != null?SystemState.Make(ctx.phase, ctx.version) : finalState;
// We need to move from previousState to finalState one step at a time
var currentState = previousState;
SystemState startState = GetStartState(finalState);
if ((currentState.version < startState.version) ||
(currentState.version == startState.version && currentState.phase < startState.phase))
{
// Fast-forward to beginning of current checkpoint cycle
currentState = startState;
}
do
{
switch (currentState.phase)
{
case Phase.PREP_INDEX_CHECKPOINT:
{
if (ctx != null)
{
if (!ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt])
{
ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = true;
}
epoch.Mark(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.PrepareForIndexCheckpt, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.INDEX_CHECKPOINT:
{
if (_checkpointType == CheckpointType.INDEX_ONLY && ctx != null)
{
// Reseting the marker for a potential FULL or INDEX_ONLY checkpoint in the future
ctx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = false;
}
if (async && !IsIndexFuzzyCheckpointCompleted())
{
clientSession?.UnsafeSuspendThread();
await IsIndexFuzzyCheckpointCompletedAsync(token);
clientSession?.UnsafeResumeThread();
}
GlobalMoveToNextCheckpointState(currentState);
break;
}
case Phase.PREPARE:
{
if (ctx != null)
{
if (!ctx.markers[EpochPhaseIdx.Prepare])
{
if (!RelaxedCPR)
{
AcquireSharedLatchesForAllPendingRequests(ctx);
}
ctx.markers[EpochPhaseIdx.Prepare] = true;
}
epoch.Mark(EpochPhaseIdx.Prepare, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.Prepare, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.IN_PROGRESS:
{
if (ctx != null)
{
// Need to be very careful here as threadCtx is changing
FasterExecutionContext _ctx;
if (previousState.phase == Phase.IN_PROGRESS)
{
_ctx = ctx.prevCtx;
}
else
{
_ctx = ctx;
}
if (!_ctx.markers[EpochPhaseIdx.InProgress])
{
AtomicSwitch(ctx, ctx.prevCtx, _ctx.version);
InitContext(ctx, ctx.prevCtx.guid, ctx.prevCtx.serialNum);
// Has to be prevCtx, not ctx
ctx.prevCtx.markers[EpochPhaseIdx.InProgress] = true;
}
epoch.Mark(EpochPhaseIdx.InProgress, currentState.version);
}
// Has to be prevCtx, not ctx
if (epoch.CheckIsComplete(EpochPhaseIdx.InProgress, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.WAIT_PENDING:
{
if (ctx != null)
{
if (!ctx.prevCtx.markers[EpochPhaseIdx.WaitPending])
{
var notify = (ctx.prevCtx.ioPendingRequests.Count == 0);
notify = notify && (ctx.prevCtx.retryRequests.Count == 0);
if (notify)
{
ctx.prevCtx.markers[EpochPhaseIdx.WaitPending] = true;
}
else
{
break;
}
}
epoch.Mark(EpochPhaseIdx.WaitPending, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.WaitPending, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.WAIT_FLUSH:
{
if (ctx == null || !ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush])
{
bool notify;
if (FoldOverSnapshot)
{
notify = (hlog.FlushedUntilAddress >= _hybridLogCheckpoint.info.finalLogicalAddress);
}
else
{
notify = (_hybridLogCheckpoint.flushedSemaphore != null) && _hybridLogCheckpoint.flushedSemaphore.CurrentCount > 0;
}
if (async && !notify)
{
Debug.Assert(_hybridLogCheckpoint.flushedSemaphore != null);
clientSession?.UnsafeSuspendThread();
await _hybridLogCheckpoint.flushedSemaphore.WaitAsync(token);
clientSession?.UnsafeResumeThread();
_hybridLogCheckpoint.flushedSemaphore.Release();
notify = true;
}
if (_checkpointType == CheckpointType.FULL)
{
notify = notify && IsIndexFuzzyCheckpointCompleted();
if (async && !notify)
{
clientSession?.UnsafeSuspendThread();
await IsIndexFuzzyCheckpointCompletedAsync(token);
clientSession?.UnsafeResumeThread();
notify = true;
}
}
if (notify)
{
if (ctx != null)
{
ctx.prevCtx.markers[EpochPhaseIdx.WaitFlush] = true;
}
}
else
{
break;
}
}
if (ctx != null)
{
epoch.Mark(EpochPhaseIdx.WaitFlush, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.WaitFlush, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.PERSISTENCE_CALLBACK:
{
if (ctx != null)
{
if (!ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback])
{
if (ctx.prevCtx.serialNum != -1)
{
var commitPoint = new CommitPoint
{
UntilSerialNo = ctx.prevCtx.serialNum,
ExcludedSerialNos = ctx.prevCtx.excludedSerialNos
};
// Thread local action
functions.CheckpointCompletionCallback(ctx.guid, commitPoint);
if (clientSession != null)
{
clientSession.LatestCommitPoint = commitPoint;
}
}
ctx.prevCtx.markers[EpochPhaseIdx.CheckpointCompletionCallback] = true;
}
epoch.Mark(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version);
}
if (epoch.CheckIsComplete(EpochPhaseIdx.CheckpointCompletionCallback, currentState.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.REST:
{
break;
}
default:
throw new FasterException("Invalid state found during checkpointing");
}
if (ctx != null)
{
// update thread local variables
ctx.phase = currentState.phase;
ctx.version = currentState.version;
}
previousState.word = currentState.word;
currentState = GetNextState(currentState, _checkpointType);
} while (previousState.word != finalState.word);
if (async)
{
clientSession?.UnsafeSuspendThread();
}
}
/// <inheritdoc />
public void GlobalAfterEnteringState <Key, Value>(
SystemState next,
FasterKV <Key, Value> faster)
{
}
/// <inheritdoc />
public ValueTask OnThreadState <Key, Value, Input, Output, Context, FasterSession>(
SystemState current, SystemState prev,
FasterKV <Key, Value> faster,
FasterKV <Key, Value> .FasterExecutionContext <Input, Output, Context> ctx,
FasterSession fasterSession,
bool async = true,
CancellationToken token = default)
where Key : new()
where Value : new()
where FasterSession : IFasterSession
{
switch (current.phase)
{
case Phase.PREPARE:
if (ctx != null)
{
if (!ctx.markers[EpochPhaseIdx.Prepare])
{
if (!faster.RelaxedCPR)
{
faster.AcquireSharedLatchesForAllPendingRequests(ctx);
}
ctx.markers[EpochPhaseIdx.Prepare] = true;
}
faster.epoch.Mark(EpochPhaseIdx.Prepare, current.version);
}
if (faster.epoch.CheckIsComplete(EpochPhaseIdx.Prepare, current.version))
{
faster.GlobalStateMachineStep(current);
}
break;
case Phase.IN_PROGRESS:
if (ctx != null)
{
// Need to be very careful here as threadCtx is changing
var _ctx = prev.phase == Phase.IN_PROGRESS ? ctx.prevCtx : ctx;
if (!_ctx.markers[EpochPhaseIdx.InProgress])
{
faster.AtomicSwitch(ctx, ctx.prevCtx, _ctx.version);
faster.InitContext(ctx, ctx.prevCtx.guid, ctx.prevCtx.serialNum);
// Has to be prevCtx, not ctx
ctx.prevCtx.markers[EpochPhaseIdx.InProgress] = true;
}
faster.epoch.Mark(EpochPhaseIdx.InProgress, current.version);
}
// Has to be prevCtx, not ctx
if (faster.epoch.CheckIsComplete(EpochPhaseIdx.InProgress, current.version))
{
faster.GlobalStateMachineStep(current);
}
break;
case Phase.WAIT_PENDING:
if (ctx != null)
{
if (!faster.RelaxedCPR && !ctx.prevCtx.markers[EpochPhaseIdx.WaitPending])
{
if (ctx.prevCtx.HasNoPendingRequests)
{
ctx.prevCtx.markers[EpochPhaseIdx.WaitPending] = true;
}
else
{
break;
}
}
faster.epoch.Mark(EpochPhaseIdx.WaitPending, current.version);
}
if (faster.epoch.CheckIsComplete(EpochPhaseIdx.WaitPending, current.version))
{
faster.GlobalStateMachineStep(current);
}
break;
case Phase.REST:
break;
}
return(default);
protected void HandleCheckpointingPhases()
{
var previousState = SystemState.Make(threadCtx.phase, threadCtx.version);
var finalState = SystemState.Copy(ref _systemState);
// Don't play around when system state is being changed
if (finalState.phase == Phase.INTERMEDIATE)
{
return;
}
// We need to move from previousState to finalState one step at a time
do
{
var currentState = default(SystemState);
if (previousState.word == finalState.word)
{
currentState.word = previousState.word;
}
else
{
currentState = GetNextState(previousState, _checkpointType);
}
switch (currentState.phase)
{
case Phase.PREP_INDEX_CHECKPOINT:
{
if (!threadCtx.markers[EpochPhaseIdx.PrepareForIndexCheckpt])
{
if (epoch.MarkAndCheckIsComplete(EpochPhaseIdx.PrepareForIndexCheckpt, threadCtx.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
threadCtx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = true;
}
break;
}
case Phase.INDEX_CHECKPOINT:
{
if (_checkpointType == CheckpointType.INDEX_ONLY)
{
// Reseting the marker for a potential FULL or INDEX_ONLY checkpoint in the future
threadCtx.markers[EpochPhaseIdx.PrepareForIndexCheckpt] = false;
}
if (IsIndexFuzzyCheckpointCompleted())
{
GlobalMoveToNextCheckpointState(currentState);
}
break;
}
case Phase.PREPARE:
{
if (!threadCtx.markers[EpochPhaseIdx.Prepare])
{
// Thread local action
AcquireSharedLatchesForAllPendingRequests();
var idx = Interlocked.Increment(ref _hybridLogCheckpoint.info.numThreads);
idx -= 1;
_hybridLogCheckpoint.info.guids[idx] = threadCtx.guid;
if (epoch.MarkAndCheckIsComplete(EpochPhaseIdx.Prepare, threadCtx.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
threadCtx.markers[EpochPhaseIdx.Prepare] = true;
}
break;
}
case Phase.IN_PROGRESS:
{
// Need to be very careful here as threadCtx is changing
ExecutionContext ctx;
if (previousState.phase == Phase.PREPARE)
{
ctx = threadCtx;
}
else
{
ctx = prevThreadCtx;
}
if (!ctx.markers[EpochPhaseIdx.InProgress])
{
prevThreadCtx = threadCtx;
InitLocalContext(ref threadCtx, prevThreadCtx.guid);
if (epoch.MarkAndCheckIsComplete(EpochPhaseIdx.InProgress, ctx.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
prevThreadCtx.markers[EpochPhaseIdx.InProgress] = true;
}
break;
}
case Phase.WAIT_PENDING:
{
if (!prevThreadCtx.markers[EpochPhaseIdx.WaitPending])
{
var notify = (prevThreadCtx.ioPendingRequests.Count == 0);
notify = notify && (prevThreadCtx.retryRequests.Count == 0);
if (notify)
{
if (epoch.MarkAndCheckIsComplete(EpochPhaseIdx.WaitPending, threadCtx.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
prevThreadCtx.markers[EpochPhaseIdx.WaitPending] = true;
}
}
break;
}
case Phase.WAIT_FLUSH:
{
if (!prevThreadCtx.markers[EpochPhaseIdx.WaitFlush])
{
var notify = false;
if (Constants.kFoldOverSnapshot)
{
notify = (hlog.FlushedUntilAddress >= _hybridLogCheckpoint.info.finalLogicalAddress);
}
else
{
notify = (_hybridLogCheckpoint.flushed != null) && _hybridLogCheckpoint.flushed.IsSet;
}
if (_checkpointType == CheckpointType.FULL)
{
notify = notify && IsIndexFuzzyCheckpointCompleted();
}
if (notify)
{
WriteHybridLogContextInfo();
if (epoch.MarkAndCheckIsComplete(EpochPhaseIdx.WaitFlush, prevThreadCtx.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
prevThreadCtx.markers[EpochPhaseIdx.WaitFlush] = true;
}
}
break;
}
case Phase.PERSISTENCE_CALLBACK:
{
if (!prevThreadCtx.markers[EpochPhaseIdx.PersistenceCallback])
{
// Thread local action
Functions.PersistenceCallback(LightEpoch.threadEntryIndex, prevThreadCtx.serialNum);
if (epoch.MarkAndCheckIsComplete(EpochPhaseIdx.PersistenceCallback, prevThreadCtx.version))
{
GlobalMoveToNextCheckpointState(currentState);
}
prevThreadCtx.markers[EpochPhaseIdx.PersistenceCallback] = true;
}
break;
}
case Phase.REST:
{
break;
}
default:
Debug.WriteLine("Error!");
break;
}
// update thread local variables
threadCtx.phase = currentState.phase;
threadCtx.version = currentState.version;
previousState.word = currentState.word;
} while (previousState.word != finalState.word);
}
internal static SystemState MakeIntermediate(SystemState state) => Make(state.Phase | Phase.INTERMEDIATE, state.Version);
protected SystemState GetNextState(SystemState start, CheckpointType type = CheckpointType.FULL)
{
var nextState = default(SystemState);
nextState.word = start.word;
switch (start.phase)
{
case Phase.REST:
switch (type)
{
case CheckpointType.HYBRID_LOG_ONLY:
nextState.phase = Phase.PREPARE;
break;
case CheckpointType.FULL:
case CheckpointType.INDEX_ONLY:
nextState.phase = Phase.PREP_INDEX_CHECKPOINT;
break;
}
break;
case Phase.PREP_INDEX_CHECKPOINT:
switch (type)
{
case CheckpointType.INDEX_ONLY:
nextState.phase = Phase.INDEX_CHECKPOINT;
break;
case CheckpointType.FULL:
nextState.phase = Phase.PREPARE;
break;
}
break;
case Phase.INDEX_CHECKPOINT:
nextState.phase = Phase.PREPARE;
break;
case Phase.PREPARE:
nextState.phase = Phase.IN_PROGRESS;
nextState.version = start.version + 1;
break;
case Phase.IN_PROGRESS:
nextState.phase = Phase.WAIT_PENDING;
break;
case Phase.WAIT_PENDING:
nextState.phase = Phase.WAIT_FLUSH;
break;
case Phase.WAIT_FLUSH:
nextState.phase = Phase.PERSISTENCE_CALLBACK;
break;
case Phase.PERSISTENCE_CALLBACK:
nextState.phase = Phase.REST;
break;
case Phase.GC:
nextState.phase = Phase.REST;
break;
case Phase.PREPARE_GROW:
nextState.phase = Phase.IN_PROGRESS_GROW;
break;
case Phase.IN_PROGRESS_GROW:
nextState.phase = Phase.REST;
break;
}
return(nextState);
}
public static SystemState MakeIntermediate(SystemState state) => Make(state.phase | Phase.INTERMEDIATE, state.version);