async Task ReplayEvents(ChannelReader <PartitionUpdateEvent> reader, StoreWorker worker)
{
await foreach (var partitionEvent in reader.ReadAllAsync(this.cancellationToken))
{
await worker.ReplayUpdate(partitionEvent);
}
}
public async Task ReplayCommitLog(long from, StoreWorker worker)
{
// this procedure is called by StoreWorker during recovery. It replays all the events
// that were committed to the log but are not reflected in the loaded store checkpoint.
try
{
// we create a pipeline where the fetch task obtains a stream of events and then duplicates the
// stream, so it can get replayed and prefetched in parallel.
var prefetchChannel = Channel.CreateBounded <TrackedObjectKey>(1000);
var replayChannel = Channel.CreateBounded <PartitionUpdateEvent>(1000);
var fetchTask = this.FetchEvents(from, replayChannel.Writer, prefetchChannel.Writer);
var replayTask = Task.Run(() => this.ReplayEvents(replayChannel.Reader, worker));
var prefetchTask = Task.Run(() => worker.RunPrefetchSession(prefetchChannel.Reader.ReadAllAsync(this.cancellationToken)));
await fetchTask;
await replayTask;
// note: we are not awaiting the prefetch task since completing the prefetches is not essential to continue.
}
catch (Exception exception)
when(this.cancellationToken.IsCancellationRequested && !Utils.IsFatal(exception))
{
throw new OperationCanceledException("Partition was terminated.", exception, this.partition.ErrorHandler.Token);
}
}
public LogWorker(BlobManager blobManager, FasterLog log, Partition partition, StoreWorker storeWorker, FasterTraceHelper traceHelper, CancellationToken cancellationToken)
: base(nameof(LogWorker), true, 500, cancellationToken)
{
partition.ErrorHandler.Token.ThrowIfCancellationRequested();
this.blobManager = blobManager;
this.log = log;
this.partition = partition;
this.storeWorker = storeWorker;
this.traceHelper = traceHelper;
this.intakeWorker = new IntakeWorker(cancellationToken, this);
this.maxFragmentSize = (1 << this.blobManager.EventLogSettings(partition.Settings.UsePremiumStorage).PageSizeBits) - 64; // faster needs some room for header, 64 bytes is conservative
}
public async Task <long> CreateOrRestoreAsync(Partition partition, IPartitionErrorHandler errorHandler, long firstInputQueuePosition)
{
this.partition = partition;
this.terminationToken = errorHandler.Token;
#if FASTER_SUPPORTS_PSF
int psfCount = partition.Settings.UsePSFQueries ? FasterKV.PSFCount : 0;
#else
int psfCount = 0;
#endif
this.blobManager = new BlobManager(
this.storageAccount,
this.pageBlobStorageAccount,
this.taskHubName,
this.logger,
this.partition.Settings.StorageLogLevelLimit,
partition.PartitionId,
errorHandler,
psfCount);
this.TraceHelper = this.blobManager.TraceHelper;
this.TraceHelper.FasterProgress("Starting BlobManager");
await this.blobManager.StartAsync().ConfigureAwait(false);
var stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
this.TraceHelper.FasterProgress("Creating FasterLog");
this.log = new FasterLog(this.blobManager, partition.Settings);
if (partition.Settings.UseAlternateObjectStore)
{
this.TraceHelper.FasterProgress("Creating FasterAlt");
this.store = new FasterAlt(this.partition, this.blobManager);
}
else
{
this.TraceHelper.FasterProgress("Creating FasterKV");
this.store = new FasterKV(this.partition, this.blobManager);
}
this.TraceHelper.FasterProgress("Creating StoreWorker");
this.storeWorker = new StoreWorker(this.store, this.partition, this.TraceHelper, this.blobManager, this.terminationToken);
this.TraceHelper.FasterProgress("Creating LogWorker");
this.logWorker = this.storeWorker.LogWorker = new LogWorker(this.blobManager, this.log, this.partition, this.storeWorker, this.TraceHelper, this.terminationToken);
if (this.log.TailAddress == this.log.BeginAddress)
{
// take an (empty) checkpoint immediately to ensure the paths are working
try
{
this.TraceHelper.FasterProgress("Creating store");
// this is a fresh partition
await this.storeWorker.Initialize(this.log.BeginAddress, firstInputQueuePosition).ConfigureAwait(false);
await this.storeWorker.TakeFullCheckpointAsync("initial checkpoint").ConfigureAwait(false);
this.TraceHelper.FasterStoreCreated(this.storeWorker.InputQueuePosition, stopwatch.ElapsedMilliseconds);
this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError(nameof(CreateOrRestoreAsync), e);
throw;
}
}
else
{
this.TraceHelper.FasterProgress("Loading checkpoint");
try
{
// we are recovering the last checkpoint of the store
(long commitLogPosition, long inputQueuePosition) = await this.store.RecoverAsync();
this.storeWorker.SetCheckpointPositionsAfterRecovery(commitLogPosition, inputQueuePosition);
// truncate the log in case the truncation did not commit after the checkpoint was taken
this.logWorker.SetLastCheckpointPosition(commitLogPosition);
this.TraceHelper.FasterCheckpointLoaded(this.storeWorker.CommitLogPosition, this.storeWorker.InputQueuePosition, this.store.StoreStats.Get(), stopwatch.ElapsedMilliseconds);
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError("loading checkpoint", e);
throw;
}
this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());
this.TraceHelper.FasterProgress($"Replaying log length={this.log.TailAddress - this.storeWorker.CommitLogPosition} range={this.storeWorker.CommitLogPosition}-{this.log.TailAddress}");
try
{
if (this.log.TailAddress > (long)this.storeWorker.CommitLogPosition)
{
// replay log as the store checkpoint lags behind the log
await this.storeWorker.ReplayCommitLog(this.logWorker).ConfigureAwait(false);
}
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError("replaying log", e);
throw;
}
// restart pending actitivities, timers, work items etc.
await this.storeWorker.RestartThingsAtEndOfRecovery().ConfigureAwait(false);
this.TraceHelper.FasterProgress("Recovery complete");
}
var ignoredTask = this.IdleLoop();
return(this.storeWorker.InputQueuePosition);
}
