public LogWorker(BlobManager blobManager, FasterLog log, Partition partition, StoreWorker storeWorker, FasterTraceHelper traceHelper, CancellationToken cancellationToken)
: base(nameof(LogWorker), true, 500, cancellationToken)
{
partition.ErrorHandler.Token.ThrowIfCancellationRequested();
this.blobManager = blobManager;
this.log = log;
this.partition = partition;
this.storeWorker = storeWorker;
this.traceHelper = traceHelper;
this.intakeWorker = new IntakeWorker(cancellationToken, this);
this.maxFragmentSize = (1 << this.blobManager.EventLogSettings(partition.Settings.UsePremiumStorage).PageSizeBits) - 64; // faster needs some room for header, 64 bytes is conservative
}
public async Task <long> CreateOrRestoreAsync(Partition partition, IPartitionErrorHandler errorHandler, long firstInputQueuePosition)
{
this.partition = partition;
this.terminationToken = errorHandler.Token;
#if FASTER_SUPPORTS_PSF
int psfCount = partition.Settings.UsePSFQueries ? FasterKV.PSFCount : 0;
#else
int psfCount = 0;
#endif
this.blobManager = new BlobManager(
this.storageAccount,
this.pageBlobStorageAccount,
this.taskHubName,
this.logger,
this.partition.Settings.StorageLogLevelLimit,
partition.PartitionId,
errorHandler,
psfCount);
this.TraceHelper = this.blobManager.TraceHelper;
this.TraceHelper.FasterProgress("Starting BlobManager");
await this.blobManager.StartAsync().ConfigureAwait(false);
var stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
this.TraceHelper.FasterProgress("Creating FasterLog");
this.log = new FasterLog(this.blobManager, partition.Settings);
if (partition.Settings.UseAlternateObjectStore)
{
this.TraceHelper.FasterProgress("Creating FasterAlt");
this.store = new FasterAlt(this.partition, this.blobManager);
}
else
{
this.TraceHelper.FasterProgress("Creating FasterKV");
this.store = new FasterKV(this.partition, this.blobManager);
}
this.TraceHelper.FasterProgress("Creating StoreWorker");
this.storeWorker = new StoreWorker(this.store, this.partition, this.TraceHelper, this.blobManager, this.terminationToken);
this.TraceHelper.FasterProgress("Creating LogWorker");
this.logWorker = this.storeWorker.LogWorker = new LogWorker(this.blobManager, this.log, this.partition, this.storeWorker, this.TraceHelper, this.terminationToken);
if (this.log.TailAddress == this.log.BeginAddress)
{
// take an (empty) checkpoint immediately to ensure the paths are working
try
{
this.TraceHelper.FasterProgress("Creating store");
// this is a fresh partition
await this.storeWorker.Initialize(this.log.BeginAddress, firstInputQueuePosition).ConfigureAwait(false);
await this.storeWorker.TakeFullCheckpointAsync("initial checkpoint").ConfigureAwait(false);
this.TraceHelper.FasterStoreCreated(this.storeWorker.InputQueuePosition, stopwatch.ElapsedMilliseconds);
this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError(nameof(CreateOrRestoreAsync), e);
throw;
}
}
else
{
this.TraceHelper.FasterProgress("Loading checkpoint");
try
{
// we are recovering the last checkpoint of the store
(long commitLogPosition, long inputQueuePosition) = await this.store.RecoverAsync();
this.storeWorker.SetCheckpointPositionsAfterRecovery(commitLogPosition, inputQueuePosition);
// truncate the log in case the truncation did not commit after the checkpoint was taken
this.logWorker.SetLastCheckpointPosition(commitLogPosition);
this.TraceHelper.FasterCheckpointLoaded(this.storeWorker.CommitLogPosition, this.storeWorker.InputQueuePosition, this.store.StoreStats.Get(), stopwatch.ElapsedMilliseconds);
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError("loading checkpoint", e);
throw;
}
this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());
this.TraceHelper.FasterProgress($"Replaying log length={this.log.TailAddress - this.storeWorker.CommitLogPosition} range={this.storeWorker.CommitLogPosition}-{this.log.TailAddress}");
try
{
if (this.log.TailAddress > (long)this.storeWorker.CommitLogPosition)
{
// replay log as the store checkpoint lags behind the log
await this.storeWorker.ReplayCommitLog(this.logWorker).ConfigureAwait(false);
}
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError("replaying log", e);
throw;
}
// restart pending actitivities, timers, work items etc.
await this.storeWorker.RestartThingsAtEndOfRecovery().ConfigureAwait(false);
this.TraceHelper.FasterProgress("Recovery complete");
}
var ignoredTask = this.IdleLoop();
return(this.storeWorker.InputQueuePosition);
}