public FasterKV(Partition partition, BlobManager blobManager)
{
this.partition = partition;
this.blobManager = blobManager;
partition.ErrorHandler.Token.ThrowIfCancellationRequested();
this.fht = new FasterKV <Key, Value>(
BlobManager.HashTableSize,
blobManager.StoreLogSettings(partition.Settings.UsePremiumStorage, partition.NumberPartitions()),
blobManager.StoreCheckpointSettings,
new SerializerSettings <Key, Value>
{
keySerializer = () => new Key.Serializer(),
valueSerializer = () => new Value.Serializer(this.StoreStats),
});
#if FASTER_SUPPORTS_PSF
if (partition.Settings.UsePSFQueries)
{
int groupOrdinal = 0;
var psfs = fht.RegisterPSF(this.blobManager.CreatePSFRegistrationSettings <PSFKey>(partition.NumberPartitions(), groupOrdinal++),
(nameof(this.RuntimeStatusPsf), (k, v) => v.Val is InstanceState state
? (PSFKey?)new PSFKey(state.OrchestrationState.OrchestrationStatus)
: null),
(nameof(this.CreatedTimePsf), (k, v) => v.Val is InstanceState state
? (PSFKey?)new PSFKey(state.OrchestrationState.CreatedTime)
: null),
(nameof(this.InstanceIdPrefixPsf), (k, v) => v.Val is InstanceState state
? (PSFKey?)new PSFKey(state.InstanceId)
: null));
this.RuntimeStatusPsf = psfs[0];
this.CreatedTimePsf = psfs[1];
this.InstanceIdPrefixPsf = psfs[2];
}
#endif
this.terminationToken = partition.ErrorHandler.Token;
var _ = this.terminationToken.Register(
() => {
try
{
this.mainSession?.Dispose();
this.fht.Dispose();
this.blobManager.HybridLogDevice.Dispose();
this.blobManager.ObjectLogDevice.Dispose();
this.blobManager.ClosePSFDevices();
}
catch (Exception e)
{
this.blobManager.TraceHelper.FasterStorageError("Disposing FasterKV", e);
}
},
useSynchronizationContext: false);
this.blobManager.TraceHelper.FasterProgress("Constructed FasterKV");
}
public LogWorker(BlobManager blobManager, FasterLog log, Partition partition, StoreWorker storeWorker, FasterTraceHelper traceHelper, CancellationToken cancellationToken)
: base(nameof(LogWorker), true, 500, cancellationToken)
{
partition.ErrorHandler.Token.ThrowIfCancellationRequested();
this.blobManager = blobManager;
this.log = log;
this.partition = partition;
this.storeWorker = storeWorker;
this.traceHelper = traceHelper;
this.intakeWorker = new IntakeWorker(cancellationToken, this);
this.maxFragmentSize = (1 << this.blobManager.EventLogSettings(partition.Settings.UsePremiumStorage).PageSizeBits) - 64; // faster needs some room for header, 64 bytes is conservative
}
public FasterAlt(Partition partition, BlobManager blobManager)
{
this.partition = partition;
this.blobManager = blobManager;
this.prefix = $"p{this.partition.PartitionId:D2}/store/";
this.terminationToken = partition.ErrorHandler.Token;
this.traceHelper = blobManager.TraceHelper;
this.detailTracer = this.traceHelper.IsTracingAtMostDetailedLevel ? this.traceHelper : null;
var _ = this.terminationToken.Register(
() => {
// nothing so far
},
useSynchronizationContext: false);
this.blobManager.TraceHelper.FasterProgress("Constructed FasterAlt");
}
public FasterLog(BlobManager blobManager, NetheriteOrchestrationServiceSettings settings)
{
this.log = new FASTER.core.FasterLog(blobManager.EventLogSettings(settings.UsePremiumStorage));
this.terminationToken = blobManager.PartitionErrorHandler.Token;
var _ = this.terminationToken.Register(
() => {
try
{
this.log.Dispose();
blobManager.EventLogDevice.Dispose();
}
catch (Exception e)
{
blobManager.TraceHelper.FasterStorageError("Disposing FasterLog", e);
}
},
useSynchronizationContext: false);
}
const long MAX_PAGEBLOB_SIZE = 512L * 1024 * 1024 * 1024; // set this at 512 GB for now TODO consider implications
/// <summary>
/// Constructs a new AzureStorageDevice instance, backed by Azure Page Blobs
/// </summary>
/// <param name="blobName">A descriptive name that will be the prefix of all segments created</param>
/// <param name="blockBlobDirectory">the directory containing the block blobs</param>
/// <param name="pageBlobDirectory">the directory containing the page blobs</param>
/// <param name="blobManager">the blob manager handling the leases</param>
/// <param name="underLease">whether this device needs to be protected by the lease</param>
public AzureStorageDevice(string blobName, CloudBlobDirectory blockBlobDirectory, CloudBlobDirectory pageBlobDirectory, BlobManager blobManager, bool underLease)
: base($"{blockBlobDirectory}\\{blobName}", PAGE_BLOB_SECTOR_SIZE, Devices.CAPACITY_UNSPECIFIED)
{
this.blobs = new ConcurrentDictionary <int, BlobEntry>();
this.blockBlobDirectory = blockBlobDirectory;
this.pageBlobDirectory = pageBlobDirectory;
this.blobName = blobName;
this.PartitionErrorHandler = blobManager.PartitionErrorHandler;
this.BlobManager = blobManager;
this.underLease = underLease;
}
public async Task PerformWithRetriesAsync(
SemaphoreSlim semaphore,
bool requireLease,
string name,
string intent,
string data,
string target,
int expectedLatencyBound,
bool isCritical,
Func <int, Task <long> > operation)
{
try
{
if (semaphore != null)
{
await semaphore.WaitAsync();
}
Stopwatch stopwatch = new Stopwatch();
int numAttempts = 0;
while (true) // retry loop
{
numAttempts++;
try
{
if (requireLease)
{
await this.ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false);
}
this.StorageTracer?.FasterStorageProgress($"storage operation {name} started attempt {numAttempts}; target={target} {data}");
stopwatch.Restart();
long size = await operation(numAttempts).ConfigureAwait(false);
stopwatch.Stop();
this.StorageTracer?.FasterStorageProgress($"storage operation {name} ({intent}) succeeded on attempt {numAttempts}; target={target} latencyMs={stopwatch.Elapsed.TotalMilliseconds:F1} {data} ");
if (stopwatch.ElapsedMilliseconds > expectedLatencyBound)
{
this.TraceHelper.FasterPerfWarning($"storage operation {name} ({intent}) took {stopwatch.Elapsed.TotalSeconds:F1}s on attempt {numAttempts}, which is excessive; {data}");
}
this.TraceHelper.FasterAzureStorageAccessCompleted(intent, size, name, target, stopwatch.Elapsed.TotalMilliseconds, numAttempts);
return;
}
catch (StorageException e) when(BlobUtils.IsTransientStorageError(e, this.PartitionErrorHandler.Token) && numAttempts < BlobManager.MaxRetries)
{
stopwatch.Stop();
if (BlobUtils.IsTimeout(e))
{
this.TraceHelper.FasterPerfWarning($"storage operation {name} ({intent}) timed out on attempt {numAttempts} after {stopwatch.Elapsed.TotalSeconds:F1}s, retrying now; target={target} {data}");
}
else
{
TimeSpan nextRetryIn = BlobManager.GetDelayBetweenRetries(numAttempts);
this.HandleStorageError(name, $"storage operation {name} ({intent}) failed transiently on attempt {numAttempts}, retry in {nextRetryIn}s", target, e, false, true);
await Task.Delay(nextRetryIn);
}
continue;
}
catch (Exception exception) when(!Utils.IsFatal(exception))
{
this.HandleStorageError(name, $"storage operation {name} ({intent}) failed on attempt {numAttempts}", target, exception, isCritical, this.PartitionErrorHandler.IsTerminated);
throw;
}
}
}
finally
{
if (semaphore != null)
{
semaphore.Release();
}
}
}
public async Task <long> CreateOrRestoreAsync(Partition partition, IPartitionErrorHandler errorHandler, long firstInputQueuePosition)
{
this.partition = partition;
this.terminationToken = errorHandler.Token;
#if FASTER_SUPPORTS_PSF
int psfCount = partition.Settings.UsePSFQueries ? FasterKV.PSFCount : 0;
#else
int psfCount = 0;
#endif
this.blobManager = new BlobManager(
this.storageAccount,
this.pageBlobStorageAccount,
this.taskHubName,
this.logger,
this.partition.Settings.StorageLogLevelLimit,
partition.PartitionId,
errorHandler,
psfCount);
this.TraceHelper = this.blobManager.TraceHelper;
this.TraceHelper.FasterProgress("Starting BlobManager");
await this.blobManager.StartAsync().ConfigureAwait(false);
var stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
this.TraceHelper.FasterProgress("Creating FasterLog");
this.log = new FasterLog(this.blobManager, partition.Settings);
if (partition.Settings.UseAlternateObjectStore)
{
this.TraceHelper.FasterProgress("Creating FasterAlt");
this.store = new FasterAlt(this.partition, this.blobManager);
}
else
{
this.TraceHelper.FasterProgress("Creating FasterKV");
this.store = new FasterKV(this.partition, this.blobManager);
}
this.TraceHelper.FasterProgress("Creating StoreWorker");
this.storeWorker = new StoreWorker(this.store, this.partition, this.TraceHelper, this.blobManager, this.terminationToken);
this.TraceHelper.FasterProgress("Creating LogWorker");
this.logWorker = this.storeWorker.LogWorker = new LogWorker(this.blobManager, this.log, this.partition, this.storeWorker, this.TraceHelper, this.terminationToken);
if (this.log.TailAddress == this.log.BeginAddress)
{
// take an (empty) checkpoint immediately to ensure the paths are working
try
{
this.TraceHelper.FasterProgress("Creating store");
// this is a fresh partition
await this.storeWorker.Initialize(this.log.BeginAddress, firstInputQueuePosition).ConfigureAwait(false);
await this.storeWorker.TakeFullCheckpointAsync("initial checkpoint").ConfigureAwait(false);
this.TraceHelper.FasterStoreCreated(this.storeWorker.InputQueuePosition, stopwatch.ElapsedMilliseconds);
this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError(nameof(CreateOrRestoreAsync), e);
throw;
}
}
else
{
this.TraceHelper.FasterProgress("Loading checkpoint");
try
{
// we are recovering the last checkpoint of the store
(long commitLogPosition, long inputQueuePosition) = await this.store.RecoverAsync();
this.storeWorker.SetCheckpointPositionsAfterRecovery(commitLogPosition, inputQueuePosition);
// truncate the log in case the truncation did not commit after the checkpoint was taken
this.logWorker.SetLastCheckpointPosition(commitLogPosition);
this.TraceHelper.FasterCheckpointLoaded(this.storeWorker.CommitLogPosition, this.storeWorker.InputQueuePosition, this.store.StoreStats.Get(), stopwatch.ElapsedMilliseconds);
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError("loading checkpoint", e);
throw;
}
this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());
this.TraceHelper.FasterProgress($"Replaying log length={this.log.TailAddress - this.storeWorker.CommitLogPosition} range={this.storeWorker.CommitLogPosition}-{this.log.TailAddress}");
try
{
if (this.log.TailAddress > (long)this.storeWorker.CommitLogPosition)
{
// replay log as the store checkpoint lags behind the log
await this.storeWorker.ReplayCommitLog(this.logWorker).ConfigureAwait(false);
}
}
catch (Exception e)
{
this.TraceHelper.FasterStorageError("replaying log", e);
throw;
}
// restart pending actitivities, timers, work items etc.
await this.storeWorker.RestartThingsAtEndOfRecovery().ConfigureAwait(false);
this.TraceHelper.FasterProgress("Recovery complete");
}
var ignoredTask = this.IdleLoop();
return(this.storeWorker.InputQueuePosition);
}
public static Task DeleteTaskhubStorageAsync(string connectionString, string taskHubName)
{
var storageAccount = (connectionString != LocalFileStorageConnectionString) ? CloudStorageAccount.Parse(connectionString) : null;
return(BlobManager.DeleteTaskhubStorageAsync(storageAccount, taskHubName));
}
internal PsfBlobCheckpointManager(BlobManager blobMan, int groupOrd)
{
this.blobManager = blobMan;
this.groupOrdinal = groupOrd;
}