public FasterKV(Partition partition, BlobManager blobManager) { this.partition = partition; this.blobManager = blobManager; partition.ErrorHandler.Token.ThrowIfCancellationRequested(); this.fht = new FasterKV <Key, Value>( BlobManager.HashTableSize, blobManager.StoreLogSettings(partition.Settings.UsePremiumStorage, partition.NumberPartitions()), blobManager.StoreCheckpointSettings, new SerializerSettings <Key, Value> { keySerializer = () => new Key.Serializer(), valueSerializer = () => new Value.Serializer(this.StoreStats), }); #if FASTER_SUPPORTS_PSF if (partition.Settings.UsePSFQueries) { int groupOrdinal = 0; var psfs = fht.RegisterPSF(this.blobManager.CreatePSFRegistrationSettings <PSFKey>(partition.NumberPartitions(), groupOrdinal++), (nameof(this.RuntimeStatusPsf), (k, v) => v.Val is InstanceState state ? (PSFKey?)new PSFKey(state.OrchestrationState.OrchestrationStatus) : null), (nameof(this.CreatedTimePsf), (k, v) => v.Val is InstanceState state ? (PSFKey?)new PSFKey(state.OrchestrationState.CreatedTime) : null), (nameof(this.InstanceIdPrefixPsf), (k, v) => v.Val is InstanceState state ? (PSFKey?)new PSFKey(state.InstanceId) : null)); this.RuntimeStatusPsf = psfs[0]; this.CreatedTimePsf = psfs[1]; this.InstanceIdPrefixPsf = psfs[2]; } #endif this.terminationToken = partition.ErrorHandler.Token; var _ = this.terminationToken.Register( () => { try { this.mainSession?.Dispose(); this.fht.Dispose(); this.blobManager.HybridLogDevice.Dispose(); this.blobManager.ObjectLogDevice.Dispose(); this.blobManager.ClosePSFDevices(); } catch (Exception e) { this.blobManager.TraceHelper.FasterStorageError("Disposing FasterKV", e); } }, useSynchronizationContext: false); this.blobManager.TraceHelper.FasterProgress("Constructed FasterKV"); }
public LogWorker(BlobManager blobManager, FasterLog log, Partition partition, StoreWorker storeWorker, FasterTraceHelper traceHelper, CancellationToken cancellationToken) : base(nameof(LogWorker), true, 500, cancellationToken) { partition.ErrorHandler.Token.ThrowIfCancellationRequested(); this.blobManager = blobManager; this.log = log; this.partition = partition; this.storeWorker = storeWorker; this.traceHelper = traceHelper; this.intakeWorker = new IntakeWorker(cancellationToken, this); this.maxFragmentSize = (1 << this.blobManager.EventLogSettings(partition.Settings.UsePremiumStorage).PageSizeBits) - 64; // faster needs some room for header, 64 bytes is conservative }
public FasterAlt(Partition partition, BlobManager blobManager) { this.partition = partition; this.blobManager = blobManager; this.prefix = $"p{this.partition.PartitionId:D2}/store/"; this.terminationToken = partition.ErrorHandler.Token; this.traceHelper = blobManager.TraceHelper; this.detailTracer = this.traceHelper.IsTracingAtMostDetailedLevel ? this.traceHelper : null; var _ = this.terminationToken.Register( () => { // nothing so far }, useSynchronizationContext: false); this.blobManager.TraceHelper.FasterProgress("Constructed FasterAlt"); }
public FasterLog(BlobManager blobManager, NetheriteOrchestrationServiceSettings settings) { this.log = new FASTER.core.FasterLog(blobManager.EventLogSettings(settings.UsePremiumStorage)); this.terminationToken = blobManager.PartitionErrorHandler.Token; var _ = this.terminationToken.Register( () => { try { this.log.Dispose(); blobManager.EventLogDevice.Dispose(); } catch (Exception e) { blobManager.TraceHelper.FasterStorageError("Disposing FasterLog", e); } }, useSynchronizationContext: false); }
const long MAX_PAGEBLOB_SIZE = 512L * 1024 * 1024 * 1024; // set this at 512 GB for now TODO consider implications /// <summary> /// Constructs a new AzureStorageDevice instance, backed by Azure Page Blobs /// </summary> /// <param name="blobName">A descriptive name that will be the prefix of all segments created</param> /// <param name="blockBlobDirectory">the directory containing the block blobs</param> /// <param name="pageBlobDirectory">the directory containing the page blobs</param> /// <param name="blobManager">the blob manager handling the leases</param> /// <param name="underLease">whether this device needs to be protected by the lease</param> public AzureStorageDevice(string blobName, CloudBlobDirectory blockBlobDirectory, CloudBlobDirectory pageBlobDirectory, BlobManager blobManager, bool underLease) : base($"{blockBlobDirectory}\\{blobName}", PAGE_BLOB_SECTOR_SIZE, Devices.CAPACITY_UNSPECIFIED) { this.blobs = new ConcurrentDictionary <int, BlobEntry>(); this.blockBlobDirectory = blockBlobDirectory; this.pageBlobDirectory = pageBlobDirectory; this.blobName = blobName; this.PartitionErrorHandler = blobManager.PartitionErrorHandler; this.BlobManager = blobManager; this.underLease = underLease; }
public async Task PerformWithRetriesAsync( SemaphoreSlim semaphore, bool requireLease, string name, string intent, string data, string target, int expectedLatencyBound, bool isCritical, Func <int, Task <long> > operation) { try { if (semaphore != null) { await semaphore.WaitAsync(); } Stopwatch stopwatch = new Stopwatch(); int numAttempts = 0; while (true) // retry loop { numAttempts++; try { if (requireLease) { await this.ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false); } this.StorageTracer?.FasterStorageProgress($"storage operation {name} started attempt {numAttempts}; target={target} {data}"); stopwatch.Restart(); long size = await operation(numAttempts).ConfigureAwait(false); stopwatch.Stop(); this.StorageTracer?.FasterStorageProgress($"storage operation {name} ({intent}) succeeded on attempt {numAttempts}; target={target} latencyMs={stopwatch.Elapsed.TotalMilliseconds:F1} {data} "); if (stopwatch.ElapsedMilliseconds > expectedLatencyBound) { this.TraceHelper.FasterPerfWarning($"storage operation {name} ({intent}) took {stopwatch.Elapsed.TotalSeconds:F1}s on attempt {numAttempts}, which is excessive; {data}"); } this.TraceHelper.FasterAzureStorageAccessCompleted(intent, size, name, target, stopwatch.Elapsed.TotalMilliseconds, numAttempts); return; } catch (StorageException e) when(BlobUtils.IsTransientStorageError(e, this.PartitionErrorHandler.Token) && numAttempts < BlobManager.MaxRetries) { stopwatch.Stop(); if (BlobUtils.IsTimeout(e)) { this.TraceHelper.FasterPerfWarning($"storage operation {name} ({intent}) timed out on attempt {numAttempts} after {stopwatch.Elapsed.TotalSeconds:F1}s, retrying now; target={target} {data}"); } else { TimeSpan nextRetryIn = BlobManager.GetDelayBetweenRetries(numAttempts); this.HandleStorageError(name, $"storage operation {name} ({intent}) failed transiently on attempt {numAttempts}, retry in {nextRetryIn}s", target, e, false, true); await Task.Delay(nextRetryIn); } continue; } catch (Exception exception) when(!Utils.IsFatal(exception)) { this.HandleStorageError(name, $"storage operation {name} ({intent}) failed on attempt {numAttempts}", target, exception, isCritical, this.PartitionErrorHandler.IsTerminated); throw; } } } finally { if (semaphore != null) { semaphore.Release(); } } }
public async Task <long> CreateOrRestoreAsync(Partition partition, IPartitionErrorHandler errorHandler, long firstInputQueuePosition) { this.partition = partition; this.terminationToken = errorHandler.Token; #if FASTER_SUPPORTS_PSF int psfCount = partition.Settings.UsePSFQueries ? FasterKV.PSFCount : 0; #else int psfCount = 0; #endif this.blobManager = new BlobManager( this.storageAccount, this.pageBlobStorageAccount, this.taskHubName, this.logger, this.partition.Settings.StorageLogLevelLimit, partition.PartitionId, errorHandler, psfCount); this.TraceHelper = this.blobManager.TraceHelper; this.TraceHelper.FasterProgress("Starting BlobManager"); await this.blobManager.StartAsync().ConfigureAwait(false); var stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); this.TraceHelper.FasterProgress("Creating FasterLog"); this.log = new FasterLog(this.blobManager, partition.Settings); if (partition.Settings.UseAlternateObjectStore) { this.TraceHelper.FasterProgress("Creating FasterAlt"); this.store = new FasterAlt(this.partition, this.blobManager); } else { this.TraceHelper.FasterProgress("Creating FasterKV"); this.store = new FasterKV(this.partition, this.blobManager); } this.TraceHelper.FasterProgress("Creating StoreWorker"); this.storeWorker = new StoreWorker(this.store, this.partition, this.TraceHelper, this.blobManager, this.terminationToken); this.TraceHelper.FasterProgress("Creating LogWorker"); this.logWorker = this.storeWorker.LogWorker = new LogWorker(this.blobManager, this.log, this.partition, this.storeWorker, this.TraceHelper, this.terminationToken); if (this.log.TailAddress == this.log.BeginAddress) { // take an (empty) checkpoint immediately to ensure the paths are working try { this.TraceHelper.FasterProgress("Creating store"); // this is a fresh partition await this.storeWorker.Initialize(this.log.BeginAddress, firstInputQueuePosition).ConfigureAwait(false); await this.storeWorker.TakeFullCheckpointAsync("initial checkpoint").ConfigureAwait(false); this.TraceHelper.FasterStoreCreated(this.storeWorker.InputQueuePosition, stopwatch.ElapsedMilliseconds); this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected()); } catch (Exception e) { this.TraceHelper.FasterStorageError(nameof(CreateOrRestoreAsync), e); throw; } } else { this.TraceHelper.FasterProgress("Loading checkpoint"); try { // we are recovering the last checkpoint of the store (long commitLogPosition, long inputQueuePosition) = await this.store.RecoverAsync(); this.storeWorker.SetCheckpointPositionsAfterRecovery(commitLogPosition, inputQueuePosition); // truncate the log in case the truncation did not commit after the checkpoint was taken this.logWorker.SetLastCheckpointPosition(commitLogPosition); this.TraceHelper.FasterCheckpointLoaded(this.storeWorker.CommitLogPosition, this.storeWorker.InputQueuePosition, this.store.StoreStats.Get(), stopwatch.ElapsedMilliseconds); } catch (Exception e) { this.TraceHelper.FasterStorageError("loading checkpoint", e); throw; } this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected()); this.TraceHelper.FasterProgress($"Replaying log length={this.log.TailAddress - this.storeWorker.CommitLogPosition} range={this.storeWorker.CommitLogPosition}-{this.log.TailAddress}"); try { if (this.log.TailAddress > (long)this.storeWorker.CommitLogPosition) { // replay log as the store checkpoint lags behind the log await this.storeWorker.ReplayCommitLog(this.logWorker).ConfigureAwait(false); } } catch (Exception e) { this.TraceHelper.FasterStorageError("replaying log", e); throw; } // restart pending actitivities, timers, work items etc. await this.storeWorker.RestartThingsAtEndOfRecovery().ConfigureAwait(false); this.TraceHelper.FasterProgress("Recovery complete"); } var ignoredTask = this.IdleLoop(); return(this.storeWorker.InputQueuePosition); }
public static Task DeleteTaskhubStorageAsync(string connectionString, string taskHubName) { var storageAccount = (connectionString != LocalFileStorageConnectionString) ? CloudStorageAccount.Parse(connectionString) : null; return(BlobManager.DeleteTaskhubStorageAsync(storageAccount, taskHubName)); }
internal PsfBlobCheckpointManager(BlobManager blobMan, int groupOrd) { this.blobManager = blobMan; this.groupOrdinal = groupOrd; }