public FasterKV(Partition partition, BlobManager blobManager)
        {
            this.partition   = partition;
            this.blobManager = blobManager;

            partition.ErrorHandler.Token.ThrowIfCancellationRequested();

            this.fht = new FasterKV <Key, Value>(
                BlobManager.HashTableSize,
                blobManager.StoreLogSettings(partition.Settings.UsePremiumStorage, partition.NumberPartitions()),
                blobManager.StoreCheckpointSettings,
                new SerializerSettings <Key, Value>
            {
                keySerializer   = () => new Key.Serializer(),
                valueSerializer = () => new Value.Serializer(this.StoreStats),
            });

#if FASTER_SUPPORTS_PSF
            if (partition.Settings.UsePSFQueries)
            {
                int groupOrdinal = 0;
                var psfs         = fht.RegisterPSF(this.blobManager.CreatePSFRegistrationSettings <PSFKey>(partition.NumberPartitions(), groupOrdinal++),
                                                   (nameof(this.RuntimeStatusPsf), (k, v) => v.Val is InstanceState state
                                                                                ? (PSFKey?)new PSFKey(state.OrchestrationState.OrchestrationStatus)
                                                                                : null),
                                                   (nameof(this.CreatedTimePsf), (k, v) => v.Val is InstanceState state
                                                                                ? (PSFKey?)new PSFKey(state.OrchestrationState.CreatedTime)
                                                                                : null),
                                                   (nameof(this.InstanceIdPrefixPsf), (k, v) => v.Val is InstanceState state
                                                                                ? (PSFKey?)new PSFKey(state.InstanceId)
                                                                                : null));

                this.RuntimeStatusPsf    = psfs[0];
                this.CreatedTimePsf      = psfs[1];
                this.InstanceIdPrefixPsf = psfs[2];
            }
#endif
            this.terminationToken = partition.ErrorHandler.Token;

            var _ = this.terminationToken.Register(
                () => {
                try
                {
                    this.mainSession?.Dispose();
                    this.fht.Dispose();
                    this.blobManager.HybridLogDevice.Dispose();
                    this.blobManager.ObjectLogDevice.Dispose();
                    this.blobManager.ClosePSFDevices();
                }
                catch (Exception e)
                {
                    this.blobManager.TraceHelper.FasterStorageError("Disposing FasterKV", e);
                }
            },
                useSynchronizationContext: false);

            this.blobManager.TraceHelper.FasterProgress("Constructed FasterKV");
        }
        public LogWorker(BlobManager blobManager, FasterLog log, Partition partition, StoreWorker storeWorker, FasterTraceHelper traceHelper, CancellationToken cancellationToken)
            : base(nameof(LogWorker), true, 500, cancellationToken)
        {
            partition.ErrorHandler.Token.ThrowIfCancellationRequested();

            this.blobManager  = blobManager;
            this.log          = log;
            this.partition    = partition;
            this.storeWorker  = storeWorker;
            this.traceHelper  = traceHelper;
            this.intakeWorker = new IntakeWorker(cancellationToken, this);

            this.maxFragmentSize = (1 << this.blobManager.EventLogSettings(partition.Settings.UsePremiumStorage).PageSizeBits) - 64; // faster needs some room for header, 64 bytes is conservative
        }
        public FasterAlt(Partition partition, BlobManager blobManager)
        {
            this.partition   = partition;
            this.blobManager = blobManager;
            this.prefix      = $"p{this.partition.PartitionId:D2}/store/";

            this.terminationToken = partition.ErrorHandler.Token;
            this.traceHelper      = blobManager.TraceHelper;
            this.detailTracer     = this.traceHelper.IsTracingAtMostDetailedLevel ? this.traceHelper : null;

            var _ = this.terminationToken.Register(
                () => {
                // nothing so far
            },
                useSynchronizationContext: false);

            this.blobManager.TraceHelper.FasterProgress("Constructed FasterAlt");
        }
Beispiel #4
0
        public FasterLog(BlobManager blobManager, NetheriteOrchestrationServiceSettings settings)
        {
            this.log = new FASTER.core.FasterLog(blobManager.EventLogSettings(settings.UsePremiumStorage));
            this.terminationToken = blobManager.PartitionErrorHandler.Token;

            var _ = this.terminationToken.Register(
                () => {
                try
                {
                    this.log.Dispose();
                    blobManager.EventLogDevice.Dispose();
                }
                catch (Exception e)
                {
                    blobManager.TraceHelper.FasterStorageError("Disposing FasterLog", e);
                }
            },
                useSynchronizationContext: false);
        }
        const long MAX_PAGEBLOB_SIZE = 512L * 1024 * 1024 * 1024; // set this at 512 GB for now TODO consider implications

        /// <summary>
        /// Constructs a new AzureStorageDevice instance, backed by Azure Page Blobs
        /// </summary>
        /// <param name="blobName">A descriptive name that will be the prefix of all segments created</param>
        /// <param name="blockBlobDirectory">the directory containing the block blobs</param>
        /// <param name="pageBlobDirectory">the directory containing the page blobs</param>
        /// <param name="blobManager">the blob manager handling the leases</param>
        /// <param name="underLease">whether this device needs to be protected by the lease</param>
        public AzureStorageDevice(string blobName, CloudBlobDirectory blockBlobDirectory, CloudBlobDirectory pageBlobDirectory, BlobManager blobManager, bool underLease)
            : base($"{blockBlobDirectory}\\{blobName}", PAGE_BLOB_SECTOR_SIZE, Devices.CAPACITY_UNSPECIFIED)
        {
            this.blobs = new ConcurrentDictionary <int, BlobEntry>();
            this.blockBlobDirectory    = blockBlobDirectory;
            this.pageBlobDirectory     = pageBlobDirectory;
            this.blobName              = blobName;
            this.PartitionErrorHandler = blobManager.PartitionErrorHandler;
            this.BlobManager           = blobManager;
            this.underLease            = underLease;
        }
Beispiel #6
0
        public async Task PerformWithRetriesAsync(
            SemaphoreSlim semaphore,
            bool requireLease,
            string name,
            string intent,
            string data,
            string target,
            int expectedLatencyBound,
            bool isCritical,
            Func <int, Task <long> > operation)
        {
            try
            {
                if (semaphore != null)
                {
                    await semaphore.WaitAsync();
                }

                Stopwatch stopwatch   = new Stopwatch();
                int       numAttempts = 0;

                while (true) // retry loop
                {
                    numAttempts++;
                    try
                    {
                        if (requireLease)
                        {
                            await this.ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false);
                        }

                        this.StorageTracer?.FasterStorageProgress($"storage operation {name} started attempt {numAttempts}; target={target} {data}");

                        stopwatch.Restart();

                        long size = await operation(numAttempts).ConfigureAwait(false);

                        stopwatch.Stop();
                        this.StorageTracer?.FasterStorageProgress($"storage operation {name} ({intent}) succeeded on attempt {numAttempts}; target={target} latencyMs={stopwatch.Elapsed.TotalMilliseconds:F1} {data} ");

                        if (stopwatch.ElapsedMilliseconds > expectedLatencyBound)
                        {
                            this.TraceHelper.FasterPerfWarning($"storage operation {name} ({intent}) took {stopwatch.Elapsed.TotalSeconds:F1}s on attempt {numAttempts}, which is excessive; {data}");
                        }

                        this.TraceHelper.FasterAzureStorageAccessCompleted(intent, size, name, target, stopwatch.Elapsed.TotalMilliseconds, numAttempts);

                        return;
                    }
                    catch (StorageException e) when(BlobUtils.IsTransientStorageError(e, this.PartitionErrorHandler.Token) && numAttempts < BlobManager.MaxRetries)
                    {
                        stopwatch.Stop();
                        if (BlobUtils.IsTimeout(e))
                        {
                            this.TraceHelper.FasterPerfWarning($"storage operation {name} ({intent}) timed out on attempt {numAttempts} after {stopwatch.Elapsed.TotalSeconds:F1}s, retrying now; target={target} {data}");
                        }
                        else
                        {
                            TimeSpan nextRetryIn = BlobManager.GetDelayBetweenRetries(numAttempts);
                            this.HandleStorageError(name, $"storage operation {name} ({intent}) failed transiently on attempt {numAttempts}, retry in {nextRetryIn}s", target, e, false, true);
                            await Task.Delay(nextRetryIn);
                        }
                        continue;
                    }
                    catch (Exception exception) when(!Utils.IsFatal(exception))
                    {
                        this.HandleStorageError(name, $"storage operation {name} ({intent}) failed on attempt {numAttempts}", target, exception, isCritical, this.PartitionErrorHandler.IsTerminated);
                        throw;
                    }
                }
            }
            finally
            {
                if (semaphore != null)
                {
                    semaphore.Release();
                }
            }
        }
        public async Task <long> CreateOrRestoreAsync(Partition partition, IPartitionErrorHandler errorHandler, long firstInputQueuePosition)
        {
            this.partition        = partition;
            this.terminationToken = errorHandler.Token;

#if FASTER_SUPPORTS_PSF
            int psfCount = partition.Settings.UsePSFQueries ? FasterKV.PSFCount : 0;
#else
            int psfCount = 0;
#endif

            this.blobManager = new BlobManager(
                this.storageAccount,
                this.pageBlobStorageAccount,
                this.taskHubName,
                this.logger,
                this.partition.Settings.StorageLogLevelLimit,
                partition.PartitionId,
                errorHandler,
                psfCount);

            this.TraceHelper = this.blobManager.TraceHelper;

            this.TraceHelper.FasterProgress("Starting BlobManager");
            await this.blobManager.StartAsync().ConfigureAwait(false);

            var stopwatch = new System.Diagnostics.Stopwatch();
            stopwatch.Start();

            this.TraceHelper.FasterProgress("Creating FasterLog");
            this.log = new FasterLog(this.blobManager, partition.Settings);

            if (partition.Settings.UseAlternateObjectStore)
            {
                this.TraceHelper.FasterProgress("Creating FasterAlt");
                this.store = new FasterAlt(this.partition, this.blobManager);
            }
            else
            {
                this.TraceHelper.FasterProgress("Creating FasterKV");
                this.store = new FasterKV(this.partition, this.blobManager);
            }

            this.TraceHelper.FasterProgress("Creating StoreWorker");
            this.storeWorker = new StoreWorker(this.store, this.partition, this.TraceHelper, this.blobManager, this.terminationToken);

            this.TraceHelper.FasterProgress("Creating LogWorker");
            this.logWorker = this.storeWorker.LogWorker = new LogWorker(this.blobManager, this.log, this.partition, this.storeWorker, this.TraceHelper, this.terminationToken);

            if (this.log.TailAddress == this.log.BeginAddress)
            {
                // take an (empty) checkpoint immediately to ensure the paths are working
                try
                {
                    this.TraceHelper.FasterProgress("Creating store");

                    // this is a fresh partition
                    await this.storeWorker.Initialize(this.log.BeginAddress, firstInputQueuePosition).ConfigureAwait(false);

                    await this.storeWorker.TakeFullCheckpointAsync("initial checkpoint").ConfigureAwait(false);

                    this.TraceHelper.FasterStoreCreated(this.storeWorker.InputQueuePosition, stopwatch.ElapsedMilliseconds);

                    this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());
                }
                catch (Exception e)
                {
                    this.TraceHelper.FasterStorageError(nameof(CreateOrRestoreAsync), e);
                    throw;
                }
            }
            else
            {
                this.TraceHelper.FasterProgress("Loading checkpoint");

                try
                {
                    // we are recovering the last checkpoint of the store
                    (long commitLogPosition, long inputQueuePosition) = await this.store.RecoverAsync();

                    this.storeWorker.SetCheckpointPositionsAfterRecovery(commitLogPosition, inputQueuePosition);

                    // truncate the log in case the truncation did not commit after the checkpoint was taken
                    this.logWorker.SetLastCheckpointPosition(commitLogPosition);

                    this.TraceHelper.FasterCheckpointLoaded(this.storeWorker.CommitLogPosition, this.storeWorker.InputQueuePosition, this.store.StoreStats.Get(), stopwatch.ElapsedMilliseconds);
                }
                catch (Exception e)
                {
                    this.TraceHelper.FasterStorageError("loading checkpoint", e);
                    throw;
                }

                this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected());

                this.TraceHelper.FasterProgress($"Replaying log length={this.log.TailAddress - this.storeWorker.CommitLogPosition} range={this.storeWorker.CommitLogPosition}-{this.log.TailAddress}");

                try
                {
                    if (this.log.TailAddress > (long)this.storeWorker.CommitLogPosition)
                    {
                        // replay log as the store checkpoint lags behind the log
                        await this.storeWorker.ReplayCommitLog(this.logWorker).ConfigureAwait(false);
                    }
                }
                catch (Exception e)
                {
                    this.TraceHelper.FasterStorageError("replaying log", e);
                    throw;
                }

                // restart pending actitivities, timers, work items etc.
                await this.storeWorker.RestartThingsAtEndOfRecovery().ConfigureAwait(false);

                this.TraceHelper.FasterProgress("Recovery complete");
            }

            var ignoredTask = this.IdleLoop();

            return(this.storeWorker.InputQueuePosition);
        }
        public static Task DeleteTaskhubStorageAsync(string connectionString, string taskHubName)
        {
            var storageAccount = (connectionString != LocalFileStorageConnectionString) ? CloudStorageAccount.Parse(connectionString) : null;

            return(BlobManager.DeleteTaskhubStorageAsync(storageAccount, taskHubName));
        }
 internal PsfBlobCheckpointManager(BlobManager blobMan, int groupOrd)
 {
     this.blobManager  = blobMan;
     this.groupOrdinal = groupOrd;
 }