const long MAX_PAGEBLOB_SIZE = 512L * 1024 * 1024 * 1024; // set this at 512 GB for now TODO consider implications /// <summary> /// Constructs a new AzureStorageDevice instance, backed by Azure Page Blobs /// </summary> /// <param name="blobName">A descriptive name that will be the prefix of all segments created</param> /// <param name="blockBlobDirectory">the directory containing the block blobs</param> /// <param name="pageBlobDirectory">the directory containing the page blobs</param> /// <param name="blobManager">the blob manager handling the leases</param> /// <param name="underLease">whether this device needs to be protected by the lease</param> public AzureStorageDevice(string blobName, CloudBlobDirectory blockBlobDirectory, CloudBlobDirectory pageBlobDirectory, BlobManager blobManager, bool underLease) : base($"{blockBlobDirectory}\\{blobName}", PAGE_BLOB_SECTOR_SIZE, Devices.CAPACITY_UNSPECIFIED) { this.blobs = new ConcurrentDictionary <int, BlobEntry>(); this.blockBlobDirectory = blockBlobDirectory; this.pageBlobDirectory = pageBlobDirectory; this.blobName = blobName; this.PartitionErrorHandler = blobManager.PartitionErrorHandler; this.BlobManager = blobManager; this.underLease = underLease; }
public Task <long> CreateOrRestoreAsync(Partition partition, IPartitionErrorHandler termination, long initialInputQueuePosition) { this.partition = partition; foreach (var trackedObject in this.trackedObjects.Values) { trackedObject.Partition = partition; if (trackedObject.Key.IsSingleton) { trackedObject.OnFirstInitialization(); } } this.commitPosition = 1; return(Task.FromResult(1L)); }
public async Task <long> CreateOrRestoreAsync(IPartitionErrorHandler errorHandler, long firstInputQueuePosition) { EventTraceContext.Clear(); this.ErrorHandler = errorHandler; this.TraceHelper.TraceProgress("Starting partition"); await MaxConcurrentStarts.WaitAsync(); // create or restore partition state from last snapshot try { // create the state this.State = ((IStorageProvider)this.host).CreatePartitionState(); // initialize timer for this partition this.PendingTimers = new BatchTimer <PartitionEvent>(this.ErrorHandler.Token, this.TimersFired); // goes to storage to create or restore the partition state this.TraceHelper.TraceProgress("Loading partition state"); var inputQueuePosition = await this.State.CreateOrRestoreAsync(this, this.ErrorHandler, firstInputQueuePosition).ConfigureAwait(false); // start processing the timers this.PendingTimers.Start($"Timer{this.PartitionId:D2}"); // start processing the worker queues this.State.StartProcessing(); this.TraceHelper.TraceProgress($"Started partition, nextInputQueuePosition={inputQueuePosition}"); return(inputQueuePosition); } catch (Exception e) when(!Utils.IsFatal(e)) { this.ErrorHandler.HandleError(nameof(CreateOrRestoreAsync), "Could not start partition", e, true, false); throw; } finally { MaxConcurrentStarts.Release(); } }
public async Task <long> CreateOrRestoreAsync(Partition partition, IPartitionErrorHandler errorHandler, long firstInputQueuePosition) { this.partition = partition; this.terminationToken = errorHandler.Token; #if FASTER_SUPPORTS_PSF int psfCount = partition.Settings.UsePSFQueries ? FasterKV.PSFCount : 0; #else int psfCount = 0; #endif this.blobManager = new BlobManager( this.storageAccount, this.pageBlobStorageAccount, this.taskHubName, this.logger, this.partition.Settings.StorageLogLevelLimit, partition.PartitionId, errorHandler, psfCount); this.TraceHelper = this.blobManager.TraceHelper; this.TraceHelper.FasterProgress("Starting BlobManager"); await this.blobManager.StartAsync().ConfigureAwait(false); var stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); this.TraceHelper.FasterProgress("Creating FasterLog"); this.log = new FasterLog(this.blobManager, partition.Settings); if (partition.Settings.UseAlternateObjectStore) { this.TraceHelper.FasterProgress("Creating FasterAlt"); this.store = new FasterAlt(this.partition, this.blobManager); } else { this.TraceHelper.FasterProgress("Creating FasterKV"); this.store = new FasterKV(this.partition, this.blobManager); } this.TraceHelper.FasterProgress("Creating StoreWorker"); this.storeWorker = new StoreWorker(this.store, this.partition, this.TraceHelper, this.blobManager, this.terminationToken); this.TraceHelper.FasterProgress("Creating LogWorker"); this.logWorker = this.storeWorker.LogWorker = new LogWorker(this.blobManager, this.log, this.partition, this.storeWorker, this.TraceHelper, this.terminationToken); if (this.log.TailAddress == this.log.BeginAddress) { // take an (empty) checkpoint immediately to ensure the paths are working try { this.TraceHelper.FasterProgress("Creating store"); // this is a fresh partition await this.storeWorker.Initialize(this.log.BeginAddress, firstInputQueuePosition).ConfigureAwait(false); await this.storeWorker.TakeFullCheckpointAsync("initial checkpoint").ConfigureAwait(false); this.TraceHelper.FasterStoreCreated(this.storeWorker.InputQueuePosition, stopwatch.ElapsedMilliseconds); this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected()); } catch (Exception e) { this.TraceHelper.FasterStorageError(nameof(CreateOrRestoreAsync), e); throw; } } else { this.TraceHelper.FasterProgress("Loading checkpoint"); try { // we are recovering the last checkpoint of the store (long commitLogPosition, long inputQueuePosition) = await this.store.RecoverAsync(); this.storeWorker.SetCheckpointPositionsAfterRecovery(commitLogPosition, inputQueuePosition); // truncate the log in case the truncation did not commit after the checkpoint was taken this.logWorker.SetLastCheckpointPosition(commitLogPosition); this.TraceHelper.FasterCheckpointLoaded(this.storeWorker.CommitLogPosition, this.storeWorker.InputQueuePosition, this.store.StoreStats.Get(), stopwatch.ElapsedMilliseconds); } catch (Exception e) { this.TraceHelper.FasterStorageError("loading checkpoint", e); throw; } this.partition.Assert(!FASTER.core.LightEpoch.AnyInstanceProtected()); this.TraceHelper.FasterProgress($"Replaying log length={this.log.TailAddress - this.storeWorker.CommitLogPosition} range={this.storeWorker.CommitLogPosition}-{this.log.TailAddress}"); try { if (this.log.TailAddress > (long)this.storeWorker.CommitLogPosition) { // replay log as the store checkpoint lags behind the log await this.storeWorker.ReplayCommitLog(this.logWorker).ConfigureAwait(false); } } catch (Exception e) { this.TraceHelper.FasterStorageError("replaying log", e); throw; } // restart pending actitivities, timers, work items etc. await this.storeWorker.RestartThingsAtEndOfRecovery().ConfigureAwait(false); this.TraceHelper.FasterProgress("Recovery complete"); } var ignoredTask = this.IdleLoop(); return(this.storeWorker.InputQueuePosition); }
public GeoDrEventConsumerFactory(bool useCheckpointing, IPartitionErrorHandler errorHandler) { this.useCheckpointing = useCheckpointing; this.errorHandler = errorHandler; }