public BlobLeaseManager( AzureStorageOrchestrationServiceSettings settings, string leaseContainerName, string blobPrefix, string leaseType, CloudBlobClient storageClient, bool skipBlobContainerCreation, AzureStorageOrchestrationServiceStats stats) { this.settings = settings; this.storageAccountName = storageClient.Credentials.AccountName; this.taskHubName = settings.TaskHubName; this.workerName = settings.WorkerId; this.leaseContainerName = leaseContainerName; this.blobPrefix = blobPrefix; this.leaseType = leaseType; this.storageClient = storageClient; this.leaseInterval = settings.LeaseInterval; this.skipBlobContainerCreation = skipBlobContainerCreation; this.renewRequestOptions = new BlobRequestOptions { ServerTimeout = settings.LeaseRenewInterval }; this.stats = stats ?? new AzureStorageOrchestrationServiceStats(); this.Initialize(); }
public BlobLeaseManager( string taskHubName, string workerName, string leaseContainerName, string blobPrefix, string consumerGroupName, CloudBlobClient storageClient, TimeSpan leaseInterval, TimeSpan renewInterval, bool skipBlobContainerCreation, AzureStorageOrchestrationServiceStats stats) { this.storageAccountName = storageClient.Credentials.AccountName; this.taskHubName = taskHubName; this.workerName = workerName; this.leaseContainerName = leaseContainerName; this.blobPrefix = blobPrefix; this.consumerGroupName = consumerGroupName; this.storageClient = storageClient; this.leaseInterval = leaseInterval; this.renewInterval = renewInterval; this.skipBlobContainerCreation = skipBlobContainerCreation; this.renewRequestOptions = new BlobRequestOptions { ServerTimeout = renewInterval }; this.stats = stats ?? new AzureStorageOrchestrationServiceStats(); this.Initialize(); }
public LegacyPartitionManager( AzureStorageOrchestrationService service, AzureStorageOrchestrationServiceSettings settings, CloudStorageAccount account, AzureStorageOrchestrationServiceStats stats) { this.service = service; this.settings = settings; this.stats = stats; this.leaseManager = new BlobLeaseManager( settings, settings.TaskHubName.ToLowerInvariant() + "-leases", string.Empty, "default", account.CreateCloudBlobClient(), skipBlobContainerCreation: false, stats); this.leaseCollectionManager = new LeaseCollectionBalancer <BlobLease>( "default", settings, account.Credentials.AccountName, leaseManager, new LeaseCollectionBalancerOptions { AcquireInterval = settings.LeaseAcquireInterval, RenewInterval = settings.LeaseRenewInterval, LeaseInterval = settings.LeaseInterval, ShouldStealLeases = true, }); }
public AppLeaseManager( AzureStorageOrchestrationServiceSettings settings, string accountName, CloudBlobClient storageClient, string appLeaseContainerName, AppLeaseOptions options, AzureStorageOrchestrationServiceStats stats) { this.settings = settings; this.accountName = accountName; this.taskHub = settings.TaskHubName; this.workerName = settings.WorkerId; this.storageClient = storageClient; this.appLeaseContainerName = appLeaseContainerName; this.appName = settings.AppName; this.options = options; this.stats = stats ?? new AzureStorageOrchestrationServiceStats(); this.isLeaseOwner = false; this.appLeaseContainer = this.storageClient.GetContainerReference(this.appLeaseContainerName); using (MD5 md5 = MD5.Create()) { byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(this.appName)); this.appLeaseId = new Guid(hash).ToString(); } }
internal async Task DeleteLargeMessageBlobs(string instanceId, AzureStorageOrchestrationServiceStats stats) { var blobForDeletionTaskList = new List <Task>(); if (!await this.cloudBlobContainer.ExistsAsync()) { return; } CloudBlobDirectory instanceDirectory = this.cloudBlobContainer.GetDirectoryReference(instanceId); BlobContinuationToken blobContinuationToken = null; while (true) { BlobResultSegment segment = await instanceDirectory.ListBlobsSegmentedAsync(blobContinuationToken); stats.StorageRequests.Increment(); foreach (IListBlobItem blobListItem in segment.Results) { var cloudBlockBlob = blobListItem as CloudBlockBlob; CloudBlockBlob blob = this.cloudBlobContainer.GetBlockBlobReference(cloudBlockBlob?.Name); blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync()); } await Task.WhenAll(blobForDeletionTaskList); stats.StorageRequests.Increment(blobForDeletionTaskList.Count); if (blobContinuationToken == null) { break; } } }
public AppLeaseManager( IPartitionManager partitionManager, AzureStorageOrchestrationServiceSettings settings, string accountName, CloudBlobClient storageClient, string appLeaseContainerName, string appLeaseInfoBlobName, AppLeaseOptions options, AzureStorageOrchestrationServiceStats stats) { this.partitionManager = partitionManager; this.settings = settings; this.accountName = accountName; this.storageClient = storageClient; this.appLeaseContainerName = appLeaseContainerName; this.appLeaseInfoBlobName = appLeaseInfoBlobName; this.options = options; this.stats = stats ?? new AzureStorageOrchestrationServiceStats(); this.taskHub = settings.TaskHubName; this.workerName = settings.WorkerId; this.appName = settings.AppName; this.appLeaseIsEnabled = this.settings.UseAppLease; this.appLeaseContainer = this.storageClient.GetContainerReference(this.appLeaseContainerName); this.appLeaseInfoBlob = this.appLeaseContainer.GetBlockBlobReference(this.appLeaseInfoBlobName); using (MD5 md5 = MD5.Create()) { byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(this.appName)); this.appLeaseId = new Guid(hash).ToString(); } this.isLeaseOwner = false; this.shutdownCompletedEvent = new AsyncManualResetEvent(); }
internal AzureTableTrackingStore( AzureStorageOrchestrationServiceStats stats, CloudTable instancesTable ) { this.stats = stats; this.InstancesTable = instancesTable; }
public WorkItemQueue( CloudQueue storageQueue, AzureStorageOrchestrationServiceSettings settings, AzureStorageOrchestrationServiceStats stats, MessageManager messageManager) : base(storageQueue, settings, stats, messageManager) { }
public Queue(AzureStorageClient azureStorageClient, CloudQueueClient queueClient, string queueName) { this.azureStorageClient = azureStorageClient; this.queueClient = queueClient; this.stats = this.azureStorageClient.Stats; this.Name = queueName; this.cloudQueue = this.queueClient.GetQueueReference(this.Name); }
public Table(AzureStorageClient azureStorageClient, CloudTableClient tableClient, string tableName) { this.azureStorageClient = azureStorageClient; this.tableClient = tableClient; this.Name = tableName; this.stats = this.azureStorageClient.Stats; this.cloudTable = this.tableClient.GetTableReference(this.Name); }
public ControlQueue( CloudQueue storageQueue, AzureStorageOrchestrationServiceSettings settings, AzureStorageOrchestrationServiceStats stats, MessageManager messageManager) : base(storageQueue, settings, stats, messageManager) { this.releaseTokenSource = new CancellationTokenSource(); this.releaseCancellationToken = this.releaseTokenSource.Token; }
public ControlQueue( AzureStorageClient azureStorageClient, string queueName, MessageManager messageManager) : base(azureStorageClient, queueName, messageManager) { this.releaseTokenSource = new CancellationTokenSource(); this.releaseCancellationToken = this.releaseTokenSource.Token; this.stats = this.azureStorageClient.Stats; }
public BlobLease( string partitionId, CloudBlobDirectory leaseDirectory, string accountName, AzureStorageOrchestrationServiceStats stats, AzureStorageOrchestrationServiceSettings settings) { this.PartitionId = partitionId; this.Blob = leaseDirectory.GetBlockBlobReference(partitionId); this.accountName = accountName; this.stats = stats; this.settings = settings; }
public OrchestrationSessionManager( string storageAccountName, AzureStorageOrchestrationServiceSettings settings, AzureStorageOrchestrationServiceStats stats, ITrackingStore trackingStore) { this.storageAccountName = storageAccountName; this.settings = settings; this.stats = stats; this.trackingStore = trackingStore; this.fetchRuntimeStateQueue = new DispatchQueue(this.settings.MaxStorageOperationConcurrency); }
internal async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, AzureStorageOrchestrationServiceStats stats) { var blobForDeletionTaskList = new List <Task>(); if (!await this.cloudBlobContainer.ExistsAsync()) { return; } CloudBlobDirectory instanceDirectory = this.cloudBlobContainer.GetDirectoryReference(sanitizedInstanceId); BlobContinuationToken blobContinuationToken = null; while (true) { BlobResultSegment segment = await TimeoutHandler.ExecuteWithTimeout( operationName : "DeleteLargeMessageBlobs", account : cloudBlobContainer?.ServiceClient?.Credentials?.AccountName, settings : this.settings, operation : (context, timeoutToken) => { return(instanceDirectory.ListBlobsSegmentedAsync( useFlatBlobListing: true, blobListingDetails: BlobListingDetails.Metadata, maxResults: null, currentToken: blobContinuationToken, options: null, operationContext: context, cancellationToken: timeoutToken)); }); stats.StorageRequests.Increment(); foreach (IListBlobItem blobListItem in segment.Results) { var cloudBlockBlob = blobListItem as CloudBlockBlob; CloudBlockBlob blob = this.cloudBlobContainer.GetBlockBlobReference(cloudBlockBlob?.Name); blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync()); } await Task.WhenAll(blobForDeletionTaskList); stats.StorageRequests.Increment(blobForDeletionTaskList.Count); if (blobContinuationToken == null) { break; } } }
public TaskHubQueue( CloudQueue storageQueue, AzureStorageOrchestrationServiceSettings settings, AzureStorageOrchestrationServiceStats stats, MessageManager messageManager) { this.storageQueue = storageQueue; this.storageAccountName = storageQueue.ServiceClient.Credentials.AccountName; this.settings = settings; this.stats = stats; this.messageManager = messageManager; TimeSpan minPollingDelay = TimeSpan.FromMilliseconds(50); TimeSpan maxPollingDelay = AzureStorageOrchestrationService.MaxQueuePollingDelay; this.backoffHelper = new BackoffPollingHelper(minPollingDelay, maxPollingDelay); }
public AzureTableTrackingStore( AzureStorageOrchestrationServiceSettings settings, MessageManager messageManager, AzureStorageOrchestrationServiceStats stats) { this.settings = settings; this.messageManager = messageManager; this.stats = stats; this.tableEntityConverter = new TableEntityConverter(); this.taskHubName = settings.TaskHubName; CloudStorageAccount account = CloudStorageAccount.Parse(settings.StorageConnectionString); this.storageAccountName = account.Credentials.AccountName; CloudTableClient tableClient = account.CreateCloudTableClient(); tableClient.BufferManager = SimpleBufferManager.Shared; string historyTableName = $"{taskHubName}History"; NameValidator.ValidateTableName(historyTableName); string instancesTableName = $"{taskHubName}Instances"; NameValidator.ValidateTableName(instancesTableName); this.historyTable = tableClient.GetTableReference(historyTableName); this.instancesTable = tableClient.GetTableReference(instancesTableName); this.StorageTableRequestOptions = settings.HistoryTableRequestOptions; this.eTagValues = new ConcurrentDictionary <string, string>(); // Use reflection to learn all the different event types supported by DTFx. // This could have been hardcoded, but I generally try to avoid hardcoding of point-in-time DTFx knowledge. Type historyEventType = typeof(HistoryEvent); IEnumerable <Type> historyEventTypes = historyEventType.Assembly.GetTypes().Where( t => !t.IsAbstract && t.IsSubclassOf(historyEventType)); PropertyInfo eventTypeProperty = historyEventType.GetProperty(nameof(HistoryEvent.EventType)); this.eventTypeMap = historyEventTypes.ToDictionary( type => ((HistoryEvent)FormatterServices.GetUninitializedObject(type)).EventType); }
public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, AzureStorageOrchestrationServiceStats stats) { if (!await this.blobContainer.ExistsAsync()) { return; } IEnumerable <Blob> blobList = await this.blobContainer.ListBlobsAsync(sanitizedInstanceId); var blobForDeletionTaskList = new List <Task>(); foreach (Blob blob in blobList) { blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync()); } await Task.WhenAll(blobForDeletionTaskList); }
internal async Task DeleteLargeMessageBlobs(string instanceId, AzureStorageOrchestrationServiceStats stats) { var blobForDeletionTaskList = new List <Task>(); if (!await this.cloudBlobContainer.ExistsAsync()) { return; } CloudBlobDirectory instanceDirectory = this.cloudBlobContainer.GetDirectoryReference(instanceId); BlobContinuationToken blobContinuationToken = null; while (true) { OperationContext context = new OperationContext { ClientRequestID = Guid.NewGuid().ToString() }; BlobResultSegment segment = await TimeoutHandler.ExecuteWithTimeout("DeleteLargeMessageBlobs", context.ClientRequestID, cloudBlobContainer?.ServiceClient?.Credentials?.AccountName, null, () => { return(instanceDirectory.ListBlobsSegmentedAsync(blobContinuationToken)); }); stats.StorageRequests.Increment(); foreach (IListBlobItem blobListItem in segment.Results) { var cloudBlockBlob = blobListItem as CloudBlockBlob; CloudBlockBlob blob = this.cloudBlobContainer.GetBlockBlobReference(cloudBlockBlob?.Name); blobForDeletionTaskList.Add(blob.DeleteIfExistsAsync()); } await Task.WhenAll(blobForDeletionTaskList); stats.StorageRequests.Increment(blobForDeletionTaskList.Count); if (blobContinuationToken == null) { break; } } }
public static async Task <T> ExecuteWithTimeout <T>( string operationName, string account, AzureStorageOrchestrationServiceSettings settings, Func <OperationContext, CancellationToken, Task <T> > operation, AzureStorageOrchestrationServiceStats stats = null, string clientRequestId = null) { OperationContext context = new OperationContext() { ClientRequestID = clientRequestId ?? Guid.NewGuid().ToString() }; if (Debugger.IsAttached) { // ignore long delays while debugging return(await operation(context, CancellationToken.None)); } while (true) { using (var cts = new CancellationTokenSource()) { Task timeoutTask = Task.Delay(DefaultTimeout, cts.Token); Task <T> operationTask = operation(context, cts.Token); if (stats != null) { stats.StorageRequests.Increment(); } Task completedTask = await Task.WhenAny(timeoutTask, operationTask); if (Equals(timeoutTask, completedTask)) { // If more less than DefaultTimeoutCooldown passed, increase timeouts count // otherwise, reset the count to 1, since this is the first timeout we receive // after a long (enough) while if (LastTimeoutHit + DefaultTimeoutCooldown > DateTime.UtcNow) { NumTimeoutsHit++; } else { NumTimeoutsHit = 1; } LastTimeoutHit = DateTime.UtcNow; string taskHubName = settings?.TaskHubName; if (NumTimeoutsHit < MaxNumberOfTimeoutsBeforeRecycle) { string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit {NumTimeoutsHit} out of {MaxNumberOfTimeoutsBeforeRecycle} allowed timeouts. Retrying the operation."; settings.Logger.GeneralWarning(account ?? "", taskHubName ?? "", message); cts.Cancel(); continue; } else { string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit maximum number ({MaxNumberOfTimeoutsBeforeRecycle}) of timeouts. Terminating the process to mitigate potential deadlock."; settings.Logger.GeneralError(account ?? "", taskHubName ?? "", message); // Delay to ensure the ETW event gets written await Task.Delay(TimeSpan.FromSeconds(3)); bool executeFailFast = true; Task <bool> gracefulShutdownTask = Task.Run(async() => { try { return(await settings.OnImminentFailFast(message)); } catch (Exception) { return(true); } }); await Task.WhenAny(gracefulShutdownTask, Task.Delay(TimeSpan.FromSeconds(35))); if (gracefulShutdownTask.IsCompleted) { executeFailFast = gracefulShutdownTask.Result; } if (executeFailFast) { TimeoutHandler.ProcessKillAction(message); } else { // Technically we don't need else as the action above would have killed the process. // However tests don't kill the process so putting in else. throw new TimeoutException(message); } } } cts.Cancel(); return(await operationTask); } } }
private void SetupTrackingStore() { var stats = new AzureStorageOrchestrationServiceStats(); this.TrackingStore = new AzureTableTrackingStore(stats, this.TableMock); }
public SafePartitionManager( AzureStorageOrchestrationService service, OrchestrationSessionManager sessionManager, AzureStorageOrchestrationServiceSettings settings, CloudStorageAccount account, AzureStorageOrchestrationServiceStats stats) { this.service = service; this.settings = settings; this.stats = stats; this.sessionManager = sessionManager; string storageAccountName = account.Credentials.AccountName; this.intentLeaseManager = new BlobLeaseManager( settings, settings.TaskHubName.ToLowerInvariant() + "-leases", string.Empty, "intent", account.CreateCloudBlobClient(), skipBlobContainerCreation: false, stats); this.intentLeaseCollectionManager = new LeaseCollectionBalancer <BlobLease>( "intent", settings, storageAccountName, this.intentLeaseManager, new LeaseCollectionBalancerOptions { AcquireInterval = settings.LeaseAcquireInterval, RenewInterval = settings.LeaseRenewInterval, LeaseInterval = settings.LeaseInterval, }); var currentlyOwnedIntentLeases = this.intentLeaseCollectionManager.GetCurrentlyOwnedLeases(); this.ownershipLeaseManager = new BlobLeaseManager( settings, settings.TaskHubName.ToLowerInvariant() + "-leases", string.Empty, "ownership", account.CreateCloudBlobClient(), skipBlobContainerCreation: false, stats); this.ownershipLeaseCollectionManager = new LeaseCollectionBalancer <BlobLease>( "ownership", this.settings, storageAccountName, this.ownershipLeaseManager, new LeaseCollectionBalancerOptions { AcquireInterval = TimeSpan.FromSeconds(5), RenewInterval = TimeSpan.FromSeconds(10), LeaseInterval = TimeSpan.FromSeconds(15), ShouldStealLeases = false }, shouldAquireLeaseDelegate: leaseKey => currentlyOwnedIntentLeases.ContainsKey(leaseKey), shouldRenewLeaseDelegate: leaseKey => currentlyOwnedIntentLeases.ContainsKey(leaseKey) || this.sessionManager.IsControlQueueReceivingMessages(leaseKey) || this.sessionManager.IsControlQueueProcessingMessages(leaseKey)); }