public OrchestrationSessionManager( string storageAccountName, AzureStorageOrchestrationServiceSettings settings, AzureStorageOrchestrationServiceStats stats, ITrackingStore trackingStore) { this.storageAccountName = storageAccountName; this.settings = settings; this.stats = stats; this.trackingStore = trackingStore; this.fetchRuntimeStateQueue = new DispatchQueue(this.settings.MaxStorageOperationConcurrency); }
public static async Task <T> ExecuteWithTimeout <T>( string operationName, string clientRequestId, string account, AzureStorageOrchestrationServiceSettings settings, Func <Task <T> > operation) { if (Debugger.IsAttached) { // ignore long delays while debugging return(await operation()); } using (var cts = new CancellationTokenSource()) { Task timeoutTask = Task.Delay(DefaultTimeout, cts.Token); Task <T> operationTask = operation(); Task completedTask = await Task.WhenAny(timeoutTask, operationTask); if (Equals(timeoutTask, completedTask)) { string taskHubName = settings.TaskHubName; string message = $"The operation '{operationName}' with id '{clientRequestId}' did not complete in '{DefaultTimeout}'. Terminating the process to mitigate potential deadlock."; settings.Logger.GeneralError(account ?? "", taskHubName ?? "", message); // Delay to ensure the ETW event gets written await Task.Delay(TimeSpan.FromSeconds(3)); Environment.FailFast(message); return(default(T)); } cts.Cancel(); return(await operationTask); } }
/// <summary> /// The message manager. /// </summary> public MessageManager( AzureStorageOrchestrationServiceSettings settings, CloudBlobClient cloudBlobClient, string blobContainerName) { this.settings = settings; this.blobContainerName = blobContainerName; this.cloudBlobContainer = cloudBlobClient.GetContainerReference(blobContainerName); this.taskMessageSerializerSettings = new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects, #if NETSTANDARD2_0 SerializationBinder = new TypeNameSerializationBinder(), #else Binder = new TypeNameSerializationBinder(), #endif }; if (this.settings.UseDataContractSerialization) { this.taskMessageSerializerSettings.Converters.Add(new DataContractJsonConverter()); } }
public static async Task <T> ExecuteWithTimeout <T>( string operationName, string account, AzureStorageOrchestrationServiceSettings settings, Func <OperationContext, CancellationToken, Task <T> > operation, AzureStorageOrchestrationServiceStats stats = null, string clientRequestId = null) { OperationContext context = new OperationContext() { ClientRequestID = clientRequestId ?? Guid.NewGuid().ToString() }; if (Debugger.IsAttached) { // ignore long delays while debugging return(await operation(context, CancellationToken.None)); } while (true) { using (var cts = new CancellationTokenSource()) { Task timeoutTask = Task.Delay(DefaultTimeout, cts.Token); Task <T> operationTask = operation(context, cts.Token); if (stats != null) { stats.StorageRequests.Increment(); } Task completedTask = await Task.WhenAny(timeoutTask, operationTask); if (Equals(timeoutTask, completedTask)) { // If more less than DefaultTimeoutCooldown passed, increase timeouts count // otherwise, reset the count to 1, since this is the first timeout we receive // after a long (enough) while if (LastTimeoutHit + DefaultTimeoutCooldown > DateTime.UtcNow) { NumTimeoutsHit++; } else { NumTimeoutsHit = 1; } LastTimeoutHit = DateTime.UtcNow; string taskHubName = settings?.TaskHubName; if (NumTimeoutsHit < MaxNumberOfTimeoutsBeforeRecycle) { string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit {NumTimeoutsHit} out of {MaxNumberOfTimeoutsBeforeRecycle} allowed timeouts. Retrying the operation."; settings.Logger.GeneralWarning(account ?? "", taskHubName ?? "", message); cts.Cancel(); continue; } else { string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit maximum number ({MaxNumberOfTimeoutsBeforeRecycle}) of timeouts. Terminating the process to mitigate potential deadlock."; settings.Logger.GeneralError(account ?? "", taskHubName ?? "", message); // Delay to ensure the ETW event gets written await Task.Delay(TimeSpan.FromSeconds(3)); bool executeFailFast = true; Task <bool> gracefulShutdownTask = Task.Run(async() => { try { return(await settings.OnImminentFailFast(message)); } catch (Exception) { return(true); } }); await Task.WhenAny(gracefulShutdownTask, Task.Delay(TimeSpan.FromSeconds(35))); if (gracefulShutdownTask.IsCompleted) { executeFailFast = gracefulShutdownTask.Result; } if (executeFailFast) { TimeoutHandler.ProcessKillAction(message); } else { // Technically we don't need else as the action above would have killed the process. // However tests don't kill the process so putting in else. throw new TimeoutException(message); } } } cts.Cancel(); return(await operationTask); } } }