public OrchestrationSessionManager(
string storageAccountName,
AzureStorageOrchestrationServiceSettings settings,
AzureStorageOrchestrationServiceStats stats,
ITrackingStore trackingStore)
{
this.storageAccountName = storageAccountName;
this.settings = settings;
this.stats = stats;
this.trackingStore = trackingStore;
this.fetchRuntimeStateQueue = new DispatchQueue(this.settings.MaxStorageOperationConcurrency);
}
public static async Task <T> ExecuteWithTimeout <T>(
string operationName,
string clientRequestId,
string account,
AzureStorageOrchestrationServiceSettings settings,
Func <Task <T> > operation)
{
if (Debugger.IsAttached)
{
// ignore long delays while debugging
return(await operation());
}
using (var cts = new CancellationTokenSource())
{
Task timeoutTask = Task.Delay(DefaultTimeout, cts.Token);
Task <T> operationTask = operation();
Task completedTask = await Task.WhenAny(timeoutTask, operationTask);
if (Equals(timeoutTask, completedTask))
{
string taskHubName = settings.TaskHubName;
string message = $"The operation '{operationName}' with id '{clientRequestId}' did not complete in '{DefaultTimeout}'. Terminating the process to mitigate potential deadlock.";
settings.Logger.GeneralError(account ?? "", taskHubName ?? "", message);
// Delay to ensure the ETW event gets written
await Task.Delay(TimeSpan.FromSeconds(3));
Environment.FailFast(message);
return(default(T));
}
cts.Cancel();
return(await operationTask);
}
}
/// <summary>
/// The message manager.
/// </summary>
public MessageManager(
AzureStorageOrchestrationServiceSettings settings,
CloudBlobClient cloudBlobClient,
string blobContainerName)
{
this.settings = settings;
this.blobContainerName = blobContainerName;
this.cloudBlobContainer = cloudBlobClient.GetContainerReference(blobContainerName);
this.taskMessageSerializerSettings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Objects,
#if NETSTANDARD2_0
SerializationBinder = new TypeNameSerializationBinder(),
#else
Binder = new TypeNameSerializationBinder(),
#endif
};
if (this.settings.UseDataContractSerialization)
{
this.taskMessageSerializerSettings.Converters.Add(new DataContractJsonConverter());
}
}
public static async Task <T> ExecuteWithTimeout <T>(
string operationName,
string account,
AzureStorageOrchestrationServiceSettings settings,
Func <OperationContext, CancellationToken, Task <T> > operation,
AzureStorageOrchestrationServiceStats stats = null,
string clientRequestId = null)
{
OperationContext context = new OperationContext()
{
ClientRequestID = clientRequestId ?? Guid.NewGuid().ToString()
};
if (Debugger.IsAttached)
{
// ignore long delays while debugging
return(await operation(context, CancellationToken.None));
}
while (true)
{
using (var cts = new CancellationTokenSource())
{
Task timeoutTask = Task.Delay(DefaultTimeout, cts.Token);
Task <T> operationTask = operation(context, cts.Token);
if (stats != null)
{
stats.StorageRequests.Increment();
}
Task completedTask = await Task.WhenAny(timeoutTask, operationTask);
if (Equals(timeoutTask, completedTask))
{
// If more less than DefaultTimeoutCooldown passed, increase timeouts count
// otherwise, reset the count to 1, since this is the first timeout we receive
// after a long (enough) while
if (LastTimeoutHit + DefaultTimeoutCooldown > DateTime.UtcNow)
{
NumTimeoutsHit++;
}
else
{
NumTimeoutsHit = 1;
}
LastTimeoutHit = DateTime.UtcNow;
string taskHubName = settings?.TaskHubName;
if (NumTimeoutsHit < MaxNumberOfTimeoutsBeforeRecycle)
{
string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit {NumTimeoutsHit} out of {MaxNumberOfTimeoutsBeforeRecycle} allowed timeouts. Retrying the operation.";
settings.Logger.GeneralWarning(account ?? "", taskHubName ?? "", message);
cts.Cancel();
continue;
}
else
{
string message = $"The operation '{operationName}' with id '{context.ClientRequestID}' did not complete in '{DefaultTimeout}'. Hit maximum number ({MaxNumberOfTimeoutsBeforeRecycle}) of timeouts. Terminating the process to mitigate potential deadlock.";
settings.Logger.GeneralError(account ?? "", taskHubName ?? "", message);
// Delay to ensure the ETW event gets written
await Task.Delay(TimeSpan.FromSeconds(3));
bool executeFailFast = true;
Task <bool> gracefulShutdownTask = Task.Run(async() =>
{
try
{
return(await settings.OnImminentFailFast(message));
}
catch (Exception)
{
return(true);
}
});
await Task.WhenAny(gracefulShutdownTask, Task.Delay(TimeSpan.FromSeconds(35)));
if (gracefulShutdownTask.IsCompleted)
{
executeFailFast = gracefulShutdownTask.Result;
}
if (executeFailFast)
{
TimeoutHandler.ProcessKillAction(message);
}
else
{
// Technically we don't need else as the action above would have killed the process.
// However tests don't kill the process so putting in else.
throw new TimeoutException(message);
}
}
}
cts.Cancel();
return(await operationTask);
}
}
}
