public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
JObject obj = JObject.Load(reader);
JToken firstRetryIntervalToken;
if (!obj.TryGetValue(FirstRetryIntervalField, out firstRetryIntervalToken))
{
throw new ArgumentException($"Invalid JSON. Must contain field {FirstRetryIntervalField}", "reader");
}
JToken maxIntervalAttemptsToken;
if (!obj.TryGetValue(MaxAttemptsField, out maxIntervalAttemptsToken))
{
throw new ArgumentException($"Invalid JSON. Must contain field {MaxAttemptsField}", "reader");
}
var firstRetryIntervalInMilliseconds = firstRetryIntervalToken.Value <int>();
var maxNumberOfAttempts = maxIntervalAttemptsToken.Value <int>();
var target = new RetryOptions(
TimeSpan.FromMilliseconds(firstRetryIntervalInMilliseconds),
maxNumberOfAttempts);
JToken backoffCoefficientToken;
if (obj.TryGetValue(BackoffCoefficientField, out backoffCoefficientToken))
{
target.BackoffCoefficient = backoffCoefficientToken.Value <double>();
}
JToken maxRetryIntervalToken;
if (obj.TryGetValue(MaxRetryIntervalField, out maxRetryIntervalToken))
{
int maxRetryIntervalMilliseconds = maxRetryIntervalToken.Value <int>();
target.MaxRetryInterval = TimeSpan.FromMilliseconds(maxRetryIntervalMilliseconds);
}
JToken retryTimeoutToken;
if (obj.TryGetValue(RetryTimeoutField, out retryTimeoutToken))
{
int retryTimeoutMilliseconds = retryTimeoutToken.Value <int>();
target.RetryTimeout = TimeSpan.FromMilliseconds(retryTimeoutMilliseconds);
}
return(target);
}
public static async Task <List <string> > ChildOrchestrator(
[OrchestrationTrigger] IDurableOrchestrationContext context)
{
var outputs = new List <string>();
var tasks = new Task <string> [3];
var option = new RetryOptions(TimeSpan.FromSeconds(5), 3);
tasks[0] = context.CallActivityWithRetryAsync <string>(nameof(MultiLayerOrchestrationWithRetry_Hello), option, "Osaka");
tasks[1] = context.CallActivityWithRetryAsync <string>(nameof(MultiLayerOrchestrationWithRetry_Hello), option, "Seattle");
tasks[2] = context.CallActivityWithRetryAsync <string>(nameof(MultiLayerOrchestrationWithRetry_Hello), option, "Atlanta");
await Task.WhenAll(tasks);
return(tasks.Select((i) => i.Result).ToList());
}
internal async Task <TResult> CallDurableTaskFunctionAsync <TResult>(
string functionName,
FunctionType functionType,
bool oneWay,
string instanceId,
string operation,
RetryOptions retryOptions,
object input)
{
this.ThrowIfInvalidAccess();
if (retryOptions != null)
{
if (!this.durabilityProvider.ValidateDelayTime(retryOptions.MaxRetryInterval, out string errorMessage))
{
throw new ArgumentException(errorMessage, nameof(retryOptions.MaxRetryInterval));
}
if (!this.durabilityProvider.ValidateDelayTime(retryOptions.FirstRetryInterval, out errorMessage))
{
throw new ArgumentException(errorMessage, nameof(retryOptions.FirstRetryInterval));
}
}
// TODO: Support for versioning
string version = DefaultVersion;
this.Config.ThrowIfFunctionDoesNotExist(functionName, functionType);
Task <TResult> callTask = null;
EntityId? lockToUse = null;
string operationId = string.Empty;
string operationName = string.Empty;
switch (functionType)
{
case FunctionType.Activity:
System.Diagnostics.Debug.Assert(instanceId == null, "The instanceId parameter should not be used for activity functions.");
System.Diagnostics.Debug.Assert(operation == null, "The operation parameter should not be used for activity functions.");
System.Diagnostics.Debug.Assert(!oneWay, "The oneWay parameter should not be used for activity functions.");
if (retryOptions == null)
{
this.IncrementActionsOrThrowException();
callTask = this.InnerContext.ScheduleTask <TResult>(functionName, version, input);
}
else
{
this.IncrementActionsOrThrowException();
callTask = this.InnerContext.ScheduleWithRetry <TResult>(
functionName,
version,
retryOptions.GetRetryOptions(),
input);
}
break;
case FunctionType.Orchestrator:
// Instance IDs should not be reused when creating sub-orchestrations. This is a best-effort
// check. We cannot easily check the full hierarchy, so we just look at the current orchestration
// and the immediate parent.
if (string.Equals(instanceId, this.InstanceId, StringComparison.OrdinalIgnoreCase) ||
(this.ParentInstanceId != null && string.Equals(instanceId, this.ParentInstanceId, StringComparison.OrdinalIgnoreCase)))
{
throw new ArgumentException("The instance ID of a sub-orchestration must be different than the instance ID of a parent orchestration.");
}
System.Diagnostics.Debug.Assert(operation == null, "The operation parameter should not be used for activity functions.");
if (instanceId != null && instanceId.StartsWith("@"))
{
throw new ArgumentException(nameof(instanceId), "Orchestration instance ids must not start with @");
}
if (oneWay)
{
this.IncrementActionsOrThrowException();
var dummyTask = this.InnerContext.CreateSubOrchestrationInstance <TResult>(
functionName,
version,
instanceId,
input,
new Dictionary <string, string>()
{
{ OrchestrationTags.FireAndForget, "" }
});
System.Diagnostics.Debug.Assert(dummyTask.IsCompleted, "task should be fire-and-forget");
}
else
{
if (this.ContextLocks != null)
{
throw new LockingRulesViolationException("While holding locks, cannot call suborchestrators.");
}
if (retryOptions == null)
{
this.IncrementActionsOrThrowException();
callTask = this.InnerContext.CreateSubOrchestrationInstance <TResult>(
functionName,
version,
instanceId,
input);
}
else
{
this.IncrementActionsOrThrowException();
callTask = this.InnerContext.CreateSubOrchestrationInstanceWithRetry <TResult>(
functionName,
version,
instanceId,
retryOptions.GetRetryOptions(),
input);
}
}
break;
case FunctionType.Entity:
System.Diagnostics.Debug.Assert(operation != null, "The operation parameter is required.");
System.Diagnostics.Debug.Assert(retryOptions == null, "Retries are not supported for entity calls.");
System.Diagnostics.Debug.Assert(instanceId != null, "Entity calls need to specify the target entity.");
if (this.ContextLocks != null)
{
lockToUse = EntityId.GetEntityIdFromSchedulerId(instanceId);
if (oneWay)
{
if (this.ContextLocks.Contains(lockToUse.Value))
{
throw new LockingRulesViolationException("While holding locks, cannot signal entities whose lock is held.");
}
}
else
{
if (!this.ContextLocks.Remove(lockToUse.Value))
{
throw new LockingRulesViolationException("While holding locks, cannot call entities whose lock is not held.");
}
}
}
var guid = this.NewGuid(); // deterministically replayable unique id for this request
var target = new OrchestrationInstance()
{
InstanceId = instanceId
};
operationId = guid.ToString();
operationName = operation;
var request = new RequestMessage()
{
ParentInstanceId = this.InstanceId,
Id = guid,
IsSignal = oneWay,
Operation = operation,
};
if (input != null)
{
request.SetInput(input);
}
this.SendEntityMessage(target, "op", request);
if (!oneWay)
{
callTask = this.WaitForEntityResponse <TResult>(guid, lockToUse);
}
break;
default:
throw new InvalidOperationException($"Unexpected function type '{functionType}'.");
}
string sourceFunctionId = this.FunctionName;
this.Config.TraceHelper.FunctionScheduled(
this.Config.Options.HubName,
functionName,
this.InstanceId,
reason: sourceFunctionId,
functionType: functionType,
isReplay: this.IsReplaying);
TResult output;
Exception exception = null;
if (oneWay)
{
return(default(TResult));
}
System.Diagnostics.Debug.Assert(callTask != null, "Two-way operations are asynchronous, so callTask must not be null.");
try
{
output = await callTask;
}
catch (TaskFailedException e)
{
exception = e;
string message = string.Format(
"The {0} function '{1}' failed: \"{2}\". See the function execution logs for additional details.",
functionType.ToString().ToLowerInvariant(),
functionName,
e.InnerException?.Message);
throw new FunctionFailedException(message, e.InnerException);
}
catch (SubOrchestrationFailedException e)
{
exception = e;
string message = string.Format(
"The {0} function '{1}' failed: \"{2}\". See the function execution logs for additional details.",
functionType.ToString().ToLowerInvariant(),
functionName,
e.InnerException?.Message);
throw new FunctionFailedException(message, e.InnerException);
}
catch (Exception e)
{
exception = e;
throw;
}
finally
{
if (exception != null && this.IsReplaying)
{
// If this were not a replay, then the orchestrator/activity/entity function trigger would have already
// emitted a FunctionFailed trace with the full exception details.
if (functionType == FunctionType.Entity)
{
this.Config.TraceHelper.OperationFailed(
this.Config.Options.HubName,
functionName,
this.InstanceId,
operationId,
operationName,
input: "(replayed)",
exception: "(replayed)",
duration: 0,
isReplay: true);
}
else
{
this.Config.TraceHelper.FunctionFailed(
this.Config.Options.HubName,
functionName,
this.InstanceId,
reason: $"(replayed {exception.GetType().Name})",
functionType: functionType,
isReplay: true);
}
}
}
if (this.IsReplaying)
{
// If this were not a replay, then the orchestrator/activity/entity function trigger would have already
// emitted a FunctionCompleted trace with the actual output details.
if (functionType == FunctionType.Entity)
{
this.Config.TraceHelper.OperationCompleted(
this.Config.Options.HubName,
functionName,
this.InstanceId,
operationId,
operationName,
input: "(replayed)",
output: "(replayed)",
duration: 0,
isReplay: true);
}
else
{
this.Config.TraceHelper.FunctionCompleted(
this.Config.Options.HubName,
functionName,
this.InstanceId,
output: "(replayed)",
continuedAsNew: false,
functionType: functionType,
isReplay: true);
}
}
return(output);
}
/// <inheritdoc />
Task <TResult> IDurableOrchestrationContext.CallActivityWithRetryAsync <TResult>(string functionName, RetryOptions retryOptions, object input)
{
if (retryOptions == null)
{
throw new ArgumentNullException(nameof(retryOptions));
}
return(this.CallDurableTaskFunctionAsync <TResult>(functionName, FunctionType.Activity, false, null, null, retryOptions, input));
}
/// <inheritdoc />
Task <TResult> IDurableOrchestrationContext.CallSubOrchestratorWithRetryAsync <TResult>(string functionName, RetryOptions retryOptions, string instanceId, object input)
{
if (retryOptions == null)
{
throw new ArgumentNullException(nameof(retryOptions));
}
return(this.CallDurableTaskFunctionAsync <TResult>(functionName, FunctionType.Orchestrator, false, instanceId, null, retryOptions, input, null));
}
/// <summary>
/// Schedules an activity function named <paramref name="functionName"/> for execution with retry options.
/// </summary>
/// <param name="context">The context object.</param>
/// <param name="functionName">The name of the activity function to call.</param>
/// <param name="retryOptions">The retry option for the activity function.</param>
/// <param name="input">The JSON-serializeable input to pass to the activity function.</param>
/// <returns>A durable task that completes when the called activity function completes or fails.</returns>
/// <exception cref="ArgumentNullException">
/// The retry option object is null.
/// </exception>
/// <exception cref="ArgumentException">
/// The specified function does not exist, is disabled, or is not an orchestrator function.
/// </exception>
/// <exception cref="InvalidOperationException">
/// The current thread is different than the thread which started the orchestrator execution.
/// </exception>
/// <exception cref="FunctionFailedException">
/// The activity function failed with an unhandled exception.
/// </exception>
public static Task CallActivityWithRetryAsync(this IDurableOrchestrationContext context, string functionName, RetryOptions retryOptions, object input)
{
return(context.CallActivityWithRetryAsync <object>(functionName, retryOptions, input));
}
/// <summary>
/// Schedules an orchestrator function named <paramref name="functionName"/> for execution with retry options.
/// </summary>
/// <typeparam name="TResult">The return type of the scheduled orchestrator function.</typeparam>
/// <param name="context">The context object.</param>
/// <param name="functionName">The name of the orchestrator function to call.</param>
/// <param name="retryOptions">The retry option for the orchestrator function.</param>
/// <param name="input">The JSON-serializeable input to pass to the orchestrator function.</param>
/// <returns>A durable task that completes when the called orchestrator function completes or fails.</returns>
/// <exception cref="ArgumentNullException">
/// The retry option object is null.
/// </exception>
/// <exception cref="ArgumentException">
/// The specified function does not exist, is disabled, or is not an orchestrator function.
/// </exception>
/// <exception cref="InvalidOperationException">
/// The current thread is different than the thread which started the orchestrator execution.
/// </exception>
/// <exception cref="FunctionFailedException">
/// The activity function failed with an unhandled exception.
/// </exception>
public static Task <TResult> CallSubOrchestratorWithRetryAsync <TResult>(this IDurableOrchestrationContext context, string functionName, RetryOptions retryOptions, object input)
{
return(context.CallSubOrchestratorWithRetryAsync <TResult>(functionName, retryOptions, null, input));
}
async Task <TResult> CallSubOrchestratorWithRetryAsync <TResult>(string functionName, RetryOptions retryOptions, string instanceId, object input, DateTime firstAttempt, int attempt)
{
try
{
return(await CallSubOrchestratorAsync <TResult>(functionName, instanceId, input));
}
catch (Exception ex)
{
var nextDelay = ComputeNextDelay(attempt, firstAttempt, ex, retryOptions);
if (!nextDelay.HasValue)
{
throw;
}
History.Add(new GenericEvent(History.Count, $"Delaying {nextDelay.Value.TotalSeconds:0.###} seconds before retry attempt {attempt} for {functionName}"));
if (nextDelay.Value > TimeSpan.Zero)
{
await CreateTimer(CurrentUtcDateTime.Add(nextDelay.Value), (object)null, CancellationToken.None);
}
return(await CallActivityWithRetryAsync <TResult>(functionName, retryOptions, input, firstAttempt, attempt + 1));
}
}
public Task <TResult> CallSubOrchestratorWithRetryAsync <TResult>(string functionName, RetryOptions retryOptions, string instanceId,
object input)
{
var firstAttempt = CurrentUtcDateTime;
return(CallSubOrchestratorWithRetryAsync <TResult>(functionName, retryOptions, instanceId, input, firstAttempt, 1));
}
public Task <TResult> CallSubOrchestratorWithRetryAsync <TResult>(string functionName, RetryOptions retryOptions, object input) => CallSubOrchestratorWithRetryAsync <TResult>(functionName, retryOptions, null, input);
public Task CallSubOrchestratorWithRetryAsync(
string functionName,
RetryOptions retryOptions,
string instanceId,
object input) => CallSubOrchestratorWithRetryAsync <object>(functionName, retryOptions, instanceId, input);
