/// <summary>
/// Executes the given actionTask and commits the ActionResponse to forgeState on success.
/// </summary>
/// <param name="treeNodeKey">The TreeNode's key where the actions are taking place.</param>
/// <param name="treeActionKey">The TreeAction's key of the action taking place.</param>
/// <param name="treeAction">The TreeAction object that holds properties of the action.</param>
/// <param name="actionDefinition">The object holding definitions for the action to execute.</param>
/// <param name="actionTimeoutTask">The delay task tied to the action timeout.</param>
/// <param name="token">The cancellation token.</param>
/// <exception cref="ActionTimeoutException">If the action-level timeout was hit.</exception>
/// <exception cref="OperationCanceledException">If the cancellation token was triggered.</exception>
/// <returns>
/// RanToCompletion if the action was completed successfully.
/// Exceptions are thrown on timeout, cancellation, or retriable failures.
/// </returns>
internal async Task ExecuteAction(
string treeNodeKey,
string treeActionKey,
TreeAction treeAction,
ActionDefinition actionDefinition,
Task actionTimeoutTask,
CancellationToken token)
{
// Set up a linked cancellation token to trigger on timeout if ContinuationOnTimeout is set.
// This ensures the runActionTask gets canceled when Forge timeout is hit.
CancellationTokenSource actionCts = CancellationTokenSource.CreateLinkedTokenSource(token);
token = treeAction.ContinuationOnTimeout ? actionCts.Token : token;
// Evaluate the dynamic properties that are used by the actionTask.
ActionContext actionContext = new ActionContext(
this.Parameters.SessionId,
treeNodeKey,
treeActionKey,
treeAction.Action,
await this.EvaluateDynamicProperty(treeAction.Input, actionDefinition.InputType).ConfigureAwait(false),
await this.EvaluateDynamicProperty(treeAction.Properties, null).ConfigureAwait(false),
this.Parameters.UserContext,
token,
this.Parameters.ForgeState
);
// Instantiate the BaseAction-derived ActionType class and invoke the RunAction method on it.
var actionObject = Activator.CreateInstance(actionDefinition.ActionType);
MethodInfo method = typeof(BaseAction).GetMethod("RunAction");
Task <ActionResponse> runActionTask = (Task <ActionResponse>)method.Invoke(actionObject, new object[] { actionContext });
// Await for the first completed task between our runActionTask and the timeout task.
// This allows us to continue without awaiting the runActionTask upon timeout.
var completedTask = await Task.WhenAny(runActionTask, actionTimeoutTask).ConfigureAwait(false);
if (completedTask == actionTimeoutTask)
{
// Throw on cancellation requested if that's the reason the timeout task completed.
token.ThrowIfCancellationRequested();
// If the timeout is hit and the ContinuationOnTimeout flag is set, commit a new ActionResponse
// with the status set to TimeoutOnAction and return.
if (treeAction.ContinuationOnTimeout)
{
// Trigger linked cancellation token before continuing to ensure the runActionTask gets cancelled.
actionCts.Cancel();
ActionResponse timeoutResponse = new ActionResponse
{
Status = "TimeoutOnAction"
};
await this.CommitActionResponse(treeActionKey, timeoutResponse).ConfigureAwait(false);
return;
}
// ActionTimeout has been hit. Throw special exception to indicate this.
throw new ActionTimeoutException(string.Format(
"ActionTimeoutTask timed out before Action could complete. TreeNodeKey: {0}, TreeActionKey: {1}, ActionName: {2}.",
treeNodeKey,
treeActionKey,
treeAction.Action));
}
else
{
// Handle the completed runActionTask.
if (runActionTask.Status == TaskStatus.RanToCompletion)
{
await this.CommitActionResponse(treeActionKey, await runActionTask).ConfigureAwait(false);
}
// Await the completed task to propagate any exceptions.
// Exceptions are thrown here if the action hit a timeout, was cancelled, or failed.
await runActionTask;
}
}
/// <summary>
/// Executes the given action. Attempts retries according to the retry policy and timeout.
/// Returns without throwing exception if the action was completed successfully.
/// </summary>
/// <param name="treeNodeKey">The TreeNode's key where the actions are taking place.</param>
/// <param name="treeActionKey">The TreeAction's key of the action taking place.</param>
/// <param name="treeAction">The TreeAction object that holds properties of the action.</param>
/// <param name="actionDefinition">The object holding definitions for the action to execute.</param>
/// <param name="token">The cancellation token.</param>
/// <exception cref="ActionTimeoutException">If the action-level timeout was hit.</exception>
/// <exception cref="OperationCanceledException">If the cancellation token was triggered.</exception>
internal async Task ExecuteActionWithRetry(
string treeNodeKey,
string treeActionKey,
TreeAction treeAction,
ActionDefinition actionDefinition,
CancellationToken token)
{
// Initialize values. Default infinite timeout. Default RetryPolicyType.None.
int retryCount = 0;
Exception innerException = null;
Stopwatch stopwatch = new Stopwatch();
int actionTimeout = (int)await this.EvaluateDynamicProperty(treeAction.Timeout ?? -1, typeof(int)).ConfigureAwait(false);
RetryPolicyType retryPolicyType = treeAction.RetryPolicy != null ? treeAction.RetryPolicy.Type : RetryPolicyType.None;
TimeSpan waitTime = treeAction.RetryPolicy != null?TimeSpan.FromMilliseconds(treeAction.RetryPolicy.MinBackoffMs) : new TimeSpan();
// Kick off timers.
Task actionTimeoutTask = Task.Delay(actionTimeout, token);
stopwatch.Start();
// Attmpt to ExecuteAction based on RetryPolicy and Timeout.
// Throw on non-retriable exceptions.
while (actionTimeout == -1 || stopwatch.ElapsedMilliseconds < actionTimeout)
{
token.ThrowIfCancellationRequested();
try
{
await this.ExecuteAction(treeNodeKey, treeActionKey, treeAction, actionDefinition, actionTimeoutTask, token).ConfigureAwait(false);
return; // success!
}
catch (OperationCanceledException)
{
throw; // non-retriable exception
}
catch (ActionTimeoutException)
{
throw; // non-retriable exception
}
catch (EvaluateDynamicPropertyException)
{
throw; // non-retriable exception
}
catch (Exception e)
{
// Cache exception as innerException in case we need to throw ActionTimeoutException.
innerException = e;
// Hit retriable exception. Retry according to RetryPolicy.
// When retries are exhausted, throw ActionTimeoutException with Exception e as the innerException.
switch (retryPolicyType)
{
case RetryPolicyType.FixedInterval:
{
// FixedInterval retries every MinBackoffMs until the timeout.
// Ex) 200ms, 200ms, 200ms...
waitTime = TimeSpan.FromMilliseconds(treeAction.RetryPolicy.MinBackoffMs);
break;
}
case RetryPolicyType.ExponentialBackoff:
{
// ExponentialBackoff retries every Math.Min(MinBackoffMs * 2^(retryCount), MaxBackoffMs) until the timeout.
// Ex) 100ms, 200ms, 400ms...
waitTime = TimeSpan.FromMilliseconds(Math.Min(treeAction.RetryPolicy.MaxBackoffMs, waitTime.TotalMilliseconds * 2));
break;
}
case RetryPolicyType.None:
default:
{
// No retries. Break out below to throw non-retriable exception.
break;
}
}
}
// Break out if no retry policy set.
if (retryPolicyType == RetryPolicyType.None)
{
// If the retries have exhausted and the ContinuationOnRetryExhaustion flag is set, commit a new ActionResponse
// with the status set to RetryExhaustedOnAction and return.
if (treeAction.ContinuationOnRetryExhaustion)
{
ActionResponse timeoutResponse = new ActionResponse
{
Status = "RetryExhaustedOnAction"
};
await this.CommitActionResponse(treeActionKey, timeoutResponse).ConfigureAwait(false);
return;
}
break;
}
// Break out early if we would hit timeout before next retry.
if (actionTimeout != -1 && stopwatch.ElapsedMilliseconds + waitTime.TotalMilliseconds >= actionTimeout)
{
// If the timeout is hit and the ContinuationOnTimeout flag is set, commit a new ActionResponse
// with the status set to TimeoutOnAction and return.
if (treeAction.ContinuationOnTimeout)
{
ActionResponse timeoutResponse = new ActionResponse
{
Status = "TimeoutOnAction"
};
await this.CommitActionResponse(treeActionKey, timeoutResponse).ConfigureAwait(false);
return;
}
break;
}
token.ThrowIfCancellationRequested();
await Task.Delay(waitTime, token).ConfigureAwait(false);
retryCount++;
}
// Retries are exhausted. Throw ActionTimeoutException with executeAction exception as innerException.
throw new ActionTimeoutException(
string.Format(
"Action did not complete successfully. TreeNodeKey: {0}, TreeActionKey: {1}, ActionName: {2}, RetryCount: {3}, RetryPolicy: {4}",
treeNodeKey,
treeActionKey,
treeAction.Action,
retryCount,
retryPolicyType),
innerException);
}