private async Task CleanupStepAsync(
FabricClient fabricClient,
FabricTestAction action,
ActionStateBase actionState,
CancellationToken cancellationToken,
ServiceInternalFaultInfo serviceInternalFaultInfo)
{
StepStateNames state = actionState.StateProgress.Peek();
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Cleaning up state={0}, name={1}, key={2}", state, actionState.ActionType, actionState.OperationId);
StepBase actionUnit = null;
actionUnit = action.GetStep(fabricClient, actionState, state, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Cleaning up {1}", actionState.OperationId, actionUnit.StepName);
try
{
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.InnerCleanupLoop).ConfigureAwait(false);
if (readRollbackState == RollbackState.RollingBackDueToUserCancel)
{
// Do nothing, already rolling back
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner cleanup loop read RollingBackDueToUserCancel", actionState.OperationId);
}
else if (readRollbackState == RollbackState.RollingBackForce)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner cleanup loop read RollingBackForce", actionState.OperationId);
break;
}
try
{
await actionUnit.CleanupAsync(cancellationToken).ConfigureAwait(false);
actionState.StateProgress.Pop();
break;
}
catch (Exception cleanupException)
{
TestabilityTrace.TraceSource.WriteWarning(
TraceType,
"{0} - Cleanup of action type={1}, failed with {2}, retrying",
actionState.OperationId,
actionState.ActionType,
cleanupException);
}
await Task.Delay(TimeSpan.FromSeconds(this.commandStepRetryBackoffInSeconds), cancellationToken).ConfigureAwait(false);
}
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - CleanupStepAsync, error: {1}", actionState.OperationId, e.ToString());
throw;
}
}
private async Task RunStepAsync(
FabricClient fabricClient,
FabricTestAction action,
ActionStateBase actionState,
CancellationToken cancellationToken,
ServiceInternalFaultInfo serviceInternalFaultInfo)
{
StepStateNames state = actionState.StateProgress.Peek();
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Running state={0}, name={1}, key={2}", state, actionState.ActionType, actionState.OperationId);
StepBase actionUnit = null;
actionUnit = action.GetStep(fabricClient, actionState, state, cancellationToken);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Running {1}", actionState.OperationId, actionUnit.StepName);
try
{
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.InnerForwardLoop).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - readRollbackState={1}", actionState.OperationId, readRollbackState);
// If RetryStepWithoutRollingbackOnFailure == true, then don't allow graceful user cancel
if (!actionState.RetryStepWithoutRollingBackOnFailure && (readRollbackState == RollbackState.RollingBackDueToUserCancel))
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - read RollingBackDueToUserCancel breaking from Run loop inside RunStepAsync()", actionState.OperationId);
break;
}
// RollingBackForce always stops execution
if (readRollbackState == RollbackState.RollingBackForce)
{
break;
}
Exception runException = null;
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0}, {1} - calling Step.Run()", actionState.OperationId, actionState.ActionType);
ActionStateBase newContext = await actionUnit.RunAsync(cancellationToken, serviceInternalFaultInfo).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0}, {1} - calling break after run", actionState.OperationId, actionState.ActionType);
break;
}
catch (Exception runExceptionTemp)
{
runException = runExceptionTemp;
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0}, {1} - runException {2}", actionState.OperationId, actionState.ActionType, runException);
if (actionState.RetryStepWithoutRollingBackOnFailure)
{
// trace and loop. Should have /backoff/?
TestabilityTrace.TraceSource.WriteWarning(
TraceType,
"{0}, {1} has RetryStepWithoutRollingbackOnFailure set to true, retrying step name='{2}'. Caught exception: {3}",
actionState.OperationId,
actionState.ActionType,
actionUnit.StepName,
runException);
this.ProcessRetryStepExceptions(actionState.OperationId, runException);
}
else
{
throw;
}
}
if (runException != null)
{
await Task.Delay(TimeSpan.FromSeconds(this.commandStepRetryBackoffInSeconds), cancellationToken).ConfigureAwait(false);
}
}
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - RunState, error: {1}", actionState.OperationId, e.ToString());
throw;
}
}
