private async Task CleanupStepAsync( FabricClient fabricClient, FabricTestAction action, ActionStateBase actionState, CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo) { StepStateNames state = actionState.StateProgress.Peek(); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Cleaning up state={0}, name={1}, key={2}", state, actionState.ActionType, actionState.OperationId); StepBase actionUnit = null; actionUnit = action.GetStep(fabricClient, actionState, state, cancellationToken); TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Cleaning up {1}", actionState.OperationId, actionUnit.StepName); try { while (true) { cancellationToken.ThrowIfCancellationRequested(); RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.InnerCleanupLoop).ConfigureAwait(false); if (readRollbackState == RollbackState.RollingBackDueToUserCancel) { // Do nothing, already rolling back TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner cleanup loop read RollingBackDueToUserCancel", actionState.OperationId); } else if (readRollbackState == RollbackState.RollingBackForce) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Inner cleanup loop read RollingBackForce", actionState.OperationId); break; } try { await actionUnit.CleanupAsync(cancellationToken).ConfigureAwait(false); actionState.StateProgress.Pop(); break; } catch (Exception cleanupException) { TestabilityTrace.TraceSource.WriteWarning( TraceType, "{0} - Cleanup of action type={1}, failed with {2}, retrying", actionState.OperationId, actionState.ActionType, cleanupException); } await Task.Delay(TimeSpan.FromSeconds(this.commandStepRetryBackoffInSeconds), cancellationToken).ConfigureAwait(false); } } catch (Exception e) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - CleanupStepAsync, error: {1}", actionState.OperationId, e.ToString()); throw; } }
private async Task RunStepAsync( FabricClient fabricClient, FabricTestAction action, ActionStateBase actionState, CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo) { StepStateNames state = actionState.StateProgress.Peek(); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Running state={0}, name={1}, key={2}", state, actionState.ActionType, actionState.OperationId); StepBase actionUnit = null; actionUnit = action.GetStep(fabricClient, actionState, state, cancellationToken); TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Running {1}", actionState.OperationId, actionUnit.StepName); try { while (true) { cancellationToken.ThrowIfCancellationRequested(); RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.InnerForwardLoop).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - readRollbackState={1}", actionState.OperationId, readRollbackState); // If RetryStepWithoutRollingbackOnFailure == true, then don't allow graceful user cancel if (!actionState.RetryStepWithoutRollingBackOnFailure && (readRollbackState == RollbackState.RollingBackDueToUserCancel)) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - read RollingBackDueToUserCancel breaking from Run loop inside RunStepAsync()", actionState.OperationId); break; } // RollingBackForce always stops execution if (readRollbackState == RollbackState.RollingBackForce) { break; } Exception runException = null; try { TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0}, {1} - calling Step.Run()", actionState.OperationId, actionState.ActionType); ActionStateBase newContext = await actionUnit.RunAsync(cancellationToken, serviceInternalFaultInfo).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0}, {1} - calling break after run", actionState.OperationId, actionState.ActionType); break; } catch (Exception runExceptionTemp) { runException = runExceptionTemp; TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0}, {1} - runException {2}", actionState.OperationId, actionState.ActionType, runException); if (actionState.RetryStepWithoutRollingBackOnFailure) { // trace and loop. Should have /backoff/? TestabilityTrace.TraceSource.WriteWarning( TraceType, "{0}, {1} has RetryStepWithoutRollingbackOnFailure set to true, retrying step name='{2}'. Caught exception: {3}", actionState.OperationId, actionState.ActionType, actionUnit.StepName, runException); this.ProcessRetryStepExceptions(actionState.OperationId, runException); } else { throw; } } if (runException != null) { await Task.Delay(TimeSpan.FromSeconds(this.commandStepRetryBackoffInSeconds), cancellationToken).ConfigureAwait(false); } } } catch (Exception e) { TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - RunState, error: {1}", actionState.OperationId, e.ToString()); throw; } }