public async Task RunAsync(FabricClient fc, FabricTestAction action, ActionStateBase actionState, ServiceInternalFaultInfo serviceInternalFaultInfo, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Inside RunAsync of Engine, entering state machine", actionState.OperationId);
try
{
do
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.OuterLoop).ConfigureAwait(false);
// For the non-force case we need to cleanup, so that is why there's no break statement in that case.
if (readRollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
break;
}
await this.RunStateMachineAsync(fc, action, actionState, serviceInternalFaultInfo, cancellationToken).ConfigureAwait(false);
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction)
{
actionState.ErrorCausingRollback = 0;
int pauseTime = this.random.Next(10, 60);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Pausing for {1} seconds before retrying", actionState.OperationId, pauseTime);
// Clear the rollback state so it will go forward when it resumes.
actionState.RollbackState = RollbackState.NotRollingBack;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(pauseTime), cancellationToken).ConfigureAwait(false);
}
}while (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully &&
actionState.StateProgress.Peek() != StepStateNames.Failed);
}
catch (FabricNotPrimaryException notPrimary)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary);
}
catch (FabricObjectClosedException objectClosed)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} caught exception - {1}", actionState.OperationId, e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Exiting state machine", actionState.OperationId);
}
// This method should only run one at a time (controlled by timer)
public async Task TruncateCallbackInnerAsync()
{
bool observedException = false;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter truncate callback");
try
{
long count = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.GetActionCountAsync(true),
this.partition,
"GetActionCountAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action store size is {0}", count);
if (count > this.maxStoredActionCount)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.TruncateAsync(count),
this.partition,
"TruncateCallbackInnerAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp);
observedException = true;
}
catch (FabricObjectClosedException foc)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(Guid.Empty, foc);
observedException = true;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Error: {0}", e.ToString());
observedException = true;
throw;
}
if (!observedException)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Rescheduling timer for {0} seconds", this.storedActionCleanupIntervalInSeconds);
this.truncateTimer.Change(TimeSpan.FromSeconds(this.storedActionCleanupIntervalInSeconds), Timeout.InfiniteTimeSpan);
}
}
// Launched as a task
private async Task ConsumeAndRunActionsAsync(CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter ConsumeAndRunActions");
using (SemaphoreSlim semaphore = new SemaphoreSlim(this.concurrentRequests, this.concurrentRequests))
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Waiting for semaphore, max count={0}", this.concurrentRequests);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Done waiting for semaphore");
}
catch (OperationCanceledException e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString());
break;
}
ActionStateBase actionState = null;
bool wasSuccessful = false;
try
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Trying to Dequeue");
wasSuccessful = this.queue.TryTake(out actionState);
if (!wasSuccessful)
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Queue was empty");
semaphore.Release();
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
continue;
}
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Dequeued Name={0} Key={1}", actionState.ActionType, actionState.OperationId);
}
catch (OperationCanceledException e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString());
break;
}
catch (Exception ex)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "TryDequeue failed with={0}", ex.ToString());
semaphore.Release();
continue;
}
Task t = null;
try
{
System.Fabric.FaultAnalysis.Service.Actions.FabricTestAction action = await this.ConstructActionAsync(actionState.ActionType, actionState);
t = this.engine.RunAsync(this.fabricClient, action, actionState, actionState.ServiceInternalFaultInfo, cancellationToken);
}
catch (FabricNotPrimaryException notPrimary)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary);
break;
}
catch (FabricObjectClosedException objectClosed)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed);
break;
}
if (!this.pendingTasks.TryAdd(actionState.OperationId, new ActionCompletionInfo(actionState.OperationId, t, semaphore)))
{
TestabilityTrace.TraceSource.WriteError(TraceType, "Add of key={0} failed, was it already added?", actionState.OperationId);
ReleaseAssert.Failfast("Add of key={0} failed, was it already added?", actionState.OperationId);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action type={0}, key={1} started", actionState.ActionType, actionState.OperationId);
}
Interlocked.Increment(ref this.isShuttingDown);
}
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Cancellation was requested");
return;
}
}