public async Task <long> GetActionCountAsync(bool excludeRunningActions)
{
long count = 0;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}, operationId.GetHashCode={2}, tx.TranslationId={3}, tx.GetHashCode={4}", "GetActionCountAsync", Guid.Empty, 0, tx.TransactionId, tx.GetHashCode());
if (!excludeRunningActions)
{
count = await this.actionTable.GetCountAsync(tx).ConfigureAwait(false);
}
count += await this.historyTable.GetCountAsync(tx).ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}, operationId.GetHashCode={2}", "GetActionCountAsync", Guid.Empty, 0);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteError(TraceType, "GetActionCountAsync: {0}", e.ToString());
throw;
}
return(count);
}
public async Task RunAsync(FabricClient fc, FabricTestAction action, ActionStateBase actionState, ServiceInternalFaultInfo serviceInternalFaultInfo, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Inside RunAsync of Engine, entering state machine", actionState.OperationId);
try
{
do
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.OuterLoop).ConfigureAwait(false);
// For the non-force case we need to cleanup, so that is why there's no break statement in that case.
if (readRollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
break;
}
await this.RunStateMachineAsync(fc, action, actionState, serviceInternalFaultInfo, cancellationToken).ConfigureAwait(false);
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction)
{
actionState.ErrorCausingRollback = 0;
int pauseTime = this.random.Next(10, 60);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Pausing for {1} seconds before retrying", actionState.OperationId, pauseTime);
// Clear the rollback state so it will go forward when it resumes.
actionState.RollbackState = RollbackState.NotRollingBack;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(pauseTime), cancellationToken).ConfigureAwait(false);
}
}while (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully &&
actionState.StateProgress.Peek() != StepStateNames.Failed);
}
catch (FabricNotPrimaryException notPrimary)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary);
}
catch (FabricObjectClosedException objectClosed)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} caught exception - {1}", actionState.OperationId, e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Exiting state machine", actionState.OperationId);
}
// This method should only run one at a time (controlled by timer)
public async Task TruncateCallbackInnerAsync()
{
bool observedException = false;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter truncate callback");
try
{
long count = await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.GetActionCountAsync(true),
this.partition,
"GetActionCountAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action store size is {0}", count);
if (count > this.maxStoredActionCount)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
Guid.Empty,
() => this.TruncateAsync(count),
this.partition,
"TruncateCallbackInnerAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(Guid.Empty, fnp);
observedException = true;
}
catch (FabricObjectClosedException foc)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(Guid.Empty, foc);
observedException = true;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Error: {0}", e.ToString());
observedException = true;
throw;
}
if (!observedException)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Rescheduling timer for {0} seconds", this.storedActionCleanupIntervalInSeconds);
this.truncateTimer.Change(TimeSpan.FromSeconds(this.storedActionCleanupIntervalInSeconds), Timeout.InfiniteTimeSpan);
}
}
public async Task <ActionStateBase> FindByOperationIdAsync(Guid operationId, CancellationToken cancellationToken)
{
ActionStateBase state = null;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, actiontable has value", operationId);
state = this.ReadData(data.Value);
}
else
{
data = await this.historyTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, history table has value", operationId);
state = this.ReadData(data.Value);
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error);
}
}
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "FindOperationId: {0}", e.ToString());
throw;
}
return(state);
}
// Called to initialize state when a new message is received.
private async Task InitializeNewActionInnerAsync(ActionStateBase actionState)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter InitializeNewActionAsync");
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId);
bool keyAlreadyExists = await this.actionTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (!keyAlreadyExists)
{
keyAlreadyExists = await this.historyTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
}
if (keyAlreadyExists)
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' already exists", actionState.OperationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TEST_COMMAND_OPERATION_ID_ALREADY_EXISTS, error);
}
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Could not add new action: {0}", e.ToString());
throw;
}
await this.PersistAsync(actionState, true).ConfigureAwait(false);
}
public async Task CancelTestCommandAsync(Guid operationId, bool force)
{
ActionStateBase state = null;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "CancelTestCommandAsync", operationId);
var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
state = this.ReadData(data.Value);
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "CancelActionAsync: operationId={0}, force={1}", operationId, force);
if (force)
{
if (state.RollbackState != RollbackState.RollingBackDueToUserCancel)
{
string error = string.Format(CultureInfo.InvariantCulture, "CancelActionAsync: operationId={0}, force is not allowed for something in {1}", operationId, state.RollbackState.ToString());
TestabilityTrace.TraceSource.WriteError(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_INVALID_TEST_COMMAND_STATE, error);
}
else
{
state.RollbackState = RollbackState.RollingBackForce;
}
}
else
{
// if it's in force, don't allow it back down to graceful
if (state.RollbackState == RollbackState.RollingBackForce)
{
string error = string.Format(
CultureInfo.InvariantCulture,
"CancelActionAsync: operationId={0}, CancelTestCommandAsync has already been called with force==true. Switching to force==false is not valid.",
operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_INVALID_TEST_COMMAND_STATE, error);
}
state.RollbackState = RollbackState.RollingBackDueToUserCancel;
}
byte[] stateToPersist = state.ToBytes();
await this.actionTable.AddOrUpdateAsync(tx, state.OperationId, stateToPersist, (k, v) => stateToPersist, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "CancelActionAsync: operationId={0}, force={1} committed", operationId, force);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "CancelTestCommandAsync", operationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteError(TraceType, "CancelActionAsync: {0}", e.ToString());
throw;
}
}
// Launched as a task
private async Task ConsumeAndRunActionsAsync(CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter ConsumeAndRunActions");
using (SemaphoreSlim semaphore = new SemaphoreSlim(this.concurrentRequests, this.concurrentRequests))
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Waiting for semaphore, max count={0}", this.concurrentRequests);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteNoise(TraceType, "DEBUG Done waiting for semaphore");
}
catch (OperationCanceledException e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString());
break;
}
ActionStateBase actionState = null;
bool wasSuccessful = false;
try
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Trying to Dequeue");
wasSuccessful = this.queue.TryTake(out actionState);
if (!wasSuccessful)
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Queue was empty");
semaphore.Release();
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
continue;
}
TestabilityTrace.TraceSource.WriteNoise(TraceType, "Dequeued Name={0} Key={1}", actionState.ActionType, actionState.OperationId);
}
catch (OperationCanceledException e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, e.ToString());
break;
}
catch (Exception ex)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "TryDequeue failed with={0}", ex.ToString());
semaphore.Release();
continue;
}
Task t = null;
try
{
System.Fabric.FaultAnalysis.Service.Actions.FabricTestAction action = await this.ConstructActionAsync(actionState.ActionType, actionState);
t = this.engine.RunAsync(this.fabricClient, action, actionState, actionState.ServiceInternalFaultInfo, cancellationToken);
}
catch (FabricNotPrimaryException notPrimary)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary);
break;
}
catch (FabricObjectClosedException objectClosed)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed);
break;
}
if (!this.pendingTasks.TryAdd(actionState.OperationId, new ActionCompletionInfo(actionState.OperationId, t, semaphore)))
{
TestabilityTrace.TraceSource.WriteError(TraceType, "Add of key={0} failed, was it already added?", actionState.OperationId);
ReleaseAssert.Failfast("Add of key={0} failed, was it already added?", actionState.OperationId);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Action type={0}, key={1} started", actionState.ActionType, actionState.OperationId);
}
Interlocked.Increment(ref this.isShuttingDown);
}
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Cancellation was requested");
return;
}
}