public static StepBase GetStep(
StepStateNames stateName,
FabricClient fabricClient,
ActionStateBase actionState,
InvokeDataLossAction action,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
StepBase step = null;
InvokeDataLossState invokeDataLossState = Convert(actionState);
StepStateNames prevContext = actionState.StateProgress.Peek();
if (stateName == StepStateNames.LookingUpState)
{
step = new DataLossStepsFactory.LookingUpState(fabricClient, invokeDataLossState, requestTimeout, operationTimeout, action.PartitionSelector);
}
else if (stateName == StepStateNames.PerformingActions)
{
step = new DataLossStepsFactory.PerformingActions(fabricClient, invokeDataLossState, requestTimeout, operationTimeout, action.PartitionSelector, action.DataLossCheckWaitDurationInSeconds, action.DataLossCheckPollIntervalInSeconds, action.ReplicaDropWaitDurationInSeconds);
}
else if (stateName == StepStateNames.CompletedSuccessfully)
{
// done - but then this method should not have been called
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "{0} - GetStep() should not have been called when the state name is CompletedSuccessfully"), actionState.OperationId);
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "{0} - Unexpected state name={1}", actionState.OperationId, stateName));
}
return(step);
}
public static StepBase GetStep(
StepStateNames stateName,
FabricClient fabricClient,
ActionStateBase actionState,
InvokeQuorumLossAction action,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
StepBase step = null;
InvokeQuorumLossState state = Convert(actionState);
if (stateName == StepStateNames.LookingUpState)
{
step = new QuorumLossStepsFactory.LookingUpState(fabricClient, state, requestTimeout, operationTimeout, action.PartitionSelector, action.QuorumLossMode);
}
else if (stateName == StepStateNames.PerformingActions)
{
step = new QuorumLossStepsFactory.PerformingActions(fabricClient, state, requestTimeout, operationTimeout, action.PartitionSelector);
}
else if (stateName == StepStateNames.CompletedSuccessfully)
{
// done - but then this method should not have been called
TestabilityTrace.TraceSource.WriteError(StepBase.TraceType, "{0} - GetStep() should not have been called when the state nme is CompletedSuccessfully", actionState.OperationId);
ReleaseAssert.Failfast("GetStep() should not have been called when the state nme is CompletedSuccessfully");
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "{0} - Unexpected state name={1}", actionState.OperationId, stateName);
TestabilityTrace.TraceSource.WriteError(StepBase.TraceType, "{0}", error);
ReleaseAssert.Failfast(error);
}
return(step);
}
public static StepBase GetStep(
StepStateNames stateName,
FabricClient fabricClient,
ActionStateBase actionState,
RestartPartitionAction action,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
StepBase step = null;
RestartPartitionState restartPartitionState = Convert(actionState);
if (stateName == StepStateNames.LookingUpState)
{
step = new LookingUpStep(fabricClient, restartPartitionState, requestTimeout, operationTimeout, action.PartitionSelector, action.RestartPartitionMode);
}
else if (stateName == StepStateNames.PerformingActions)
{
step = new RestartingSelectedReplicas(fabricClient, restartPartitionState, requestTimeout, operationTimeout, action.RestartPartitionMode);
}
else if (stateName == StepStateNames.CompletedSuccessfully)
{
// done - but then this method should not have been called
ReleaseAssert.Failfast("GetStep() should not have been called when the state nme is CompletedSuccessfully");
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "Unexpected state name={0}", stateName));
}
return(step);
}
public static StepBase GetStep(
StepStateNames stateName,
FabricClient fabricClient,
ActionStateBase actionState,
StartNodeFromFASAction action,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
StepBase step = null;
NodeCommandState startNodeState = Convert(actionState);
StepStateNames prevContext = actionState.StateProgress.Peek();
if (stateName == StepStateNames.LookingUpState)
{
step = new StartNodeStepsFactory.LookingUpState(fabricClient, startNodeState, action, requestTimeout, operationTimeout);
}
else if (stateName == StepStateNames.PerformingActions)
{
step = new StartNodeStepsFactory.PerformingActions(fabricClient, startNodeState, action, requestTimeout, operationTimeout);
}
else if (stateName == StepStateNames.CompletedSuccessfully)
{
// done - but then this method should not have been called
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "{0} - GetStep() should not have been called when the state name is CompletedSuccessfully"), actionState.OperationId);
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "{0} - Unexpected state name={1}", actionState.OperationId, stateName));
}
return(step);
}
public static async Task RemoveUnreliableTransportAsync(ActionStateBase state, FabricClient fabricClient, TimeSpan requestTimeout, TimeSpan operationTimeout, CancellationToken cancellationToken)
{
RestartPartitionState restartPartitionState = Convert(state);
if (restartPartitionState.Info.UnreliableTransportInfo != null)
{
string behaviorName = restartPartitionState.Info.UnreliableTransportInfo.First().Item2;
string failoverManagerPrimaryNodeName = restartPartitionState.Info.NodeName;
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - Enter Cleanup for '{1}', fmPrimaryNodeName={2}, behavior name={3}", state.OperationId, behaviorName, failoverManagerPrimaryNodeName, behaviorName);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => fabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
failoverManagerPrimaryNodeName,
behaviorName,
requestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
operationTimeout,
cancellationToken).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
}
}
private static TestCommandType ConvertType(ActionStateBase action)
{
ActionType type = action.ActionType;
switch (type)
{
case ActionType.InvokeDataLoss:
return(TestCommandType.PartitionDataLoss);
case ActionType.InvokeQuorumLoss:
return(TestCommandType.PartitionQuorumLoss);
case ActionType.RestartPartition:
return(TestCommandType.PartitionRestart);
case ActionType.StartNode:
return(TestCommandType.NodeTransition);
case ActionType.StopNode:
return(TestCommandType.NodeTransition);
default:
return(TestCommandType.Invalid);
}
}
public Task InitializeNewActionAsync(ActionStateBase actionState, TimeSpan timeout)
{
return(FaultAnalysisServiceUtility.RetryOnTimeout(
actionState.OperationId,
() => this.InitializeNewActionInnerAsync(actionState),
timeout,
this.cancellationToken));
}
public void WaitForState(Guid operationId, StepStateNames targetState, RollbackState targetRollbackState = RollbackState.NotRollingBack)
{
StepStateNames currentState = StepStateNames.IntentSaved;
RollbackState rollbackState = RollbackState.NotRollingBack;
TimeoutHelper timeoutHelper = new TimeoutHelper(TimeSpan.FromMinutes(3));
do
{
bool callSucceeded = false;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} waiting for state '{1}'", operationId, targetState);
ActionStateBase state = null;
try
{
state = this.messageProcessor.ProcessGetProgressAsync(operationId, FASConstants.DefaultTestTimeout, CancellationToken.None).GetAwaiter().GetResult();
callSucceeded = true;
}
catch (OperationCanceledException oce)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, oce.ToString());
}
catch (TransactionFaultedException tfe)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, tfe.ToString());
}
catch (FabricTransientException fte)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, fte.ToString());
}
if (callSucceeded)
{
currentState = state.StateProgress.Peek();
rollbackState = state.RollbackState;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} current state='{1}', rollbackState='{2}'", operationId, currentState, rollbackState);
}
Task.Delay(TimeSpan.FromSeconds(5)).Wait();
}while ((currentState != targetState || rollbackState != targetRollbackState) &&
timeoutHelper.GetRemainingTime() > TimeSpan.Zero);
if ((currentState != targetState) ||
(rollbackState != targetRollbackState))
{
string error = string.Format(
CultureInfo.InvariantCulture,
"{0} Did not reach expected situation: current state='{1}', expected state='{2}', current rollbackState='{3}', expected rollbackState='{4}'",
operationId,
currentState,
targetState,
rollbackState,
targetRollbackState);
TestabilityTrace.TraceSource.WriteError(TraceType, error);
ReleaseAssert.Failfast(error);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} reached expected state '{1}' and rollbackState '{2}'", operationId, targetState, targetRollbackState);
}
public async Task UpdateActionStateAsync(ActionStateBase actionState, bool wasCalledFromCancelApiPath)
{
await FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
actionState.OperationId,
() => this.UpdateActionStateInnerAsync(actionState, wasCalledFromCancelApiPath),
this.partition,
"UpdateActionStateAsync",
ActionStore.MaxRetries,
this.cancellationToken).ConfigureAwait(false);
}
private Task <RollbackState> CheckUserCancellationAndUpdateIfNeededAsync(ActionStateBase actionState, CancellationToken cancellationToken, string location)
{
return(FaultAnalysisServiceUtility.RunAndReportFaultOnRepeatedFailure(
actionState.OperationId,
() => this.CheckUserCancellationAndUpdateIfNeededInnerAsync(actionState, cancellationToken, location),
this.partition,
"ReliableFaultsEngine.CheckUserCancellationAndUpdateIfNeededAsync",
3,
cancellationToken));
}
internal async Task <RollbackState> CheckUserCancellationAndUpdateIfNeededAsync(ActionStateBase actionState, CancellationToken cancellationToken, string location)
{
ActionStateBase readActionState = null;
readActionState = await this.FindByOperationIdAsync(actionState.OperationId, cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteNoise(TraceType, "CheckUserCancellationAndUpdateIfNeededAsync() {0} location='{1}' read rollbackstate={2}", actionState.OperationId, location, readActionState.RollbackState);
return(readActionState.RollbackState);
}
public static InvokeQuorumLossState Convert(ActionStateBase actionState)
{
InvokeQuorumLossState invokeQuorumLossState = actionState as InvokeQuorumLossState;
if (invokeQuorumLossState == null)
{
throw new InvalidCastException("State object could not be converted");
}
return(invokeQuorumLossState);
}
public static NodeCommandState Convert(ActionStateBase actionState)
{
NodeCommandState startNodeState = actionState as NodeCommandState;
if (startNodeState == null)
{
throw new InvalidCastException("State object could not be converted");
}
return(startNodeState);
}
public static RestartPartitionState Convert(ActionStateBase actionState)
{
RestartPartitionState restartPartitionState = actionState as RestartPartitionState;
if (restartPartitionState == null)
{
throw new InvalidCastException("State object could not be converted");
}
return(restartPartitionState);
}
private void Enqueue(ActionStateBase actionState)
{
try
{
this.queue.Add(actionState);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Error while enqueuing request={0}, error={1}", actionState.OperationId, e.ToString());
throw;
}
}
public async Task RunAsync(FabricClient fc, FabricTestAction action, ActionStateBase actionState, ServiceInternalFaultInfo serviceInternalFaultInfo, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Inside RunAsync of Engine, entering state machine", actionState.OperationId);
try
{
do
{
cancellationToken.ThrowIfCancellationRequested();
RollbackState readRollbackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, cancellationToken, FASConstants.OuterLoop).ConfigureAwait(false);
// For the non-force case we need to cleanup, so that is why there's no break statement in that case.
if (readRollbackState == RollbackState.RollingBackForce)
{
actionState.StateProgress.Push(StepStateNames.Failed);
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
break;
}
await this.RunStateMachineAsync(fc, action, actionState, serviceInternalFaultInfo, cancellationToken).ConfigureAwait(false);
if (actionState.RollbackState == RollbackState.RollingBackAndWillRetryAction)
{
actionState.ErrorCausingRollback = 0;
int pauseTime = this.random.Next(10, 60);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Pausing for {1} seconds before retrying", actionState.OperationId, pauseTime);
// Clear the rollback state so it will go forward when it resumes.
actionState.RollbackState = RollbackState.NotRollingBack;
await this.actionStore.UpdateActionStateAsync(actionState).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(pauseTime), cancellationToken).ConfigureAwait(false);
}
}while (actionState.StateProgress.Peek() != StepStateNames.CompletedSuccessfully &&
actionState.StateProgress.Peek() != StepStateNames.Failed);
}
catch (FabricNotPrimaryException notPrimary)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, notPrimary);
}
catch (FabricObjectClosedException objectClosed)
{
FaultAnalysisServiceUtility.TraceFabricObjectClosed(actionState.OperationId, objectClosed);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} caught exception - {1}", actionState.OperationId, e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Exiting state machine", actionState.OperationId);
}
public async Task <PartitionDataLossProgress> GetInvokeDataLossProgressAsync(
Guid operationId,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
PartitionDataLossProgress progress = null;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Inside GetInvokeDataLossProgressAsync, operationId = {0}", operationId);
try
{
ActionStateBase actionState = await this.MessageProcessor.ProcessGetProgressAsync(operationId, timeout, cancellationToken);
StepStateNames stateName = actionState.StateProgress.Peek();
TestCommandProgressState state = FaultAnalysisServiceUtility.ConvertState(actionState, TraceType);
InvokeDataLossState invokeDataLossState = actionState as InvokeDataLossState;
if (invokeDataLossState == null)
{
throw new InvalidCastException("State object could not be converted");
}
StepStateNames stepState = actionState.StateProgress.Peek();
var selectedPartition = new SelectedPartition
{
ServiceName = invokeDataLossState.Info.PartitionSelector.ServiceName,
PartitionId = invokeDataLossState.Info.PartitionId
};
PartitionDataLossResult result = new PartitionDataLossResult(selectedPartition, actionState.ErrorCausingRollback);
progress = new PartitionDataLossProgress(state, result);
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"{0} - {1} progress - {2}, Exception - {3}",
operationId,
ActionType.InvokeDataLoss,
progress.Result != null ? progress.Result.SelectedPartition.ToString() : FASConstants.UnavailableMessage,
(progress.Result != null && progress.Result.Exception != null) ? progress.Result.Exception.ToString() : FASConstants.UnavailableMessage);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, e.ToString());
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
return(progress);
}
public async Task <PartitionRestartProgress> GetRestartPartitionProgressAsync(
Guid operationId,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
PartitionRestartProgress progress = null;
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "GetRestartPartitionProgressAsync calling message processor");
ActionStateBase actionState = await this.MessageProcessor.ProcessGetProgressAsync(operationId, timeout, cancellationToken);
StepStateNames stateName = actionState.StateProgress.Peek();
TestCommandProgressState state = FaultAnalysisServiceUtility.ConvertState(actionState, TraceType);
RestartPartitionState restartPartitionState = actionState as RestartPartitionState;
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"RestartPartition - serviceName={0}, partitionId={1}",
restartPartitionState.Info.PartitionSelector.ServiceName.ToString(),
restartPartitionState.Info.PartitionId);
var selectedPartition = new SelectedPartition
{
ServiceName = restartPartitionState.Info.PartitionSelector.ServiceName,
PartitionId = restartPartitionState.Info.PartitionId
};
PartitionRestartResult result = new PartitionRestartResult(selectedPartition, actionState.ErrorCausingRollback);
progress = new PartitionRestartProgress(state, result);
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"{0} - {1} progress - {2}, Exception - {3}",
operationId,
ActionType.RestartPartition,
progress.Result != null ? progress.Result.SelectedPartition.ToString() : FASConstants.UnavailableMessage,
(progress.Result != null && progress.Result.Exception != null) ? progress.Result.Exception.ToString() : FASConstants.UnavailableMessage);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, e.ToString());
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
return(progress);
}
public async Task <ActionStateBase> FindByOperationIdAsync(Guid operationId, CancellationToken cancellationToken)
{
ActionStateBase state = null;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, actiontable has value", operationId);
state = this.ReadData(data.Value);
}
else
{
data = await this.historyTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, history table has value", operationId);
state = this.ReadData(data.Value);
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error);
}
}
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "FindOperationId: {0}", e.ToString());
throw;
}
return(state);
}
public async Task UpdateActionStateInnerAsync(ActionStateBase actionState, bool wasCalledFromCancelApiPath)
{
if (!wasCalledFromCancelApiPath)
{
// This is from an engine update. In this case it needs to check to see if the user called the cancel api before writing. That call inside overwrite the flag, but does not persist
RollbackState readRollBackState = await this.CheckUserCancellationAndUpdateIfNeededAsync(actionState, CancellationToken.None, string.Empty).ConfigureAwait(false);
if (readRollBackState == RollbackState.RollingBackDueToUserCancel ||
readRollBackState == RollbackState.RollingBackForce)
{
actionState.RollbackState = readRollBackState;
}
}
await this.PersistAsync(actionState).ConfigureAwait(false);
}
// This method should only run one at a time (controlled by timer)
private async Task TruncateAsync(long totalActionCount)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter TruncateAsync, count is {0}", totalActionCount);
List <ActionStateBase> completedActionsMatchingFilter = new List <ActionStateBase>();
long numberToTruncate = totalActionCount - this.maxStoredActionCount;
using (var tx = this.stateManager.CreateTransaction())
{
var enumerable = await this.historyTable.CreateEnumerableAsync(tx).ConfigureAwait(false);
var enumerator = enumerable.GetAsyncEnumerator();
while (await enumerator.MoveNextAsync(this.cancellationToken).ConfigureAwait(false))
{
ActionStateBase actionState = this.ReadData(enumerator.Current.Value);
StepStateNames stateName = actionState.StateProgress.Peek();
if (stateName == StepStateNames.CompletedSuccessfully || stateName == StepStateNames.Failed)
{
if (DateTime.UtcNow - actionState.TimeStopped > TimeSpan.FromSeconds(this.completedActionKeepDurationInSeconds))
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "DEBUG - Adding action {0} which ended at {1}, time now is {2}", actionState.OperationId, actionState.TimeStopped, DateTime.UtcNow);
completedActionsMatchingFilter.Add(actionState);
}
}
}
if (completedActionsMatchingFilter.Count > numberToTruncate)
{
// Sort by completed date and cut 'numberToCut' of those.
completedActionsMatchingFilter.Sort((a, b) => { return(a.TimeStopped.CompareTo(b.TimeStopped)); });
int numToTruncateAsInt = (int)numberToTruncate;
completedActionsMatchingFilter.RemoveRange(numToTruncateAsInt, completedActionsMatchingFilter.Count - numToTruncateAsInt);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "TruncateAsync Removing '{0}' completed actions", completedActionsMatchingFilter.Count);
foreach (ActionStateBase a in completedActionsMatchingFilter)
{
await this.historyTable.TryRemoveAsync(tx, a.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
}
await tx.CommitAsync().ConfigureAwait(false);
}
}
private ActionStateBase ReadData(byte[] bytes)
{
ActionStateBase result = null;
using (BinaryReader br = new BinaryReader(new MemoryStream(bytes)))
{
// The first 4 bytes are the command type
ActionType a = ActionStateBase.ReadCommandType(br);
if (a == ActionType.InvokeDataLoss)
{
result = InvokeDataLossState.FromBytes(br);
}
else if (a == ActionType.InvokeQuorumLoss)
{
result = InvokeQuorumLossState.FromBytes(br);
}
else if (a == ActionType.RestartPartition)
{
result = RestartPartitionState.FromBytes(br);
}
else if (a == ActionType.TestStuck)
{
result = StuckState.FromBytes(br);
}
else if (a == ActionType.TestRetryStep)
{
result = TestRetryStepState.FromBytes(br);
}
else if (a == ActionType.StartNode)
{
result = NodeCommandState.FromBytes(br, a);
}
else if (a == ActionType.StopNode)
{
result = NodeCommandState.FromBytes(br, a);
}
}
return(result);
}
private void TestModeSerialize(ActionStateBase state)
{
if (this.isTestMode)
{
// DEBUG/////////////////////////////////////////////
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Begin test");
try
{
byte[] bytes = state.ToBytes();
ActionStateBase deserializedData = this.ReadData(bytes);
if (!state.VerifyEquals(deserializedData))
{
ReleaseAssert.Failfast("{0} data should match but does not", state.OperationId);
}
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteError(TraceType, "{0} - TestModeSerialize exception {1}", state.OperationId.ToString(), e.ToString());
throw;
}
}
}
// Called to initialize state when a new message is received.
private async Task InitializeNewActionInnerAsync(ActionStateBase actionState)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter InitializeNewActionAsync");
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId);
bool keyAlreadyExists = await this.actionTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (!keyAlreadyExists)
{
keyAlreadyExists = await this.historyTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
}
if (keyAlreadyExists)
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' already exists", actionState.OperationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TEST_COMMAND_OPERATION_ID_ALREADY_EXISTS, error);
}
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Could not add new action: {0}", e.ToString());
throw;
}
await this.PersistAsync(actionState, true).ConfigureAwait(false);
}
private async Task AddActionsMatchingFilterAsync(IReliableDictionary <Guid, byte[]> dictionary, List <ActionStateBase> selectedActions, TestCommandStateFilter stateFilter, TestCommandTypeFilter typeFilter)
{
using (var tx = this.stateManager.CreateTransaction())
{
var enumerable = await dictionary.CreateEnumerableAsync(tx).ConfigureAwait(false);
var enumerator = enumerable.GetAsyncEnumerator();
while (await enumerator.MoveNextAsync(CancellationToken.None))
{
ActionStateBase actionState = this.ReadData(enumerator.Current.Value);
StepStateNames stateName = actionState.StateProgress.Peek();
if (MatchesStateFilter(actionState, stateFilter) && MatchesTypeFilter(actionState.ActionType, typeFilter))
{
selectedActions.Add(actionState);
}
else
{
TestabilityTrace.TraceSource.WriteNoise(TraceType, "{0} - Current action does not match filter, not adding", actionState.OperationId);
}
}
}
}
public static async Task RemoveUTAsync(FabricClient fabricClient, ActionStateBase state, TimeSpan requestTimeout, TimeSpan operationTimeout, CancellationToken cancellationToken)
{
InvokeQuorumLossState invokeQuorumLossState = Convert(state);
Guid partitionId = invokeQuorumLossState.Info.PartitionId;
List <Task> tasks = new List <Task>();
if (invokeQuorumLossState.Info.UnreliableTransportInfo != null)
{
foreach (Tuple <string, string> info in invokeQuorumLossState.Info.UnreliableTransportInfo)
{
UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "StatefulServiceReopen");
behavior.AddFilterForPartitionId(partitionId);
string nodeName = info.Item1;
string behaviorName = info.Item2;
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - Cleaning up behavior={1}", state.OperationId, behaviorName);
Task task = FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => fabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync(
nodeName,
behaviorName,
requestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value,
operationTimeout,
cancellationToken);
tasks.Add(task);
}
await Task.WhenAll(tasks).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
}
}
public async Task <NodeTransitionProgress> GetNodeTransitionProgressAsync(
Guid operationId,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
NodeTransitionProgress progress = null;
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "GetNodeTransitionProgressAsync got operation id {0}", operationId);
ActionStateBase actionState = await this.MessageProcessor.ProcessGetProgressAsync(operationId, timeout, cancellationToken);
StepStateNames stateName = actionState.StateProgress.Peek();
TestCommandProgressState state = FaultAnalysisServiceUtility.ConvertState(actionState, TraceType);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - GetNodeTransitionProgressAsync reading nodecommandstate", operationId);
NodeCommandState nodeState = actionState as NodeCommandState;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} NodeCommandState is null={1}", operationId, nodeState == null);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - GetNodeTransitionProgressAsync node name={1}, node instance id={2}", operationId, nodeState.Info.NodeName, nodeState.Info.InputNodeInstanceId);
NodeResult nodeResult = new NodeResult(nodeState.Info.NodeName, nodeState.Info.InputNodeInstanceId);
NodeCommandResult result = new NodeCommandResult(nodeResult, actionState.ErrorCausingRollback);
progress = new NodeTransitionProgress(state, result);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, e.ToString());
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
return(progress);
}
// "throwException" - if this call originated from a client call, the exception should be thrown back.
private async Task PersistAsync(ActionStateBase actionState, bool throwException)
{
StepStateNames currentStateName = actionState.StateProgress.Peek();
byte[] stateToPersist = actionState.ToBytes();
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "PersistAsync", actionState.OperationId);
if (actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully ||
actionState.StateProgress.Peek() == StepStateNames.Failed)
{
await this.actionTable.TryRemoveAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"Adding {0} to history table state={1}, rollbackState={2}",
actionState.OperationId,
actionState.StateProgress.Peek(),
actionState.RollbackState.ToString());
await this.historyTable.AddOrUpdateAsync(tx, actionState.OperationId, stateToPersist, (k, v) => stateToPersist, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
}
else
{
await this.actionTable.AddOrUpdateAsync(tx, actionState.OperationId, stateToPersist, (k, v) => stateToPersist, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
}
await tx.CommitAsync().ConfigureAwait(false);
}
try
{
if (actionState.StateProgress.Peek() == StepStateNames.CompletedSuccessfully)
{
await actionState.AfterSuccessfulCompletion().ConfigureAwait(false);
}
else if (actionState.StateProgress.Peek() == StepStateNames.Failed)
{
await actionState.AfterFatalRollback().ConfigureAwait(false);
}
}
catch (Exception ex)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} afterX threw {1}, reporting fault", actionState.OperationId, ex);
this.partition.ReportFault(FaultType.Transient);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "PersistAsync", actionState.OperationId);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Error in persisting in state " + currentStateName + " " + e.ToString());
throw;
}
if (this.isTestMode)
{
this.TestModeSerialize(actionState);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} Exiting PersistAsync", actionState.OperationId);
}
private Task PersistAsync(ActionStateBase actionState)
{
return(this.PersistAsync(actionState, false));
}
private static bool MatchesStateFilter(ActionStateBase actionState, TestCommandStateFilter filter)
{
if (filter == TestCommandStateFilter.All)
{
return(true);
}
if (filter == TestCommandStateFilter.Default)
{
// future use
return(true);
}
StepStateNames stateName = actionState.StateProgress.Peek();
RollbackState rollbackState = actionState.RollbackState;
if (filter.HasFlag(TestCommandStateFilter.RollingBack))
{
if ((rollbackState == RollbackState.RollingBackAndWillFailAction ||
rollbackState == RollbackState.RollingBackForce ||
rollbackState == RollbackState.RollingBackDueToUserCancel)
&&
(stateName != StepStateNames.Failed))
{
return(true);
}
}
if (filter.HasFlag(TestCommandStateFilter.Cancelled))
{
// stateName == CompletedSuccessfully is possible if the command was in its last step, and finished the last step successfully
// before cancellation status was read. There is no point in "cancelling" since the point is to stop execution. Since it completed successfully,
// that has been achieved. However, the status should be translated here so it is not confusing to user.
if ((stateName == StepStateNames.Failed || stateName == StepStateNames.CompletedSuccessfully) &&
(rollbackState == RollbackState.RollingBackDueToUserCancel))
{
return(true);
}
}
if (filter.HasFlag(TestCommandStateFilter.ForceCancelled))
{
// See notes above in Cancelled section, which also applies here.
if ((stateName == StepStateNames.Failed || stateName == StepStateNames.CompletedSuccessfully) &&
(rollbackState == RollbackState.RollingBackForce))
{
return(true);
}
}
switch (stateName)
{
case StepStateNames.IntentSaved:
return(filter.HasFlag(TestCommandStateFilter.Running));
case StepStateNames.LookingUpState:
return(filter.HasFlag(TestCommandStateFilter.Running));
case StepStateNames.PerformingActions:
return(filter.HasFlag(TestCommandStateFilter.Running));
case StepStateNames.CompletedSuccessfully:
return(filter.HasFlag(TestCommandStateFilter.CompletedSuccessfully));
case StepStateNames.Failed:
return(filter.HasFlag(TestCommandStateFilter.Failed));
default:
return(false);
}
}
