public static StepBase GetStep(
StepStateNames stateName,
FabricClient fabricClient,
ActionStateBase actionState,
StartNodeFromFASAction action,
TimeSpan requestTimeout,
TimeSpan operationTimeout,
CancellationToken cancellationToken)
{
StepBase step = null;
NodeCommandState startNodeState = Convert(actionState);
StepStateNames prevContext = actionState.StateProgress.Peek();
if (stateName == StepStateNames.LookingUpState)
{
step = new StartNodeStepsFactory.LookingUpState(fabricClient, startNodeState, action, requestTimeout, operationTimeout);
}
else if (stateName == StepStateNames.PerformingActions)
{
step = new StartNodeStepsFactory.PerformingActions(fabricClient, startNodeState, action, requestTimeout, operationTimeout);
}
else if (stateName == StepStateNames.CompletedSuccessfully)
{
// done - but then this method should not have been called
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "{0} - GetStep() should not have been called when the state name is CompletedSuccessfully"), actionState.OperationId);
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "{0} - Unexpected state name={1}", actionState.OperationId, stateName));
}
return(step);
}
public async Task ResumePendingActionsAsync(FabricClient fc, CancellationToken cancellationToken)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Getting running actions");
IEnumerable <ActionStateBase> incompleteActions = await this.actionStore.GetRunningActionsAsync();
IEnumerable <ActionStateBase> two = incompleteActions.OrderBy(s => s.TimeReceived);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Done getting running actions");
foreach (ActionStateBase actionState in two)
{
if (actionState is NodeCommandState)
{
NodeCommandState nodeState = actionState as NodeCommandState;
nodeState.NodeSync = this.entitySynch.NodeSynchronizer;
nodeState.StoppedNodeTable = this.stoppedNodeTable;
this.entitySynch.NodeSynchronizer.Add(nodeState.Info.NodeName);
}
FabricTestAction action = await this.ConstructActionAsync(actionState.ActionType, actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Resuming action of type {1}", actionState.OperationId, actionState.ActionType);
this.Enqueue(actionState);
}
}
public static NodeCommandState Convert(ActionStateBase actionState)
{
NodeCommandState startNodeState = actionState as NodeCommandState;
if (startNodeState == null)
{
throw new InvalidCastException("State object could not be converted");
}
return(startNodeState);
}
private SuccessRetryOrFail HandleFabricException(FabricException fe, NodeCommandState state)
{
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw FabricException with ErrorCode={1}", this.State.OperationId, fe.ErrorCode);
if (fe.ErrorCode == FabricErrorCode.InstanceIdMismatch)
{
status = SuccessRetryOrFail.Fail;
}
else if (fe.ErrorCode == FabricErrorCode.NodeIsUp)
{
status = SuccessRetryOrFail.Success;
}
else if (fe.ErrorCode == FabricErrorCode.NodeHasNotStoppedYet)
{
status = SuccessRetryOrFail.RetryStep;
}
else if (fe.ErrorCode == FabricErrorCode.InvalidAddress)
{
if (state.Info.InitialQueriedNodeStatus != NodeStatus.Down && state.Info.NodeWasInitiallyInStoppedState == false)
{
// This is a (probably unlikely) case that may happen if
// 1. The request reaches the target node, which is up and is processed.
// 2. The response is dropped
// 3. The request is retried with enough delay such that the node has now transitioned from up to stopped
// So, we say that if the initial conditions were valid and we get InvalidAddress, then consider it successful.
status = SuccessRetryOrFail.Success;
}
else
{
// The preconditions passed, but the node is down now, so something out of band happened.
status = SuccessRetryOrFail.RetryStep;
}
}
else if (fe.ErrorCode == FabricErrorCode.NodeNotFound)
{
// Always fatal
string nodeNotFoundErrorMessage = string.Format(CultureInfo.InvariantCulture, "{0} - node {1} was not found", state.OperationId, state.Info.NodeName);
TestabilityTrace.TraceSource.WriteError(StepBase.TraceType, nodeNotFoundErrorMessage);
status = SuccessRetryOrFail.Fail;
}
else
{
status = SuccessRetryOrFail.RetryStep;
}
return(status);
}
private SuccessRetryOrFail HandleFabricException(FabricException fe, NodeCommandState state)
{
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
if (fe.ErrorCode == FabricErrorCode.InstanceIdMismatch)
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw InstanceIdMismatch", this.State.OperationId);
status = SuccessRetryOrFail.Fail;
}
else if (fe.ErrorCode == FabricErrorCode.InvalidAddress)
{
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - StopNode HandleFabricException InitialQueriedNodeStatus='{1}', NodeWasInitiallyInStoppedState='{2}'",
this.State.OperationId,
state.Info.InitialQueriedNodeStatus,
state.Info.NodeWasInitiallyInStoppedState);
// If the request was valid (node was up), but the response was dropped, and then the request was retried, we might now be
// sending a stop to a node that is now stopped. The request will return InvalidAddress, but this is really success.
if (state.Info.InitialQueriedNodeStatus != NodeStatus.Down && state.Info.NodeWasInitiallyInStoppedState == false)
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode api threw InvalidAddress, considering stop command successful due to preconditions", this.State.OperationId);
status = SuccessRetryOrFail.Success;
}
else
{
// Invalid address is not expected here. There may have been an out of band stop using the deprecated api. We should retry until we get something else.
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw InvalidAddress", this.State.OperationId);
status = SuccessRetryOrFail.RetryStep;
}
}
else if (fe.ErrorCode == FabricErrorCode.NodeNotFound)
{
status = SuccessRetryOrFail.Fail;
}
else
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StopNode api threw {1}", this.State.OperationId, fe);
status = SuccessRetryOrFail.RetryStep;
}
return(status);
}
private ActionStateBase ReadData(byte[] bytes)
{
ActionStateBase result = null;
using (BinaryReader br = new BinaryReader(new MemoryStream(bytes)))
{
// The first 4 bytes are the command type
ActionType a = ActionStateBase.ReadCommandType(br);
if (a == ActionType.InvokeDataLoss)
{
result = InvokeDataLossState.FromBytes(br);
}
else if (a == ActionType.InvokeQuorumLoss)
{
result = InvokeQuorumLossState.FromBytes(br);
}
else if (a == ActionType.RestartPartition)
{
result = RestartPartitionState.FromBytes(br);
}
else if (a == ActionType.TestStuck)
{
result = StuckState.FromBytes(br);
}
else if (a == ActionType.TestRetryStep)
{
result = TestRetryStepState.FromBytes(br);
}
else if (a == ActionType.StartNode)
{
result = NodeCommandState.FromBytes(br, a);
}
else if (a == ActionType.StopNode)
{
result = NodeCommandState.FromBytes(br, a);
}
}
return(result);
}
public async Task ProcessStopNodeCommandAsync(Guid operationId, string nodeName, BigInteger nodeInstanceId, int durationInSeconds, TimeSpan timeout, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - ProcessStopNodeCommandAsync, duration is {1}", operationId, durationInSeconds);
this.entitySynch.NodeSynchronizer.Add(nodeName);
NodeCommandState actionState = new NodeCommandState(ActionType.StopNode, operationId, this.entitySynch.NodeSynchronizer, serviceInternalFaultInfo, nodeName, nodeInstanceId, durationInSeconds);
try
{
// After this call finishes the intent has been persisted
await this.actionStore.InitializeNewActionAsync(actionState, timeout);
this.Enqueue(actionState);
}
catch (Exception e)
{
this.entitySynch.NodeSynchronizer.Remove(nodeName);
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Exception {1}", operationId, e);
throw;
}
}
private async Task ValidateAsync(FabricClient fc, NodeCommandState state, CancellationToken cancellationToken)
{
// It takes a few seconds for the node to shutdown + a few seconds for FM to find out.
await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken).ConfigureAwait(false);
TimeoutHelper timeoutHelper = new TimeoutHelper(this.OperationTimeout);
Node queriedNode = null;
do
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - start node validating node '{1}' is not down", this.State.OperationId, state.Info.InputNodeInstanceId);
queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
fc,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(5.0d), cancellationToken).ConfigureAwait(false);
}while (timeoutHelper.GetRemainingTime() > TimeSpan.Zero);
if (!FaultAnalysisServiceUtility.IsNodeRunning(queriedNode))
{
// something is wrong, retry
FaultAnalysisServiceUtility.ThrowEngineRetryableException(string.Format(CultureInfo.InvariantCulture, "{0} - start node validation - node is not running yet. Status={1}", state.OperationId, queriedNode.NodeStatus));
}
}
private async Task ValidateAsync(FabricClient fc, NodeCommandState state, CancellationToken cancellationToken)
{
// It takes a few seconds for the node to shutdown + a few seconds for FM to find out.
await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken).ConfigureAwait(false);
TimeoutHelper timeoutHelper = new TimeoutHelper(this.OperationTimeout);
Node queriedNode = null;
do
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - stop node validating node '{1}' is not up", this.State.OperationId, state.Info.InputNodeInstanceId);
queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
fc,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
if (!FaultAnalysisServiceUtility.IsNodeRunning(queriedNode))
{
break;
}
await Task.Delay(TimeSpan.FromSeconds(5.0d), cancellationToken).ConfigureAwait(false);
}while (timeoutHelper.GetRemainingTime() > TimeSpan.Zero);
if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode))
{
// Something is amiss. The api returned success, but we're not reaching the desired state. It might be something out of band happened. This is best effort, so don't fail.
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - node '{1}' did not reach desired state in {2}", this.State.OperationId, state.Info.InputNodeInstanceId, this.OperationTimeout);
}
}
public async Task <NodeTransitionProgress> GetNodeTransitionProgressAsync(
Guid operationId,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
NodeTransitionProgress progress = null;
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "GetNodeTransitionProgressAsync got operation id {0}", operationId);
ActionStateBase actionState = await this.MessageProcessor.ProcessGetProgressAsync(operationId, timeout, cancellationToken);
StepStateNames stateName = actionState.StateProgress.Peek();
TestCommandProgressState state = FaultAnalysisServiceUtility.ConvertState(actionState, TraceType);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - GetNodeTransitionProgressAsync reading nodecommandstate", operationId);
NodeCommandState nodeState = actionState as NodeCommandState;
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} NodeCommandState is null={1}", operationId, nodeState == null);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - GetNodeTransitionProgressAsync node name={1}, node instance id={2}", operationId, nodeState.Info.NodeName, nodeState.Info.InputNodeInstanceId);
NodeResult nodeResult = new NodeResult(nodeState.Info.NodeName, nodeState.Info.InputNodeInstanceId);
NodeCommandResult result = new NodeCommandResult(nodeResult, actionState.ErrorCausingRollback);
progress = new NodeTransitionProgress(state, result);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Caught {1}", operationId, e.ToString());
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
return(progress);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNode LookingUpState reading RD", this.State.OperationId);
bool isStopped = await FaultAnalysisServiceUtility.ReadStoppedNodeStateAsync(
this.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
cancellationToken).ConfigureAwait(false);
if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode))
{
if (!isStopped)
{
// For illustration, if you just called StartNodeUsingNodeNameAsync() in this situation w/o checking first, you'd either get instance mismatch or node has not stopped yet
// Note: this is different than the logic in the PerformingActions step (the former does not check instance id, the latter does), which is after the call to StartNodeUsingNodeNameAsync(), because
// this is a precondition check.
Exception nodeAlreadyUp = FaultAnalysisServiceUtility.CreateException(
TraceType,
NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_UP,
string.Format(CultureInfo.InvariantCulture, "Node {0} already started", state.Info.NodeName),
FabricErrorCode.NodeIsUp);
throw new FatalException("fatal", nodeAlreadyUp);
}
else
{
// The only way this can happen is OOB start. FAS should fix it's incorrect state then fail the command with
// node already up.
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNode LookingUpState setting RD entry for node {1} to not stopped", this.State.OperationId, state.Info.NodeName);
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
queriedNode.NodeName,
false,
cancellationToken).ConfigureAwait(false);
Exception nodeIsUp = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_UP,
string.Format(CultureInfo.InvariantCulture, "Node {0} is up", state.Info.NodeName));
throw new FatalException("fatal", nodeIsUp);
}
}
else if (queriedNode.NodeStatus == NodeStatus.Down && !isStopped)
{
// This is a special scenario that can happen if:
// 1) There was an OOB stop using the old api
// 2) A node went down (not stopped, down)
// Don't handle this, return node down.
Exception nodeIsDown = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_DOWN,
string.Format(CultureInfo.InvariantCulture, "Node {0} is down", state.Info.NodeName));
throw new FatalException("fatal", nodeIsDown);
}
state.Info.InitialQueriedNodeStatus = queriedNode.NodeStatus;
state.Info.NodeWasInitiallyInStoppedState = isStopped;
state.StateProgress.Push(StepStateNames.PerformingActions);
return(state);
}
public LookingUpState(FabricClient fabricClient, NodeCommandState state, StartNodeFromFASAction action, TimeSpan requestTimeout, TimeSpan operationTimeout)
: base(fabricClient, state, requestTimeout, operationTimeout)
{
this.action = action;
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
// The return value is ignored, this is just being used to check if the RemoveNodeState was called.
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - calling StartNodeUsingNodeNameAsync, ApiInputNodeInstanceId={1}", this.State.OperationId, state.Info.InputNodeInstanceId);
Exception exception = null;
try
{
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.FaultManager.StartNodeUsingNodeNameAsync(
state.Info.NodeName,
state.Info.InputNodeInstanceId,
null,
0,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.StartNodeErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw {1}", this.State.OperationId, e);
exception = e;
}
cancellationToken.ThrowIfCancellationRequested();
SuccessRetryOrFail status = SuccessRetryOrFail.Invalid;
if (exception != null)
{
FabricException fe = exception as FabricException;
if (fe != null)
{
status = this.HandleFabricException(fe, state);
}
else
{
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, "{0} - StartNodeUsingNodeNameAsync threw non-FabricException with ErrorCode={1}", this.State.OperationId, exception);
status = SuccessRetryOrFail.RetryStep;
}
}
else
{
// success
status = SuccessRetryOrFail.Success;
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
false,
cancellationToken).ConfigureAwait(false);
}
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
if (status == SuccessRetryOrFail.RetryStep)
{
throw new RetrySameStepException("retrystep", exception);
}
else if (status == SuccessRetryOrFail.Fail)
{
throw new FatalException("fatal", exception);
}
else if (status == SuccessRetryOrFail.Success)
{
// no-op
}
else
{
ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "This condition should not have been hit. OperationId: {0}", this.State.OperationId));
}
await this.ValidateAsync(this.FabricClient, state, cancellationToken).ConfigureAwait(false);
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode.LookingUpState performing node query", this.State.OperationId);
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode.LookingUpState node query completed", this.State.OperationId);
// Check for bad state
if (queriedNode == null ||
queriedNode.NodeStatus == NodeStatus.Invalid ||
queriedNode.NodeStatus == NodeStatus.Unknown ||
queriedNode.NodeStatus == NodeStatus.Removed)
{
// Fail the command
Exception nodeNotFoundException = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_NOT_FOUND,
string.Format(CultureInfo.InvariantCulture, "{0} - Node {1} does not exist", this.State.OperationId, state.Info.NodeName));
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - throwing fatal exception {1}", this.State.OperationId, nodeNotFoundException);
throw new FatalException("fatal", nodeNotFoundException);
}
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode LookingUpState reading RD", this.State.OperationId);
bool isStopped = await FaultAnalysisServiceUtility.ReadStoppedNodeStateAsync(
this.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
cancellationToken).ConfigureAwait(false);
if (queriedNode.NodeStatus == NodeStatus.Down && isStopped)
{
// Node already stopped
Exception nodeAlreadyStopped = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_ALREADY_STOPPED,
string.Format(CultureInfo.InvariantCulture, "Node {0} is already stopped", state.Info.NodeName));
throw new FatalException("fatal", nodeAlreadyStopped);
}
else if (queriedNode.NodeStatus != NodeStatus.Down && isStopped)
{
// FM says the node is up, so FAS has incorrect state, perhaps because of an out of band start from the original deprecated api.
// Correct the state, then continue to run this command normally. It is valid.
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
queriedNode.NodeName,
false,
cancellationToken).ConfigureAwait(false);
}
else if (queriedNode.NodeStatus == NodeStatus.Down && !isStopped)
{
// Node is down (as opposed to stopped)
Exception nodeIsDown = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_DOWN,
string.Format(CultureInfo.InvariantCulture, "Node {0} is down", state.Info.NodeName));
throw new FatalException("fatal", nodeIsDown);
}
state.Info.InitialQueriedNodeStatus = queriedNode.NodeStatus;
state.Info.NodeWasInitiallyInStoppedState = isStopped;
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - StopNode LookingUpState InitialQueriedNodeStatus='{1}', NodeWasInitiallyInStoppedState='{2}'",
this.State.OperationId,
state.Info.InitialQueriedNodeStatus,
state.Info.NodeWasInitiallyInStoppedState);
state.StateProgress.Push(StepStateNames.PerformingActions);
return(state);
}
public PerformingActions(FabricClient fabricClient, NodeCommandState state, StopNodeFromFASAction action, TimeSpan requestTimeout, TimeSpan operationTimeout)
: base(fabricClient, state, requestTimeout, operationTimeout)
{
this.action = action;
}
private async Task <FabricTestAction> ConstructActionAsync(ActionType actionType, ActionStateBase actionStateBase)
{
FabricTestAction action = null;
if (actionType == ActionType.InvokeDataLoss)
{
InvokeDataLossState actionState = actionStateBase as InvokeDataLossState;
StepStateNames currentState = actionState.StateProgress.Peek();
if (currentState == StepStateNames.IntentSaved)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
await this.actionStore.UpdateActionStateAsync(actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "action state has been updated");
}
action = new InvokeDataLossAction(
this.stateManager,
this.Partition,
actionState,
actionState.Info.PartitionSelector,
actionState.Info.DataLossMode,
this.dataLossCheckWaitDurationInSeconds,
this.dataLossCheckPollIntervalInSeconds,
this.replicaDropWaitDurationInSeconds,
this.requestTimeout,
this.operationTimeout);
}
else if (actionType == ActionType.InvokeQuorumLoss)
{
InvokeQuorumLossState actionState = actionStateBase as InvokeQuorumLossState;
StepStateNames currentState = actionState.StateProgress.Peek();
if (currentState == StepStateNames.IntentSaved)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
await this.actionStore.UpdateActionStateAsync(actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "action state has been updated");
}
// This is the case for resuming an action after a failover
action = new InvokeQuorumLossAction(this.stateManager, this.Partition, actionState, actionState.Info.PartitionSelector, actionState.Info.QuorumLossMode, actionState.Info.QuorumLossDuration, this.requestTimeout, this.operationTimeout);
}
else if (actionType == ActionType.RestartPartition)
{
RestartPartitionState actionState = actionStateBase as RestartPartitionState;
StepStateNames currentState = actionState.StateProgress.Peek();
if (currentState == StepStateNames.IntentSaved)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
await this.actionStore.UpdateActionStateAsync(actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "action state has been updated");
}
// This is the case for resuming an action after a failover
action = new RestartPartitionAction(this.stateManager, this.Partition, actionState, actionState.Info.PartitionSelector, actionState.Info.RestartPartitionMode, this.requestTimeout, this.operationTimeout);
}
else if (actionType == ActionType.TestStuck)
{
StuckState actionState = actionStateBase as StuckState;
StepStateNames currentState = actionState.StateProgress.Peek();
if (currentState == StepStateNames.IntentSaved)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
await this.actionStore.UpdateActionStateAsync(actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "action state has been updated");
}
action = new StuckAction(this.stateManager, this.Partition, actionState, this.requestTimeout, this.operationTimeout);
}
else if (actionType == ActionType.TestRetryStep)
{
TestRetryStepState actionState = actionStateBase as TestRetryStepState;
StepStateNames currentState = actionState.StateProgress.Peek();
if (currentState == StepStateNames.IntentSaved)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
await this.actionStore.UpdateActionStateAsync(actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "action state has been updated");
}
action = new TestRetryStepAction(this.stateManager, this.Partition, actionState, this.requestTimeout, this.operationTimeout);
}
else if (actionType == ActionType.StartNode)
{
NodeCommandState actionState = actionStateBase as NodeCommandState;
actionState.StoppedNodeTable = this.stoppedNodeTable;
StepStateNames currentState = actionState.StateProgress.Peek();
if (currentState == StepStateNames.IntentSaved)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
await this.actionStore.UpdateActionStateAsync(actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "action state has been updated");
}
action = new StartNodeFromFASAction(this.stateManager, this.Partition, actionState, this.stoppedNodeTable, this.requestTimeout, this.operationTimeout);
}
else if (actionType == ActionType.StopNode)
{
NodeCommandState actionState = actionStateBase as NodeCommandState;
actionState.StoppedNodeTable = this.stoppedNodeTable;
StepStateNames currentState = actionState.StateProgress.Peek();
if (currentState == StepStateNames.IntentSaved)
{
actionState.StateProgress.Push(StepStateNames.LookingUpState);
await this.actionStore.UpdateActionStateAsync(actionState);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "action state has been updated");
}
action = new StopNodeFromFASAction(this.stateManager, this.Partition, actionState, this.stoppedNodeTable, this.requestTimeout, this.operationTimeout);
}
else
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Unknown actionType");
}
return(action);
}
