private static void ThrowIfRestartPartitionModeInvalid(RestartPartitionMode restartPartitionMode)
{
if (restartPartitionMode == RestartPartitionMode.Invalid)
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_INVALIDARG, Strings.StringResources.Error_UnsupportedRestartPartitionMode);
}
}
public async Task <string> CallSystemService(
string action,
string inputBlob,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
string outputBlob = null;
try
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter CallSystemService for {0}.", action);
if (this.actionMapping.ContainsKey(action))
{
outputBlob = await this.actionMapping[action](inputBlob, timeout, cancellationToken).ConfigureAwait(false);
}
else
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_NOTIMPL, "{0} is not implemented", action);
}
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "CallSystemService error: {0}", e);
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit CallSystemService for {0} with output {1}.", action, outputBlob);
return(outputBlob);
}
private void ThrowIfNotReady(Func <bool> readinessCriteria = null)
{
if (this.MessageProcessor == null || (readinessCriteria != null && !readinessCriteria()))
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_ABORT, "Service is not accepting client calls yet");
}
if (this.MessageProcessor.Partition.WriteStatus != PartitionAccessStatus.Granted)
{
// not primary, reconfiguration pending, and no write quorum all get translated to E_ABORT by the gateway, so there's no point in throwing a specific exception for each here.
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_ABORT, string.Empty);
}
}
public void Add(string nodeName)
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "Enter Add '{0}'", nodeName);
lock (this.NodeCommandLock)
{
if (this.nodeCommandInProgress.Contains(nodeName))
{
throw FaultAnalysisServiceUtility.CreateException(StepBase.TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_TRANSITION_IN_PROGRESS, Strings.StringResources.Error_NodeTransitionInProgress);
}
this.nodeCommandInProgress.Add(nodeName);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "Done - Add '{0}'", nodeName);
}
}
public async Task <ActionStateBase> FindByOperationIdAsync(Guid operationId, CancellationToken cancellationToken)
{
ActionStateBase state = null;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, actiontable has value", operationId);
state = this.ReadData(data.Value);
}
else
{
data = await this.historyTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "FindOperationId: {0}, history table has value", operationId);
state = this.ReadData(data.Value);
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error);
}
}
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside {0} - operationId={1}", "FindByOperationId", operationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "FindOperationId: {0}", e.ToString());
throw;
}
return(state);
}
// Called to initialize state when a new message is received.
private async Task InitializeNewActionInnerAsync(ActionStateBase actionState)
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter InitializeNewActionAsync");
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId);
bool keyAlreadyExists = await this.actionTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (!keyAlreadyExists)
{
keyAlreadyExists = await this.historyTable.ContainsKeyAsync(tx, actionState.OperationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
}
if (keyAlreadyExists)
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' already exists", actionState.OperationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_TEST_COMMAND_OPERATION_ID_ALREADY_EXISTS, error);
}
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside InitializeNewActionAsync - operationId={0}", actionState.OperationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(actionState.OperationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "Could not add new action: {0}", e.ToString());
throw;
}
await this.PersistAsync(actionState, true).ConfigureAwait(false);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
InvokeQuorumLossState state = Convert(this.State);
// get info about the service so we can check type and trss
ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.GetServiceDescriptionAsync(
this.partitionSelector.ServiceName,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
if (result.Kind != ServiceDescriptionKind.Stateful)
{
// The message in the first arg is only for debugging, it is not returned to the user.
throw new FabricInvalidForStatelessServicesException("FabricInvalidForStatelessServicesException", FabricErrorCode.InvalidForStatelessServices);
}
StatefulServiceDescription statefulServiceDescription = result as StatefulServiceDescription;
ReleaseAssert.AssertIf(statefulServiceDescription == null, string.Format(CultureInfo.InvariantCulture, "{0} - Service is not a stateful service", this.State.OperationId));
if (!statefulServiceDescription.HasPersistedState)
{
// The message in the first arg is only for debugging, it is not returned to the user.
throw new FabricOnlyValidForStatefulPersistentServicesException("This is only valid for stateful persistent services", FabricErrorCode.OnlyValidForStatefulPersistentServices);
}
SelectedPartition targetPartition = await FaultAnalysisServiceUtility.GetSelectedPartitionStateAsync(
this.FabricClient,
this.partitionSelector,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
Guid partitionId = targetPartition.PartitionId;
// get data about replicas in that partition
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
List <StatefulServiceReplica> tempReplicas = new List <StatefulServiceReplica>();
foreach (var replica in replicasResult)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "Expected stateful replica");
tempReplicas.Add(statefulReplica);
}
List <StatefulServiceReplica> targetReplicas = null;
if (this.quorumLossMode == QuorumLossMode.AllReplicas)
{
targetReplicas = tempReplicas.Where(r => r.ReplicaRole == ReplicaRole.Primary || r.ReplicaRole == ReplicaRole.ActiveSecondary).ToList();
}
else if (this.quorumLossMode == QuorumLossMode.QuorumReplicas)
{
targetReplicas = FaultAnalysisServiceUtility.GetReplicasForPartialLoss(state.OperationId, tempReplicas);
}
else
{
throw FaultAnalysisServiceUtility.CreateException(StepBase.TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_INVALIDARG, Strings.StringResources.Error_UnsupportedQuorumLossMode);
}
if (targetReplicas == null)
{
// This will cause the command to rollback and retry
throw new FabricTransientException("The operation could not be performed, please retry", FabricErrorCode.NotReady);
}
List <string> targetNodes = new List <string>();
foreach (var replica in targetReplicas)
{
targetNodes.Add(replica.NodeName);
}
List <Tuple <string, string> > unreliableTransportInfoList = new List <Tuple <string, string> >();
foreach (string nodeName in targetNodes)
{
UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "StatefulServiceReopen");
behavior.AddFilterForPartitionId(partitionId);
// ApplyingUnreliableTransport.BehaviorNamePrefix + nodeName;
string behaviorName = this.CreateBehaviorName(nodeName);
unreliableTransportInfoList.Add(new Tuple <string, string>(nodeName, behaviorName));
}
state.StateProgress.Push(StepStateNames.PerformingActions);
state.Info.PartitionId = partitionId;
state.Info.ReplicaIds = targetReplicas.Select(r => r.Id).ToList();
state.Info.UnreliableTransportInfo = unreliableTransportInfoList;
return(state);
}
public async Task CancelTestCommandAsync(Guid operationId, bool force)
{
ActionStateBase state = null;
try
{
using (var tx = this.stateManager.CreateTransaction())
{
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "CancelTestCommandAsync", operationId);
var data = await this.actionTable.TryGetValueAsync(tx, operationId, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
if (data.HasValue)
{
state = this.ReadData(data.Value);
}
else
{
string error = string.Format(CultureInfo.InvariantCulture, "OperationId '{0}' was not found", operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_KEY_NOT_FOUND, error);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "CancelActionAsync: operationId={0}, force={1}", operationId, force);
if (force)
{
if (state.RollbackState != RollbackState.RollingBackDueToUserCancel)
{
string error = string.Format(CultureInfo.InvariantCulture, "CancelActionAsync: operationId={0}, force is not allowed for something in {1}", operationId, state.RollbackState.ToString());
TestabilityTrace.TraceSource.WriteError(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_INVALID_TEST_COMMAND_STATE, error);
}
else
{
state.RollbackState = RollbackState.RollingBackForce;
}
}
else
{
// if it's in force, don't allow it back down to graceful
if (state.RollbackState == RollbackState.RollingBackForce)
{
string error = string.Format(
CultureInfo.InvariantCulture,
"CancelActionAsync: operationId={0}, CancelTestCommandAsync has already been called with force==true. Switching to force==false is not valid.",
operationId);
TestabilityTrace.TraceSource.WriteWarning(TraceType, error);
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_INVALID_TEST_COMMAND_STATE, error);
}
state.RollbackState = RollbackState.RollingBackDueToUserCancel;
}
byte[] stateToPersist = state.ToBytes();
await this.actionTable.AddOrUpdateAsync(tx, state.OperationId, stateToPersist, (k, v) => stateToPersist, DefaultDictionaryTimeout, this.cancellationToken).ConfigureAwait(false);
await tx.CommitAsync().ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(TraceType, "CancelActionAsync: operationId={0}, force={1} committed", operationId, force);
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside {0} - operationId={1}", "CancelTestCommandAsync", operationId);
}
catch (FabricNotPrimaryException fnp)
{
FaultAnalysisServiceUtility.TraceFabricNotPrimary(operationId, fnp);
throw;
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteError(TraceType, "CancelActionAsync: {0}", e.ToString());
throw;
}
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNode LookingUpState reading RD", this.State.OperationId);
bool isStopped = await FaultAnalysisServiceUtility.ReadStoppedNodeStateAsync(
this.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
cancellationToken).ConfigureAwait(false);
if (FaultAnalysisServiceUtility.IsNodeRunning(queriedNode))
{
if (!isStopped)
{
// For illustration, if you just called StartNodeUsingNodeNameAsync() in this situation w/o checking first, you'd either get instance mismatch or node has not stopped yet
// Note: this is different than the logic in the PerformingActions step (the former does not check instance id, the latter does), which is after the call to StartNodeUsingNodeNameAsync(), because
// this is a precondition check.
Exception nodeAlreadyUp = FaultAnalysisServiceUtility.CreateException(
TraceType,
NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_UP,
string.Format(CultureInfo.InvariantCulture, "Node {0} already started", state.Info.NodeName),
FabricErrorCode.NodeIsUp);
throw new FatalException("fatal", nodeAlreadyUp);
}
else
{
// The only way this can happen is OOB start. FAS should fix it's incorrect state then fail the command with
// node already up.
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StartNode LookingUpState setting RD entry for node {1} to not stopped", this.State.OperationId, state.Info.NodeName);
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
queriedNode.NodeName,
false,
cancellationToken).ConfigureAwait(false);
Exception nodeIsUp = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_UP,
string.Format(CultureInfo.InvariantCulture, "Node {0} is up", state.Info.NodeName));
throw new FatalException("fatal", nodeIsUp);
}
}
else if (queriedNode.NodeStatus == NodeStatus.Down && !isStopped)
{
// This is a special scenario that can happen if:
// 1) There was an OOB stop using the old api
// 2) A node went down (not stopped, down)
// Don't handle this, return node down.
Exception nodeIsDown = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_DOWN,
string.Format(CultureInfo.InvariantCulture, "Node {0} is down", state.Info.NodeName));
throw new FatalException("fatal", nodeIsDown);
}
state.Info.InitialQueriedNodeStatus = queriedNode.NodeStatus;
state.Info.NodeWasInitiallyInStoppedState = isStopped;
state.StateProgress.Push(StepStateNames.PerformingActions);
return(state);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
NodeCommandState state = Convert(this.State);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode.LookingUpState performing node query", this.State.OperationId);
Node queriedNode = await FaultAnalysisServiceUtility.GetNodeInfoAsync(
this.State.OperationId,
this.FabricClient,
state.Info.NodeName,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode.LookingUpState node query completed", this.State.OperationId);
// Check for bad state
if (queriedNode == null ||
queriedNode.NodeStatus == NodeStatus.Invalid ||
queriedNode.NodeStatus == NodeStatus.Unknown ||
queriedNode.NodeStatus == NodeStatus.Removed)
{
// Fail the command
Exception nodeNotFoundException = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_NOT_FOUND,
string.Format(CultureInfo.InvariantCulture, "{0} - Node {1} does not exist", this.State.OperationId, state.Info.NodeName));
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - throwing fatal exception {1}", this.State.OperationId, nodeNotFoundException);
throw new FatalException("fatal", nodeNotFoundException);
}
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - StopNode LookingUpState reading RD", this.State.OperationId);
bool isStopped = await FaultAnalysisServiceUtility.ReadStoppedNodeStateAsync(
this.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
state.Info.NodeName,
cancellationToken).ConfigureAwait(false);
if (queriedNode.NodeStatus == NodeStatus.Down && isStopped)
{
// Node already stopped
Exception nodeAlreadyStopped = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_ALREADY_STOPPED,
string.Format(CultureInfo.InvariantCulture, "Node {0} is already stopped", state.Info.NodeName));
throw new FatalException("fatal", nodeAlreadyStopped);
}
else if (queriedNode.NodeStatus != NodeStatus.Down && isStopped)
{
// FM says the node is up, so FAS has incorrect state, perhaps because of an out of band start from the original deprecated api.
// Correct the state, then continue to run this command normally. It is valid.
await FaultAnalysisServiceUtility.SetStoppedNodeStateAsync(
this.action.State.OperationId,
this.action.Partition,
this.action.StateManager,
this.action.StoppedNodeTable,
queriedNode.NodeName,
false,
cancellationToken).ConfigureAwait(false);
}
else if (queriedNode.NodeStatus == NodeStatus.Down && !isStopped)
{
// Node is down (as opposed to stopped)
Exception nodeIsDown = FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NODE_IS_DOWN,
string.Format(CultureInfo.InvariantCulture, "Node {0} is down", state.Info.NodeName));
throw new FatalException("fatal", nodeIsDown);
}
state.Info.InitialQueriedNodeStatus = queriedNode.NodeStatus;
state.Info.NodeWasInitiallyInStoppedState = isStopped;
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - StopNode LookingUpState InitialQueriedNodeStatus='{1}', NodeWasInitiallyInStoppedState='{2}'",
this.State.OperationId,
state.Info.InitialQueriedNodeStatus,
state.Info.NodeWasInitiallyInStoppedState);
state.StateProgress.Push(StepStateNames.PerformingActions);
return(state);
}
internal static void ThrowAlreadyRunning(string message = default(string))
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_CHAOS_ALREADY_RUNNING, StringResources.ChaosError_ChaosAlreadyRunning);
}
internal static void ThrowNotReady(string message)
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_NOT_READY, message);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
InvokeDataLossState state = Convert(this.State);
PartitionSelector partitionSelector = state.Info.PartitionSelector;
DataLossMode dataLossMode = state.Info.DataLossMode;
long preDataLossNumber = state.Info.DataLossNumber;
string failoverManagerPrimaryNodeName = state.Info.NodeName;
Guid partitionId = state.Info.PartitionId;
string behaviorName = state.Info.UnreliableTransportInfo.First().Item2;
int targetReplicaSetSize = state.Info.TargetReplicaSetSize;
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - applying UT, partitionId={1}", this.State.OperationId, partitionId);
System.Fabric.Common.UnreliableTransportBehavior behavior = new System.Fabric.Common.UnreliableTransportBehavior("*", "DoReconfiguration");
behavior.AddFilterForPartitionId(partitionId);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync(
failoverManagerPrimaryNodeName,
behaviorName,
behavior,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
// TODO: Wait for some time so that the unreliable transport behavior can be read from the files.
// Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied
await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false);
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
List <StatefulServiceReplica> replicaList = new List <StatefulServiceReplica>();
foreach (var replica in replicasResult)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "Service Replica is not of stateful type even though service is stateful");
replicaList.Add(statefulReplica);
}
// Select target replicas based on the DataLosMode
List <StatefulServiceReplica> targets = null;
if (dataLossMode == DataLossMode.FullDataLoss)
{
targets = GetReplicasForFullDataLoss(replicaList);
}
else if (dataLossMode == DataLossMode.PartialDataLoss)
{
targets = FaultAnalysisServiceUtility.GetReplicasForPartialLoss(state.OperationId, replicaList);
}
else
{
throw FaultAnalysisServiceUtility.CreateException(StepBase.TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_INVALIDARG, Strings.StringResources.Error_UnsupportedDataLossMode);
}
if (targets == null)
{
// This will cause the command to rollback and retry
throw new FabricTransientException("The operation could not be performed, please retry", FabricErrorCode.NotReady);
}
foreach (var replica in targets)
{
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - Removing replica {1} in partition {2} with role {3} and status {4} to induce data loss",
this.State.OperationId,
replica.Id,
partitionId,
replica.ReplicaRole,
replica.ReplicaStatus);
await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.RemoveReplicaAsync(
replica.NodeName,
partitionId,
replica.Id,
this.RequestTimeout,
cancellationToken),
FabricClientRetryErrors.RemoveReplicaErrors.Value,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
}
ActionTest.PerformInternalServiceFaultIfRequested(this.State.OperationId, serviceInternalFaultInfo, this.State, cancellationToken, true);
await this.WaitForAllTargetReplicasToGetDroppedAsync(partitionId, targets, cancellationToken).ConfigureAwait(false);
await RemoveUnreliableTransportAsync(this.FabricClient, failoverManagerPrimaryNodeName, behaviorName, this.RequestTimeout, this.OperationTimeout, cancellationToken).ConfigureAwait(false);
bool dataLossWasSuccessful = false;
TimeoutHelper timeoutHelper = new TimeoutHelper(TimeSpan.FromSeconds(30));
do
{
ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetPartitionListAsync(
this.partitionSelector.ServiceName,
null,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
bool partitionFound = false;
long postDataLossNumber = 0;
foreach (StatefulServicePartition partition in partitionsResult)
{
if (partition.PartitionInformation.Id == partitionId)
{
postDataLossNumber = partition.PrimaryEpoch.DataLossNumber;
partitionFound = true;
break;
}
}
if (!partitionFound)
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionNotFound), FabricErrorCode.PartitionNotFound);
}
TestabilityTrace.TraceSource.WriteInfo(
StepBase.TraceType,
"{0} - Checking data loss numbers for partition {1} with remaining time {2}. Current numbers {3}:{4}",
this.State.OperationId,
partitionId,
timeoutHelper.GetRemainingTime(),
preDataLossNumber,
postDataLossNumber);
if (postDataLossNumber != preDataLossNumber)
{
dataLossWasSuccessful = true;
break;
}
await System.Fabric.Common.AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(this.dataLossCheckPollIntervalInSeconds), cancellationToken).ConfigureAwait(false);
}while (timeoutHelper.GetRemainingTime() > TimeSpan.Zero);
if (!dataLossWasSuccessful)
{
// This is only viewable internally for debug. This will cause a retry of the whole flow.
string error = string.Format(
CultureInfo.InvariantCulture,
"{0} - Service could not induce data loss for service '{1}' partition '{2}' in '{3}' Please retry",
this.State.OperationId,
partitionSelector.ServiceName,
partitionId,
this.dataLossCheckWaitDurationInSeconds);
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, error);
throw new FabricTransientException("The operation could not be performed, please retry", FabricErrorCode.NotReady);
}
state.StateProgress.Push(StepStateNames.CompletedSuccessfully);
return(state);
}
public override async Task <ActionStateBase> RunAsync(CancellationToken cancellationToken, ServiceInternalFaultInfo serviceInternalFaultInfo)
{
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "Inside CollectingState, service={0}", this.partitionSelector.ServiceName);
RestartPartitionState state = Convert(this.State);
// Get service info and validate if the parameters are valid
ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.ServiceManager.GetServiceDescriptionAsync(
this.partitionSelector.ServiceName,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
if (result.Kind != ServiceDescriptionKind.Stateful && this.restartPartitionMode == RestartPartitionMode.OnlyActiveSecondaries)
{
// The message in the first arg is only for debugging, it is not returned to the user.
string debugText = string.Format(CultureInfo.InvariantCulture, "RestartPartition: for stateless services only RestartPartitionMode.AllReplicasOrInstances is valid");
TestabilityTrace.TraceSource.WriteWarning(StepBase.TraceType, debugText);
throw FaultAnalysisServiceUtility.CreateException(StepBase.TraceType, NativeTypes.FABRIC_ERROR_CODE.E_INVALIDARG, debugText);
}
bool hasPersistedState = false;
if (result.Kind == ServiceDescriptionKind.Stateful)
{
StatefulServiceDescription statefulDescription = result as StatefulServiceDescription;
ReleaseAssert.AssertIf(statefulDescription == null, "Stateful service description is not WinFabricStatefulServiceDescription");
hasPersistedState = statefulDescription.HasPersistedState;
}
SelectedPartition targetPartition = await FaultAnalysisServiceUtility.GetSelectedPartitionStateAsync(
this.FabricClient,
this.partitionSelector,
this.RequestTimeout,
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
Guid partitionId = targetPartition.PartitionId;
// get replicas for target
ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetReplicaListAsync(
partitionId,
0,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
// get replicas for fm in order to get the primary
ServiceReplicaList failoverManagersReplicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync(
() => this.FabricClient.QueryManager.GetReplicaListAsync(
FASConstants.FmPartitionId,
0,
this.RequestTimeout,
cancellationToken),
this.OperationTimeout,
cancellationToken).ConfigureAwait(false);
string failoverManagerPrimaryNodeName = string.Empty;
var readyFMReplicas = failoverManagersReplicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray();
foreach (var replica in readyFMReplicas)
{
StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica;
ReleaseAssert.AssertIf(statefulReplica == null, "FM Replica is not a stateful replica");
if (statefulReplica.ReplicaRole == ReplicaRole.Primary)
{
failoverManagerPrimaryNodeName = replica.NodeName;
}
}
if (string.IsNullOrEmpty(failoverManagerPrimaryNodeName))
{
throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, "FailoverManager"), FabricErrorCode.NotReady);
}
TestabilityTrace.TraceSource.WriteInfo(StepBase.TraceType, "{0} - FM primary is at node={1}", this.State.OperationId, failoverManagerPrimaryNodeName);
string behaviorName = RestartingSelectedReplicas.UTBehaviorPrefixName + "_" + this.State.OperationId;
List <Tuple <string, string> > unreliableTransportInfo = new List <Tuple <string, string> >();
unreliableTransportInfo.Add(new Tuple <string, string>(failoverManagerPrimaryNodeName, behaviorName));
state.StateProgress.Push(StepStateNames.PerformingActions);
state.Info.PartitionId = partitionId;
state.Info.NodeName = failoverManagerPrimaryNodeName;
state.Info.HasPersistedState = hasPersistedState;
state.Info.UnreliableTransportInfo = unreliableTransportInfo;
return(state);
}
private void ThrowEAbort()
{
throw FaultAnalysisServiceUtility.CreateException(TraceType, Interop.NativeTypes.FABRIC_ERROR_CODE.E_ABORT, "Operation cancelled");
}
public async Task StartNodeTransitionAsync(
NodeTransitionDescription description,
TimeSpan timeout,
CancellationToken cancellationToken)
{
this.ThrowIfNotReady();
TestabilityTrace.TraceSource.WriteInfo(
TraceType,
"FaultAnalysisServiceImpl StartNodeTransitionAsync operationId={0} NodeTransitionType={1} - {2}:{3}",
description.OperationId,
description.NodeTransitionType,
description.NodeName,
description.NodeInstanceId);
try
{
if (description.NodeTransitionType == NodeTransitionType.Start)
{
NodeStartDescription translatedDescription = (NodeStartDescription)description;
await this.MessageProcessor.ProcessStartNodeCommandAsync(
translatedDescription.OperationId,
translatedDescription.NodeName,
translatedDescription.NodeInstanceId,
timeout,
null).ConfigureAwait(false);
}
else if (description.NodeTransitionType == NodeTransitionType.Stop)
{
NodeStopDescription translatedDescription = (NodeStopDescription)description;
if (translatedDescription.StopDurationInSeconds < this.minStopDurationInSeconds ||
translatedDescription.StopDurationInSeconds > this.maxStopDurationInSeconds)
{
TestabilityTrace.TraceSource.WriteWarning(
TraceType,
"Duration in seconds '{0}' is out of range. Min='{1}' Max='{2}'",
translatedDescription.StopDurationInSeconds,
this.minStopDurationInSeconds,
this.maxStopDurationInSeconds);
throw FaultAnalysisServiceUtility.CreateException(
TraceType,
Interop.NativeTypes.FABRIC_ERROR_CODE.FABRIC_E_INVALID_DURATION,
"StopDurationInSeconds is out of range");
}
await this.MessageProcessor.ProcessStopNodeCommandAsync(
translatedDescription.OperationId,
translatedDescription.NodeName,
translatedDescription.NodeInstanceId,
translatedDescription.StopDurationInSeconds,
timeout,
null).ConfigureAwait(false);
}
else
{
throw new NotImplementedException();
}
}
catch (Exception e)
{
TestabilityTrace.TraceSource.WriteWarning(TraceType, "{0} - Exception occurred: {1}", description.OperationId, e.Message);
FaultAnalysisServiceUtility.ThrowTransientExceptionIfRetryable(e);
throw;
}
TestabilityTrace.TraceSource.WriteInfo(TraceType, "{0} - Intent saved for {1}", description.OperationId, ActionType.StartNode);
}