protected PartitionSelector GetPartitionSelector()
{
if (this.ParameterSetName == "PartitionId")
{
return(PartitionSelector.PartitionIdOf(this.ServiceName, this.PartitionId));
}
else
{
switch (this.ParameterSetName)
{
case "ServiceNameRandomPartition":
return(PartitionSelector.RandomOf(this.ServiceName));
case "ServiceNamePartitionSingleton":
return(PartitionSelector.SingletonOf(this.ServiceName));
case "ServiceNamePartitionNamed":
return(PartitionSelector.PartitionKeyOf(this.ServiceName, this.PartitionKey));
case "ServiceNamePartitionUniformedInt":
long partitionKeyLong;
if (!long.TryParse(this.PartitionKey, out partitionKeyLong))
{
throw new ArgumentException(StringResources.Error_InvalidPartitionKey);
}
return(PartitionSelector.PartitionKeyOf(this.ServiceName, partitionKeyLong));
default:
throw new ArgumentException(StringResources.Error_CouldNotParsePartitionSelector);
}
}
}
/// <summary>
/// Restarts a random node.
/// </summary>
/// <param name="serviceName">Uri of the service in the format fabric:/[application]/[service name]</param>
/// <returns>Task instance.</returns>
public async Task RestartNodeAsync(Uri serviceName)
{
PartitionSelector randomPartitionSelector = PartitionSelector.RandomOf(serviceName);
ReplicaSelector primaryofReplicaSelector = ReplicaSelector.PrimaryOf(randomPartitionSelector);
// Create FabricClient with connection and security information here
await _client.FaultManager.RestartNodeAsync(primaryofReplicaSelector, CompletionMode.Verify).ConfigureAwait(false);
}
public async Task <bool> RestartGetSnowConditionsService()
{
var replicaSelector = ReplicaSelector.PrimaryOf(PartitionSelector.RandomOf(GlobalContext.ServiceName));
//PartitionSelector namedPartitionSelector = PartitionSelector.PartitionKeyOf(new Uri(GlobalContext.appName), "Partition1");
//await fabricClient.ClusterManager.(new ServicePar); ; //statefull
return(true);
}
/// <summary>
/// Executes a fail over test scenario.
/// </summary>
/// <param name="serviceName">Uri containing the service name.</param>
/// <param name="handler">Progress changed handler.</param>
/// <param name="duration">Duration of the test.</param>
/// <param name="stabilization">Duration of the stabilization period.</param>
/// <param name="token">CancellationToken instance.</param>
/// <returns>Task instance.</returns>
private async Task ExecuteFailoverTestScenarioAsync(Uri serviceName, ProgressChangedEventHandler handler, TimeSpan stabilization, CancellationToken token)
{
Console.WriteLine($"Starting Failover scenario test on {serviceName.AbsoluteUri} lasting {_duration.TotalMinutes} minutes.");
FailoverTestScenarioParameters ftsp = new FailoverTestScenarioParameters(PartitionSelector.RandomOf(serviceName), _duration, stabilization);
FailoverTestScenario fts = new FailoverTestScenario(_client, ftsp);
fts.ProgressChanged += handler;
try
{
await fts.ExecuteAsync(token);
}
catch (AggregateException ex) { CommonExceptionHandler.OutputInnerExceptions(ex); }
catch (Exception ex) { Console.WriteLine($"FaultTest.RunAsync Exception: {ex.Message} at {ex.StackTrace}"); }
}
public static PartitionSelector GetExpectedPartitionSelector(PartitionCase partitionCase)
{
PartitionSelector result = null;
switch (partitionCase)
{
case PartitionCase.PartitionId:
{
result = PartitionSelector.PartitionIdOf(ServiceName, PartitionID);
break;
}
case PartitionCase.ServiceNameRandomPartition:
{
result = PartitionSelector.RandomOf(ServiceName);
break;
}
case PartitionCase.ServiceNamePartitionSingleton:
{
result = PartitionSelector.SingletonOf(ServiceName);
break;
}
case PartitionCase.ServiceNamePartitionNamed:
{
result = PartitionSelector.PartitionKeyOf(ServiceName, PartitionKey);
break;
}
case PartitionCase.ServiceNamePartitionUniformedInt:
{
result = PartitionSelector.PartitionKeyOf(ServiceName, PartitionKeyLong);
break;
}
}
return(result);
}
static async Task RunFailoverTestScenarioAsync(string clusterConnection, Uri serviceName)
{
TimeSpan maxClusterStabilizationTimeout = TimeSpan.FromSeconds(180);
PartitionSelector randomPartitionSelector = PartitionSelector.RandomOf(serviceName);
FabricClient fabricClient = new FabricClient(clusterConnection);
TimeSpan timeToRun = TimeSpan.FromMinutes(60);
FailoverTestScenarioParameters scenarioParameters = new FailoverTestScenarioParameters(
randomPartitionSelector,
timeToRun,
maxClusterStabilizationTimeout);
FailoverTestScenario failoverScenario = new FailoverTestScenario(fabricClient, scenarioParameters);
try
{
await failoverScenario.ExecuteAsync(CancellationToken.None);
}
catch (AggregateException ae)
{
throw ae.InnerException;
}
}
public Task InsertCommandAsync(Command command)
{
Uri failoverManagerUri = new Uri("fabric:/System/FailoverManagerService");
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "****Adding command: " + command);
Task task = null;
if (command == Command.FailoverManagerDataLoss)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLoss];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, null);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeDataLossMidActionTestFatal)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFatal];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.InvokeDataLossMidActionTestTransient)
{
// rollback and retry then success
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestTransient];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeDataLossMidActionTestFailover)
{
// failover then success
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFailover];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatal)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatal];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossFailoverFaultAnalysisService)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossFailoverFaultAnalysisService];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep)
{
PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep];
task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeQuorumLossMidActionTestFatal)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFatal];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.InvokeQuorumLossMidActionTestFailover)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFailover];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.InvokeQuorumLossMidActionTestTransient)
{
Uri uri = new Uri("fabric:/System/NamingService");
PartitionSelector ps = PartitionSelector.RandomOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestTransient];
task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.RestartPartitionMidActionTestFatal)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFatal];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed);
}
else if (command == Command.RestartPartitionMidActionTestFailover)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFailover];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.RestartPartitionMidActionTestTransient)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry);
Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestTransient];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully);
}
else if (command == Command.StuckAction)
{
Guid id = MockClientCommandInfo[Command.StuckAction];
task = this.messageProcessor.ProcessStuckCommandAsync(id, null);
}
else if (command == Command.RestartPartitionCancelOuterLoopNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.None, // for this case this value does not matter
ServiceInternalFaultType.None,
RollbackState.RollingBackDueToUserCancel,
FASConstants.OuterLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelForwardNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.None,
ServiceInternalFaultType.None,
RollbackState.RollingBackDueToUserCancel,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelForwardExceptionNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.ForwardLoopExceptionBlock);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelOuterCleanupNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.OuterCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelCleanupInnerNoForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackDueToUserCancel,
FASConstants.InnerCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerNoForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel);
}
else if (command == Command.RestartPartitionCancelOuterLoopForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions, // for this case, this value does not matter
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelForwardForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelForwardExceptionForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.ForwardLoopExceptionBlock);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelOuterCleanupForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.OuterCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.RestartPartitionCancelCleanupInnerForce)
{
Uri uri = new Uri("fabric:/System/ClusterManagerService");
PartitionSelector ps = PartitionSelector.SingletonOf(uri);
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.PerformingActions,
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce,
FASConstants.InnerCleanupLoop);
Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerForce];
task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.TestRetryStepWithSuccessAfterRetries)
{
// Intentionally fail the step corresponding to StepStateNames.PerformingActions step a few times, then run it normally (pass). It should succeed.
Guid id = MockClientCommandInfo[Command.TestRetryStepWithSuccessAfterRetries];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.ThrowThreeTimes);
task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
}
else if (command == Command.TestRetryStepWithForceCancel)
{
// Force cancel a command with ActionStateBase.RetryStepWithoutRollingBackOnFailure set to true
Guid id = MockClientCommandInfo[Command.TestRetryStepWithForceCancel];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.CompletedSuccessfully, // this just has to be a late step so an earlier fault is not used before we reach the situation we want.
ServiceInternalFaultType.RollbackAction,
RollbackState.RollingBackForce, // note, the graceful one should not cause cancellation since for this type we only allow user cancellation when force is true
FASConstants.InnerForwardLoop,
StepStateNames.PerformingActions);
task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
}
else if (command == Command.StopNodeWithUnknownException)
{
Guid id = MockClientCommandInfo[Command.StopNodeWithUnknownException];
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.RollbackAction); // In this case, since start and stop node do not rollback like other commands, this exception should cause the step to retry.
Node target = ActionTest.GetNodeWithFASSecondary().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId);
task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo);
// Let the command make progress
Task.Delay(TimeSpan.FromSeconds(30)).Wait();
this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.NotRollingBack);
// This should not result in cancellation, since start and stop node have different rollback policies than the other commands.
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==false. This should not cancel the command", id);
this.messageProcessor.CancelTestCommandAsync(id, false);
this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.RollingBackDueToUserCancel);
// Now force cancel. This should cancel.
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==true. This should cancel the command", id);
this.messageProcessor.CancelTestCommandAsync(id, true);
this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce);
NodeList nodes = ActionTest.GetNodeListAsync().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - node info:", id);
foreach (Node n in nodes)
{
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, " OperationId:{0} - NodeName{1}, NodeStatus:{2}, IsStopped:{3}", id, n.NodeName, n.NodeStatus, n.IsStopped);
}
Node targetNodeAfterTest = nodes.Where(n => n.NodeName == target.NodeName).FirstOrDefault();
if (targetNodeAfterTest == null)
{
throw new InvalidOperationException("target node was not found in query after test");
}
if (targetNodeAfterTest.IsStopped == false)
{
throw new InvalidOperationException("target node should have IsStopped true, was false");
}
}
else if (command == Command.StopNodeWithExceptionAndSuccessAfterRetries)
{
Guid id = MockClientCommandInfo[Command.StopNodeWithExceptionAndSuccessAfterRetries];
// Inject a fault during the operation so that step "StepStateNames.MidPerformingActions" has to retry 3 times before succeeding
ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(
StepStateNames.MidPerformingActions,
ServiceInternalFaultType.ThrowThreeTimes);
Node target = ActionTest.GetNodeWithFASSecondary().Result;
TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId);
task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
// Start the stopped node
task = this.messageProcessor.ProcessStartNodeCommandAsync(Guid.NewGuid(), target.NodeName, target.NodeInstanceId, FASConstants.DefaultTestTimeout, faultInfo);
this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack);
}
else
{
ReleaseAssert.Failfast("Unexpected command");
}
return(task);
}
internal static ReplicaSelector GetReplicaSelector(string partitionSetName, Guid partitionId, Uri serviceName, string partitionKey, long?replicaOrInstanceId)
{
ReplicaSelector replicaSelector = null;
PartitionSelector partitionSelector = null;
if (partitionSetName.Contains("PartitionId"))
{
partitionSelector = PartitionSelector.PartitionIdOf(serviceName, partitionId);
}
else
{
if (partitionSetName.Contains("PartitionSingleton"))
{
partitionSelector = PartitionSelector.SingletonOf(serviceName);
}
else if (partitionSetName.Contains("PartitionNamed"))
{
partitionSelector = PartitionSelector.PartitionKeyOf(serviceName, partitionKey);
}
else if (partitionSetName.Contains("PartitionUniformedInt"))
{
long partitionKeyLong;
if (!long.TryParse(partitionKey, out partitionKeyLong))
{
throw new ArgumentException(StringResources.Error_InvalidPartitionKey);
}
partitionSelector = PartitionSelector.PartitionKeyOf(serviceName, partitionKeyLong);
}
else if (!partitionSetName.Contains("Partition"))
{
partitionSelector = PartitionSelector.RandomOf(serviceName);
}
}
if (partitionSelector == null)
{
throw new ArgumentException(StringResources.Error_CouldNotParsePartitionSelector);
}
if (partitionSetName.Contains("ReplicaPrimary"))
{
replicaSelector = ReplicaSelector.PrimaryOf(partitionSelector);
}
else if (partitionSetName.Contains("ReplicaRandomSecondary"))
{
replicaSelector = ReplicaSelector.RandomSecondaryOf(partitionSelector);
}
else if (partitionSetName.Contains("ReplicaId"))
{
replicaSelector = ReplicaSelector.ReplicaIdOf(partitionSelector, replicaOrInstanceId ?? 0);
}
else if (!partitionSetName.Contains("Replica"))
{
replicaSelector = ReplicaSelector.RandomOf(partitionSelector);
}
if (replicaSelector == null)
{
throw new ArgumentException(StringResources.Error_CouldNotParseReplicaSelector);
}
return(replicaSelector);
}
