protected PartitionSelector GetPartitionSelector() { if (this.ParameterSetName == "PartitionId") { return(PartitionSelector.PartitionIdOf(this.ServiceName, this.PartitionId)); } else { switch (this.ParameterSetName) { case "ServiceNameRandomPartition": return(PartitionSelector.RandomOf(this.ServiceName)); case "ServiceNamePartitionSingleton": return(PartitionSelector.SingletonOf(this.ServiceName)); case "ServiceNamePartitionNamed": return(PartitionSelector.PartitionKeyOf(this.ServiceName, this.PartitionKey)); case "ServiceNamePartitionUniformedInt": long partitionKeyLong; if (!long.TryParse(this.PartitionKey, out partitionKeyLong)) { throw new ArgumentException(StringResources.Error_InvalidPartitionKey); } return(PartitionSelector.PartitionKeyOf(this.ServiceName, partitionKeyLong)); default: throw new ArgumentException(StringResources.Error_CouldNotParsePartitionSelector); } } }
/// <summary> /// Restarts a random node. /// </summary> /// <param name="serviceName">Uri of the service in the format fabric:/[application]/[service name]</param> /// <returns>Task instance.</returns> public async Task RestartNodeAsync(Uri serviceName) { PartitionSelector randomPartitionSelector = PartitionSelector.RandomOf(serviceName); ReplicaSelector primaryofReplicaSelector = ReplicaSelector.PrimaryOf(randomPartitionSelector); // Create FabricClient with connection and security information here await _client.FaultManager.RestartNodeAsync(primaryofReplicaSelector, CompletionMode.Verify).ConfigureAwait(false); }
public async Task <bool> RestartGetSnowConditionsService() { var replicaSelector = ReplicaSelector.PrimaryOf(PartitionSelector.RandomOf(GlobalContext.ServiceName)); //PartitionSelector namedPartitionSelector = PartitionSelector.PartitionKeyOf(new Uri(GlobalContext.appName), "Partition1"); //await fabricClient.ClusterManager.(new ServicePar); ; //statefull return(true); }
/// <summary> /// Executes a fail over test scenario. /// </summary> /// <param name="serviceName">Uri containing the service name.</param> /// <param name="handler">Progress changed handler.</param> /// <param name="duration">Duration of the test.</param> /// <param name="stabilization">Duration of the stabilization period.</param> /// <param name="token">CancellationToken instance.</param> /// <returns>Task instance.</returns> private async Task ExecuteFailoverTestScenarioAsync(Uri serviceName, ProgressChangedEventHandler handler, TimeSpan stabilization, CancellationToken token) { Console.WriteLine($"Starting Failover scenario test on {serviceName.AbsoluteUri} lasting {_duration.TotalMinutes} minutes."); FailoverTestScenarioParameters ftsp = new FailoverTestScenarioParameters(PartitionSelector.RandomOf(serviceName), _duration, stabilization); FailoverTestScenario fts = new FailoverTestScenario(_client, ftsp); fts.ProgressChanged += handler; try { await fts.ExecuteAsync(token); } catch (AggregateException ex) { CommonExceptionHandler.OutputInnerExceptions(ex); } catch (Exception ex) { Console.WriteLine($"FaultTest.RunAsync Exception: {ex.Message} at {ex.StackTrace}"); } }
public static PartitionSelector GetExpectedPartitionSelector(PartitionCase partitionCase) { PartitionSelector result = null; switch (partitionCase) { case PartitionCase.PartitionId: { result = PartitionSelector.PartitionIdOf(ServiceName, PartitionID); break; } case PartitionCase.ServiceNameRandomPartition: { result = PartitionSelector.RandomOf(ServiceName); break; } case PartitionCase.ServiceNamePartitionSingleton: { result = PartitionSelector.SingletonOf(ServiceName); break; } case PartitionCase.ServiceNamePartitionNamed: { result = PartitionSelector.PartitionKeyOf(ServiceName, PartitionKey); break; } case PartitionCase.ServiceNamePartitionUniformedInt: { result = PartitionSelector.PartitionKeyOf(ServiceName, PartitionKeyLong); break; } } return(result); }
static async Task RunFailoverTestScenarioAsync(string clusterConnection, Uri serviceName) { TimeSpan maxClusterStabilizationTimeout = TimeSpan.FromSeconds(180); PartitionSelector randomPartitionSelector = PartitionSelector.RandomOf(serviceName); FabricClient fabricClient = new FabricClient(clusterConnection); TimeSpan timeToRun = TimeSpan.FromMinutes(60); FailoverTestScenarioParameters scenarioParameters = new FailoverTestScenarioParameters( randomPartitionSelector, timeToRun, maxClusterStabilizationTimeout); FailoverTestScenario failoverScenario = new FailoverTestScenario(fabricClient, scenarioParameters); try { await failoverScenario.ExecuteAsync(CancellationToken.None); } catch (AggregateException ae) { throw ae.InnerException; } }
public Task InsertCommandAsync(Command command) { Uri failoverManagerUri = new Uri("fabric:/System/FailoverManagerService"); TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "****Adding command: " + command); Task task = null; if (command == Command.FailoverManagerDataLoss) { PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); Guid id = MockClientCommandInfo[Command.FailoverManagerDataLoss]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, null); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.InvokeDataLossMidActionTestFatal) { PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction); Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFatal]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed); } else if (command == Command.InvokeDataLossMidActionTestTransient) { // rollback and retry then success PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry); Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestTransient]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.InvokeDataLossMidActionTestFailover) { // failover then success PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess); Guid id = MockClientCommandInfo[Command.InvokeDataLossMidActionTestFailover]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatal) { PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackAction); Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatal]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed); } else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry) { PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.CompletedSuccessfully, ServiceInternalFaultType.RollbackActionAndRetry); Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetry]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.FailoverManagerDataLossFailoverFaultAnalysisService) { PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.KillProcess); Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossFailoverFaultAnalysisService]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep) { PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction); Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackFatalBeforeActionStep]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed); } else if (command == Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep) { PartitionSelector ps = PartitionSelector.SingletonOf(failoverManagerUri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackActionAndRetry); Guid id = MockClientCommandInfo[Command.FailoverManagerDataLossCauseActionRollbackWithSuccessOnRetryBeforeActionStep]; task = this.messageProcessor.ProcessDataLossCommandAsync(id, ps, DataLossMode.FullDataLoss, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.InvokeQuorumLossMidActionTestFatal) { Uri uri = new Uri("fabric:/System/NamingService"); PartitionSelector ps = PartitionSelector.RandomOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction); Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFatal]; task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed); } else if (command == Command.InvokeQuorumLossMidActionTestFailover) { Uri uri = new Uri("fabric:/System/NamingService"); PartitionSelector ps = PartitionSelector.RandomOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess); Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestFailover]; task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.InvokeQuorumLossMidActionTestTransient) { Uri uri = new Uri("fabric:/System/NamingService"); PartitionSelector ps = PartitionSelector.RandomOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry); Guid id = MockClientCommandInfo[Command.InvokeQuorumLossMidActionTestTransient]; task = this.messageProcessor.ProcessQuorumLossCommandAsync(id, ps, QuorumLossMode.AllReplicas, TimeSpan.FromSeconds(10.0d), FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.RestartPartitionMidActionTestFatal) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction); Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFatal]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed); } else if (command == Command.RestartPartitionMidActionTestFailover) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.KillProcess); Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestFailover]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.RestartPartitionMidActionTestTransient) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo(StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackActionAndRetry); Guid id = MockClientCommandInfo[Command.RestartPartitionMidActionTestTransient]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully); } else if (command == Command.StuckAction) { Guid id = MockClientCommandInfo[Command.StuckAction]; task = this.messageProcessor.ProcessStuckCommandAsync(id, null); } else if (command == Command.RestartPartitionCancelOuterLoopNoForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.None, // for this case this value does not matter ServiceInternalFaultType.None, RollbackState.RollingBackDueToUserCancel, FASConstants.OuterLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopNoForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel); } else if (command == Command.RestartPartitionCancelForwardNoForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.None, ServiceInternalFaultType.None, RollbackState.RollingBackDueToUserCancel, FASConstants.ForwardLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardNoForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel); } else if (command == Command.RestartPartitionCancelForwardExceptionNoForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackDueToUserCancel, FASConstants.ForwardLoopExceptionBlock); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionNoForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel); } else if (command == Command.RestartPartitionCancelOuterCleanupNoForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackDueToUserCancel, FASConstants.OuterCleanupLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupNoForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel); } else if (command == Command.RestartPartitionCancelCleanupInnerNoForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackDueToUserCancel, FASConstants.InnerCleanupLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerNoForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackDueToUserCancel); } else if (command == Command.RestartPartitionCancelOuterLoopForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.PerformingActions, // for this case, this value does not matter ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackForce, FASConstants.ForwardLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterLoopForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce); } else if (command == Command.RestartPartitionCancelForwardForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackForce, FASConstants.ForwardLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce); } else if (command == Command.RestartPartitionCancelForwardExceptionForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackForce, FASConstants.ForwardLoopExceptionBlock); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelForwardExceptionForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce); } else if (command == Command.RestartPartitionCancelOuterCleanupForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackForce, FASConstants.OuterCleanupLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelOuterCleanupForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce); } else if (command == Command.RestartPartitionCancelCleanupInnerForce) { Uri uri = new Uri("fabric:/System/ClusterManagerService"); PartitionSelector ps = PartitionSelector.SingletonOf(uri); ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.PerformingActions, ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackForce, FASConstants.InnerCleanupLoop); Guid id = MockClientCommandInfo[Command.RestartPartitionCancelCleanupInnerForce]; task = this.messageProcessor.ProcessRestartPartitionCommandAsync(id, ps, RestartPartitionMode.AllReplicasOrInstances, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce); } else if (command == Command.TestRetryStepWithSuccessAfterRetries) { // Intentionally fail the step corresponding to StepStateNames.PerformingActions step a few times, then run it normally (pass). It should succeed. Guid id = MockClientCommandInfo[Command.TestRetryStepWithSuccessAfterRetries]; ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.MidPerformingActions, ServiceInternalFaultType.ThrowThreeTimes); task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack); } else if (command == Command.TestRetryStepWithForceCancel) { // Force cancel a command with ActionStateBase.RetryStepWithoutRollingBackOnFailure set to true Guid id = MockClientCommandInfo[Command.TestRetryStepWithForceCancel]; ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.CompletedSuccessfully, // this just has to be a late step so an earlier fault is not used before we reach the situation we want. ServiceInternalFaultType.RollbackAction, RollbackState.RollingBackForce, // note, the graceful one should not cause cancellation since for this type we only allow user cancellation when force is true FASConstants.InnerForwardLoop, StepStateNames.PerformingActions); task = this.messageProcessor.ProcessRetryStepCommandAsync(id, faultInfo); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce); } else if (command == Command.StopNodeWithUnknownException) { Guid id = MockClientCommandInfo[Command.StopNodeWithUnknownException]; ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.MidPerformingActions, ServiceInternalFaultType.RollbackAction); // In this case, since start and stop node do not rollback like other commands, this exception should cause the step to retry. Node target = ActionTest.GetNodeWithFASSecondary().Result; TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId); task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo); // Let the command make progress Task.Delay(TimeSpan.FromSeconds(30)).Wait(); this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.NotRollingBack); // This should not result in cancellation, since start and stop node have different rollback policies than the other commands. TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==false. This should not cancel the command", id); this.messageProcessor.CancelTestCommandAsync(id, false); this.WaitForState(id, StepStateNames.PerformingActions, RollbackState.RollingBackDueToUserCancel); // Now force cancel. This should cancel. TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - cancelling with force==true. This should cancel the command", id); this.messageProcessor.CancelTestCommandAsync(id, true); this.WaitForState(id, StepStateNames.Failed, RollbackState.RollingBackForce); NodeList nodes = ActionTest.GetNodeListAsync().Result; TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} - node info:", id); foreach (Node n in nodes) { TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, " OperationId:{0} - NodeName{1}, NodeStatus:{2}, IsStopped:{3}", id, n.NodeName, n.NodeStatus, n.IsStopped); } Node targetNodeAfterTest = nodes.Where(n => n.NodeName == target.NodeName).FirstOrDefault(); if (targetNodeAfterTest == null) { throw new InvalidOperationException("target node was not found in query after test"); } if (targetNodeAfterTest.IsStopped == false) { throw new InvalidOperationException("target node should have IsStopped true, was false"); } } else if (command == Command.StopNodeWithExceptionAndSuccessAfterRetries) { Guid id = MockClientCommandInfo[Command.StopNodeWithExceptionAndSuccessAfterRetries]; // Inject a fault during the operation so that step "StepStateNames.MidPerformingActions" has to retry 3 times before succeeding ServiceInternalFaultInfo faultInfo = new ServiceInternalFaultInfo( StepStateNames.MidPerformingActions, ServiceInternalFaultType.ThrowThreeTimes); Node target = ActionTest.GetNodeWithFASSecondary().Result; TestabilityTrace.TraceSource.WriteInfo(MockClient.TraceType, "{0} stopping {1}:{2}", id, target.NodeName, target.NodeInstanceId); task = this.messageProcessor.ProcessStopNodeCommandAsync(id, target.NodeName, target.NodeInstanceId, 999, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack); // Start the stopped node task = this.messageProcessor.ProcessStartNodeCommandAsync(Guid.NewGuid(), target.NodeName, target.NodeInstanceId, FASConstants.DefaultTestTimeout, faultInfo); this.WaitForState(id, StepStateNames.CompletedSuccessfully, RollbackState.NotRollingBack); } else { ReleaseAssert.Failfast("Unexpected command"); } return(task); }
internal static ReplicaSelector GetReplicaSelector(string partitionSetName, Guid partitionId, Uri serviceName, string partitionKey, long?replicaOrInstanceId) { ReplicaSelector replicaSelector = null; PartitionSelector partitionSelector = null; if (partitionSetName.Contains("PartitionId")) { partitionSelector = PartitionSelector.PartitionIdOf(serviceName, partitionId); } else { if (partitionSetName.Contains("PartitionSingleton")) { partitionSelector = PartitionSelector.SingletonOf(serviceName); } else if (partitionSetName.Contains("PartitionNamed")) { partitionSelector = PartitionSelector.PartitionKeyOf(serviceName, partitionKey); } else if (partitionSetName.Contains("PartitionUniformedInt")) { long partitionKeyLong; if (!long.TryParse(partitionKey, out partitionKeyLong)) { throw new ArgumentException(StringResources.Error_InvalidPartitionKey); } partitionSelector = PartitionSelector.PartitionKeyOf(serviceName, partitionKeyLong); } else if (!partitionSetName.Contains("Partition")) { partitionSelector = PartitionSelector.RandomOf(serviceName); } } if (partitionSelector == null) { throw new ArgumentException(StringResources.Error_CouldNotParsePartitionSelector); } if (partitionSetName.Contains("ReplicaPrimary")) { replicaSelector = ReplicaSelector.PrimaryOf(partitionSelector); } else if (partitionSetName.Contains("ReplicaRandomSecondary")) { replicaSelector = ReplicaSelector.RandomSecondaryOf(partitionSelector); } else if (partitionSetName.Contains("ReplicaId")) { replicaSelector = ReplicaSelector.ReplicaIdOf(partitionSelector, replicaOrInstanceId ?? 0); } else if (!partitionSetName.Contains("Replica")) { replicaSelector = ReplicaSelector.RandomOf(partitionSelector); } if (replicaSelector == null) { throw new ArgumentException(StringResources.Error_CouldNotParseReplicaSelector); } return(replicaSelector); }