public RunspaceGroup.Token Take() { Runspace runspace = null; bool createNew = false; lock (this.thisLock) { if (this.runspaces.IsEmpty && this.currentStackSize < GroupSizeMax) { createNew = true; this.currentStackSize++; } } if (createNew) { runspace = this.CreateRunspace(); } TimeoutHelper helper = new TimeoutHelper(MaxRunspaceWaitTime); while (runspace == null) { if (!this.runspaces.TryPop(out runspace)) { this.runSpaceAvailableEvent.WaitOne(TimeSpan.FromSeconds(5)); } ThrowIf.IsTrue(helper.GetRemainingTime() == TimeSpan.Zero, "Could not get a RunSpace in {0} minutes. Check if something is stuck", MaxRunspaceWaitTime); } return(new Token(this, runspace)); }
private static WinFabricRepairTask Convert(RepairTask repairTask) { ThrowIf.Null(repairTask, "repairTask"); return(new WinFabricRepairTask() { TaskId = repairTask.TaskId, Version = repairTask.Version, Description = repairTask.Description, State = Convert(repairTask.State), Flags = Convert(repairTask.Flags), Action = repairTask.Action, Target = Convert(repairTask.Target), Executor = repairTask.Executor, ExecutorData = repairTask.ExecutorData, Impact = Convert(repairTask.Impact), ResultStatus = Convert(repairTask.ResultStatus), ResultCode = repairTask.ResultCode, ResultDetails = repairTask.ResultDetails, CreatedTimestamp = repairTask.CreatedTimestamp, ClaimedTimestamp = repairTask.ClaimedTimestamp, PreparingTimestamp = repairTask.PreparingTimestamp, ApprovedTimestamp = repairTask.ApprovedTimestamp, ExecutingTimestamp = repairTask.ExecutingTimestamp, RestoringTimestamp = repairTask.RestoringTimestamp, CompletedTimestamp = repairTask.CompletedTimestamp }); }
private static NodeImpact Convert(WinFabricNodeImpact winFabricNodeImpact) { ThrowIf.Null(winFabricNodeImpact, "winFabricNodeImpact"); return(new NodeImpact() { NodeName = winFabricNodeImpact.NodeName, ImpactLevel = Convert(winFabricNodeImpact.ImpactLevel) }); }
private static WinFabricNodeImpact Convert(NodeImpact nodeImpact) { ThrowIf.Null(nodeImpact, "nodeImpact"); return(new WinFabricNodeImpact() { NodeName = nodeImpact.NodeName, ImpactLevel = Convert(nodeImpact.ImpactLevel) }); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, StopNodeAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.NodeName, "NodeName"); this.helper = new TimeoutHelper(action.ActionTimeout); string nodeName = action.NodeName; BigInteger nodeInstance = action.NodeInstance; if (nodeInstance == BigInteger.MinusOne) { var nodeInfo = await GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken); nodeInstance = nodeInfo.NodeInstanceId; } await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.StopNodeUsingNodeNameAsync( nodeName, nodeInstance, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (action.CompletionMode == CompletionMode.Verify) { bool success = false; while (this.helper.GetRemainingTime() > TimeSpan.Zero) { var nodeInfo = await this.GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken).ConfigureAwait(false); if (!nodeInfo.IsNodeUp) { success = true; break; } ActionTraceSource.WriteInfo(TraceSource, "NodeName = {0} not yet stopped. Retrying...", action.NodeName); await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); } if (!success) { throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "StopNode", nodeName)); } } action.Result = new StopNodeResult(nodeName, nodeInstance); ResultTraceString = StringHelper.Format("StopNodeAction succeeded for {0}:{1} with CompletionMode = {2}", nodeName, nodeInstance, action.CompletionMode); }
public static TestServicePartitionInfo Convert(ResolvedServicePartition resolvedPartition, ResolvedServiceEndpoint resolvedEndpoint, int partitionIdentifierNumber) { ThrowIf.Null(resolvedPartition, "resolvedPartition"); ThrowIf.Null(resolvedEndpoint, "resolvedEndpoint"); ServicePartitionKind keyType = ServicePartitionKind.Invalid; object rangeHighKey = null; object rangeLowKey = null; switch (resolvedPartition.Info.Kind) { case ServicePartitionKind.Singleton: keyType = ServicePartitionKind.Singleton; break; case ServicePartitionKind.Int64Range: keyType = ServicePartitionKind.Int64Range; rangeHighKey = ((System.Fabric.Int64RangePartitionInformation)resolvedPartition.Info).HighKey; rangeLowKey = ((System.Fabric.Int64RangePartitionInformation)resolvedPartition.Info).LowKey; break; case ServicePartitionKind.Named: keyType = ServicePartitionKind.Named; break; default: throw new InvalidOperationException("Unknown ServicePartitionKind " + resolvedPartition.Info.Kind.ToString() + "."); } TestServicePartitionInfo testServicePartitionInfo = new TestServicePartitionInfo() { Location = resolvedEndpoint.Address, Name = resolvedPartition.ServiceName, Id = resolvedPartition.Info.Id, KeyType = keyType, RangeHighKey = rangeHighKey, RangeLowKey = rangeLowKey, Role = resolvedEndpoint.Role, PartitionIdentifier = partitionIdentifierNumber, }; if (resolvedPartition.Endpoints.Any(e => e.Role == ServiceEndpointRole.StatefulPrimary)) { testServicePartitionInfo.IsPrimaryEndpoint = resolvedPartition.GetEndpoint().Address == resolvedEndpoint.Address; } else { testServicePartitionInfo.IsPrimaryEndpoint = false; } return(testServicePartitionInfo); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, GetSelectedPartitionStateAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.PartitionSelector, "PartitionSelector"); Guid partitionId; Uri serviceName; if (!action.PartitionSelector.TryGetPartitionIdIfNotGetServiceName(out partitionId, out serviceName)) { // TODO: make these actions which store state locally as well. ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <ServicePartitionList>( () => testContext.FabricClient.QueryManager.GetPartitionListAsync( serviceName, null, default(string), action.RequestTimeout, cancellationToken), action.ActionTimeout, cancellationToken).ConfigureAwait(false); Partition partitionResult = action.PartitionSelector.GetSelectedPartition(partitionsResult.ToArray(), testContext.Random); partitionId = partitionResult.PartitionInformation.Id; } else { // Validate the partition specified is actually from the service specified. // Intentionally do not use FabricClientRetryErrors.GetPartitionListFabricErrors. We do not want to retry "service not found". ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <ServicePartitionList>( () => testContext.FabricClient.QueryManager.GetPartitionListAsync( serviceName, null, default(string), action.RequestTimeout, cancellationToken), action.ActionTimeout, cancellationToken).ConfigureAwait(false); var guids = partitionsResult.Select(p => p.PartitionId()).ToList(); if (!guids.Contains(partitionId)) { // The message in the first arg is only for debugging, it is not returned to the user. throw new FabricException("Partition not found", FabricErrorCode.PartitionNotFound); } } action.Result = new SelectedPartition(serviceName, partitionId); ResultTraceString = StringHelper.Format("PartitionSelector Selected Partition with ID {0}", action.Result); }
internal static NodeInfo CreateNodeInfo(Node nodeResult) { ThrowIf.Null(nodeResult, "nodeResult"); NodeInfo nodeInfo = new NodeInfo(nodeResult.IpAddressOrFQDN, nodeResult.NodeName, nodeResult.NodeId, nodeResult.NodeInstanceId, nodeResult.IsSeedNode, nodeResult.HealthState) { UpgradeDomain = nodeResult.UpgradeDomain, FaultDomain = nodeResult.FaultDomain }; nodeInfo.IsNodeUp = nodeResult.IsNodeUp(); nodeInfo.NodeType = nodeResult.NodeType; return(nodeInfo); }
public static void UpdateRepairTask(WinFabricRepairTask winFabricRepairTask, RepairTask repairTask) { ThrowIf.Null(winFabricRepairTask, "winFabricRepairTask"); repairTask.Version = winFabricRepairTask.Version; repairTask.Description = winFabricRepairTask.Description; repairTask.State = Convert(winFabricRepairTask.State); repairTask.Target = Convert(winFabricRepairTask.Target); repairTask.Executor = winFabricRepairTask.Executor; repairTask.ExecutorData = winFabricRepairTask.ExecutorData; repairTask.Impact = Convert(winFabricRepairTask.Impact); repairTask.ResultStatus = Convert(winFabricRepairTask.ResultStatus); repairTask.ResultCode = winFabricRepairTask.ResultCode; repairTask.ResultDetails = winFabricRepairTask.ResultDetails; }
// Running multiple times should be safe. private async Task LoadPartitionAndReplicaCountAsync(CancellationToken ct) { // Optimization. No need for more than one thread to enter this method. await this.asyncMutex.WaitAsync(); try { if (this.isStateful.HasValue && this.partitionCount.HasValue && this.targetReplicaSetSize.HasValue) { // values already loaded return; } ServicePartitionList servicePartitions = await this.GetPartitionsAsync(ct).ConfigureAwait(false); // Make sure servicePartitions has at least one item. ThrowIf.NullOrEmpty(servicePartitions, "servicePartitions"); // set PartitionCount ReleaseAssert.AssertIfNot(this.partitionCount.TrySetValue(servicePartitions.Count), "partitionCount has already been set to a different value"); // set isStateful field Partition partition = servicePartitions[0]; bool stateful = partition is StatefulServicePartition; ReleaseAssert.AssertIfNot(this.isStateful.TrySetValue(stateful), "isStateful has already been set to a different value"); // retrieve replicaCount if (stateful) { var statefulServicePartition = partition as StatefulServicePartition; ReleaseAssert.AssertIfNot(this.targetReplicaSetSize.TrySetValue((int)statefulServicePartition.TargetReplicaSetSize), "targetReplicaSetSize has already been set to a different value"); } else { var statelessServicePartition = partition as StatelessServicePartition; ReleaseAssert.AssertIfNot(this.targetReplicaSetSize.TrySetValue((int)statelessServicePartition.InstanceCount), "targetReplicaSetSize has already been set to a different value"); } } catch (Exception e) { TestabilityTrace.TraceSource.WriteError(TraceSource, "Error while getting partitions for service {0}. Exception: {1}", this.serviceName, e.Message); throw; } finally { this.asyncMutex.Release(); } }
public static Task WaitAsync(TimeSpan duration, CancellationToken cancellationToken) { ThrowIf.OutOfRange(duration, TimeSpan.Zero, TimeSpan.MaxValue, "duration"); int durationInMilliseconds; if (duration.TotalMilliseconds > int.MaxValue) { durationInMilliseconds = -1; } else { durationInMilliseconds = (int)duration.TotalMilliseconds; } return(Task.Delay(durationInMilliseconds, cancellationToken)); }
internal static void PerformWithRetry(Action action, Type[] exceptionTypes, TimeSpan retryInterval, TimeSpan timeout) { bool firstRun = true; var timeoutHelper = new TimeoutHelper(timeout); Exception lastException = null; while (!TimeoutHelper.HasExpired(timeoutHelper)) { try { if (!firstRun) { System.Threading.Thread.Sleep(retryInterval); } action(); return; } catch (Exception ex) { bool exceptionSupported = false; foreach (Type type in exceptionTypes) { if (ex.GetType() == type) { exceptionSupported = true; lastException = ex; break; } } if (!exceptionSupported) { AppTrace.TraceSource.WriteError("PerformWithRetry", "PerformWithRetry hit unexpected exception {0}\nStack:\n{1}", ex.GetType().ToString(), ex.StackTrace); throw; } AppTrace.TraceSource.WriteWarning("PerformWithRetry", String.Format("PerformWithRetry retrying action after exception {0}\nStack:\n{1}", ex.GetType().ToString(), ex.StackTrace)); firstRun = false; } } ThrowIf.Null(lastException, "lastException"); throw lastException; }
public static RepairTask Convert(WinFabricRepairTask winFabricRepairTask) { ThrowIf.Null(winFabricRepairTask, "winFabricRepairTask"); var repairTask = new ClusterRepairTask(winFabricRepairTask.TaskId, winFabricRepairTask.Action) { Version = winFabricRepairTask.Version, Description = winFabricRepairTask.Description, State = Convert(winFabricRepairTask.State), Target = Convert(winFabricRepairTask.Target), Executor = winFabricRepairTask.Executor, ExecutorData = winFabricRepairTask.ExecutorData, Impact = Convert(winFabricRepairTask.Impact), ResultStatus = Convert(winFabricRepairTask.ResultStatus), ResultCode = winFabricRepairTask.ResultCode, ResultDetails = winFabricRepairTask.ResultDetails }; return(repairTask); }
internal static void PerformWithRetry(Action action, Type[] exceptionTypes, TimeSpan retryInterval, int retryCount = 3) { Exception lastException = null; for (int retry = 0; retry < retryCount; retry++) { try { if (retry > 0) { System.Threading.Thread.Sleep(retryInterval); } action(); return; } catch (Exception ex) { bool exceptionSupported = false; foreach (Type type in exceptionTypes) { if (ex.GetType() == type) { exceptionSupported = true; lastException = ex; break; } } if (!exceptionSupported) { AppTrace.TraceSource.WriteError("PerformWithRetry", "PerformWithRetry hit unexpected exception {0}\nStack:\n{1}", ex.GetType().ToString(), ex.StackTrace); throw; } AppTrace.TraceSource.WriteWarning("PerformWithRetry", String.Format("PerformWithRetry retrying action after exception {0}\nStack:\n{1}", ex.GetType().ToString(), ex.StackTrace)); } } ThrowIf.Null(lastException, "lastException"); throw lastException; }
public ServiceQueryClient( Uri serviceName, FabricTestContext testContext, ValidationCheckFlag checkFlags, TimeSpan operationTimeout, TimeSpan requestTimeout) { ThrowIf.Null(serviceName, "serviceName"); ThrowIf.Null(testContext, "testContext"); // initialize lock and synchronized fields. this.asyncMutex = new SemaphoreSlim(1); // only 1 thread can access CS. this.isStateful = new SynchronizedModifyOnce <bool>(); this.partitionCount = new SynchronizedModifyOnce <int>(); this.targetReplicaSetSize = new SynchronizedModifyOnce <int>(); this.TestContext = testContext; this.serviceName = serviceName; this.isSystemService = serviceName.OriginalString.StartsWith(SystemApplicationName); this.checkFlags = checkFlags; this.operationTimeout = operationTimeout; this.requestTimeout = requestTimeout; }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartNodeAction action, CancellationToken cancellationToken) { ActionTraceSource.WriteInfo(TraceSource, "Enter RestartNodeAction/ExecuteActionAsync: operationTimeout='{0}', requestTimeout='{1}'", action.ActionTimeout, action.RequestTimeout); this.helper = new TimeoutHelper(action.ActionTimeout); SelectedReplica selectedReplica = SelectedReplica.None; string nodeName = action.NodeName; BigInteger nodeInstance = action.NodeInstance; bool createFabricDump = action.CreateFabricDump; if (string.IsNullOrEmpty(nodeName)) { ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector"); var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false); var replicaStateActionResult = getReplicaStateAction.Result; ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null"); selectedReplica = replicaStateActionResult.Item1; Replica replicaStateResult = replicaStateActionResult.Item2; ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null"); nodeName = replicaStateResult.NodeName; nodeInstance = BigInteger.MinusOne; } if (nodeInstance == BigInteger.MinusOne) { var nodeInfo = await this.GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken).ConfigureAwait(false); nodeInstance = nodeInfo.NodeInstanceId; } await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.RestartNodeUsingNodeNameAsync( nodeName, nodeInstance, createFabricDump, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (action.CompletionMode == CompletionMode.Verify) { bool success = false; while (this.helper.GetRemainingTime() > TimeSpan.Zero) { var nodeInfo = await this.GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken).ConfigureAwait(false); if (nodeInfo.NodeInstanceId > nodeInstance && nodeInfo.IsNodeUp) { success = true; break; } ActionTraceSource.WriteInfo(TraceSource, "NodeName = {0} not yet restarted. '{1}' seconds remain. Retrying...", nodeName, this.helper.GetRemainingTime().TotalSeconds); await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); } if (!success) { throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "RestartNode", nodeName)); } } // create result action.Result = new RestartNodeResult(selectedReplica, new NodeResult(nodeName, nodeInstance)); ResultTraceString = StringHelper.Format("RestartNodeAction succeeded for {0}:{1} with CompletionMode = {2}", nodeName, nodeInstance, action.CompletionMode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartDeployedCodePackageAction action, CancellationToken cancellationToken) { this.helper = new TimeoutHelper(action.ActionTimeout); string nodeName = action.NodeName; Uri applicationName = action.ApplicationName; string serviceManifestName = action.ServiceManifestName; string servicePackageActivationId = action.ServicePackageActivationId; string codePackageName = action.CodePackageName; SelectedReplica replicaSelectorResult = SelectedReplica.None; ThrowIf.Null(applicationName, "ApplicationName"); if (string.IsNullOrEmpty(nodeName) || string.IsNullOrEmpty(serviceManifestName) || string.IsNullOrEmpty(codePackageName)) { ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector"); var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = this.helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false); var replicaStateActionResult = getReplicaStateAction.Result; ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null"); replicaSelectorResult = replicaStateActionResult.Item1; ReleaseAssert.AssertIf(replicaSelectorResult == null || replicaSelectorResult.SelectedPartition == null, "replicaSelectorResult cannot be null or for a non-null replicaSelectorResult, the selected partition must be non-null"); Guid partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId; Replica replicaStateResult = replicaStateActionResult.Item2; ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null"); nodeName = replicaStateResult.NodeName; var deployedReplicaListResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <DeployedServiceReplicaList>( () => testContext.FabricClient.QueryManager.GetDeployedReplicaListAsync( nodeName, applicationName, null, partitionId, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); DeployedServiceReplica selectedReplica = deployedReplicaListResult.FirstOrDefault(r => r.Partitionid == partitionId); if (selectedReplica == null) { throw new FabricException( StringHelper.Format(StringResources.Error_DidNotFindDeployedReplicaOnNode, partitionId, nodeName), FabricErrorCode.ReplicaDoesNotExist); } serviceManifestName = selectedReplica.ServiceManifestName; servicePackageActivationId = selectedReplica.ServicePackageActivationId; codePackageName = selectedReplica.CodePackageName; } ActionTraceSource.WriteInfo(TraceSource, "SelectedReplica: serviceManifestName: {0}, servicePackageActivationId: {1}, codePackageName: {2}", serviceManifestName, servicePackageActivationId, codePackageName); DeployedCodePackage deployedCodePackageListResult = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false); var codepackageEntrypointToRestart = GetCodepackageEntrypointToRestart(action, deployedCodePackageListResult); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.RestartDeployedCodePackageUsingNodeNameAsync( nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (action.CompletionMode == CompletionMode.Verify) { bool success = false; while (this.helper.GetRemainingTime() > TimeSpan.Zero) { var deployedCodePackageListResultAfterRestart = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false); if (deployedCodePackageListResultAfterRestart != null) { var entryPointAfterRestart = codepackageEntrypointToRestart.EntryPointType == EntryPointType.Main ? deployedCodePackageListResultAfterRestart.EntryPoint : deployedCodePackageListResultAfterRestart.SetupEntryPoint; if (entryPointAfterRestart != null && entryPointAfterRestart.CodePackageInstanceId > codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId && entryPointAfterRestart.EntryPointStatus == EntryPointStatus.Started) { success = true; break; } } ActionTraceSource.WriteInfo(TraceSource, "CodePackage = {0}:{1}:{2} not yet restarted. Retrying...", nodeName, applicationName, codePackageName); await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false); } if (!success) { throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "RestartDeployedCodePackage", applicationName)); } } action.Result = new RestartDeployedCodePackageResult( nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId, replicaSelectorResult); ResultTraceString = StringHelper.Format("RestartCodePackageAction succeeded for {0}:{1}:{2} with CompletionMode = {3}", nodeName, applicationName, codePackageName, action.CompletionMode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RemoveReplicaAction action, CancellationToken cancellationToken) { TimeoutHelper helper = new TimeoutHelper(action.ActionTimeout); string nodeName = action.NodeName; Guid? partitionId = action.PartitionId; long? replicaId = action.ReplicaId; SelectedReplica replicaSelectorResult = SelectedReplica.None; if (string.IsNullOrEmpty(nodeName) || !partitionId.HasValue || !replicaId.HasValue) { ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector"); var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false); var replicaStateActionResult = getReplicaStateAction.Result; ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null"); replicaSelectorResult = replicaStateActionResult.Item1; partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId; Replica replicaStateResult = replicaStateActionResult.Item2; ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null"); nodeName = replicaStateResult.NodeName; replicaId = replicaStateResult.Id; } ThrowIf.IsFalse(partitionId.HasValue, "PartitionID"); ThrowIf.IsFalse(replicaId.HasValue, "ReplicaID"); bool forceRemove = action.ForceRemove; await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.ServiceManager.RemoveReplicaAsync( nodeName, partitionId.Value, replicaId.Value, forceRemove, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RemoveReplicaErrors.Value, helper.GetRemainingTime(), cancellationToken); if (action.CompletionMode == CompletionMode.Verify) { // Check that replica on selected node has been removed i.e. the replica id does not exist anymore. bool success = false; while (helper.GetRemainingTime() > TimeSpan.Zero) { var replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( partitionId.Value, replicaId.Value, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); bool dropped = replicasResult.Count == 0; if (!dropped) { // Since we added a replica filter the result should contain the replica or none ReleaseAssert.AssertIf(replicasResult.Count > 1, "More than 1 replica returned with replica filter {0}:{1}", partitionId.Value, replicaId.Value); ReleaseAssert.AssertIf(replicasResult[0].Id != replicaId, "Incorrect replica Id {0} returned by query instead of {1}", replicasResult[0].Id, replicaId); dropped = replicasResult[0].ReplicaStatus == ServiceReplicaStatus.Dropped; } if (dropped) { success = true; break; } ActionTraceSource.WriteInfo(TraceSource, "Replica = {0}:{1} not yet completely removed. Retrying...", partitionId.Value, replicaId.Value); await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); } if (!success) { throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "RemoveReplica", StringHelper.Format("{0}:{1}", partitionId.Value, replicaId.Value))); } } action.Result = new RemoveReplicaResult(replicaSelectorResult); ResultTraceString = StringHelper.Format( "RemoveReplicaOrInstance succeeded by removing replica {0}:{1} on node {2} with CompletionMode {3}", partitionId.Value, replicaId.Value, nodeName, action.CompletionMode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, MovePrimaryAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.PartitionSelector, "PartitionSelector"); this.helper = new TimeoutHelper(action.ActionTimeout); string newPrimaryNodeName = action.NodeName; var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = this.helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken); Guid partitionId = getPartitionStateAction.Result.PartitionId; if (!action.IgnoreConstraints) { // select random node where replica's primary not present var nodesInfo = await testContext.FabricCluster.GetLatestNodeInfoAsync(action.RequestTimeout, this.helper.GetRemainingTime(), cancellationToken); if ((nodesInfo == null || nodesInfo.Count() == 0)) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_NotEnoughNodesForTestabilityAction, "MovePrimary")); } ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( partitionId, 0, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); NodeInfo currentPrimaryNodeInfo = null; string currentPrimaryNodeName = string.Empty; foreach (var replica in replicasResult) { StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica; if (statefulReplica == null) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "MovePrimary", "Stateful", action.PartitionSelector.ServiceName, "Stateless")); } if (statefulReplica.ReplicaRole == ReplicaRole.Primary) { currentPrimaryNodeInfo = nodesInfo.FirstOrDefault(n => n.NodeName == statefulReplica.NodeName); if (!string.IsNullOrEmpty(newPrimaryNodeName) && newPrimaryNodeName == statefulReplica.NodeName) { throw new FabricException( StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newPrimaryNodeName, "MovePrimary", "Primary already exists on node"), FabricErrorCode.AlreadyPrimaryReplica); } break; } } if (currentPrimaryNodeInfo == null) { throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, action.PartitionSelector + ":" + partitionId), FabricErrorCode.NotReady); } currentPrimaryNodeName = currentPrimaryNodeInfo.NodeName; if (newPrimaryNodeName == currentPrimaryNodeName) { throw new FabricException( StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newPrimaryNodeName, "MovePrimary", "Primary already exists on node"), FabricErrorCode.AlreadyPrimaryReplica); } } ActionTraceSource.WriteInfo(TraceSource, "Calling move primary with node {0}, partition {1}", string.IsNullOrEmpty(newPrimaryNodeName) ? "Random" : newPrimaryNodeName, partitionId); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.MovePrimaryUsingNodeNameAsync( newPrimaryNodeName, getPartitionStateAction.Result.ServiceName, partitionId, action.IgnoreConstraints, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.MovePrimaryFabricErrors.Value, this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); action.Result = new MovePrimaryResult(newPrimaryNodeName, getPartitionStateAction.Result); ResultTraceString = StringHelper.Format("MovePrimaryAction succeeded for moving Primary for {0} to node {1}.", partitionId, newPrimaryNodeName); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartPartitionAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.PartitionSelector, "partitionSelector"); this.helper = new TimeoutHelper(action.ActionTimeout); // get service info so we can validate if the operation is valid ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.ServiceManager.GetServiceDescriptionAsync( action.PartitionSelector.ServiceName, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (result.Kind != ServiceDescriptionKind.Stateful && action.RestartPartitionMode == RestartPartitionMode.OnlyActiveSecondaries) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "RestartPartitionMode.OnlyActiveSecondaries", "Stateful", action.PartitionSelector.ServiceName, "Stateless")); } bool hasPersistedState = false; if (result.Kind == ServiceDescriptionKind.Stateful) { StatefulServiceDescription statefulDescription = result as StatefulServiceDescription; ReleaseAssert.AssertIf(statefulDescription == null, "Stateful service description is not WinFabricStatefulServiceDescription"); hasPersistedState = statefulDescription.HasPersistedState; } // now actually select a partition var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken); Guid partitionId = getPartitionStateAction.Result.PartitionId; // get replicas for target ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( partitionId, 0, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); // get replicas for fm in order to get the primary ServiceReplicaList fmReplicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( Constants.FmPartitionId, 0, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); string fmPrimaryNodeName = string.Empty; var readyFMReplicas = fmReplicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray(); foreach (var replica in readyFMReplicas) { StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica; ReleaseAssert.AssertIf(statefulReplica == null, "FM Replica is not a stateful replica"); if (statefulReplica.ReplicaRole == ReplicaRole.Primary) { fmPrimaryNodeName = replica.NodeName; } } if (string.IsNullOrEmpty(fmPrimaryNodeName)) { throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, "FailoverManager"), FabricErrorCode.NotReady); } ////------------------------------------------------------ // target ut at the fm primary only UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "DoReconfiguration"); behavior.AddFilterForPartitionId(partitionId); string behaviorName = "BlockDoReconfiguration"; await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync( fmPrimaryNodeName, behaviorName, behavior, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); // TODO: Wait for some time so that the unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false); bool triedToRemovedBehavior = false; // inspect the actual replicas to restart, only operate on stable ones try { var stableReplicasToRestart = replicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray(); foreach (var replica in stableReplicasToRestart) { var currentReplica = replica; if (action.RestartPartitionMode == RestartPartitionMode.OnlyActiveSecondaries) { StatefulServiceReplica statefulReplica = currentReplica as StatefulServiceReplica; ReleaseAssert.AssertIf(statefulReplica == null, "Stateful service replica is not StatefulServiceReplica"); if (statefulReplica.ReplicaRole == ReplicaRole.Primary) { continue; } } if (hasPersistedState) { await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.RestartReplicaAsync( currentReplica.NodeName, partitionId, currentReplica.Id, CompletionMode.DoNotVerify, action.RequestTimeout.TotalSeconds, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); } else { await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.RemoveReplicaAsync( currentReplica.NodeName, partitionId, currentReplica.Id, CompletionMode.DoNotVerify, false, /*force remove*/ action.RequestTimeout.TotalSeconds, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); } } triedToRemovedBehavior = true; await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync( fmPrimaryNodeName, behaviorName, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value, this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); // TODO: Wait for some time so that the unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied await Task.Delay(TimeSpan.FromSeconds(5.0)).ConfigureAwait(false); } finally { // TODO: Provide a way to clear all behaviors just in case. if (!triedToRemovedBehavior) { ActionTraceSource.WriteWarning(TraceType, "Exception after adding behavior to block messages. Removing behavior synchronously"); FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync( fmPrimaryNodeName, behaviorName, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value, this.helper.GetRemainingTime(), cancellationToken).GetAwaiter().GetResult(); // TODO: Wait for some time so that the unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied Task.Delay(TimeSpan.FromSeconds(5.0)).GetAwaiter().GetResult(); } } // -- note there's no explict validation // action result action.Result = new RestartPartitionResult(getPartitionStateAction.Result); ResultTraceString = StringHelper.Format("RestartPartitionAction succeeded for {0} with RestartPartitionMode = {1}", partitionId, action.RestartPartitionMode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartReplicaAction action, CancellationToken cancellationToken) { TimeoutHelper helper = new TimeoutHelper(action.ActionTimeout); string nodeName = action.NodeName; Guid? partitionId = action.PartitionId; long? replicaId = action.ReplicaId; SelectedReplica replicaSelectorResult = SelectedReplica.None; if (string.IsNullOrEmpty(nodeName) || !partitionId.HasValue || !replicaId.HasValue) { ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector"); var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false); var replicaStateActionResult = getReplicaStateAction.Result; replicaSelectorResult = replicaStateActionResult.Item1; ReleaseAssert.AssertIf(replicaSelectorResult == null, "replicaSelectorResult cannot be null"); partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId; Replica replicaStateResult = replicaStateActionResult.Item2; ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null"); nodeName = replicaStateResult.NodeName; replicaId = replicaStateResult.Id; } ThrowIf.IsFalse(partitionId.HasValue, "PartitionID"); ThrowIf.IsFalse(replicaId.HasValue, "ReplicaID"); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.ServiceManager.RestartReplicaAsync( nodeName, partitionId.Value, replicaId.Value, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RestartReplicaErrors.Value, helper.GetRemainingTime(), cancellationToken); if (action.CompletionMode == CompletionMode.Verify) { // TODO: Check with failover team to see how to confirm that the replica actually restarted. We do not expose instance id for persisted replicas } action.Result = new RestartReplicaResult(replicaSelectorResult); this.ResultTraceString = StringHelper.Format( "RestartReplicaOrInstance succeeded by restarting replica {0}:{1} node {2} with CompletionMode {3}", partitionId.Value, replicaId.Value, nodeName, action.CompletionMode); }
public static T ConvertToEnum <T>(string name) { ThrowIf.NullOrEmpty(name, "name"); return((T)Enum.Parse(typeof(T), name)); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, MoveSecondaryAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.PartitionSelector, "PartitionSelector"); this.helper = new TimeoutHelper(action.ActionTimeout); string newSecondaryNode = action.NewSecondaryNodeName; string currentSecondaryNode = action.CurrentSecondaryNodeName; var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = this.helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken).ConfigureAwait(false); Guid partitionId = getPartitionStateAction.Result.PartitionId; if (!action.IgnoreConstraints) { // get current primary replica node name. ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( partitionId, 0, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); string currentPrimaryNodeInfo = string.Empty; List <string> currentSecReplicaNodes = new List <string>(); foreach (var replica in replicasResult) { StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica; if (statefulReplica == null) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "MoveSecondary", "Stateful", action.PartitionSelector.ServiceName, "Stateless")); } if (statefulReplica.ReplicaRole == ReplicaRole.Primary) { currentPrimaryNodeInfo = statefulReplica.NodeName; if (!string.IsNullOrEmpty(newSecondaryNode) && newSecondaryNode == statefulReplica.NodeName) { throw new FabricException( StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newSecondaryNode, "MoveSecondary", "Primary exists on node"), FabricErrorCode.AlreadyPrimaryReplica); } } else if (statefulReplica.ReplicaRole == ReplicaRole.ActiveSecondary) { currentSecReplicaNodes.Add(statefulReplica.NodeName); if (!string.IsNullOrEmpty(newSecondaryNode) && newSecondaryNode == statefulReplica.NodeName) { throw new FabricException( StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newSecondaryNode, "MoveSecondary", "Secondary exists on node"), FabricErrorCode.AlreadySecondaryReplica); } } } if (currentSecReplicaNodes.Count == 0) { throw new InvalidOperationException(StringResources.Error_NoSecondariesInReplicaSet); } if (string.IsNullOrEmpty(currentSecondaryNode)) { int num = testContext.Random.Next(currentSecReplicaNodes.Count); currentSecondaryNode = currentSecReplicaNodes.ElementAt(num); } if (!currentSecReplicaNodes.Contains(currentSecondaryNode)) { throw new FabricException( StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newSecondaryNode, "MoveSecondary", "Current node does not have a secondary replica"), FabricErrorCode.InvalidReplicaStateForReplicaOperation); } } ReleaseAssert.AssertIf(string.IsNullOrEmpty(currentSecondaryNode), "Current node name cannot be null or empty."); ReleaseAssert.AssertIf(newSecondaryNode == currentSecondaryNode, "Current and New node names are same."); ActionTraceSource.WriteInfo(TraceSource, "Calling move secondary with current node {0}, new node {1}, partition {2}", currentSecondaryNode, string.IsNullOrEmpty(newSecondaryNode) ? "Random" : newSecondaryNode, partitionId); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.MoveSecondaryUsingNodeNameAsync( currentSecondaryNode, newSecondaryNode, getPartitionStateAction.Result.ServiceName, partitionId, action.IgnoreConstraints, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.MoveSecondaryFabricErrors.Value, this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); action.Result = new MoveSecondaryResult(currentSecondaryNode, newSecondaryNode, getPartitionStateAction.Result); this.ResultTraceString = StringHelper.Format( "MoveSecondaryAction succeeded for moving Primary for {0} from {1} to {2}.", partitionId, currentSecondaryNode, newSecondaryNode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, InvokeDataLossAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.PartitionSelector, "PartitionSelector"); var helper = new TimeoutHelper(action.ActionTimeout); ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.ServiceManager.GetServiceDescriptionAsync( action.PartitionSelector.ServiceName, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (result.Kind != ServiceDescriptionKind.Stateful) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "DataLoss", "Stateful", action.PartitionSelector.ServiceName, "Stateless")); } var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken).ConfigureAwait(false); Guid partitionId = getPartitionStateAction.Result.PartitionId; long preDataLossNumber = 0; ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetPartitionListAsync( action.PartitionSelector.ServiceName, null, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); bool partitionFound = false; foreach (StatefulServicePartition partition in partitionsResult) { if (partition.PartitionInformation.Id == partitionId) { preDataLossNumber = partition.PrimaryEpoch.DataLossNumber; partitionFound = true; break; } } if (!partitionFound) { throw new FabricException(StringHelper.Format(StringResources.Error_PartitionNotFound), FabricErrorCode.PartitionNotFound); } long postDataLossNumber = preDataLossNumber; do { ActionTraceSource.WriteInfo( TraceType, "InvokeDataLossAction action pending time:{0}", helper.GetRemainingTime()); if (helper.GetRemainingTime() <= TimeSpan.Zero) { throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "InvokeDataLoss", partitionId)); } ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( partitionId, 0, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); ServiceReplicaList fmReplicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( Constants.FmPartitionId, 0, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); string fmPrimaryNodeName = string.Empty; var readyFMReplicas = fmReplicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray(); foreach (var replica in readyFMReplicas) { StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica; ReleaseAssert.AssertIf(statefulReplica == null, "FM Replica is not a stateful replica"); if (statefulReplica.ReplicaRole == ReplicaRole.Primary) { fmPrimaryNodeName = replica.NodeName; } } if (string.IsNullOrEmpty(fmPrimaryNodeName)) { throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, "FailoverManager"), FabricErrorCode.NotReady); } UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "DoReconfiguration"); behavior.AddFilterForPartitionId(partitionId); string behaviorName = "BlockDoReconfiguration"; await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync( fmPrimaryNodeName, behaviorName, behavior, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); // TODO: Wait for some time so that the unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false); bool triedToRemovedBehavior = false; try { var stableReplicasToRemove = replicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray(); ActionTraceSource.WriteInfo(TraceType, "Total number of replicas found {0}:{1}", replicasResult.Count(), stableReplicasToRemove.Count()); int replicasToRestartWithoutPrimary = action.DataLossMode == DataLossMode.FullDataLoss ? stableReplicasToRemove.Length - 1 : (stableReplicasToRemove.Length + 1) / 2 - 1; foreach (var replica in stableReplicasToRemove) { var currentReplica = replica; StatefulServiceReplica statefulReplica = currentReplica as StatefulServiceReplica; ReleaseAssert.AssertIf(statefulReplica == null, "Service Replica is not of stateful type even though service is stateful"); ActionTraceSource.WriteInfo( TraceType, "Inspecting replica {0}:{1} with role {2} and status {3} to induce data loss", currentReplica.Id, partitionId, statefulReplica.ReplicaRole, statefulReplica.ReplicaStatus); if (statefulReplica.ReplicaRole != ReplicaRole.Primary) { replicasToRestartWithoutPrimary--; } if (replicasToRestartWithoutPrimary >= 0 || statefulReplica.ReplicaRole == ReplicaRole.Primary) { ActionTraceSource.WriteInfo(TraceType, "Removing replica {0}:{1} to induce data loss", currentReplica.Id, partitionId); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.RemoveReplicaAsync( currentReplica.NodeName, partitionId, currentReplica.Id, CompletionMode.DoNotVerify, false, /*force remove*/ action.RequestTimeout.TotalSeconds, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); } } triedToRemovedBehavior = true; await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync( fmPrimaryNodeName, behaviorName, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value, helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); // TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).ConfigureAwait(false); // retry check for whether data loss number has increased 5 times else do the entire process again const int maxRetryCount = 5; int retryCount = 0; do { partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetPartitionListAsync( action.PartitionSelector.ServiceName, null, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.GetPartitionListFabricErrors.Value, helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); partitionFound = false; foreach (StatefulServicePartition partition in partitionsResult) { if (partition.PartitionInformation.Id == partitionId) { postDataLossNumber = partition.PrimaryEpoch.DataLossNumber; partitionFound = true; break; } } if (!partitionFound) { throw new FabricException(StringHelper.Format(StringResources.Error_PartitionNotFound), FabricErrorCode.PartitionNotFound); } ActionTraceSource.WriteInfo( TraceType, "Checking data loss numbers for partition {0} with retryCount {1}. Current numbers {2}:{3}", partitionId, retryCount, preDataLossNumber, postDataLossNumber); if (postDataLossNumber != preDataLossNumber) { break; } await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); ++retryCount; } while (retryCount < maxRetryCount); } finally { if (!triedToRemovedBehavior) { ActionTraceSource.WriteWarning(TraceType, "Exception after adding behavior to block messages. Removing behavior synchronously"); FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync( fmPrimaryNodeName, behaviorName, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value, helper.GetRemainingTime(), cancellationToken).GetAwaiter().GetResult(); // TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).GetAwaiter().GetResult(); } } }while (postDataLossNumber == preDataLossNumber); ActionTraceSource.WriteInfo( TraceType, "InvokeDataLossAction action completed postDataLossNumber:{0}, preDataLossNumber:{1}", postDataLossNumber, preDataLossNumber); action.Result = new InvokeDataLossResult(getPartitionStateAction.Result); this.ResultTraceString = StringHelper.Format("InvokeDataLossAction succeeded for {0} with DatalossMode = {1}", partitionId, action.DataLossMode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, InvokeQuorumLossAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.PartitionSelector, "PartitionSelector"); var helper = new TimeoutHelper(action.ActionTimeout); // get info about the service so we can check type and trss ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.ServiceManager.GetServiceDescriptionAsync( action.PartitionSelector.ServiceName, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (result.Kind != ServiceDescriptionKind.Stateful) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "QuorumLoss", "Stateful", action.PartitionSelector.ServiceName, "Stateless")); } StatefulServiceDescription statefulServiceDescription = result as StatefulServiceDescription; ReleaseAssert.AssertIf(statefulServiceDescription == null, "Service is not a stateful service"); if (!statefulServiceDescription.HasPersistedState) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "QuorumLoss", "Stateful Persistent", action.PartitionSelector.ServiceName, "Stateful In-Memory Only")); } // figure out /which/ partition to select var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken); Guid partitionId = getPartitionStateAction.Result.PartitionId; // get data about replicas in that partition ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( partitionId, 0, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); var removeUTRequestList = new List <Tuple <string, string> >(); Dictionary <Tuple <string, string>, Task> removeUTTaskDictionary = new Dictionary <Tuple <string, string>, Task>(); try { var stableReplicas = replicasResult.Where(r => r.ReplicaStatus == ServiceReplicaStatus.Ready).ToArray(); var stableReplicasToRemove = new List <StatefulServiceReplica>(); long replicasToRestartWithoutPrimary = action.QuorumLossMode == QuorumLossMode.AllReplicas ? stableReplicas.Length - 1 : FabricCluster.GetWriteQuorumSize(replicasResult.Count); foreach (var replica in stableReplicas) { StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica; ReleaseAssert.AssertIf(statefulReplica == null, "Service Replica is not of stateful type even though service is stateful"); if (statefulReplica.ReplicaRole != ReplicaRole.Primary) { replicasToRestartWithoutPrimary--; } if (replicasToRestartWithoutPrimary >= 0 || statefulReplica.ReplicaRole == ReplicaRole.Primary) { stableReplicasToRemove.Add(statefulReplica); } } // for selected replicas, block reopen so that when we restart the replica (NOT remove the replica) it doesn't come up var utTaskList = new List <Task>(); foreach (var statefulReplica in stableReplicasToRemove) { string nodeName = statefulReplica.NodeName; UnreliableTransportBehavior behavior = new UnreliableTransportBehavior("*", "StatefulServiceReopen"); behavior.AddFilterForPartitionId(partitionId); string behaviorName = "BlockStatefulServiceReopen_" + nodeName; removeUTRequestList.Add(new Tuple <string, string>(nodeName, behaviorName)); utTaskList.Add( FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.AddUnreliableTransportBehaviorAsync( nodeName, behaviorName, behavior, action.RequestTimeout, cancellationToken), helper.GetRemainingTime(), cancellationToken)); } await Task.WhenAll(utTaskList).ConfigureAwait(false); // TODO: Wait for some time so that the unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successfully applied await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken); var restartReplicaTaskList = new List <Task>(); foreach (var statefulReplica in stableReplicasToRemove) { ReplicaSelector replicaSelector = ReplicaSelector.ReplicaIdOf(PartitionSelector.PartitionIdOf(action.PartitionSelector.ServiceName, partitionId), statefulReplica.Id); var restartReplicaAction = new RestartReplicaAction(replicaSelector) { CompletionMode = CompletionMode.DoNotVerify, RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; restartReplicaTaskList.Add(testContext.ActionExecutor.RunAsync(restartReplicaAction, cancellationToken)); } await Task.WhenAll(restartReplicaTaskList).ConfigureAwait(false); await AsyncWaiter.WaitAsync(action.QuorumLossDuration, cancellationToken).ConfigureAwait(false); // validate ServicePartitionList partitionsResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetPartitionListAsync( action.PartitionSelector.ServiceName, null, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.GetPartitionListFabricErrors.Value, helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); foreach (StatefulServicePartition partition in partitionsResult) { if (partition.PartitionInformation.Id == partitionId) { ReleaseAssert.AssertIf(partition.PartitionStatus != ServicePartitionStatus.InQuorumLoss, "Partition failed to be in Quorum Loss."); break; } } foreach (var removeUTParams in removeUTRequestList) { var currentParams = removeUTParams; Task task = FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync( currentParams.Item1, /*nodeName*/ currentParams.Item2, /*behaviorName*/ action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value, helper.GetRemainingTime(), cancellationToken); removeUTTaskDictionary[currentParams] = task; } await Task.WhenAll(removeUTTaskDictionary.Values).ConfigureAwait(false); // TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied await Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken); } finally { var removeUTTaskList = new List <Task>(); foreach (var removeUTRequest in removeUTTaskDictionary) { var currentRemoveUTRequest = removeUTRequest; if (currentRemoveUTRequest.Value == null || currentRemoveUTRequest.Value.IsFaulted) { removeUTTaskList.Add( FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.TestManager.RemoveUnreliableTransportBehaviorAsync( currentRemoveUTRequest.Key.Item1, /*nodeName*/ currentRemoveUTRequest.Key.Item2, /*behaviorName*/ action.RequestTimeout, cancellationToken), FabricClientRetryErrors.RemoveUnreliableTransportBehaviorErrors.Value, helper.GetRemainingTime(), cancellationToken)); } } Task.WhenAll(removeUTTaskList).Wait(cancellationToken); // TODO: Wait for some time so that the removal of this unreliable transport behavior can be read from the files. // Bug#2271465 - Unreliable transport through API should return only once the behavior has been successully applied Task.Delay(TimeSpan.FromSeconds(5.0), cancellationToken).GetAwaiter().GetResult(); } action.Result = new InvokeQuorumLossResult(getPartitionStateAction.Result); this.ResultTraceString = StringHelper.Format("InvokeQuorumLossAction succeeded for {0} with QuorumLossMode = {1}", partitionId, action.QuorumLossMode); }
public async Task <ValidationReport> EnsureStabilityWithReportAsync(TimeSpan maximumStabilizationTimeout, TimeSpan retryWait, CancellationToken ct) { TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Ensuring that '{0}' is online with timeout '{1}'.", this.serviceName, maximumStabilizationTimeout); bool checkQuorumLoss = (this.checkFlags & ValidationCheckFlag.CheckQuorumLoss) != 0; // Load basic information about this service. TestabilityTrace.TraceSource.WriteNoise(TraceSource, "Querying basic information for {0}.", this.serviceName); await this.LoadPartitionAndReplicaCountAsync(ct); DateTime startTime = DateTime.Now; TimeoutHelper timer = new TimeoutHelper(maximumStabilizationTimeout); bool success = false; List <Guid> partitionsInQuorumLoss = new List <Guid>(); StringBuilder errorString = new StringBuilder(); int retryCount = 1; while (!success && timer.GetRemainingTime() > TimeSpan.Zero) { TestabilityTrace.TraceSource.WriteInfo(TraceSource, "EnsureStabilityWithReportAsync(): retryCount='{0}', timer.GetRemainingTime()='{1}'", retryCount, timer.GetRemainingTime()); var nodes = await this.TestContext.FabricCluster.GetLatestNodeInfoAsync(this.requestTimeout, this.operationTimeout, ct); // Empty error string and list of partitions in quorum loss partitionsInQuorumLoss.Clear(); errorString.Clear(); success = true; int totalPartitionsFound = 0; bool stateful; ReleaseAssert.AssertIfNot(this.isStateful.TryGetValue(out stateful), "isStateful flag is not available"); bool checkTarget = (this.checkFlags & ValidationCheckFlag.CheckTargetReplicaSetSize) != 0; bool checkInBuild = (this.checkFlags & ValidationCheckFlag.CheckInBuildReplica) != 0; if (stateful) { var partitionDictionary = await this.QueryPartitionAndReplicaResultAsyncStateful(ct); totalPartitionsFound = partitionDictionary.Count(); foreach (KeyValuePair <Partition, StatefulServiceReplica[]> partition in partitionDictionary) { bool partitionIsReady = partition.Key.PartitionStatus == ServicePartitionStatus.Ready; if (!partitionIsReady) { var message = StringHelper.Format("Partition '{0}' is not Ready", partition.Key.PartitionId()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); } if (partition.Key.PartitionStatus != ServicePartitionStatus.InQuorumLoss) { int validCount = 0; int inBuildReplicas = 0; foreach (StatefulServiceReplica replica in partition.Value) { if (replica.ReplicaStatus == ServiceReplicaStatus.Ready && (replica.ReplicaRole == ReplicaRole.Primary || replica.ReplicaRole == ReplicaRole.ActiveSecondary)) { ++validCount; } if (replica.ReplicaStatus == ServiceReplicaStatus.InBuild) { ++inBuildReplicas; var message = StringHelper.Format("Replica {0} for partition '{1}' is InBuild", replica.Id, partition.Key.PartitionId()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); } } bool targetAchieved = this.CheckReplicaSetSize(partition.Key.PartitionInformation.Id, validCount, startTime, nodes, errorString); if (!partitionIsReady || (checkInBuild && inBuildReplicas > 0) || (checkTarget && !targetAchieved)) { success = false; } } else { partitionsInQuorumLoss.Add(partition.Key.PartitionInformation.Id); } } } else { int targetInstanceCount = 0; ReleaseAssert.AssertIf(!this.targetReplicaSetSize.TryGetValue(out targetInstanceCount), "targetReplicaSetSize for service: {0} should have been populated at this point.", this.serviceName); bool placementConstraintsDefined = false; try { // Get the service description to find out if there are placement constraints on the service ServiceDescription result = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.TestContext.FabricClient.ServiceManager.GetServiceDescriptionAsync( this.serviceName, this.requestTimeout, ct), this.operationTimeout, ct).ConfigureAwait(false); ThrowIf.IsTrue(result == null, "A description must be associated with the service: {0}", this.serviceName); placementConstraintsDefined = !string.IsNullOrEmpty(result.PlacementConstraints); } catch (UnauthorizedAccessException) { ServiceGroupDescription groupDescription = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.TestContext.FabricClient.ServiceGroupManager.GetServiceGroupDescriptionAsync( this.serviceName, this.requestTimeout, ct), this.operationTimeout, ct).ConfigureAwait(false); ThrowIf.IsTrue(groupDescription == null, "A description must be associated with the service group: {0}", this.serviceName); placementConstraintsDefined = !string.IsNullOrEmpty(groupDescription.ServiceDescription.PlacementConstraints); } // If a stateless service has instance count == -1 and it has placement constraints such // that the possible number of instances cannot match the total number of nodes, // we need to find out the number of eligible nodes for the service which is tracked by RDBug 8993319. // Until RDBug 8993319 is fixed, we take the presence of placement constraints into consideration to make the // validation more accurate. if (targetInstanceCount == -1 && placementConstraintsDefined) { checkTarget = false; } var partitionDictionary = await this.QueryPartitionAndReplicaResultAsyncStateless(timer.GetRemainingTime(), ct); totalPartitionsFound = partitionDictionary.Count(); foreach (KeyValuePair <Partition, StatelessServiceInstance[]> partition in partitionDictionary) { bool partitionIsReady = partition.Key.PartitionStatus == ServicePartitionStatus.Ready; if (!partitionIsReady) { var message = StringHelper.Format("Partition '{0}' is not Ready", partition.Key.PartitionId()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); } int validCount = 0; foreach (StatelessServiceInstance instance in partition.Value) { if (instance.ReplicaStatus == ServiceReplicaStatus.Ready) { ++validCount; } } bool targetAchieved = this.CheckReplicaSetSize(partition.Key.PartitionInformation.Id, validCount, startTime, nodes, errorString); if (!partitionIsReady || (checkTarget && !targetAchieved)) { success = false; } } } if (!this.ValidatePartitionCount(totalPartitionsFound)) { success = false; } if (partitionsInQuorumLoss.Count > 0 && checkQuorumLoss) { string paritionIds = string.Join(",", partitionsInQuorumLoss.ToArray()); var message = StringHelper.Format("Partitions '{0}' in quorum loss for service {1}", paritionIds, this.serviceName); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "{0}", message); errorString.AppendLine(message); success = false; } if (!success) { if (retryCount % 10 == 0) { TestabilityTrace.TraceSource.WriteWarning(TraceSource, "Service {0} validation failed due to issues below, will retry: \n{1}", this.serviceName, errorString); } // Delay before querying again so we allow some time for state to change - don't spam the node await AsyncWaiter.WaitAsync(retryWait, ct).ConfigureAwait(false); } retryCount++; } if (partitionsInQuorumLoss.Count > 0) { string partitionIds = string.Join(",", partitionsInQuorumLoss.ToArray()); TestabilityTrace.TraceSource.WriteInfo(TraceSource, "Partitions in quorum loss for service {0} are '{1}'", this.serviceName, partitionIds); if (checkQuorumLoss) { throw new FabricValidationException(StringHelper.Format(StringResources.Error_PartitionsInQuorumLoss, partitionIds, this.serviceName)); } } if (!success) { return(new ValidationReport( true, StringHelper.Format(StringResources.Error_ServiceNotStable, this.serviceName, maximumStabilizationTimeout, errorString))); } else { return(ValidationReport.Default); } }