internal static async Task <Node> GetNodeWithFASSecondary() { NodeList nodeList = ActionTest.GetNodeListAsync().Result; ServiceReplicaList list = null; FabricClient fc = new FabricClient(); System.Fabric.Common.TimeoutHelper timeoutHelper = new System.Fabric.Common.TimeoutHelper(TimeSpan.FromMinutes(2)); do { try { list = await fc.QueryManager.GetReplicaListAsync(new Guid("00000000-0000-0000-0000-000000005000")); } catch (Exception) { Task.Delay(TimeSpan.FromSeconds(1)).Wait(); } }while (list == null && timeoutHelper.GetRemainingTime() > TimeSpan.Zero); if (list == null) { throw new InvalidOperationException("Could not resolve FAS primary"); } Replica replica = list.Where(r => ((StatefulServiceReplica)r).ReplicaRole == ReplicaRole.ActiveSecondary).FirstOrDefault(); return(nodeList.Where(n => n.NodeName == replica.NodeName).FirstOrDefault()); }
private void WaitForActionCount(long targetCount) { long count = 0; System.Fabric.Common.TimeoutHelper timeoutHelper = new System.Fabric.Common.TimeoutHelper(TimeSpan.FromMinutes(3)); do { count = this.actionStore.GetActionCountAsync(false).GetAwaiter().GetResult(); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Current action count='{0}', target action count='{1}'", count, targetCount); if (count == targetCount) { break; } Task.Delay(TimeSpan.FromSeconds(5)).Wait(); }while (count != targetCount && timeoutHelper.GetRemainingTime() > TimeSpan.Zero); if (count != targetCount) { string error = string.Format(CultureInfo.InvariantCulture, "Did not reach expected target action count='{0}', current action count='{1}'", targetCount, count); TestabilityTrace.TraceSource.WriteError(TraceType, error); System.Fabric.Common.ReleaseAssert.Failfast(error); } }
internal static async Task <bool> IsUriReachableAsync( Uri uri, string requestMethod = DMConstants.HttpMethodHead, int operationTimeoutInMs = DMConstants.UriReachableTimeoutInMs, int requestTimeoutInMs = DMConstants.UriRequestTimeoutInMs, int retryIntervalInMs = DMConstants.UriReachableRetryIntervalInMs) { ReleaseAssert.AssertIf(uri == null, "uri cannot be null for IsUriReachableAsync."); if (uri.IsFile) { FileInfo fi = new FileInfo(uri.LocalPath); return(fi.Exists); } else { if (string.IsNullOrWhiteSpace(uri.Host)) { return(false); } var timeout = new System.Fabric.Common.TimeoutHelper(TimeSpan.FromMilliseconds(operationTimeoutInMs)); while (!System.Fabric.Common.TimeoutHelper.HasExpired(timeout)) { WebRequest request = WebRequest.Create(uri); #if !DotNetCoreClrLinux request.Timeout = requestTimeoutInMs; #endif request.Method = requestMethod; try { using (WebResponse response = await request.GetResponseAsync().ConfigureAwait(false)) { if (response is HttpWebResponse) { if (((HttpWebResponse)response).StatusCode == HttpStatusCode.OK) { return(true); } return(false); } else { return(response.ContentLength > 0); } } } catch (WebException ex) { SFDeployerTrace.WriteNoise(StringResources.Error_SFUriUnreachable_Formatted, uri, ex.Message); } System.Threading.Thread.Sleep(retryIntervalInMs); } } return(false); }
// This doesn't run in automation, but it is being kept here so it can be run as a small test. // See FaultAnalysisServiceTruncate.test for a test on this code path. private void TestTruncate() { this.StartActionIfItHasNotBeenStarted(Command.StuckAction); this.StartActionIfItHasNotBeenStarted(Command.FailoverManagerDataLoss); this.StartActionIfItHasNotBeenStarted(Command.InvokeDataLossMidActionTestFatal); this.StartActionIfItHasNotBeenStarted(Command.InvokeDataLossMidActionTestTransient); this.StartActionIfItHasNotBeenStarted(Command.RestartPartitionMidActionTestFatal); this.StartActionIfItHasNotBeenStarted(Command.RestartPartitionMidActionTestTransient); this.WaitForActionCount(FASConstants.TestMaxStoredActionCountValue); // Confirm this action is still stuck - ie that an action not in terminal state is not removed this.mockClient.WaitForState(MockClient.MockClientCommandInfo[Command.StuckAction], Actions.Steps.StepStateNames.LookingUpState); this.mockClient.WaitForState(MockClient.MockClientCommandInfo[Command.RestartPartitionMidActionTestTransient], Actions.Steps.StepStateNames.CompletedSuccessfully); // At this point there should be 1 command in the actionTable, StuckAction, and somewhere between 2 (Constants.TestMaxStoredActionCountValue) and 5 (the total number of possible // completed commands) commands in the historyTable. In steady state, after truncates have run, the historyTable should have 2 (Constants.TestMaxStoredActionCountValue) commands remaining, // and they should be the ones that completed last. Since this test only allows 1 action at a time, this will always be the 2 that were started last - // the RestartPartition ones. bool conditionSatisfied = false; var timeoutHelper = new System.Fabric.Common.TimeoutHelper(TimeSpan.FromSeconds(3 * FASConstants.TestStoredActionCleanupIntervalInSeconds)); do { TestCommandListDescription queryDescription = new TestCommandListDescription(Query.TestCommandStateFilter.CompletedSuccessfully, Query.TestCommandTypeFilter.PartitionRestart); TestCommandQueryResult queryResult = this.mockClient.GetTestCommandListAsync(queryDescription).GetAwaiter().GetResult(); List <TestCommandStatus> result = queryResult.Items; if (result.Count < FASConstants.TestMaxStoredActionCountValue) { string error = string.Format( CultureInfo.InvariantCulture, "Number of commands in the historyTable {0} is below TestMaxStoredActionCountValue (config 'DefaultMaxStoredActionCount')", result.Count); TestabilityTrace.TraceSource.WriteError(TraceType, error); System.Fabric.Common.ReleaseAssert.Failfast(error); } if (result.Where(c => c.TestCommandType == TestCommandType.PartitionRestart).Count() != FASConstants.TestMaxStoredActionCountValue) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Number of PartitionRestart results is {0}, expecting {1}, retrying", result.Count, FASConstants.TestMaxStoredActionCountValue); continue; } conditionSatisfied = true; }while (!conditionSatisfied && timeoutHelper.GetRemainingTime() > TimeSpan.Zero); System.Fabric.Common.ReleaseAssert.Failfast(string.Format(CultureInfo.InvariantCulture, "Did not reach expected target action, see traces above filtered by type~ActionTest'")); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exiting TestTruncate"); }
private async Task PollAddedNodesActivatedAsync(List <NodeDescription> addedNodes) { System.Fabric.Common.TimeoutHelper timeoutHelper = new System.Fabric.Common.TimeoutHelper(Constants.FabricPollActivationNodesTimeoutInMinutes); bool isActivationComplete = true; while (!System.Fabric.Common.TimeoutHelper.HasExpired(timeoutHelper) && !this.cancellationToken.IsCancellationRequested) { UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Checking activation status of added nodes."); try { System.Fabric.Query.NodeList nodes = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.fabricClient.QueryManager.GetNodeListAsync( null, Constants.FabricQueryTimeoutInMinutes, this.cancellationToken), Constants.FabricQueryRetryTimeoutInMinutes).ConfigureAwait(false); isActivationComplete = true; for (int i = 0; i < nodes.Count; ++i) { if (addedNodes.Any(addedNode => addedNode.NodeName == nodes[i].NodeName) && nodes[i].NodeStatus != System.Fabric.Query.NodeStatus.Up) { UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Node {0} has not been activated", nodes[i].IpAddressOrFQDN); isActivationComplete = false; break; } } if (isActivationComplete) { UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "All nodes to be added are up. Continuing with upgrade"); break; } else { UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Waiting for five seconds before polling activation status again."); } await Task.Delay(TimeSpan.FromSeconds(5), this.cancellationToken).ConfigureAwait(false); } catch (FabricTransientException fte) { UpgradeOrchestrationTrace.TraceSource.WriteError(TraceType, "Retrying Polling Activated Node because of retryable exception {0}", fte); } } timeoutHelper.ThrowIfExpired(); }
internal static async Task <string> GetContentsFromUriAsyncWithRetry(Uri uri, TimeSpan retryInterval, TimeSpan operationTimeout, CancellationToken cancellationToken) { string downloadedContent = null; var timeoutHelper = new System.Fabric.Common.TimeoutHelper(operationTimeout); while (!System.Fabric.Common.TimeoutHelper.HasExpired(timeoutHelper)) { cancellationToken.ThrowIfCancellationRequested(); try { if (uri.IsFile) { downloadedContent = File.ReadAllText(uri.LocalPath); } else { using (var wc = new WebClient()) { downloadedContent = await wc.DownloadStringTaskAsync(uri).ConfigureAwait(false); } } return(downloadedContent); } catch (Exception e) { SFDeployerTrace.WriteWarning(StringResources.Error_SFUriNotDownloaded, uri, e.ToString()); } await Task.Delay(retryInterval, cancellationToken).ConfigureAwait(false); } SFDeployerTrace.WriteError(StringResources.Error_SFTimedOut, operationTimeout); throw new FabricValidationException(string.Format(StringResources.Error_SFTimedOut, operationTimeout), FabricErrorCode.OperationCanceled); }
internal async Task ExecuteIterationsWithPauseAsync() { TestabilityTrace.TraceSource.WriteNoise("StartTrek", "Enter ExecuteIterationsWithPauseAsync, datetimeutc={0}", DateTime.UtcNow); this.stopwatch = new Stopwatch(); this.stopwatch.Start(); Exception capturedException = null; TimeSpan waitTime = this.ChaosParameters.WaitTimeBetweenIterations; while (!this.cancellationToken.IsCancellationRequested) { // If this is not the beginning of a fresh Chaos run, before starting a new iteration, // consult the NextIterationTimeStampRD to find out if there is some // residual wait time from the previous iteration, if there is then wait that amount var nextIterationTimeStampUtc = await this.StateManager.GetUtcTimeStampAsync( FASConstants.NextItearationTimeStampRDName, FASConstants.NextItearationTimeStampKey, this.partition, this.cancellationToken).ConfigureAwait(false); var residualWaitTime = nextIterationTimeStampUtc.Subtract(DateTime.UtcNow); if (residualWaitTime > TimeSpan.Zero) { await this.StateManager.RegisterChaosEventAndUpdateChaosStatusAsync( new WaitingEvent(DateTime.UtcNow, StringHelper.Format(StringResources.ChaosInfo_ResidualWaitingFromPreviousIteration, residualWaitTime)), ChaosStatus.Running, this.partition, this.cancellationToken, () => { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Registering WaitingEvent for waiting '{0}' left over from the previous iteration.", residualWaitTime); }).ConfigureAwait(false); await Task.Delay(residualWaitTime, this.cancellationToken).ConfigureAwait(false); } try { if (await this.IsClusterReadyForFaultsAsync(this.cancellationToken).ConfigureAwait(false)) { System.Fabric.Common.TimeoutHelper timer = new System.Fabric.Common.TimeoutHelper(this.ChaosParameters.MaxClusterStabilizationTimeout); StringBuilder validationReport = new StringBuilder(); var clusterReport = await this.validationHelper.ValidateClusterHealthAsync( timer.GetRemainingTime(), this.cancellationToken).ConfigureAwait(false); if (clusterReport.ValidationFailed) { // quadratic with an upper bound of DefaultMaximumBackoffForChaosIterations waitTime += (waitTime >= FASConstants.DefaultMaximumBackoffForChaosIterations) ? TimeSpan.Zero : waitTime; var serviceReport = await this.validationHelper.ValidateAllServicesAsync( timer.GetRemainingTime(), this.cancellationToken).ConfigureAwait(false); if (serviceReport.ValidationFailed) { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Even though some services are unhealthy or unstable, going to induce faults, because the cluster is healthy."); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Failure reason: \n'{0}'", serviceReport.FailureReason); validationReport.Append(serviceReport.FailureReason); } StringBuilder reportBuilder = new StringBuilder(); reportBuilder.Append(StringHelper.Format(StringResources.ChaosInfo_WaitingNotice, waitTime)); reportBuilder.AppendLine(); reportBuilder.AppendLine(clusterReport.FailureReason); validationReport.Insert(0, reportBuilder.ToString()); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Cluster validation failed for '{0}'.", clusterReport.FailureReason); var validationFailedEvent = new ValidationFailedEvent(DateTime.UtcNow, validationReport.ToString()); // record validation failed event await this.StateManager.RegisterChaosEventAndUpdateChaosStatusAsync( validationFailedEvent, ChaosStatus.Running, this.partition, this.cancellationToken, () => { FabricEvents.Events.ChaosValidationFailed( Guid.NewGuid().ToString(), validationFailedEvent.TimeStampUtc.Ticks, validationFailedEvent.Reason); }).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Pausing for '{0}' before performing next check.", waitTime); } else { waitTime = this.ChaosParameters.WaitTimeBetweenIterations; await this.StateManager.RegisterCurrentStatusAsync( ChaosStatus.Running, this.partition, this.cancellationToken).ConfigureAwait(false); var timestampOfNextIteration = DateTime.UtcNow.Add(waitTime); await this.StateManager.SetUtcTimeStampAsync( FASConstants.NextItearationTimeStampRDName, FASConstants.NextItearationTimeStampKey, timestampOfNextIteration, this.partition, this.cancellationToken).ConfigureAwait(false); await this.ExecuteFaultIterationAsync(this.cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Pausing for '{0}' before executing next iteration.", waitTime); } } } catch (Exception exception) { capturedException = exception; // quadratic with an upper bound of DefaultMaximumBackoffForChaosIterations waitTime += (waitTime >= FASConstants.DefaultMaximumBackoffForChaosIterations) ? TimeSpan.Zero : waitTime; var timestampOfNextIteration = DateTime.UtcNow.Add(waitTime); await this.StateManager.SetUtcTimeStampAsync( FASConstants.NextItearationTimeStampRDName, FASConstants.NextItearationTimeStampKey, timestampOfNextIteration, this.partition, this.cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Exception occurred in the ChaosTestScenario loop: {0}, cancellationreq: {1}", capturedException, this.cancellationToken.IsCancellationRequested); } if (capturedException != null) { var ae = capturedException as AggregateException; if (ae != null) { capturedException = ae.Flatten().InnerException; } if (!ChaosUtil.IsExpectedException(capturedException)) { string testErrorEventMessage = capturedException.Message; if (capturedException is FabricChaosEngineException) { testErrorEventMessage = StringResources.ChaosError_UnexpectedInChaosEngine; } await this.StateManager.RegisterChaosEventAndUpdateChaosStatusAsync( new TestErrorEvent(DateTime.UtcNow, testErrorEventMessage), ChaosStatus.Running, this.partition, this.cancellationToken).ConfigureAwait(false); TestabilityTrace.TraceSource.WriteInfo(TraceType, "Unexpected exception '{0}' was turned into TestErrorEvent.", capturedException); } // handled the exception, now clear it capturedException = null; } if (this.testMode && (this.ChaosParameters.Context != null && this.ChaosParameters.Context.ContainsKey(ChaosConstants.FailoverAfterChaosFaultsKey))) { this.partition.ReportFault(FaultType.Transient); } await this.StateManager.RegisterChaosEventAndUpdateChaosStatusAsync( new WaitingEvent(DateTime.UtcNow, StringHelper.Format(StringResources.ChaosInfo_WaitingBetweenIterations, waitTime)), ChaosStatus.Running, this.partition, this.cancellationToken, () => { TestabilityTrace.TraceSource.WriteInfo(TraceType, "Registering WaitingEvent for waiting '{0}' between iterations.", waitTime); }).ConfigureAwait(false); await Task.Delay(waitTime, this.cancellationToken).ConfigureAwait(false); this.IterationsCompleted++; } TestabilityTrace.TraceSource.WriteInfo(TraceType, "Session has completed. \nTotal iterations: {0}. Total elapsed time: {1}", this.IterationsCompleted, this.GetElapsedTime()); }
private static bool CheckRPCAccess(MachineHealthContainer machineHealthContainer) { var retryTimeout = new System.Fabric.Common.TimeoutHelper(DMConstants.BpaRpcRetryTimeout); SFDeployerTrace.WriteNoise(StringResources.Info_SFRpcInfo); Parallel.ForEach <string>( machineHealthContainer.GetHealthyMachineNames(), (string machine) => { bool result = true; bool willRetry; do { willRetry = false; try { Utility.GetTempPath(machine); } catch (Exception ex) { string message; if (ex is System.IO.IOException) { switch (ex.HResult) { // If new failures are discovered: https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx case 53: // ERROR_BAD_NETPATH message = string.Format(StringResources.Error_SFRpcIoNetpath, machine, ex.HResult); willRetry = true; break; case 1723: // RPC_S_SERVER_TOO_BUSY message = string.Format(StringResources.Error_SFRpcIoTooBusy, machine, ex.HResult); willRetry = true; break; case 1727: // RPC_S_CALL_FAILED_DNE message = string.Format(StringResources.Error_SFRpcIoFailedDne, machine, ex.HResult); break; default: message = string.Format(StringResources.Error_SFRpcIoGeneric, machine, ex.HResult); break; } } else if (ex is System.Security.SecurityException) { switch (ex.HResult) { case -2146233078: // COR_E_SECURITY message = string.Format(StringResources.Error_SFRpcSecAccess, machine, ex.HResult); break; default: message = string.Format(StringResources.Error_SFRpcSecGeneric, machine, ex.HResult); break; } } else if (ex is NullReferenceException) { switch (ex.HResult) { case -2146232828: // COR_E_TARGETINVOCATION message = string.Format(StringResources.Error_SFRpcNullRegAccess, machine, ex.HResult); break; default: message = string.Format(StringResources.Error_SFRpcNullGeneric, machine, ex.HResult); break; } } else { // This is to catch coding errors. message = string.Format(StringResources.Error_SFRpcGeneric, machine, ex.HResult); } willRetry &= !System.Fabric.Common.TimeoutHelper.HasExpired(retryTimeout); if (willRetry) { SFDeployerTrace.WriteWarning(message); StandaloneUtility.OpenRemoteRegistryNamedPipe(machine, retryTimeout.GetRemainingTime()); Thread.Sleep(TimeSpan.FromSeconds(5)); } else { SFDeployerTrace.WriteError(message); result = false; } } }while (willRetry); if (!result) { machineHealthContainer.MarkMachineAsUnhealthy(machine); } }); return(machineHealthContainer.EnoughHealthyMachines()); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartDeployedCodePackageAction action, CancellationToken cancellationToken) { this.helper = new TimeoutHelper(action.ActionTimeout); string nodeName = action.NodeName; Uri applicationName = action.ApplicationName; string serviceManifestName = action.ServiceManifestName; string servicePackageActivationId = action.ServicePackageActivationId; string codePackageName = action.CodePackageName; SelectedReplica replicaSelectorResult = SelectedReplica.None; ThrowIf.Null(applicationName, "ApplicationName"); if (string.IsNullOrEmpty(nodeName) || string.IsNullOrEmpty(serviceManifestName) || string.IsNullOrEmpty(codePackageName)) { ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector"); var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = this.helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false); var replicaStateActionResult = getReplicaStateAction.Result; ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null"); replicaSelectorResult = replicaStateActionResult.Item1; ReleaseAssert.AssertIf(replicaSelectorResult == null || replicaSelectorResult.SelectedPartition == null, "replicaSelectorResult cannot be null or for a non-null replicaSelectorResult, the selected partition must be non-null"); Guid partitionId = replicaStateActionResult.Item1.SelectedPartition.PartitionId; Replica replicaStateResult = replicaStateActionResult.Item2; ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null"); nodeName = replicaStateResult.NodeName; var deployedReplicaListResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <DeployedServiceReplicaList>( () => testContext.FabricClient.QueryManager.GetDeployedReplicaListAsync( nodeName, applicationName, null, partitionId, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); DeployedServiceReplica selectedReplica = deployedReplicaListResult.FirstOrDefault(r => r.Partitionid == partitionId); if (selectedReplica == null) { throw new FabricException( StringHelper.Format(StringResources.Error_DidNotFindDeployedReplicaOnNode, partitionId, nodeName), FabricErrorCode.ReplicaDoesNotExist); } serviceManifestName = selectedReplica.ServiceManifestName; servicePackageActivationId = selectedReplica.ServicePackageActivationId; codePackageName = selectedReplica.CodePackageName; } ActionTraceSource.WriteInfo(TraceSource, "SelectedReplica: serviceManifestName: {0}, servicePackageActivationId: {1}, codePackageName: {2}", serviceManifestName, servicePackageActivationId, codePackageName); DeployedCodePackage deployedCodePackageListResult = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false); var codepackageEntrypointToRestart = GetCodepackageEntrypointToRestart(action, deployedCodePackageListResult); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.RestartDeployedCodePackageUsingNodeNameAsync( nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (action.CompletionMode == CompletionMode.Verify) { bool success = false; while (this.helper.GetRemainingTime() > TimeSpan.Zero) { var deployedCodePackageListResultAfterRestart = await this.GetCodePackageInfoAsync(testContext, nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, action, cancellationToken).ConfigureAwait(false); if (deployedCodePackageListResultAfterRestart != null) { var entryPointAfterRestart = codepackageEntrypointToRestart.EntryPointType == EntryPointType.Main ? deployedCodePackageListResultAfterRestart.EntryPoint : deployedCodePackageListResultAfterRestart.SetupEntryPoint; if (entryPointAfterRestart != null && entryPointAfterRestart.CodePackageInstanceId > codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId && entryPointAfterRestart.EntryPointStatus == EntryPointStatus.Started) { success = true; break; } } ActionTraceSource.WriteInfo(TraceSource, "CodePackage = {0}:{1}:{2} not yet restarted. Retrying...", nodeName, applicationName, codePackageName); await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false); } if (!success) { throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "RestartDeployedCodePackage", applicationName)); } } action.Result = new RestartDeployedCodePackageResult( nodeName, applicationName, serviceManifestName, servicePackageActivationId, codePackageName, codepackageEntrypointToRestart.EntryPoint.CodePackageInstanceId, replicaSelectorResult); ResultTraceString = StringHelper.Format("RestartCodePackageAction succeeded for {0}:{1}:{2} with CompletionMode = {3}", nodeName, applicationName, codePackageName, action.CompletionMode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, RestartNodeAction action, CancellationToken cancellationToken) { ActionTraceSource.WriteInfo(TraceSource, "Enter RestartNodeAction/ExecuteActionAsync: operationTimeout='{0}', requestTimeout='{1}'", action.ActionTimeout, action.RequestTimeout); this.helper = new TimeoutHelper(action.ActionTimeout); SelectedReplica selectedReplica = SelectedReplica.None; string nodeName = action.NodeName; BigInteger nodeInstance = action.NodeInstance; bool createFabricDump = action.CreateFabricDump; if (string.IsNullOrEmpty(nodeName)) { ThrowIf.Null(action.ReplicaSelector, "ReplicaSelector"); var getReplicaStateAction = new GetSelectedReplicaStateAction(action.ReplicaSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getReplicaStateAction, cancellationToken).ConfigureAwait(false); var replicaStateActionResult = getReplicaStateAction.Result; ReleaseAssert.AssertIf(replicaStateActionResult == null, "replicaStateActionResult cannot be null"); selectedReplica = replicaStateActionResult.Item1; Replica replicaStateResult = replicaStateActionResult.Item2; ReleaseAssert.AssertIf(replicaStateResult == null, "replicaStateResult cannot be null"); nodeName = replicaStateResult.NodeName; nodeInstance = BigInteger.MinusOne; } if (nodeInstance == BigInteger.MinusOne) { var nodeInfo = await this.GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken).ConfigureAwait(false); nodeInstance = nodeInfo.NodeInstanceId; } await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.RestartNodeUsingNodeNameAsync( nodeName, nodeInstance, createFabricDump, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); if (action.CompletionMode == CompletionMode.Verify) { bool success = false; while (this.helper.GetRemainingTime() > TimeSpan.Zero) { var nodeInfo = await this.GetCurrentNodeInfoAsync(testContext, nodeName, action, cancellationToken).ConfigureAwait(false); if (nodeInfo.NodeInstanceId > nodeInstance && nodeInfo.IsNodeUp) { success = true; break; } ActionTraceSource.WriteInfo(TraceSource, "NodeName = {0} not yet restarted. '{1}' seconds remain. Retrying...", nodeName, this.helper.GetRemainingTime().TotalSeconds); await AsyncWaiter.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); } if (!success) { throw new TimeoutException(StringHelper.Format(StringResources.Error_TestabilityActionTimeout, "RestartNode", nodeName)); } } // create result action.Result = new RestartNodeResult(selectedReplica, new NodeResult(nodeName, nodeInstance)); ResultTraceString = StringHelper.Format("RestartNodeAction succeeded for {0}:{1} with CompletionMode = {2}", nodeName, nodeInstance, action.CompletionMode); }
protected override async Task ExecuteActionAsync(FabricTestContext testContext, MovePrimaryAction action, CancellationToken cancellationToken) { ThrowIf.Null(action.PartitionSelector, "PartitionSelector"); this.helper = new TimeoutHelper(action.ActionTimeout); string newPrimaryNodeName = action.NodeName; var getPartitionStateAction = new GetSelectedPartitionStateAction(action.PartitionSelector) { RequestTimeout = action.RequestTimeout, ActionTimeout = this.helper.GetRemainingTime() }; await testContext.ActionExecutor.RunAsync(getPartitionStateAction, cancellationToken); Guid partitionId = getPartitionStateAction.Result.PartitionId; if (!action.IgnoreConstraints) { // select random node where replica's primary not present var nodesInfo = await testContext.FabricCluster.GetLatestNodeInfoAsync(action.RequestTimeout, this.helper.GetRemainingTime(), cancellationToken); if ((nodesInfo == null || nodesInfo.Count() == 0)) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_NotEnoughNodesForTestabilityAction, "MovePrimary")); } ServiceReplicaList replicasResult = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.QueryManager.GetReplicaListAsync( partitionId, 0, action.RequestTimeout, cancellationToken), this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); NodeInfo currentPrimaryNodeInfo = null; string currentPrimaryNodeName = string.Empty; foreach (var replica in replicasResult) { StatefulServiceReplica statefulReplica = replica as StatefulServiceReplica; if (statefulReplica == null) { throw new InvalidOperationException(StringHelper.Format(StringResources.Error_InvalidServiceTypeTestability, "MovePrimary", "Stateful", action.PartitionSelector.ServiceName, "Stateless")); } if (statefulReplica.ReplicaRole == ReplicaRole.Primary) { currentPrimaryNodeInfo = nodesInfo.FirstOrDefault(n => n.NodeName == statefulReplica.NodeName); if (!string.IsNullOrEmpty(newPrimaryNodeName) && newPrimaryNodeName == statefulReplica.NodeName) { throw new FabricException( StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newPrimaryNodeName, "MovePrimary", "Primary already exists on node"), FabricErrorCode.AlreadyPrimaryReplica); } break; } } if (currentPrimaryNodeInfo == null) { throw new FabricException(StringHelper.Format(StringResources.Error_PartitionPrimaryNotReady, action.PartitionSelector + ":" + partitionId), FabricErrorCode.NotReady); } currentPrimaryNodeName = currentPrimaryNodeInfo.NodeName; if (newPrimaryNodeName == currentPrimaryNodeName) { throw new FabricException( StringHelper.Format(StringResources.Error_InvalidNodeNameProvided, newPrimaryNodeName, "MovePrimary", "Primary already exists on node"), FabricErrorCode.AlreadyPrimaryReplica); } } ActionTraceSource.WriteInfo(TraceSource, "Calling move primary with node {0}, partition {1}", string.IsNullOrEmpty(newPrimaryNodeName) ? "Random" : newPrimaryNodeName, partitionId); await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => testContext.FabricClient.FaultManager.MovePrimaryUsingNodeNameAsync( newPrimaryNodeName, getPartitionStateAction.Result.ServiceName, partitionId, action.IgnoreConstraints, action.RequestTimeout, cancellationToken), FabricClientRetryErrors.MovePrimaryFabricErrors.Value, this.helper.GetRemainingTime(), cancellationToken).ConfigureAwait(false); action.Result = new MovePrimaryResult(newPrimaryNodeName, getPartitionStateAction.Result); ResultTraceString = StringHelper.Format("MovePrimaryAction succeeded for moving Primary for {0} to node {1}.", partitionId, newPrimaryNodeName); }