private static NodeRepairImpactDescription GetImpactFromDetails(ITenantJob tenantJob, CoordinatorEnvironment environment) { environment.Validate("environment"); var translator = new ImpactTranslator(environment); var impact = new NodeRepairImpactDescription(); if (tenantJob.JobStep == null || tenantJob.JobStep.CurrentlyImpactedRoleInstances == null) { return(impact); } foreach (var roleInstance in tenantJob.JobStep.CurrentlyImpactedRoleInstances) { string nodeName = roleInstance.RoleInstanceName.TranslateRoleInstanceToNodeName(); NodeImpactLevel impactLevel = translator.TranslateImpactDetailToNodeImpactLevel( tenantJob.ImpactDetail.ImpactAction, roleInstance.ExpectedImpact); if (impactLevel != NodeImpactLevel.None) { var nodeImpact = new NodeImpact(nodeName, impactLevel); impact.ImpactedNodes.Add(nodeImpact); } } return(impact); }
/// <inheritdoc/> protected override void ProcessRecordInternal() { RepairTargetDescriptionBase repairTargetDescriptionBase = null; if (this.Node.IsPresent) { repairTargetDescriptionBase = new NodeRepairTargetDescription( nodeNames: this.NodeNames); } RepairImpactDescriptionBase repairImpactDescriptionBase = null; if (this.Node.IsPresent) { repairImpactDescriptionBase = new NodeRepairImpactDescription( nodeImpactList: this.NodeImpactList); } var repairTaskHistory = new RepairTaskHistory( createdUtcTimestamp: this.CreatedUtcTimestamp, claimedUtcTimestamp: this.ClaimedUtcTimestamp, preparingUtcTimestamp: this.PreparingUtcTimestamp, approvedUtcTimestamp: this.ApprovedUtcTimestamp, executingUtcTimestamp: this.ExecutingUtcTimestamp, restoringUtcTimestamp: this.RestoringUtcTimestamp, completedUtcTimestamp: this.CompletedUtcTimestamp, preparingHealthCheckStartUtcTimestamp: this.PreparingHealthCheckStartUtcTimestamp, preparingHealthCheckEndUtcTimestamp: this.PreparingHealthCheckEndUtcTimestamp, restoringHealthCheckStartUtcTimestamp: this.RestoringHealthCheckStartUtcTimestamp, restoringHealthCheckEndUtcTimestamp: this.RestoringHealthCheckEndUtcTimestamp); var repairTask = new RepairTask( taskId: this.TaskId, state: this.State, action: this.Action, version: this.Version, description: this.Description, flags: this.Flags, target: repairTargetDescriptionBase, executor: this.Executor, executorData: this.ExecutorData, impact: repairImpactDescriptionBase, resultStatus: this.ResultStatus, resultCode: this.ResultCode, resultDetails: this.ResultDetails, history: repairTaskHistory, preparingHealthCheckState: this.PreparingHealthCheckState, restoringHealthCheckState: this.RestoringHealthCheckState, performPreparingHealthCheck: this.PerformPreparingHealthCheck, performRestoringHealthCheck: this.PerformRestoringHealthCheck); var result = this.ServiceFabricClient.Repairs.UpdateRepairExecutionStateAsync( repairTask: repairTask, cancellationToken: this.CancellationToken).GetAwaiter().GetResult(); if (result != null) { this.WriteObject(this.FormatOutput(result)); } }
public async Task NodeRemovalTest_UnexpectedException_RecentTask() { var impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode)); var task = new MockRepairTask("id", "action") { Impact = impact, }; var action = CreateMoveToRestoringAction(task); task.ExecutingTimestamp = DateTime.UtcNow.AddMinutes(-5); try { await action.ExecuteAsync(Guid.Empty); Assert.Fail(); } catch (FabricException e) { Assert.AreEqual(FabricErrorCode.OperationNotComplete, e.ErrorCode); } }
private static RepairImpactDescription Convert(WinFabricRepairImpactDescription winFabricRepairImpactDescription) { // Impact can be null, so we just return null here if (winFabricRepairImpactDescription == null) { return(null); } switch (winFabricRepairImpactDescription.Kind) { case WinFabricRepairImpactKind.Node: var result = new NodeRepairImpactDescription(); var impactedNodes = (winFabricRepairImpactDescription as WinFabricNodeRepairImpactDescription).ImpactedNodes; if (impactedNodes != null) { foreach (var winFabricNodeImpact in impactedNodes) { result.ImpactedNodes.Add(Convert(winFabricNodeImpact)); } } return(result); case WinFabricRepairImpactKind.Invalid: default: throw new ArgumentOutOfRangeException("Invalid WinFabricRepairImpactKind value " + winFabricRepairImpactDescription.Kind, "winFabricRepairImpactDescription.Kind"); } }
/// <summary> /// Serializes the object to JSON. /// </summary> /// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param> /// <param name="obj">The object to serialize to JSON.</param> internal static void Serialize(JsonWriter writer, NodeRepairImpactDescription obj) { // Required properties are always serialized, optional properties are serialized when not null. writer.WriteStartObject(); writer.WriteProperty(obj.Kind, "Kind", RepairImpactKindConverter.Serialize); if (obj.NodeImpactList != null) { writer.WriteEnumerableProperty(obj.NodeImpactList, "NodeImpactList", NodeImpactConverter.Serialize); } writer.WriteEndObject(); }
public async Task NodeRemovalTest_NothingToRemove() { var impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveData)); var task = new MockRepairTask("id", "action") { Impact = impact, }; var action = CreateMoveToRestoringAction(task); await action.ExecuteAsync(Guid.Empty); }
public async Task NodeRemovalTest_UnexpectedException_OldTask() { var impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode)); var task = new MockRepairTask("id", "action") { Impact = impact, }; var action = CreateMoveToRestoringAction(task); task.ExecutingTimestamp = DateTime.UtcNow.AddMinutes(-60); await action.ExecuteAsync(Guid.Empty); }
public async Task NodeRemovalTest_IgnoreNodeNotFound_MultipleNodes() { var impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode)); impact.ImpactedNodes.Add(new NodeImpact("Node.1", NodeImpactLevel.RemoveNode)); impact.ImpactedNodes.Add(new NodeImpact("Node.2", NodeImpactLevel.RemoveNode)); var task = new MockRepairTask("id", "action") { Impact = impact, }; var action = CreateMoveToRestoringAction(task); // Retry on NodeIsUp int callCount = 0; int failCallCount = 2; rm.RemoveNodeStateAsyncHandler = (n, timeout, token) => { Constants.TraceType.WriteInfo("RemoveNodeState({0})", n); ++callCount; if (n == "Node.1") { if (failCallCount-- > 0) { throw new FabricException(FabricErrorCode.NodeIsUp); } } return(Task.FromResult(0)); }; await action.ExecuteAsync(Guid.Empty); Assert.AreEqual(5, callCount); // Multiple nodes, eventually give up callCount = 0; failCallCount = 1000; await action.ExecuteAsync(Guid.Empty); Assert.AreEqual(8, callCount); }
public async Task NodeRemovalTest_IgnoreNodeNotFound() { var impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode)); var task = new MockRepairTask("id", "action") { Impact = impact, }; var action = CreateMoveToRestoringAction(task); rm.RemoveNodeStateAsyncHandler = (n, timeout, token) => { throw new FabricException(FabricErrorCode.NodeNotFound); }; await action.ExecuteAsync(Guid.Empty); }
/// <summary> /// Puts a repair task in Preparing state. /// By doing this RM will do health check (if applicable) and start disabling the node. Post disable task would be moved to Approved state by RM /// </summary> /// <param name="task">Repair task to be put to preparing state</param> internal void StartPreparingRepairTask(RepairTask task) { // Update the health policy for the repair task Task <long> resultTask = this.fabricClient.RepairManager.UpdateRepairTaskHealthPolicyAsync(task.TaskId, task.Version, true, null); resultTask.Wait(); ServiceEventSource.Current.VerboseMessage("Updated health policy for repair task {0} successfully, moving to preparing state", task.TaskId); // Put the repair task in preparing state string nodeName = this.GetNodeNameFromRepairTask(task); task.State = RepairTaskState.Preparing; NodeRepairImpactDescription impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact(nodeName, NodeImpactLevel.Restart)); task.Impact = impact; task.PerformPreparingHealthCheck = true; // Use the latest version or else next Update call would fail task.Version = resultTask.Result; this.fabricClient.RepairManager.UpdateRepairExecutionStateAsync(task); }
public async Task ExecuteAsync(IRepairManager repairManager) { string taskId = GenerateTaskId(this.record.RecordId, DateTime.UtcNow); var repairTask = repairManager.NewRepairTask(taskId, this.record.RepairType.ToString()); repairTask.State = RepairTaskState.Preparing; repairTask.Executor = this.serviceName; var target = new NodeRepairTargetDescription(this.record.MachineName); repairTask.Target = target; var impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact(this.record.MachineName, this.impactLevel)); repairTask.Impact = impact; repairTask.PerformPreparingHealthCheck = this.performHealthCheck; await repairManager.CreateRepairTaskAsync(Guid.Empty, repairTask); }
public async Task NodeRemovalTest_RetryNodeIsUp() { var impact = new NodeRepairImpactDescription(); impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode)); var task = new MockRepairTask("id", "action") { Impact = impact, }; var action = CreateMoveToRestoringAction(task); // Retry on NodeIsUp int callCount = 0; int failCallCount = 2; rm.RemoveNodeStateAsyncHandler = (n, timeout, token) => { ++callCount; if (failCallCount-- > 0) { throw new FabricException(FabricErrorCode.NodeIsUp); } return(Task.FromResult(0)); }; await action.ExecuteAsync(Guid.Empty); Assert.AreEqual(3, callCount); // Eventually give up callCount = 0; failCallCount = 1000; await action.ExecuteAsync(Guid.Empty); Assert.AreEqual(6, callCount); }
private RepairTask CreateRepairTask() { string taskId = string.Format( CultureInfo.InvariantCulture, DefaultTaskIdFormat, Guid.NewGuid()); if (!string.IsNullOrEmpty(this.TaskId)) { taskId = this.TaskId; } switch (this.ParameterSetName) { case ParamSetNodeBuiltInAuto: { var repairTask = new ClusterRepairTask( taskId, SystemRepairActionHelper.GetActionString(this.NodeAction)); repairTask.Description = this.Description; repairTask.Target = new NodeRepairTargetDescription(this.NodeName); return(repairTask); } case ParamSetNodeCustomAuto: { var repairTask = new ClusterRepairTask(taskId, this.CustomAction); repairTask.Description = this.Description; repairTask.Target = new NodeRepairTargetDescription(this.NodeNames); return(repairTask); } case ParamSetNodeManual: { var repairTask = new ClusterRepairTask( taskId, SystemRepairActionHelper.ManualRepairAction); repairTask.Description = this.Description; repairTask.State = RepairTaskState.Preparing; repairTask.Executor = "Manual"; // Informational only repairTask.Target = new NodeRepairTargetDescription(this.NodeNames); var impact = new NodeRepairImpactDescription(); foreach (var nodeName in this.NodeNames) { impact.ImpactedNodes.Add(new NodeImpact(nodeName, this.NodeImpact)); } repairTask.Impact = impact; return(repairTask); } default: // Unsupported parameter set throw new NotSupportedException(); } }