private static NodeRepairImpactDescription GetImpactFromDetails(ITenantJob tenantJob, CoordinatorEnvironment environment)
{
environment.Validate("environment");
var translator = new ImpactTranslator(environment);
var impact = new NodeRepairImpactDescription();
if (tenantJob.JobStep == null || tenantJob.JobStep.CurrentlyImpactedRoleInstances == null)
{
return(impact);
}
foreach (var roleInstance in tenantJob.JobStep.CurrentlyImpactedRoleInstances)
{
string nodeName = roleInstance.RoleInstanceName.TranslateRoleInstanceToNodeName();
NodeImpactLevel impactLevel = translator.TranslateImpactDetailToNodeImpactLevel(
tenantJob.ImpactDetail.ImpactAction,
roleInstance.ExpectedImpact);
if (impactLevel != NodeImpactLevel.None)
{
var nodeImpact = new NodeImpact(nodeName, impactLevel);
impact.ImpactedNodes.Add(nodeImpact);
}
}
return(impact);
}
/// <inheritdoc/>
protected override void ProcessRecordInternal()
{
RepairTargetDescriptionBase repairTargetDescriptionBase = null;
if (this.Node.IsPresent)
{
repairTargetDescriptionBase = new NodeRepairTargetDescription(
nodeNames: this.NodeNames);
}
RepairImpactDescriptionBase repairImpactDescriptionBase = null;
if (this.Node.IsPresent)
{
repairImpactDescriptionBase = new NodeRepairImpactDescription(
nodeImpactList: this.NodeImpactList);
}
var repairTaskHistory = new RepairTaskHistory(
createdUtcTimestamp: this.CreatedUtcTimestamp,
claimedUtcTimestamp: this.ClaimedUtcTimestamp,
preparingUtcTimestamp: this.PreparingUtcTimestamp,
approvedUtcTimestamp: this.ApprovedUtcTimestamp,
executingUtcTimestamp: this.ExecutingUtcTimestamp,
restoringUtcTimestamp: this.RestoringUtcTimestamp,
completedUtcTimestamp: this.CompletedUtcTimestamp,
preparingHealthCheckStartUtcTimestamp: this.PreparingHealthCheckStartUtcTimestamp,
preparingHealthCheckEndUtcTimestamp: this.PreparingHealthCheckEndUtcTimestamp,
restoringHealthCheckStartUtcTimestamp: this.RestoringHealthCheckStartUtcTimestamp,
restoringHealthCheckEndUtcTimestamp: this.RestoringHealthCheckEndUtcTimestamp);
var repairTask = new RepairTask(
taskId: this.TaskId,
state: this.State,
action: this.Action,
version: this.Version,
description: this.Description,
flags: this.Flags,
target: repairTargetDescriptionBase,
executor: this.Executor,
executorData: this.ExecutorData,
impact: repairImpactDescriptionBase,
resultStatus: this.ResultStatus,
resultCode: this.ResultCode,
resultDetails: this.ResultDetails,
history: repairTaskHistory,
preparingHealthCheckState: this.PreparingHealthCheckState,
restoringHealthCheckState: this.RestoringHealthCheckState,
performPreparingHealthCheck: this.PerformPreparingHealthCheck,
performRestoringHealthCheck: this.PerformRestoringHealthCheck);
var result = this.ServiceFabricClient.Repairs.UpdateRepairExecutionStateAsync(
repairTask: repairTask,
cancellationToken: this.CancellationToken).GetAwaiter().GetResult();
if (result != null)
{
this.WriteObject(this.FormatOutput(result));
}
}
public async Task NodeRemovalTest_UnexpectedException_RecentTask()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
task.ExecutingTimestamp = DateTime.UtcNow.AddMinutes(-5);
try
{
await action.ExecuteAsync(Guid.Empty);
Assert.Fail();
}
catch (FabricException e)
{
Assert.AreEqual(FabricErrorCode.OperationNotComplete, e.ErrorCode);
}
}
private static RepairImpactDescription Convert(WinFabricRepairImpactDescription winFabricRepairImpactDescription)
{
// Impact can be null, so we just return null here
if (winFabricRepairImpactDescription == null)
{
return(null);
}
switch (winFabricRepairImpactDescription.Kind)
{
case WinFabricRepairImpactKind.Node:
var result = new NodeRepairImpactDescription();
var impactedNodes = (winFabricRepairImpactDescription as WinFabricNodeRepairImpactDescription).ImpactedNodes;
if (impactedNodes != null)
{
foreach (var winFabricNodeImpact in impactedNodes)
{
result.ImpactedNodes.Add(Convert(winFabricNodeImpact));
}
}
return(result);
case WinFabricRepairImpactKind.Invalid:
default:
throw new ArgumentOutOfRangeException("Invalid WinFabricRepairImpactKind value " + winFabricRepairImpactDescription.Kind, "winFabricRepairImpactDescription.Kind");
}
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, NodeRepairImpactDescription obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
writer.WriteProperty(obj.Kind, "Kind", RepairImpactKindConverter.Serialize);
if (obj.NodeImpactList != null)
{
writer.WriteEnumerableProperty(obj.NodeImpactList, "NodeImpactList", NodeImpactConverter.Serialize);
}
writer.WriteEndObject();
}
public async Task NodeRemovalTest_NothingToRemove()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveData));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
await action.ExecuteAsync(Guid.Empty);
}
public async Task NodeRemovalTest_UnexpectedException_OldTask()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
task.ExecutingTimestamp = DateTime.UtcNow.AddMinutes(-60);
await action.ExecuteAsync(Guid.Empty);
}
public async Task NodeRemovalTest_IgnoreNodeNotFound_MultipleNodes()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
impact.ImpactedNodes.Add(new NodeImpact("Node.1", NodeImpactLevel.RemoveNode));
impact.ImpactedNodes.Add(new NodeImpact("Node.2", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
// Retry on NodeIsUp
int callCount = 0;
int failCallCount = 2;
rm.RemoveNodeStateAsyncHandler = (n, timeout, token) =>
{
Constants.TraceType.WriteInfo("RemoveNodeState({0})", n);
++callCount;
if (n == "Node.1")
{
if (failCallCount-- > 0)
{
throw new FabricException(FabricErrorCode.NodeIsUp);
}
}
return(Task.FromResult(0));
};
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(5, callCount);
// Multiple nodes, eventually give up
callCount = 0;
failCallCount = 1000;
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(8, callCount);
}
public async Task NodeRemovalTest_IgnoreNodeNotFound()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
rm.RemoveNodeStateAsyncHandler = (n, timeout, token) =>
{
throw new FabricException(FabricErrorCode.NodeNotFound);
};
await action.ExecuteAsync(Guid.Empty);
}
/// <summary>
/// Puts a repair task in Preparing state.
/// By doing this RM will do health check (if applicable) and start disabling the node. Post disable task would be moved to Approved state by RM
/// </summary>
/// <param name="task">Repair task to be put to preparing state</param>
internal void StartPreparingRepairTask(RepairTask task)
{
// Update the health policy for the repair task
Task <long> resultTask = this.fabricClient.RepairManager.UpdateRepairTaskHealthPolicyAsync(task.TaskId, task.Version, true, null);
resultTask.Wait();
ServiceEventSource.Current.VerboseMessage("Updated health policy for repair task {0} successfully, moving to preparing state", task.TaskId);
// Put the repair task in preparing state
string nodeName = this.GetNodeNameFromRepairTask(task);
task.State = RepairTaskState.Preparing;
NodeRepairImpactDescription impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact(nodeName, NodeImpactLevel.Restart));
task.Impact = impact;
task.PerformPreparingHealthCheck = true;
// Use the latest version or else next Update call would fail
task.Version = resultTask.Result;
this.fabricClient.RepairManager.UpdateRepairExecutionStateAsync(task);
}
public async Task ExecuteAsync(IRepairManager repairManager)
{
string taskId = GenerateTaskId(this.record.RecordId, DateTime.UtcNow);
var repairTask = repairManager.NewRepairTask(taskId, this.record.RepairType.ToString());
repairTask.State = RepairTaskState.Preparing;
repairTask.Executor = this.serviceName;
var target = new NodeRepairTargetDescription(this.record.MachineName);
repairTask.Target = target;
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact(this.record.MachineName, this.impactLevel));
repairTask.Impact = impact;
repairTask.PerformPreparingHealthCheck = this.performHealthCheck;
await repairManager.CreateRepairTaskAsync(Guid.Empty, repairTask);
}
public async Task NodeRemovalTest_RetryNodeIsUp()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
// Retry on NodeIsUp
int callCount = 0;
int failCallCount = 2;
rm.RemoveNodeStateAsyncHandler = (n, timeout, token) =>
{
++callCount;
if (failCallCount-- > 0)
{
throw new FabricException(FabricErrorCode.NodeIsUp);
}
return(Task.FromResult(0));
};
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(3, callCount);
// Eventually give up
callCount = 0;
failCallCount = 1000;
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(6, callCount);
}
private RepairTask CreateRepairTask()
{
string taskId = string.Format(
CultureInfo.InvariantCulture,
DefaultTaskIdFormat,
Guid.NewGuid());
if (!string.IsNullOrEmpty(this.TaskId))
{
taskId = this.TaskId;
}
switch (this.ParameterSetName)
{
case ParamSetNodeBuiltInAuto:
{
var repairTask = new ClusterRepairTask(
taskId,
SystemRepairActionHelper.GetActionString(this.NodeAction));
repairTask.Description = this.Description;
repairTask.Target = new NodeRepairTargetDescription(this.NodeName);
return(repairTask);
}
case ParamSetNodeCustomAuto:
{
var repairTask = new ClusterRepairTask(taskId, this.CustomAction);
repairTask.Description = this.Description;
repairTask.Target = new NodeRepairTargetDescription(this.NodeNames);
return(repairTask);
}
case ParamSetNodeManual:
{
var repairTask = new ClusterRepairTask(
taskId,
SystemRepairActionHelper.ManualRepairAction);
repairTask.Description = this.Description;
repairTask.State = RepairTaskState.Preparing;
repairTask.Executor = "Manual";
// Informational only
repairTask.Target = new NodeRepairTargetDescription(this.NodeNames);
var impact = new NodeRepairImpactDescription();
foreach (var nodeName in this.NodeNames)
{
impact.ImpactedNodes.Add(new NodeImpact(nodeName, this.NodeImpact));
}
repairTask.Impact = impact;
return(repairTask);
}
default:
// Unsupported parameter set
throw new NotSupportedException();
}
}
