public async Task NodeRemovalTest_UnexpectedException_RecentTask()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
task.ExecutingTimestamp = DateTime.UtcNow.AddMinutes(-5);
try
{
await action.ExecuteAsync(Guid.Empty);
Assert.Fail();
}
catch (FabricException e)
{
Assert.AreEqual(FabricErrorCode.OperationNotComplete, e.ErrorCode);
}
}
protected override void AfterUpdateRepairExecutionState(Guid activityId, MockRepairTask repairTask)
{
base.AfterUpdateRepairExecutionState(activityId, repairTask);
if (repairTask.State == RepairTaskState.Completed)
{
CompletedEvent.Set();
}
}
public void RepairTaskSerDeTest()
{
var repairTask = new MockRepairTask("taskA", "System.Azure.Reboot");
string json = JsonConvert.SerializeObject(repairTask, internalSerializationSettings);
var repairTask2 = json.FromJson <MockRepairTask>();
string json3 = JsonConvert.SerializeObject(repairTask2, internalSerializationSettings);
Assert.AreEqual(json, json3);
}
private IAction CreateMoveToRestoringAction(MockRepairTask task)
{
return(new MoveToRestoringAction(
env,
rm,
task,
RepairTaskResult.Succeeded,
null,
surpriseJob: false,
cancelRestoringHealthCheck: false,
processRemovedNodes: true));
}
protected virtual void AfterCreateRepairTask(
Guid activityId,
MockRepairTask repairTask)
{
// the "real" repair manager requires repair tasks to be created with the initial state as Preparing
if (repairTask.State != RepairTaskState.Preparing)
{
return;
}
repairTask.State = RepairTaskState.Approved;
UpdateRepairExecutionStateAsync(activityId, repairTask).GetAwaiter().GetResult();
}
private static IRepairTask CreateRepairTask()
{
var repairTask = new MockRepairTask(
"rt1",
Parallel.Constants.RepairActionFormat.ToString(RepairActionTypeEnum.Reboot))
{
Target = new NodeRepairTargetDescription("Role.0"),
State = RepairTaskState.Created,
Description = "Creating a repair task externally and not via IS",
};
return(repairTask);
}
protected override void AfterUpdateRepairExecutionState(Guid activityId, MockRepairTask repairTask)
{
base.AfterUpdateRepairExecutionState(activityId, repairTask);
if (repairTask.State == RepairTaskState.Claimed && !repairTask.ClaimedTimestamp.HasValue)
{
repairTask.ClaimedTimestamp = DateTime.UtcNow;
}
if (repairTask.State == RepairTaskState.Completed && repairTask.ResultStatus == RepairTaskResult.Cancelled)
{
CompletedEvent.Set();
}
}
public async Task NodeRemovalTest_NothingToRemove()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveData));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
await action.ExecuteAsync(Guid.Empty);
}
public async Task NodeRemovalTest_UnexpectedException_OldTask()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
task.ExecutingTimestamp = DateTime.UtcNow.AddMinutes(-60);
await action.ExecuteAsync(Guid.Empty);
}
public async Task NodeRemovalTest_IgnoreNodeNotFound_MultipleNodes()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
impact.ImpactedNodes.Add(new NodeImpact("Node.1", NodeImpactLevel.RemoveNode));
impact.ImpactedNodes.Add(new NodeImpact("Node.2", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
// Retry on NodeIsUp
int callCount = 0;
int failCallCount = 2;
rm.RemoveNodeStateAsyncHandler = (n, timeout, token) =>
{
Constants.TraceType.WriteInfo("RemoveNodeState({0})", n);
++callCount;
if (n == "Node.1")
{
if (failCallCount-- > 0)
{
throw new FabricException(FabricErrorCode.NodeIsUp);
}
}
return(Task.FromResult(0));
};
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(5, callCount);
// Multiple nodes, eventually give up
callCount = 0;
failCallCount = 1000;
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(8, callCount);
}
public async Task NodeRemovalTest_IgnoreNodeNotFound()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
rm.RemoveNodeStateAsyncHandler = (n, timeout, token) =>
{
throw new FabricException(FabricErrorCode.NodeNotFound);
};
await action.ExecuteAsync(Guid.Empty);
}
/// <summary>
/// TODO, consider providing overloads in case we need timeout and cancellationToken parameters.
/// </summary>
protected virtual void AfterUpdateRepairExecutionState(
Guid activityId,
MockRepairTask repairTask)
{
bool stateChanged = false;
switch (repairTask.State)
{
case RepairTaskState.Claimed:
repairTask.ClaimedTimestamp = DateTime.UtcNow;
break;
case RepairTaskState.Preparing:
repairTask.PreparingTimestamp = DateTime.UtcNow;
repairTask.ApprovedTimestamp = DateTime.UtcNow;
repairTask.State = RepairTaskState.Approved;
stateChanged = true;
break;
case RepairTaskState.Executing:
repairTask.ExecutingTimestamp = DateTime.UtcNow;
break;
case RepairTaskState.Restoring:
repairTask.RestoringTimestamp = DateTime.UtcNow;
repairTask.CompletedTimestamp = DateTime.UtcNow;
repairTask.State = RepairTaskState.Completed;
stateChanged = true;
break;
default:
return;
}
if (stateChanged)
{
UpdateRepairExecutionStateAsync(activityId, repairTask).GetAwaiter().GetResult();
}
}
public async Task NodeRemovalTest_RetryNodeIsUp()
{
var impact = new NodeRepairImpactDescription();
impact.ImpactedNodes.Add(new NodeImpact("Node.0", NodeImpactLevel.RemoveNode));
var task = new MockRepairTask("id", "action")
{
Impact = impact,
};
var action = CreateMoveToRestoringAction(task);
// Retry on NodeIsUp
int callCount = 0;
int failCallCount = 2;
rm.RemoveNodeStateAsyncHandler = (n, timeout, token) =>
{
++callCount;
if (failCallCount-- > 0)
{
throw new FabricException(FabricErrorCode.NodeIsUp);
}
return(Task.FromResult(0));
};
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(3, callCount);
// Eventually give up
callCount = 0;
failCallCount = 1000;
await action.ExecuteAsync(Guid.Empty);
Assert.AreEqual(6, callCount);
}
