public RequestRepairAction(CoordinatorEnvironment environment, IPolicyAgentClient policyAgentClient, IRepairTask repairTask, RepairActionProvider repairActionProvider)
: base(environment, ActionType.RequestRepair)
{
this.policyAgentClient = policyAgentClient.Validate("policyAgentClient");
this.repairTask = repairTask.Validate("repairTask");
this.repairActionProvider = repairActionProvider.Validate("repairActionProvider");
}
public async Task <long> CancelRepairTaskAsync(Guid activityId, IRepairTask repairTask)
{
repairTask.Validate("repairTask");
//var startTime = DateTimeOffset.UtcNow;
//var oldState = repairTask.State;
//var duration = repairTask.GetTimeInCurrentState(startTime);
try
{
var commitVersion = await repairManager.CancelRepairTaskAsync(repairTask.TaskId, repairTask.Version, false).ConfigureAwait(false);
traceType.WriteInfo(
"Canceled repair task: {0}",
repairTask.ToJson());
//activityLogger.LogChangeState(activityId, repairTask, oldState, RepairTaskState.Invalid, duration, repairTask.ToString(), repairTask.TaskId);
//activityLogger.LogOperation(activityId, startTime, OperationResult.Success, null, repairTask.ToJson());
return(commitVersion);
}
catch (Exception ex)
{
traceType.WriteWarning(
"Unable to cancel repair task. Continuing. Repair task: {0}{1}Errors: {2}",
repairTask.ToJson(), Environment.NewLine, ex);
//activityLogger.LogOperation(activityId, startTime, OperationResult.Failure, ex, repairTask.ToJson());
throw;
}
}
public async Task <long> UpdateRepairExecutionStateAsync(Guid activityId, IRepairTask repairTask)
{
repairTask.Validate("repairTask");
//var startTime = DateTimeOffset.UtcNow;
try
{
var commitVersion = await repairManager.UpdateRepairExecutionStateAsync(Unwrap(repairTask)).ConfigureAwait(false);
traceType.WriteInfo(
"Successfully updated repair execution state. Commit version: {0}, repair task: {1}",
commitVersion,
repairTask.ToJson());
//activityLogger.LogOperation(activityId, startTime, OperationResult.Success, null, repairTask.ToJson());
return(commitVersion);
}
catch (Exception ex)
{
traceType.WriteWarning("Unable to update repair execution state: Error: {0}", ex);
//activityLogger.LogOperation(activityId, startTime, OperationResult.Failure, ex, repairTask.ToJson());
throw;
}
}
public static RepairTaskExecutorData GetExecutorData(this IRepairTask repairTask)
{
repairTask.Validate("repairTask");
if (string.IsNullOrWhiteSpace(repairTask.ExecutorData))
{
return(null);
}
try
{
var executorData = repairTask.ExecutorData.FromJson <RepairTaskExecutorData>();
return(executorData);
}
catch (Exception ex)
{
Constants.TraceType.WriteWarning(
"Error while getting executor data for repair task. Returning null. Repair task: {0}{1}Exception: {2}",
repairTask.ToJson(),
Environment.NewLine,
ex);
return(null);
}
}
private static bool IsOldCompletedRepairTask(IRepairTask task, DateTime now, TimeSpan ageThreshold)
{
return
((task.State == RepairTaskState.Completed) &&
task.CompletedTimestamp.HasValue &&
((now - task.CompletedTimestamp) > ageThreshold));
}
// Does repairTask represent a VendorRepair, via action/impact match?
// This is used to check if a VRE matches a particular repair task
public static bool MatchesVendorRepairEndJob(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
if (!repairTask.IsVendorRepair())
{
return(false);
}
// 2. check impact
if (repairTask.Impact == null || repairTask.Impact.Kind != RepairImpactKind.Node)
{
return(false);
}
var impactedNodesFromRepairTask = ((NodeRepairImpactDescription)repairTask.Impact).ImpactedNodes;
var impactedNodesFromTenantJob = new HashSet <string>(
tenantJob.JobStep.CurrentlyImpactedRoleInstances.Select(i => i.RoleInstanceName.TranslateRoleInstanceToNodeName()),
StringComparer.OrdinalIgnoreCase);
// If all impacted nodes in this VendorRepair task are covered by the job, it's a match
foreach (var nodeFromRepairTask in impactedNodesFromRepairTask)
{
if (!impactedNodesFromTenantJob.Contains(nodeFromRepairTask.NodeName))
{
return(false);
}
}
return(true);
}
public async Task <long> CreateRepairTaskAsync(Guid activityId, IRepairTask repairTask)
{
repairTask.Validate("repairTask");
//var startTime = DateTimeOffset.UtcNow;
try
{
// TODO, using the overload without timeout and cancellation token for now since there is some max timeout limit
// being exercised somewhere. if timeout provided is more than that, repair task creation fails
long result = await repairManager.CreateRepairTaskAsync(Unwrap(repairTask)).ConfigureAwait(false);
traceType.WriteInfo(
"Created repair task. Result: {0}, repair task: {1}",
result,
repairTask.ToJson());
//activityLogger.LogChangeState(activityId, repairTask, RepairTaskState.Invalid, repairTask.State, TimeSpan.Zero, repairTask.ToJson(), repairTask.TaskId);
//activityLogger.LogOperation(activityId, startTime, OperationResult.Success, null, repairTask.ToJson());
return(result);
}
catch (Exception ex)
{
traceType.WriteWarning("Unable to create repair task. Errors: {0}", ex.GetMessage());
//activityLogger.LogOperation(activityId, startTime, OperationResult.Failure, ex, repairTask.ToJson());
throw;
}
}
public MoveToExecutingAction(CoordinatorEnvironment environment, IRepairManager repairManager, IRepairTask repairTask, string warning = null)
: base(environment, ActionType.Execute)
{
this.repairManager = repairManager.Validate("repairManager");
this.repairTask = repairTask.Validate("repairTask");
this.warning = warning;
}
public IAction NewMoveToCompletedAction(
IRepairTask repairTask,
RepairTaskResult repairTaskResult,
string resultDetails,
bool surpriseJob)
{
return(new MoveToCompletedAction(environment, repairManager, repairTask, repairTaskResult, resultDetails, surpriseJob));
}
public Task <long> CancelRepairTaskAsync(Guid activityId, IRepairTask repairTask)
{
VerifyMockRepairTask(repairTask);
// TODO - mock a safe cancellation (not a request abort)
repairTask.ResultStatus = RepairTaskResult.Cancelled;
repairTask.State = RepairTaskState.Completed;
return(Task.FromResult(0L));
}
public CancelRestoringHealthCheckAction(
CoordinatorEnvironment environment,
IRepairManager repairManager,
IRepairTask repairTask)
: base(environment, ActionType.Restore)
{
this.repairManager = repairManager.Validate("repairManager");
this.repairTask = repairTask.Validate("repairTask");
}
public Task <long> UpdateRepairTaskHealthPolicyAsync(
Guid activityId,
IRepairTask repairTask,
bool?performPreparingHealthCheck,
bool?performRestoringHealthCheck)
{
VerifyMockRepairTask(repairTask);
// Note: no implementation. For now, set the corresponding properties accordingly
return(Task.FromResult(0L));
}
public static bool IsContextIdMatch(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
var contextId = repairTask.GetTenantJobContext();
bool contextIdMatch = string.Equals(contextId, tenantJob.ContextStringGivenByTenant, StringComparison.OrdinalIgnoreCase);
return(contextIdMatch);
}
public Task <long> UpdateRepairExecutionStateAsync(Guid activityId, IRepairTask repairTask)
{
var mockTask = VerifyMockRepairTask(repairTask);
repairTasks[repairTask.TaskId] = repairTask;
AfterUpdateRepairExecutionState(activityId, mockTask);
return(Task.FromResult(0L));
}
protected override void Run(CancellationToken cancellationToken)
{
IRepairTask repairTask = CreateRepairTask();
RepairManager.CreateRepairTaskAsync(Guid.NewGuid(), repairTask).GetAwaiter().GetResult();
RepairManager.CompletedEvent.WaitOne();
PolicyAgentService.CompletedEvent.WaitOne();
CompletedEvent.Set();
}
public IAction NewMoveToRestoringAction(
IRepairTask repairTask,
RepairTaskResult repairTaskResult,
string resultDetails,
bool surpriseJob = false,
bool cancelRestoringHealthCheck = false,
bool processRemovedNodes = false)
{
return(new MoveToRestoringAction(environment, repairManager, repairTask, repairTaskResult, resultDetails, surpriseJob, cancelRestoringHealthCheck, processRemovedNodes));
}
public static IEnumerable <string> GetTargetRoleInstances(this IRepairTask repairTask)
{
var nodeNames = repairTask.GetTargetNodeNames();
if (nodeNames == null)
{
return(null);
}
return(nodeNames.Select(e => e.TranslateNodeNameToRoleInstance()));
}
private static string GetJobStepId(this IRepairTask repairTask)
{
var executorData = repairTask.GetExecutorData();
if (executorData == null)
{
return(null);
}
return(executorData.StepId);
}
protected override void Run(CancellationToken cancellationToken)
{
IRepairTask repairTask = CreateRepairTask();
RepairManager.CreateRepairTaskAsync(Guid.NewGuid(), repairTask).GetAwaiter().GetResult();
((MockRepairTask)repairTask).CreatedTimestamp = DateTime.UtcNow;
RepairManager.CompletedEvent.WaitOne();
CompletedEvent.Set();
}
public static bool IsJobIdMatch(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
var jobId = repairTask.GetJobId();
bool isMatch = (jobId != null) && string.Equals(jobId, tenantJob.Id.ToString());
return(isMatch);
}
public static bool IsJobKeyMatch(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
var jobId = repairTask.GetJobId();
var jobKey = repairTask.GetJobKey();
bool jobKeyMatch = jobId != null && jobKey != null && string.Equals(jobKey, tenantJob.GetJobKey());
return(jobKeyMatch);
}
public static bool IsVendorRepair(this IRepairTask repairTask)
{
repairTask.Validate("repairTask");
var executorData = repairTask.GetExecutorData();
if (executorData == null)
{
return(false);
}
return(executorData.HasFlag(RepairTaskExecutorData.VendorRepairFlag));
}
public ExecuteJobAction(
CoordinatorEnvironment environment,
Action <Guid> approveJobAction,
IRepairManager repairManager,
ITenantJob tenantJob,
IRepairTask repairTask)
: base(environment, ActionType.None)
{
this.approveJobAction = approveJobAction.Validate("approveJobAction");
this.repairManager = repairManager.Validate("repairManager");
this.tenantJob = tenantJob.Validate("tenantJob");
this.repairTask = repairTask.Validate("repairTask");
}
public ClaimedRepairTaskAction(
CoordinatorEnvironment environment,
IRepairManager repairManager,
IRepairTask repairTask,
IList <string> roleInstanceNames,
RepairActionProvider repairActionProvider)
: base(environment, ActionType.None)
{
this.repairManager = repairManager.Validate("repairManager");
this.repairTask = repairTask.Validate("repairTask");
this.roleInstanceNames = roleInstanceNames.Validate("roleInstanceName");
this.repairActionProvider = repairActionProvider.Validate("repairActionProvider");
}
/// <summary>
/// Gets the target node name for repair tasks.
/// </summary>
/// <param name="repairTask"></param>
/// <returns>The target node name or <c>null</c> if the repair task didn't match the expected format.</returns>
public static IList <string> GetTargetNodeNames(this IRepairTask repairTask)
{
repairTask.Validate("repairTask");
if (repairTask.Target == null || repairTask.Target.Kind != RepairTargetKind.Node)
{
Constants.TraceType.WriteInfo("Repair task {0}: ignoring unknown target type {1}", repairTask.TaskId, repairTask.Target);
return(null);
}
NodeRepairTargetDescription targetDescription = (NodeRepairTargetDescription)repairTask.Target;
return(targetDescription.Nodes);
}
private void ReconcileUnmatchedRepairTask(IRepairTask repairTask)
{
if (repairTask.State != RepairTaskState.Completed)
{
traceType.WriteInfo(
"Repair task {0} in state {1} does not match any pending or active repairs",
repairTask.TaskId,
repairTask.State);
}
if (this.doNotCancelMachineNames.Contains(GetMachineNameFromTaskId(repairTask.TaskId)))
{
// e.g. once in SoftWipe Proceed, the machine is in C, and *ImageBurnin tasks proceed without waiting for ask mode
traceType.WriteInfo(
"Repair task {0} will not be cancelled/completed at this time because the current machine state allows additional repairs to proceed without approval",
repairTask.TaskId);
return;
}
switch (repairTask.State)
{
case RepairTaskState.Preparing:
EmitActionCancelRepairTask(repairTask);
break;
case RepairTaskState.Approved:
EmitActionMoveRepairTaskToRestoring(repairTask);
break;
case RepairTaskState.Executing:
// Dampen cancellation for repair tasks that recently changed to Executing,
// to give the repair record time to show up in the active repairs list.
if (IsTimestampOld(repairTask.ExecutingTimestamp, this.configReader.MinimumRepairExecutionTime, unknownIsOld: true))
{
EmitActionMoveRepairTaskToRestoring(repairTask);
}
else
{
traceType.WriteInfo(
"Delaying completion of repair task {0} because it was moved to Executing recently (at {1:O})",
repairTask.TaskId,
repairTask.ExecutingTimestamp);
}
break;
}
// Nothing to do in other cases
}
private async Task <bool> CompleteRepairTaskAsync(Guid activityId, IRepairTask task)
{
traceType.WriteInfo(
"Cancelling repair task {0} in state {1}:{2}{3}",
task.TaskId,
task.State,
Environment.NewLine,
task.ToJson());
bool succeeded = true;
try
{
switch (task.State)
{
case RepairTaskState.Claimed:
case RepairTaskState.Preparing:
await repairManager.CancelRepairTaskAsync(activityId, task);
break;
case RepairTaskState.Approved:
case RepairTaskState.Executing:
task.State = RepairTaskState.Restoring;
task.ResultStatus = RepairTaskResult.Interrupted;
task.ResultDetails = "Repair task was cancelled because Azure coordination is disabled";
await repairManager.UpdateRepairExecutionStateAsync(activityId, task);
break;
case RepairTaskState.Restoring:
await repairManager.UpdateRepairTaskHealthPolicyAsync(activityId, task, null, performRestoringHealthCheck : false);
break;
}
}
catch (Exception e)
{
traceType.WriteWarning(
"Failed to cancel repair task {0} in state {1}: {2}",
task.TaskId,
task.State,
e);
succeeded = false;
}
return(succeeded);
}
/// <summary>
/// Gets the job key which is a Azure JobId/UD combination corresponding to the repair task.
/// </summary>
private static string GetJobKey(this IRepairTask repairTask)
{
repairTask.Validate("repairTask");
var id = repairTask.GetJobId();
var stepId = repairTask.GetJobStepId();
if (string.IsNullOrWhiteSpace(id) || stepId == null)
{
return(null);
}
var key = Constants.JobKeyFormat.ToString(id, stepId);
return(key);
}
public MoveToCompletedAction(
CoordinatorEnvironment environment,
IRepairManager repairManager,
IRepairTask repairTask,
RepairTaskResult repairTaskResult,
string repairTaskResultDetails,
bool surpriseJob)
: base(environment, ActionType.None)
{
this.repairManager = repairManager.Validate("repairManager");
this.repairTask = repairTask.Validate("repairTask");
this.repairTaskResult = repairTaskResult;
this.repairTaskResultDetails = repairTaskResultDetails;
this.surpriseJob = surpriseJob;
}
public MoveToPreparingAction(
CoordinatorEnvironment environment,
IRepairManager repairManager,
IRepairTask repairTask,
ITenantJob tenantJob,
bool surpriseJob,
RepairTaskPrepareArgs args)
: base(environment, ActionType.Prepare)
{
this.repairManager = repairManager.Validate("repairManager");
this.repairTask = repairTask.Validate("repairTask");
this.tenantJob = tenantJob.Validate("tenantJob");
this.surpriseJob = surpriseJob;
this.args = args.Validate("args");
}