private static NodeRepairImpactDescription GetImpactFromDetails(ITenantJob tenantJob, CoordinatorEnvironment environment)
{
environment.Validate("environment");
var translator = new ImpactTranslator(environment);
var impact = new NodeRepairImpactDescription();
if (tenantJob.JobStep == null || tenantJob.JobStep.CurrentlyImpactedRoleInstances == null)
{
return(impact);
}
foreach (var roleInstance in tenantJob.JobStep.CurrentlyImpactedRoleInstances)
{
string nodeName = roleInstance.RoleInstanceName.TranslateRoleInstanceToNodeName();
NodeImpactLevel impactLevel = translator.TranslateImpactDetailToNodeImpactLevel(
tenantJob.ImpactDetail.ImpactAction,
roleInstance.ExpectedImpact);
if (impactLevel != NodeImpactLevel.None)
{
var nodeImpact = new NodeImpact(nodeName, impactLevel);
impact.ImpactedNodes.Add(nodeImpact);
}
}
return(impact);
}
// Does repairTask represent a VendorRepair, via action/impact match?
// This is used to check if a VRE matches a particular repair task
public static bool MatchesVendorRepairEndJob(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
if (!repairTask.IsVendorRepair())
{
return(false);
}
// 2. check impact
if (repairTask.Impact == null || repairTask.Impact.Kind != RepairImpactKind.Node)
{
return(false);
}
var impactedNodesFromRepairTask = ((NodeRepairImpactDescription)repairTask.Impact).ImpactedNodes;
var impactedNodesFromTenantJob = new HashSet <string>(
tenantJob.JobStep.CurrentlyImpactedRoleInstances.Select(i => i.RoleInstanceName.TranslateRoleInstanceToNodeName()),
StringComparer.OrdinalIgnoreCase);
// If all impacted nodes in this VendorRepair task are covered by the job, it's a match
foreach (var nodeFromRepairTask in impactedNodesFromRepairTask)
{
if (!impactedNodesFromTenantJob.Contains(nodeFromRepairTask.NodeName))
{
return(false);
}
}
return(true);
}
private static bool IsJobWaitingForApproval(ITenantJob job)
{
return
((job.JobStatus == JobStatusEnum.Executing) &&
(job.JobStep != null) &&
(job.JobStep.AcknowledgementStatus == AcknowledgementStatusEnum.WaitingForAcknowledgement));
}
public static bool IsPlatformUpdateJobType(this ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
var action = tenantJob.GetImpactAction();
return(action == ImpactActionEnum.PlatformUpdate);
}
public static bool IsRepairJobType(this ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
var action = tenantJob.GetImpactAction();
return(action == ImpactActionEnum.PlatformMaintenance || action == ImpactActionEnum.TenantMaintenance);
}
public MappedTenantJob(ITenantJob tenantJob)
{
TenantJob = tenantJob.Validate("tenantJob");
Id = tenantJob.Id;
Actions = new List <IAction>();
ImpactedNodeCount = UnknownNodeCount;
MatchedTasks = new List <RepairTaskSummary>();
}
public static bool DoesJobRequirePreparingHealthCheck(this ITenantJob tenantJob, IConfigSection configSection)
{
return(DoesJobRequireHealthCheck(
tenantJob,
configSection,
Constants.ConfigKeys.EnablePreparingHealthCheckFormat,
false));
}
public static bool DoesJobRequireRestoringHealthCheck(this ITenantJob tenantJob, IConfigSection configSection)
{
return(DoesJobRequireHealthCheck(
tenantJob,
configSection,
Constants.ConfigKeys.EnableRestoringHealthCheckFormat,
tenantJob.IsUpdateJobType()));
}
public static bool IsContextIdMatch(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
var contextId = repairTask.GetTenantJobContext();
bool contextIdMatch = string.Equals(contextId, tenantJob.ContextStringGivenByTenant, StringComparison.OrdinalIgnoreCase);
return(contextIdMatch);
}
private static bool DoesJobRequireHealthCheck(ITenantJob tenantJob, IConfigSection configSection, string keyFormat, bool defaultValue)
{
tenantJob.Validate("tenantJob");
configSection.Validate("configSection");
var impactAction = tenantJob.GetImpactAction();
var key = keyFormat.ToString(impactAction);
return(configSection.ReadConfigValue(key, defaultValue));
}
public void CreateTenantJob(ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
lock (locker)
{
tenantJobs.Add(tenantJob.Id, tenantJob);
jobDocIncarnation++;
}
}
public void UpdateTenantJob(ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
lock (locker)
{
tenantJobs[tenantJob.Id] = tenantJob;
jobDocIncarnation++;
}
}
public static bool IsJobIdMatch(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
var jobId = repairTask.GetJobId();
bool isMatch = (jobId != null) && string.Equals(jobId, tenantJob.Id.ToString());
return(isMatch);
}
public bool CanAddActiveJob(ITenantJob job, out JobCount count)
{
var jobType = job.GetImpactAction();
if (map.TryGetValue(jobType, out count) && (count.ActiveCount < count.MaxCount))
{
return(true);
}
return(false);
}
public void AddActiveJob(ITenantJob job)
{
var jobType = job.GetImpactAction();
JobCount count;
if (map.TryGetValue(jobType, out count))
{
count.ActiveCount++;
map[jobType] = count;
}
}
public static bool IsJobKeyMatch(this IRepairTask repairTask, ITenantJob tenantJob)
{
repairTask.Validate("repairTask");
tenantJob.Validate("tenantJob");
var jobId = repairTask.GetJobId();
var jobKey = repairTask.GetJobKey();
bool jobKeyMatch = jobId != null && jobKey != null && string.Equals(jobKey, tenantJob.GetJobKey());
return(jobKeyMatch);
}
public CreateInPreparingAction(
CoordinatorEnvironment environment,
IRepairManager repairManager,
ITenantJob tenantJob,
bool surpriseJob,
RepairTaskPrepareArgs args)
: base(environment, ActionType.Prepare)
{
this.repairManager = repairManager.Validate("repairManager");
this.tenantJob = tenantJob.Validate("tenantJob");
this.surpriseJob = surpriseJob;
this.args = args.Validate("args");
}
public ExecuteJobAction(
CoordinatorEnvironment environment,
Action <Guid> approveJobAction,
IRepairManager repairManager,
ITenantJob tenantJob,
IRepairTask repairTask)
: base(environment, ActionType.None)
{
this.approveJobAction = approveJobAction.Validate("approveJobAction");
this.repairManager = repairManager.Validate("repairManager");
this.tenantJob = tenantJob.Validate("tenantJob");
this.repairTask = repairTask.Validate("repairTask");
}
public JobCategory GetJobCategory(ITenantJob job)
{
if (job.IsVendorRepairBegin())
{
return(JobCategory.VendorRepairBegin);
}
if (job.IsVendorRepairEnd())
{
return(JobCategory.VendorRepairEnd);
}
return(JobCategory.Normal);
}
public static bool IsWaitingForImpactEndAck(this ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
if (tenantJob.JobStep == null)
{
// we have noticed that some jobs (probably when they are created initially have no job step
return(false);
}
bool ready =
tenantJob.JobStep.AcknowledgementStatus == AcknowledgementStatusEnum.WaitingForAcknowledgement &&
tenantJob.JobStep.ImpactStep == ImpactStepEnum.ImpactEnd;
return(ready);
}
/// <summary>
/// Gets the job key which is a JobId/UD combination.
/// A job key forms a 1:1 mapping with a repair task. A single job may walk multiple UDs. With each UD, it
/// creates a new repair task. The description field of the repair task is associated with this job key.
/// </summary>
/// <remarks>
/// The job key can be null when the jobstep is null (probably because the job has just been created,
/// or has completed) and there is no UD information.
/// </remarks>
private static string GetJobKey(this ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
var id = tenantJob.Id;
var stepId = tenantJob.GetJobStepId();
if (stepId == null)
{
return(null);
}
var key = Constants.JobKeyFormat.ToString(id, stepId);
return(key);
}
public JobPhase GetJobPhase(ITenantJob job)
{
// Versioned to allow fallback to previous job
// classification logic in case live site issues
// require it.
int version = this.config.ReadConfigValue(
Constants.ConfigKeys.JobClassifierVersion,
defaultValue: 2);
switch (version)
{
case 2:
return(GetJobPhase_V2(job));
default:
return(GetJobPhase_V1(job));
}
}
private static JobPhase GetJobPhase_V1(ITenantJob job)
{
if (job.IsWaitingForImpactStartAck())
{
return(JobPhase.ImpactStartWaitingForAck);
}
if (job.IsImpactStartAcked())
{
return(JobPhase.ImpactStartAcked);
}
if (job.IsWaitingForImpactEndAck())
{
return(JobPhase.ImpactEndWaitingForAck);
}
return(JobPhase.Inactive);
}
public static uint?GetJobUD(this ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
if (tenantJob.JobStep == null || tenantJob.JobStep.CurrentlyImpactedRoleInstances == null)
{
return(null);
}
var uds = tenantJob.JobStep.CurrentlyImpactedRoleInstances.Select(e => e.UpdateDomain).Distinct().ToList();
if (uds.Count != 1)
{
// There is not a single consistent UD for this job; perhaps it is an FD-based maintenance job
return(null);
}
return(uds[0]);
}
public static bool IsImpactStartAcked(this ITenantJob tenantJob)
{
tenantJob.Validate("tenantJob");
if (tenantJob.JobStep == null)
{
// we have noticed that some jobs (probably when they are created initially have no job step
return(false);
}
// TODO consider adding JobStatus == JobStatusEnum.Executing check or deleting this comment once MR team
// fixes Icm 24737274
bool acked =
(tenantJob.JobStep.AcknowledgementStatus == AcknowledgementStatusEnum.Acknowledged || tenantJob.JobStep.AcknowledgementStatus == AcknowledgementStatusEnum.Timedout) &&
tenantJob.JobStep.ImpactStep == ImpactStepEnum.ImpactStart /* &&
* tenantJob.JobStatus == JobStatusEnum.Executing */;
return(acked);
}
private JobPhase GetJobPhase_V2(ITenantJob job)
{
// First classify by top-level job status. Only active jobs
// require further classification.
JobStatusCategory jobStatusCategory = this.config.ReadConfigValue(
Constants.ConfigKeys.JobClassifierJobStatusCategoryFormat.ToString(job.JobStatus),
GetDefaultJobStatusCategory(job.JobStatus));
switch (jobStatusCategory)
{
case JobStatusCategory.Inactive:
return(JobPhase.Inactive);
case JobStatusCategory.Active:
return(GetJobPhaseForActiveStep(job.JobStep));
case JobStatusCategory.Unknown:
default:
return(JobPhase.Unknown);
}
}
public static string GetJobStepId(this ITenantJob tenantJob)
{
string jobStepId = null;
if (tenantJob.JobStep != null)
{
// An empty impacted role instance list will result in a job step ID with the following string
jobStepId = "-";
if (tenantJob.JobStep.CurrentlyImpactedRoleInstances != null)
{
var canonicalRoleInstance = tenantJob.JobStep.CurrentlyImpactedRoleInstances.OrderBy(i => i.RoleInstanceName, StringComparer.Ordinal).FirstOrDefault();
if (canonicalRoleInstance != null)
{
jobStepId = canonicalRoleInstance.RoleInstanceName;
}
}
}
return(jobStepId);
}
private static void TakeSomeActionOnJob(ITenantJob tenantJob)
{
// pretend that FC actually reboots the VM or takes some action
// since the job is approved
}
public static RepairTaskPrepareArgs FromTenantJob(
ITenantJob tenantJob,
int jobDocIncarnation,
CoordinatorEnvironment environment,
bool isVendorRepair,
bool restoringHealthCheckOnly,
string description = null)
{
tenantJob.Validate("tenantJob");
environment.Validate("environment");
var jobId = tenantJob.Id;
var ud = tenantJob.GetJobUD();
string jobStepId = tenantJob.GetJobStepId();
if (jobStepId == null)
{
environment.DefaultTraceType.WriteWarning(
"RepairTaskPrepareArgs.FromTenantJob: not continuing since job step ID is null in job: {0}",
tenantJob.ToJson());
return(null);
}
// use the role instance names from the JobStep. Don't use tenantJob.RoleInstancesToBeImpacted
// since that lists all the role instances that will be impacted.
// E.g. in a tenant update job, where multiple UDs are walked, if there are 8 role instances,
// tenantJob.RoleInstancesToBeImpacted will list all 8, whereas
// tenantJob.JobStep.CurrentlyImpactedRoleInstances will list only those in the current UD of the jobstep
var nodeNames = new List <string>();
if (tenantJob.JobStep.CurrentlyImpactedRoleInstances != null)
{
nodeNames.AddRange(tenantJob.JobStep.CurrentlyImpactedRoleInstances.Select(
e => e.RoleInstanceName.TranslateRoleInstanceToNodeName()));
}
var executorData = new RepairTaskExecutorData
{
JobId = jobId.ToString(),
UD = ud,
StepId = jobStepId,
};
if (isVendorRepair)
{
executorData.Flags = RepairTaskExecutorData.VendorRepairFlag;
}
string repairTaskId = GenerateRepairTaskId(
tenantJob.GetImpactAction(),
jobId,
ud,
jobDocIncarnation);
string repairAction = GenerateRepairAction(tenantJob.GetImpactAction());
var args = new RepairTaskPrepareArgs()
{
TaskId = repairTaskId,
Description = description,
Action = repairAction,
ExecutorData = executorData,
Target = new NodeRepairTargetDescription(nodeNames),
};
if (restoringHealthCheckOnly)
{
args.Impact = new NodeRepairImpactDescription();
args.PerformPreparingHealthCheck = false;
args.PerformRestoringHealthCheck = true;
}
else
{
args.Impact = GetImpactFromDetails(tenantJob, environment);
args.PerformPreparingHealthCheck = tenantJob.DoesJobRequirePreparingHealthCheck(environment.Config);
args.PerformRestoringHealthCheck = tenantJob.DoesJobRequireRestoringHealthCheck(environment.Config);
}
if (tenantJob.IsTenantUpdateJobType() && nodeNames.Count == 0)
{
// Never perform health checks on TenantUpdate job steps that have zero role
// instances listed. These occur at the end of each UD walk when the tenant
// setting Tenant.PolicyAgent.TenantUpdateUdCleanupApprovalRequired == true.
args.PerformPreparingHealthCheck = false;
args.PerformRestoringHealthCheck = false;
}
return(args);
}
public AckJobAction(CoordinatorEnvironment environment, Action <Guid> approveJobAction, ITenantJob tenantJob)
: base(environment, ActionType.None)
{
this.approveJobAction = approveJobAction.Validate("approveJobAction");
this.tenantJob = tenantJob.Validate("tenantJob");
}
