private async Task ApproveAllJobsAsync(Guid activityId, CancellationToken cancellationToken)
{
// Create the PA client; this will fail when MR is off, which it is expected to be
IPolicyAgentClient policyAgentClient = await this.policyAgentClientAsyncFactory();
IPolicyAgentDocumentForTenant doc = await policyAgentClient.GetDocumentAsync(activityId, cancellationToken);
if (doc == null)
{
// Parallel job mode is not enabled
throw new InvalidOperationException("Failed to get job info. Verify that job parallelism is enabled for this tenant.");
}
string docString = CoordinatorHelper.FormatDocumentForTracing(doc);
traceType.WriteInfo("Received document: {0}", docString);
Dictionary <Guid, JobStepResponseEnum> jobResponses = new Dictionary <Guid, JobStepResponseEnum>();
foreach (var job in doc.Jobs)
{
string message;
if (IsJobWaitingForApproval(job))
{
message = $"Auto-approving job {job.Id}";
jobResponses.Add(job.Id, JobStepResponseEnum.Acknowledged);
}
else
{
message = $"Ignoring job {job.Id} in state that does not require tenant approval";
}
traceType.WriteInfo(
"{0}: {1} {2} ({3}/{4})",
message,
job.GetImpactAction(),
job.JobStatus,
job.JobStep == null ? "-" : job.JobStep.ImpactStep.ToString(),
job.JobStep == null ? "-" : job.JobStep.AcknowledgementStatus.ToString());
}
if (jobResponses.Count > 0)
{
// Send all job responses (if any) as a single batch response
await policyAgentClient.SendJobResponseAsync(
activityId,
doc.JobDocumentIncarnation,
jobResponses,
"Auto-approved by Service Fabric because coordination is disabled",
cancellationToken);
}
else
{
traceType.WriteInfo("No jobs to approve");
}
}
private async Task ProcessJobInfoAsync(Guid activityId, IPolicyAgentDocumentForTenant doc, CancellationToken cancellationToken)
{
DateTime now = DateTime.UtcNow;
TimeSpan completedTaskAgeThreshold = this.GetCompletedTaskAgeThreshold();
// get the ones that are claimed by this service (which is the Repair Executor (RE))
var repairTasks = await repairManager
.GetRepairTaskListAsync(activityId, null, RepairTaskStateFilter.Default, this.environment.ServiceName)
.ConfigureAwait(false);
// Count completed tasks that are going to be ignored because they are too old
int oldCompletedTaskCount = repairTasks.Count(t => IsOldCompletedRepairTask(t, now, completedTaskAgeThreshold));
// Keep all of the rest (active or recent)
repairTasks = repairTasks.Where(t => !IsOldCompletedRepairTask(t, now, completedTaskAgeThreshold)).ToList();
// get all the unclaimed repair tasks too... some may not match, we'll evaluate and ignore them later
var unclaimedRepairTasks = await repairManager
.GetRepairTaskListAsync(activityId, null, RepairTaskStateFilter.Created, null)
.ConfigureAwait(false);
foreach (var unclaimedRepairTask in unclaimedRepairTasks)
{
repairTasks.Add(unclaimedRepairTask);
}
traceType.WriteInfo(
"Processing repair tasks (count: {0}; ignored completed tasks older than {1}: {2})",
repairTasks.Count,
completedTaskAgeThreshold,
oldCompletedTaskCount);
// Dump each task separately to avoid exceeding the ETW event size limit
for (int i = 0; i < repairTasks.Count; ++i)
{
traceType.WriteInfo("Repair task {0} of {1}:{2}{3}", i + 1, repairTasks.Count, Environment.NewLine, repairTasks[i].ToJson());
}
var coordinatorContext = new CoordinatorContext {
Doc = doc
};
ActionHelper.CreateMappedWorkItems(doc.Jobs, repairTasks, coordinatorContext);
var reconciler = new Reconciler(
this.environment,
this.policyAgentClient,
this.repairManager,
this.repairActionProvider);
await reconciler.ReconcileAsync(activityId, coordinatorContext).ConfigureAwait(false);
await ActionHelper.ApplyPoliciesAsync(activityId, actionPolicies, coordinatorContext).ConfigureAwait(false);
// Publish state after reconcile + policy phases have completed
coordinatorContext.MarkFinished();
coordinatorCommandProcessor.CoordinatorContext = coordinatorContext;
var blockingPolicy = await this.jobBlockingPolicyManager.GetPolicyAsync();
if (blockingPolicy == JobBlockingPolicy.BlockAllJobs)
{
traceType.WriteInfo("All actions are blocked by job blocking policy");
}
else
{
var actionsToExecute = ActionHelper.GetActions(activityId, coordinatorContext);
await ActionHelper.ExecuteActionsAsync(activityId, this.actionTraceType, actionsToExecute).ConfigureAwait(false);
// Send all job responses (if any) as a single batch response
await policyAgentClient.SendJobResponseAsync(
activityId,
doc.JobDocumentIncarnation,
reconciler.GetJobStepResponses(),
"Automated response by Service Fabric Infrastructure Service",
cancellationToken).ConfigureAwait(false);
}
}