private async Task ApproveAllJobsAsync(Guid activityId, CancellationToken cancellationToken)
{
// Create the PA client; this will fail when MR is off, which it is expected to be
IPolicyAgentClient policyAgentClient = await this.policyAgentClientAsyncFactory();
IPolicyAgentDocumentForTenant doc = await policyAgentClient.GetDocumentAsync(activityId, cancellationToken);
if (doc == null)
{
// Parallel job mode is not enabled
throw new InvalidOperationException("Failed to get job info. Verify that job parallelism is enabled for this tenant.");
}
string docString = CoordinatorHelper.FormatDocumentForTracing(doc);
traceType.WriteInfo("Received document: {0}", docString);
Dictionary <Guid, JobStepResponseEnum> jobResponses = new Dictionary <Guid, JobStepResponseEnum>();
foreach (var job in doc.Jobs)
{
string message;
if (IsJobWaitingForApproval(job))
{
message = $"Auto-approving job {job.Id}";
jobResponses.Add(job.Id, JobStepResponseEnum.Acknowledged);
}
else
{
message = $"Ignoring job {job.Id} in state that does not require tenant approval";
}
traceType.WriteInfo(
"{0}: {1} {2} ({3}/{4})",
message,
job.GetImpactAction(),
job.JobStatus,
job.JobStep == null ? "-" : job.JobStep.ImpactStep.ToString(),
job.JobStep == null ? "-" : job.JobStep.AcknowledgementStatus.ToString());
}
if (jobResponses.Count > 0)
{
// Send all job responses (if any) as a single batch response
await policyAgentClient.SendJobResponseAsync(
activityId,
doc.JobDocumentIncarnation,
jobResponses,
"Auto-approved by Service Fabric because coordination is disabled",
cancellationToken);
}
else
{
traceType.WriteInfo("No jobs to approve");
}
}
private bool IsInstanceRemoval(ImpactActionEnum jobType, AffectedResourceImpact impactDetail)
{
bool isInstanceRemovalImpact =
jobType == ImpactActionEnum.TenantUpdate &&
CoordinatorHelper.AllImpactsEqual(impactDetail, Impact.Wipe);
bool isInstanceRemovalLegacy = false;
if (impactDetail.ListOfImpactTypes != null)
{
isInstanceRemovalLegacy = impactDetail.ListOfImpactTypes.Contains(ImpactTypeEnum.InstanceRemoval);
}
if (isInstanceRemovalImpact != isInstanceRemovalLegacy)
{
tracer.WriteWarning(
"Mismatch between IsInstanceRemoval result from resource impacts ({0}) and ImpactTypeEnum ({1}); using {0}",
isInstanceRemovalImpact,
isInstanceRemovalLegacy);
}
return(isInstanceRemovalImpact);
}
private async Task ProcessPolicyAgentDocumentAsync(CancellationToken cancellationToken)
{
traceType.WriteInfo("Starting to process policy agent documents.");
var lastSuccessfulRunAt = DateTimeOffset.UtcNow;
while (!cancellationToken.IsCancellationRequested)
{
traceType.WriteInfo(
"Azure coordinator: service name = {0}, partition ID = {1}, tenant ID = {2}, code version = {3}, start time = {4:O}",
this.environment.ServiceName,
this.partitionId,
this.tenantId,
this.assemblyFileVersion,
this.coordinatorRunAsyncStartTime);
TimeSpan timerPeriod = this.GetPollingInterval(Constants.ConfigKeys.JobPollingIntervalInSeconds,
DefaultJobPollingIntervalInSeconds,
MaxJobPollingInterval);
try
{
var activityId = Guid.NewGuid();
IPolicyAgentDocumentForTenant doc = await policyAgentClient.GetDocumentAsync(activityId, cancellationToken).ConfigureAwait(false);
if (doc == null)
{
// Parallel job mode is not enabled
throw new InvalidOperationException("Failed to get job info. Verify that job parallelism is enabled for this tenant.");
}
else
{
// If the incarnation stops changing for a long time, the MR channel may be unhealthy,
// due to the physical node being in HI/OFR state. Exit the process to cause a failover,
// hopefully to a healthier node.
this.CheckChannelHealth(activityId, doc);
if (!this.isNotificationAvailable)
{
traceType.WriteInfo("MR channel notification is not available; waiting for channel health check to pass.");
}
else
{
string docString = CoordinatorHelper.FormatDocumentForTracing(doc);
traceType.WriteInfo("Received document: {0}", docString);
Task healthProcessingTask = ProcessHealthAsync(activityId, doc, timerPeriod, cancellationToken);
Task jobProcessingTask = ProcessJobInfoAsync(activityId, doc, cancellationToken);
await healthProcessingTask.ConfigureAwait(false);
await jobProcessingTask.ConfigureAwait(false);
}
}
lastSuccessfulRunAt = DateTimeOffset.UtcNow;
UpdateCoordinatorHealthStatus(HealthState.Ok, "Azure coordinator is operating normally.");
}
catch (Exception ex)
{
if (!HandleCoordinatorException(ex, lastSuccessfulRunAt))
{
throw;
}
}
traceType.WriteInfo("Next run will occur in {0} at {1:O}", timerPeriod, DateTimeOffset.UtcNow + timerPeriod);
await Task.Delay(timerPeriod, cancellationToken);
}
traceType.WriteInfo("Cancellation requested. Returning from processing policy agent documents.");
}
