private void SetBlockedActions(string callerName, TraceType traceType, ActionType blockedActions)
{
ActionType oldAllowedActions = this.AllowedActions;
this.blockedActions = blockedActions;
if (oldAllowedActions != this.AllowedActions)
{
traceType.WriteInfo("{0}: changed allowed actions for repair task {1} from '{2}' to '{3}'",
callerName,
this.Id,
oldAllowedActions,
this.AllowedActions);
}
}
/// <summary>
/// Translates the impact details to an RM node impact level.
/// </summary>
/// <param name="impactDetail">The impact details to be translated.</param>
/// <returns>The node impact level to be sent to the RM.</returns>
public NodeImpactLevel TranslateImpactDetailToNodeImpactLevel(ImpactActionEnum jobType, AffectedResourceImpact impactDetail)
{
impactDetail.Validate("impactDetail");
ImpactCategory category;
if (IsInstanceRemoval(jobType, impactDetail))
{
// Handle instance removal as a special case
category = ImpactCategory.Decommission;
tracer.WriteInfo("Overall impact set to {0} due to instance removal", category);
}
else
{
category = TranslateImpactDetailToCategory(impactDetail);
}
return(TranslateImpactCategoryToNodeImpactLevel(category));
}
public override async Task ApplyAsync(Guid activityId, CoordinatorContext coordinatorContext)
{
coordinatorContext.Validate("coordinatorContext");
if (coordinatorContext.MappedTenantJobs.Count == 0)
{
return;
}
var policy = await jobBlockingPolicyManager.GetPolicyAsync();
traceType.WriteInfo("Current job blocking policy is {0}", policy);
foreach (var mappedTenantJob in coordinatorContext.MappedTenantJobs.Values)
{
if (IsBlocked(policy, mappedTenantJob))
{
mappedTenantJob.DenyActions(this.traceType, ActionType.Prepare);
}
}
}
private void ApplyActionPolicy(IMappedTenantJob job, ActionType actionType)
{
if (job.AllowedActions.HasFlag(actionType))
{
var jobType = job.TenantJob.GetImpactAction();
// e.g. Azure.AutoExecute.TenantUpdate=false
string configKey = Constants.ConfigKeys.AutoActionTypeHandlingFormat.ToString(actionType, jobType);
if (!this.config.ReadConfigValue(configKey, true))
{
traceType.WriteInfo(
"Job {0}: automated processing of {1} action for {2} jobs is disallowed by configuration ({3})",
job.Id,
actionType,
jobType,
configKey);
job.DenyActions(this.traceType, actionType);
}
}
}
public async Task RunAsync(int primaryEpoch, CancellationToken token)
{
if (primaryEpoch < 0)
{
throw new ArgumentOutOfRangeException("primaryEpoch");
}
if (!this.TryStartRunAsync())
{
throw new InvalidOperationException("Coordinator has already been started");
}
try
{
if (primaryEpoch < this.primaryEpoch)
{
throw new ArgumentException("Primary epoch must be non-decreasing", "primaryEpoch");
}
if (this.primaryEpoch < primaryEpoch)
{
this.nextSequenceNumber = 0;
}
coordinatorCommandProcessor.Reset();
coordinatorRunAsyncStartTime = DateTimeOffset.UtcNow;
coordinatorCommandProcessor.CoordinatorDiagnosticInfo = new CoordinatorDiagnosticInfo
{
CoordinatorStartTime = "{0:o}".ToString(coordinatorRunAsyncStartTime),
AzureTenantId = this.tenantId,
AssemblyFileVersion = this.assemblyFileVersion,
};
this.primaryEpoch = primaryEpoch;
traceType.WriteInfo("RunAsync: primaryEpoch = 0x{0:X}, nextSequenceNumber = 0x{1:X}",
this.primaryEpoch,
this.nextSequenceNumber);
traceType.WriteAggregatedEvent(
InfrastructureService.Constants.CoordinatorModeKey,
this.tenantId,
"{0}",
new CoordinatorStateData().Mode);
this.isNotificationAvailable = false;
this.lastHealthIncarnation = 0;
this.lastHealthIncarnationUpdatedAt = null;
this.repairActionProvider = new RepairActionProvider(configSection);
ActionHelper.ResetActionPolicies(actionPolicies);
// Clear old AzureSerial health property
ClearLegacyNotificationApprovalHealthStatus();
await StartJobTaskAsync(token).ConfigureAwait(false);
}
finally
{
this.CompleteRunAsync();
}
}
public Task <string> RunCommandAsync(bool isAdminCommand, string command, TimeSpan timeout, CancellationToken cancellationToken)
{
traceType.WriteInfo("Ignoring RunCommand(isAdminCommand = {0}, command = '{1}')", isAdminCommand, command);
return(Task.FromResult(string.Empty));
}
public async Task RunAsync(int primaryEpoch, CancellationToken token)
{
if (primaryEpoch < 0)
{
throw new ArgumentOutOfRangeException("primaryEpoch");
}
if (!this.TryStartRunAsync())
{
throw new InvalidOperationException("Coordinator has already been started");
}
try
{
if (primaryEpoch < this.primaryEpoch)
{
throw new ArgumentException("Primary epoch must be non-decreasing", "primaryEpoch");
}
if (this.primaryEpoch < primaryEpoch)
{
this.nextSequenceNumber = 0;
}
coordinatorRunAsyncStartTime = DateTimeOffset.UtcNow;
this.primaryEpoch = primaryEpoch;
traceType.WriteInfo("RunAsync: primaryEpoch = 0x{0:X}, nextSequenceNumber = 0x{1:X}",
this.primaryEpoch,
this.nextSequenceNumber);
await ProcessAskModeAsync(token).ConfigureAwait(false);
}
finally
{
this.CompleteRunAsync();
}
}
// It is easier to see the JSON via TraceViewer.
// It may be harder to parse JSON and post-process it via Cosmos. If that is the case,
// then we may need another sink (observer) that dumps key/value formats for Cosmos to process
// Anyway, additional observers are required if we do the following
// a. pipe all activities from customer clusters to SFRP
// b. pipe to MDM or operationalinsights.
public static void Log(this IActivityEvent activityEvent, TraceType traceType)
{
activityEvent.Validate("activityEvent");
traceType.WriteInfo("{0}", activityEvent.ToJson());
}
// TODO, ensure stability
// i.e. on re-execution, the same things should re-execute (if the state hasn't changed)
public override Task ApplyAsync(Guid activityId, CoordinatorContext coordinatorContext)
{
coordinatorContext.Validate("coordinatorContext");
if (coordinatorContext.MappedTenantJobs.Count == 0)
{
return(Task.FromResult(0));
}
var allJobs = coordinatorContext.MappedTenantJobs.Values;
var activeJobs = allJobs.Where(j => j.IsActive).ToList();
// Count all active jobs
int totalActiveJobCount = activeJobs.Count;
// Count active jobs by type
int activeUpdateJobCount = 0;
JobTypeCounter jobTypeCounter = new JobTypeCounter(this.configSection);
foreach (var job in activeJobs)
{
traceType.WriteInfo("Active job {0} ({1})", job.Id, job.TenantJob.GetImpactAction());
jobTypeCounter.AddActiveJob(job.TenantJob);
if (job.TenantJob.IsUpdateJobType())
{
++activeUpdateJobCount;
}
}
int maxParallelJobCount = configSection.ReadConfigValue(
Constants.ConfigKeys.MaxParallelJobCountTotal,
DefaultMaxParallelJobCountTotal);
int maxParallelUpdateJobCount = configSection.ReadConfigValue(
Constants.ConfigKeys.MaxParallelJobCountUpdate,
DefaultMaxParallelJobCountUpdate);
traceType.WriteInfo(
"Active/max job counts: Total: {0}/{1}, Updates: {2}/{3}, {4}",
totalActiveJobCount,
maxParallelJobCount,
activeUpdateJobCount,
maxParallelUpdateJobCount,
jobTypeCounter);
// Find all jobs that are waiting to prepare
var pendingJobs = allJobs
.Where(j => !j.IsActive && ((j.AllowedActions & ActionType.Prepare) == ActionType.Prepare))
.OrderBy(j => j.TenantJob.GetImpactAction()) // Apply default static priority based on job type
.ToList();
// TODO, ensure that we don't ack too many in the 2nd pass just after acking once
// choose the simplest logic for now. In future, we will pick based on oldest document incarnation number etc.
foreach (var pendingJob in pendingJobs)
{
// Fall through the checks, so that all blocking reasons are logged
bool allowJob = true;
if (totalActiveJobCount >= maxParallelJobCount)
{
traceType.WriteInfo(
"Not starting job {0} because it would exceed max total parallel job count ({1}/{2})",
pendingJob.Id,
totalActiveJobCount,
maxParallelJobCount);
allowJob = false;
}
JobCount count;
if (!jobTypeCounter.CanAddActiveJob(pendingJob.TenantJob, out count))
{
traceType.WriteInfo(
"Not starting job {0} because it would exceed max parallel job count for type {1} ({2})",
pendingJob.Id,
pendingJob.TenantJob.GetImpactAction(),
count);
allowJob = false;
}
if (pendingJob.TenantJob.IsUpdateJobType() && (activeUpdateJobCount >= maxParallelUpdateJobCount))
{
traceType.WriteInfo(
"Not starting job {0} because it would exceed max parallel update job count ({1}/{2})",
pendingJob.Id,
activeUpdateJobCount,
maxParallelUpdateJobCount);
allowJob = false;
}
if (allowJob)
{
++totalActiveJobCount;
jobTypeCounter.AddActiveJob(pendingJob.TenantJob);
if (pendingJob.TenantJob.IsUpdateJobType())
{
++activeUpdateJobCount;
}
traceType.WriteInfo("Allowing job {0} to start", pendingJob.Id);
}
else
{
pendingJob.DenyActions(traceType, ActionType.Prepare);
}
}
return(Task.FromResult(0));
}
