private void SetBlockedActions(string callerName, TraceType traceType, ActionType blockedActions) { ActionType oldAllowedActions = this.AllowedActions; this.blockedActions = blockedActions; if (oldAllowedActions != this.AllowedActions) { traceType.WriteInfo("{0}: changed allowed actions for repair task {1} from '{2}' to '{3}'", callerName, this.Id, oldAllowedActions, this.AllowedActions); } }
/// <summary> /// Translates the impact details to an RM node impact level. /// </summary> /// <param name="impactDetail">The impact details to be translated.</param> /// <returns>The node impact level to be sent to the RM.</returns> public NodeImpactLevel TranslateImpactDetailToNodeImpactLevel(ImpactActionEnum jobType, AffectedResourceImpact impactDetail) { impactDetail.Validate("impactDetail"); ImpactCategory category; if (IsInstanceRemoval(jobType, impactDetail)) { // Handle instance removal as a special case category = ImpactCategory.Decommission; tracer.WriteInfo("Overall impact set to {0} due to instance removal", category); } else { category = TranslateImpactDetailToCategory(impactDetail); } return(TranslateImpactCategoryToNodeImpactLevel(category)); }
public override async Task ApplyAsync(Guid activityId, CoordinatorContext coordinatorContext) { coordinatorContext.Validate("coordinatorContext"); if (coordinatorContext.MappedTenantJobs.Count == 0) { return; } var policy = await jobBlockingPolicyManager.GetPolicyAsync(); traceType.WriteInfo("Current job blocking policy is {0}", policy); foreach (var mappedTenantJob in coordinatorContext.MappedTenantJobs.Values) { if (IsBlocked(policy, mappedTenantJob)) { mappedTenantJob.DenyActions(this.traceType, ActionType.Prepare); } } }
private void ApplyActionPolicy(IMappedTenantJob job, ActionType actionType) { if (job.AllowedActions.HasFlag(actionType)) { var jobType = job.TenantJob.GetImpactAction(); // e.g. Azure.AutoExecute.TenantUpdate=false string configKey = Constants.ConfigKeys.AutoActionTypeHandlingFormat.ToString(actionType, jobType); if (!this.config.ReadConfigValue(configKey, true)) { traceType.WriteInfo( "Job {0}: automated processing of {1} action for {2} jobs is disallowed by configuration ({3})", job.Id, actionType, jobType, configKey); job.DenyActions(this.traceType, actionType); } } }
public async Task RunAsync(int primaryEpoch, CancellationToken token) { if (primaryEpoch < 0) { throw new ArgumentOutOfRangeException("primaryEpoch"); } if (!this.TryStartRunAsync()) { throw new InvalidOperationException("Coordinator has already been started"); } try { if (primaryEpoch < this.primaryEpoch) { throw new ArgumentException("Primary epoch must be non-decreasing", "primaryEpoch"); } if (this.primaryEpoch < primaryEpoch) { this.nextSequenceNumber = 0; } coordinatorCommandProcessor.Reset(); coordinatorRunAsyncStartTime = DateTimeOffset.UtcNow; coordinatorCommandProcessor.CoordinatorDiagnosticInfo = new CoordinatorDiagnosticInfo { CoordinatorStartTime = "{0:o}".ToString(coordinatorRunAsyncStartTime), AzureTenantId = this.tenantId, AssemblyFileVersion = this.assemblyFileVersion, }; this.primaryEpoch = primaryEpoch; traceType.WriteInfo("RunAsync: primaryEpoch = 0x{0:X}, nextSequenceNumber = 0x{1:X}", this.primaryEpoch, this.nextSequenceNumber); traceType.WriteAggregatedEvent( InfrastructureService.Constants.CoordinatorModeKey, this.tenantId, "{0}", new CoordinatorStateData().Mode); this.isNotificationAvailable = false; this.lastHealthIncarnation = 0; this.lastHealthIncarnationUpdatedAt = null; this.repairActionProvider = new RepairActionProvider(configSection); ActionHelper.ResetActionPolicies(actionPolicies); // Clear old AzureSerial health property ClearLegacyNotificationApprovalHealthStatus(); await StartJobTaskAsync(token).ConfigureAwait(false); } finally { this.CompleteRunAsync(); } }
public Task <string> RunCommandAsync(bool isAdminCommand, string command, TimeSpan timeout, CancellationToken cancellationToken) { traceType.WriteInfo("Ignoring RunCommand(isAdminCommand = {0}, command = '{1}')", isAdminCommand, command); return(Task.FromResult(string.Empty)); }
public async Task RunAsync(int primaryEpoch, CancellationToken token) { if (primaryEpoch < 0) { throw new ArgumentOutOfRangeException("primaryEpoch"); } if (!this.TryStartRunAsync()) { throw new InvalidOperationException("Coordinator has already been started"); } try { if (primaryEpoch < this.primaryEpoch) { throw new ArgumentException("Primary epoch must be non-decreasing", "primaryEpoch"); } if (this.primaryEpoch < primaryEpoch) { this.nextSequenceNumber = 0; } coordinatorRunAsyncStartTime = DateTimeOffset.UtcNow; this.primaryEpoch = primaryEpoch; traceType.WriteInfo("RunAsync: primaryEpoch = 0x{0:X}, nextSequenceNumber = 0x{1:X}", this.primaryEpoch, this.nextSequenceNumber); await ProcessAskModeAsync(token).ConfigureAwait(false); } finally { this.CompleteRunAsync(); } }
// It is easier to see the JSON via TraceViewer. // It may be harder to parse JSON and post-process it via Cosmos. If that is the case, // then we may need another sink (observer) that dumps key/value formats for Cosmos to process // Anyway, additional observers are required if we do the following // a. pipe all activities from customer clusters to SFRP // b. pipe to MDM or operationalinsights. public static void Log(this IActivityEvent activityEvent, TraceType traceType) { activityEvent.Validate("activityEvent"); traceType.WriteInfo("{0}", activityEvent.ToJson()); }
// TODO, ensure stability // i.e. on re-execution, the same things should re-execute (if the state hasn't changed) public override Task ApplyAsync(Guid activityId, CoordinatorContext coordinatorContext) { coordinatorContext.Validate("coordinatorContext"); if (coordinatorContext.MappedTenantJobs.Count == 0) { return(Task.FromResult(0)); } var allJobs = coordinatorContext.MappedTenantJobs.Values; var activeJobs = allJobs.Where(j => j.IsActive).ToList(); // Count all active jobs int totalActiveJobCount = activeJobs.Count; // Count active jobs by type int activeUpdateJobCount = 0; JobTypeCounter jobTypeCounter = new JobTypeCounter(this.configSection); foreach (var job in activeJobs) { traceType.WriteInfo("Active job {0} ({1})", job.Id, job.TenantJob.GetImpactAction()); jobTypeCounter.AddActiveJob(job.TenantJob); if (job.TenantJob.IsUpdateJobType()) { ++activeUpdateJobCount; } } int maxParallelJobCount = configSection.ReadConfigValue( Constants.ConfigKeys.MaxParallelJobCountTotal, DefaultMaxParallelJobCountTotal); int maxParallelUpdateJobCount = configSection.ReadConfigValue( Constants.ConfigKeys.MaxParallelJobCountUpdate, DefaultMaxParallelJobCountUpdate); traceType.WriteInfo( "Active/max job counts: Total: {0}/{1}, Updates: {2}/{3}, {4}", totalActiveJobCount, maxParallelJobCount, activeUpdateJobCount, maxParallelUpdateJobCount, jobTypeCounter); // Find all jobs that are waiting to prepare var pendingJobs = allJobs .Where(j => !j.IsActive && ((j.AllowedActions & ActionType.Prepare) == ActionType.Prepare)) .OrderBy(j => j.TenantJob.GetImpactAction()) // Apply default static priority based on job type .ToList(); // TODO, ensure that we don't ack too many in the 2nd pass just after acking once // choose the simplest logic for now. In future, we will pick based on oldest document incarnation number etc. foreach (var pendingJob in pendingJobs) { // Fall through the checks, so that all blocking reasons are logged bool allowJob = true; if (totalActiveJobCount >= maxParallelJobCount) { traceType.WriteInfo( "Not starting job {0} because it would exceed max total parallel job count ({1}/{2})", pendingJob.Id, totalActiveJobCount, maxParallelJobCount); allowJob = false; } JobCount count; if (!jobTypeCounter.CanAddActiveJob(pendingJob.TenantJob, out count)) { traceType.WriteInfo( "Not starting job {0} because it would exceed max parallel job count for type {1} ({2})", pendingJob.Id, pendingJob.TenantJob.GetImpactAction(), count); allowJob = false; } if (pendingJob.TenantJob.IsUpdateJobType() && (activeUpdateJobCount >= maxParallelUpdateJobCount)) { traceType.WriteInfo( "Not starting job {0} because it would exceed max parallel update job count ({1}/{2})", pendingJob.Id, activeUpdateJobCount, maxParallelUpdateJobCount); allowJob = false; } if (allowJob) { ++totalActiveJobCount; jobTypeCounter.AddActiveJob(pendingJob.TenantJob); if (pendingJob.TenantJob.IsUpdateJobType()) { ++activeUpdateJobCount; } traceType.WriteInfo("Allowing job {0} to start", pendingJob.Id); } else { pendingJob.DenyActions(traceType, ActionType.Prepare); } } return(Task.FromResult(0)); }