protected override async Task <bool> CheckSavedJobStatus(JobModel job, CancellationToken cancellationToken) { if (job.Status == JobStatus.InProgress && job.LearnerCount > 0) { return(false); } if (job.DcJobSucceeded.HasValue && !job.DcJobSucceeded.Value && job.Status != JobStatus.DcTasksFailed) { job.Status = JobStatus.DcTasksFailed; await JobStorageService.SaveJobStatus(job.DcJobId.Value, JobStatus.DcTasksFailed, job.StartTime, cancellationToken).ConfigureAwait(false); } if (job.Status != JobStatus.InProgress && job.DcJobSucceeded.HasValue) { Logger.LogWarning($"Job {job.DcJobId} has already finished. Status: {job.Status}"); await EventPublisher.SubmissionFinished(job.DcJobSucceeded.Value, job.DcJobId.Value, job.Ukprn.Value, job.AcademicYear, job.CollectionPeriod, job.IlrSubmissionTime.Value).ConfigureAwait(false); return(true); } if (job.LearnerCount.HasValue && job.LearnerCount.Value == 0) { await JobStorageService.SaveJobStatus(job.DcJobId.Value, JobStatus.Completed, job.StartTime, cancellationToken).ConfigureAwait(false); } return(false); }
protected virtual async Task <(bool hasFailedMessages, DateTimeOffset?endTime)> UpdateJobStatus(long jobId, List <CompletedMessage> completedItems, CancellationToken cancellationToken) { var statusChanged = false; var currentJobStatus = await JobStorageService.GetJobStatus(jobId, cancellationToken).ConfigureAwait(false); var completedItemsEndTime = completedItems.Max(item => item.CompletedTime); if (!currentJobStatus.endTime.HasValue || completedItemsEndTime > currentJobStatus.endTime) { currentJobStatus.endTime = completedItemsEndTime; statusChanged = true; } if (currentJobStatus.hasFailedMessages || completedItems.Any(item => !item.Succeeded)) { currentJobStatus.hasFailedMessages = true; statusChanged = true; } if (statusChanged) { Logger.LogVerbose($"Detected change in job status for job: {jobId}. Has failed messages: {currentJobStatus.hasFailedMessages}, End time: {currentJobStatus.endTime}"); await JobStorageService.StoreJobStatus(jobId, currentJobStatus.hasFailedMessages, currentJobStatus.endTime, cancellationToken) .ConfigureAwait(false); } return(currentJobStatus); }
public async Task RecordDcJobCompleted(long jobId, bool succeeded, CancellationToken cancellationToken) { Logger.LogDebug($"Now storing the completion status of the submission job. Id: {jobId}, succeeded: {succeeded}"); await JobStorageService.StoreDcJobStatus(jobId, succeeded, cancellationToken).ConfigureAwait(false); Logger.LogInfo($"Finished storing the completion status of the submission job. Id: {jobId}, succeeded: {succeeded}"); }
protected virtual async Task ManageMessageStatus(long jobId, List <CompletedMessage> completedMessages, List <InProgressMessage> inProgressMessages, CancellationToken cancellationToken) { await JobStorageService.RemoveInProgressMessages(jobId, completedMessages.Select(item => item.MessageId).ToList(), cancellationToken) .ConfigureAwait(false); await JobStorageService.RemoveCompletedMessages(jobId, completedMessages.Select(item => item.MessageId).ToList(), cancellationToken) .ConfigureAwait(false); }
private async Task <List <CompletedMessage> > GetCompletedMessages(long jobId, List <InProgressMessage> inProgressMessages, CancellationToken cancellationToken) { var completedMessages = await JobStorageService.GetCompletedMessages(jobId, cancellationToken) .ConfigureAwait(false); var completedItems = completedMessages .Where(completedMessage => inProgressMessages.Any(inProgress => inProgress.MessageId == completedMessage.MessageId)).ToList(); return(completedItems); }
protected virtual async Task <bool> CompleteJob(JobModel job, JobStatus status, DateTimeOffset endTime, CancellationToken cancellationToken) { job.Status = status; job.EndTime = endTime; await JobStorageService.SaveJobStatus(job.DcJobId.Value, status, endTime, cancellationToken).ConfigureAwait(false); SendTelemetry(job); Logger.LogInfo($"Finished recording completion status of job. Job: {job.Id}, status: {job.Status}, end time: {job.EndTime}"); return(true); }
protected async Task RecordJobInProgressMessages(long jobId, List <GeneratedMessage> generatedMessages, CancellationToken cancellationToken) { await JobStorageService.StoreInProgressMessages(jobId, generatedMessages.Select(message => new InProgressMessage { MessageId = message.MessageId, JobId = jobId, MessageName = message.MessageName }).ToList(), cancellationToken); }
private async Task <bool> CompleteJob(long jobId, JobStatus status, DateTimeOffset endTime, CancellationToken cancellationToken) { var job = await JobStorageService.GetJob(jobId, cancellationToken); if (job == null) { Logger.LogWarning($"Attempting to record completion status for job {jobId} but the job has not been persisted to database."); return(false); } return(await CompleteJob(job, status, endTime, cancellationToken).ConfigureAwait(false)); }
protected virtual async Task RecordNewJob(JobModel jobDetails, List <GeneratedMessage> generatedMessages, CancellationToken cancellationToken = default(CancellationToken)) { if (!jobDetails.DcJobId.HasValue) { throw new InvalidOperationException("No DcJob Id found on the job."); } var isNewJob = await JobStorageService.StoreNewJob(jobDetails, CancellationToken.None); Logger.LogDebug($"Finished storing new job: {jobDetails.Id} for dc job id: {jobDetails.DcJobId}. Now storing the job messages."); await RecordJobInProgressMessages(jobDetails.DcJobId.Value, generatedMessages, cancellationToken).ConfigureAwait(false); Logger.LogDebug($"Finished storing new job messages for job: {jobDetails.Id} for dc job id: {jobDetails.DcJobId}."); if (isNewJob) { SendTelemetry(jobDetails); } Logger.LogInfo($"Finished saving the job info. DC Job Id: {jobDetails.DcJobId}"); }
private async Task CompleteDataLocks(long jobId, List <CompletedMessage> completedMessages, List <InProgressMessage> inProgressMessages, CancellationToken cancellationToken) { var job = await JobStorageService.GetJob(jobId, cancellationToken).ConfigureAwait(false); if (job == null) { return; } var dataLocksMessages = new[] { nameof(FunctionalSkillEarningFailedDataLockMatching), nameof(PayableFunctionalSkillEarningEvent), nameof(PayableEarningEvent), nameof(EarningFailedDataLockMatching), nameof(ProcessLearnerCommand), nameof(Act1FunctionalSkillEarningsEvent), nameof(ApprenticeshipContractType1EarningEvent), nameof(EarningFailedDataLockMatching), }; var inProgressDataLocks = inProgressMessages .Where(inProgress => dataLocksMessages.Any(dlockType => inProgress.MessageName?.Contains(dlockType) ?? false)) .ToList(); if (!inProgressDataLocks.Any()) { return; } if (!inProgressDataLocks.All(inProgress => completedMessages.Any(completed => completed.MessageId == inProgress.MessageId))) { return; } var completionTime = completedMessages .Where(completed => inProgressDataLocks.Exists(inProgress => inProgress.MessageId == completed.MessageId)) .Max(completed => completed.CompletedTime); await JobStorageService.SaveDataLocksCompletionTime(job.DcJobId.Value, completionTime, cancellationToken) .ConfigureAwait(false); var properties = new Dictionary <string, string> { { TelemetryKeys.JobId, job.DcJobId.ToString() }, { TelemetryKeys.JobType, job.JobType.ToString("G") }, { TelemetryKeys.Ukprn, job.Ukprn?.ToString() ?? string.Empty }, { TelemetryKeys.InternalJobId, job.Id.ToString() }, { TelemetryKeys.CollectionPeriod, job.CollectionPeriod.ToString() }, { TelemetryKeys.AcademicYear, job.AcademicYear.ToString() }, { TelemetryKeys.Status, job.Status.ToString("G") } }; var metrics = new Dictionary <string, double> { { TelemetryKeys.Duration, (completionTime - job.StartTime).TotalMilliseconds }, { "Learner Count", job.LearnerCount ?? 0 } }; Telemetry.TrackEvent("Finished DataLocks", properties, metrics); Logger.LogInfo( $"Recorded DataLocks completion time for Job: {job.DcJobId}, completion time: {completionTime}"); }
public virtual async Task <bool> ManageStatus(long jobId, CancellationToken cancellationToken) { Logger.LogVerbose($"Now determining if job {jobId} has finished."); var job = await JobStorageService.GetJob(jobId, cancellationToken).ConfigureAwait(false); if (job != null) { if (await CheckSavedJobStatus(job, cancellationToken)) { return(true); } if (await IsJobTimedOut(job, cancellationToken)) { return(true); } } var additionalJobChecksResult = await PerformAdditionalJobChecks(job, cancellationToken); if (!additionalJobChecksResult.IsComplete) { return(false); } var inProgressMessages = await JobStorageService.GetInProgressMessages(jobId, cancellationToken).ConfigureAwait(false); var completedItems = await GetCompletedMessages(jobId, inProgressMessages, cancellationToken).ConfigureAwait(false); //TODO: make a little more elegant Logger.LogDebug($"Inprogress count: {inProgressMessages.Count}, completed count: {completedItems.Count}. Job: {jobId}"); inProgressMessages.GroupBy(message => message.MessageName).ToList().ForEach(group => Logger.LogDebug($"Inprogress message type: {group.Key}, count: {group.Count()}. Job: {jobId}")); if (!completedItems.Any()) { Logger.LogVerbose($"Found no completed messages for job: {jobId}"); return(false); } cancellationToken.ThrowIfCancellationRequested(); await ManageMessageStatus(jobId, completedItems, inProgressMessages, cancellationToken).ConfigureAwait(false); var currentJobStatus = await UpdateJobStatus(jobId, completedItems, cancellationToken).ConfigureAwait(false); if (!inProgressMessages.TrueForAll(inProgress => completedItems.Any(item => item.MessageId == inProgress.MessageId))) { Logger.LogDebug($"Found {inProgressMessages.Count} in progress messages for job id: {jobId}. Cannot set status for job."); return(false); } var jobStatus = additionalJobChecksResult.OverriddenJobStatus ?? (currentJobStatus.hasFailedMessages ? JobStatus.CompletedWithErrors : JobStatus.Completed); var endTime = (additionalJobChecksResult.completionTime.HasValue && additionalJobChecksResult.completionTime > currentJobStatus.endTime.Value) ? additionalJobChecksResult.completionTime.Value : currentJobStatus.endTime.Value; return(await CompleteJob(jobId, jobStatus, endTime, cancellationToken).ConfigureAwait(false)); }