protected override async Task <bool> CheckSavedJobStatus(JobModel job, CancellationToken cancellationToken)
{
if (job.Status == JobStatus.InProgress && job.LearnerCount > 0)
{
return(false);
}
if (job.DcJobSucceeded.HasValue && !job.DcJobSucceeded.Value && job.Status != JobStatus.DcTasksFailed)
{
job.Status = JobStatus.DcTasksFailed;
await JobStorageService.SaveJobStatus(job.DcJobId.Value, JobStatus.DcTasksFailed,
job.StartTime, cancellationToken).ConfigureAwait(false);
}
if (job.Status != JobStatus.InProgress && job.DcJobSucceeded.HasValue)
{
Logger.LogWarning($"Job {job.DcJobId} has already finished. Status: {job.Status}");
await EventPublisher.SubmissionFinished(job.DcJobSucceeded.Value, job.DcJobId.Value, job.Ukprn.Value,
job.AcademicYear, job.CollectionPeriod, job.IlrSubmissionTime.Value).ConfigureAwait(false);
return(true);
}
if (job.LearnerCount.HasValue && job.LearnerCount.Value == 0)
{
await JobStorageService.SaveJobStatus(job.DcJobId.Value, JobStatus.Completed,
job.StartTime, cancellationToken).ConfigureAwait(false);
}
return(false);
}
protected virtual async Task <(bool hasFailedMessages, DateTimeOffset?endTime)> UpdateJobStatus(long jobId, List <CompletedMessage> completedItems,
CancellationToken cancellationToken)
{
var statusChanged = false;
var currentJobStatus = await JobStorageService.GetJobStatus(jobId, cancellationToken).ConfigureAwait(false);
var completedItemsEndTime = completedItems.Max(item => item.CompletedTime);
if (!currentJobStatus.endTime.HasValue || completedItemsEndTime > currentJobStatus.endTime)
{
currentJobStatus.endTime = completedItemsEndTime;
statusChanged = true;
}
if (currentJobStatus.hasFailedMessages || completedItems.Any(item => !item.Succeeded))
{
currentJobStatus.hasFailedMessages = true;
statusChanged = true;
}
if (statusChanged)
{
Logger.LogVerbose($"Detected change in job status for job: {jobId}. Has failed messages: {currentJobStatus.hasFailedMessages}, End time: {currentJobStatus.endTime}");
await JobStorageService.StoreJobStatus(jobId, currentJobStatus.hasFailedMessages, currentJobStatus.endTime, cancellationToken)
.ConfigureAwait(false);
}
return(currentJobStatus);
}
public async Task RecordDcJobCompleted(long jobId, bool succeeded, CancellationToken cancellationToken)
{
Logger.LogDebug($"Now storing the completion status of the submission job. Id: {jobId}, succeeded: {succeeded}");
await JobStorageService.StoreDcJobStatus(jobId, succeeded, cancellationToken).ConfigureAwait(false);
Logger.LogInfo($"Finished storing the completion status of the submission job. Id: {jobId}, succeeded: {succeeded}");
}
protected virtual async Task ManageMessageStatus(long jobId, List <CompletedMessage> completedMessages,
List <InProgressMessage> inProgressMessages, CancellationToken cancellationToken)
{
await JobStorageService.RemoveInProgressMessages(jobId, completedMessages.Select(item => item.MessageId).ToList(), cancellationToken)
.ConfigureAwait(false);
await JobStorageService.RemoveCompletedMessages(jobId, completedMessages.Select(item => item.MessageId).ToList(), cancellationToken)
.ConfigureAwait(false);
}
private async Task <List <CompletedMessage> > GetCompletedMessages(long jobId, List <InProgressMessage> inProgressMessages, CancellationToken cancellationToken)
{
var completedMessages = await JobStorageService.GetCompletedMessages(jobId, cancellationToken)
.ConfigureAwait(false);
var completedItems = completedMessages
.Where(completedMessage => inProgressMessages.Any(inProgress => inProgress.MessageId == completedMessage.MessageId)).ToList();
return(completedItems);
}
protected virtual async Task <bool> CompleteJob(JobModel job, JobStatus status, DateTimeOffset endTime, CancellationToken cancellationToken)
{
job.Status = status;
job.EndTime = endTime;
await JobStorageService.SaveJobStatus(job.DcJobId.Value, status, endTime, cancellationToken).ConfigureAwait(false);
SendTelemetry(job);
Logger.LogInfo($"Finished recording completion status of job. Job: {job.Id}, status: {job.Status}, end time: {job.EndTime}");
return(true);
}
protected async Task RecordJobInProgressMessages(long jobId, List <GeneratedMessage> generatedMessages, CancellationToken cancellationToken)
{
await JobStorageService.StoreInProgressMessages(jobId,
generatedMessages.Select(message => new InProgressMessage
{
MessageId = message.MessageId,
JobId = jobId,
MessageName = message.MessageName
}).ToList(), cancellationToken);
}
private async Task <bool> CompleteJob(long jobId, JobStatus status, DateTimeOffset endTime, CancellationToken cancellationToken)
{
var job = await JobStorageService.GetJob(jobId, cancellationToken);
if (job == null)
{
Logger.LogWarning($"Attempting to record completion status for job {jobId} but the job has not been persisted to database.");
return(false);
}
return(await CompleteJob(job, status, endTime, cancellationToken).ConfigureAwait(false));
}
protected virtual async Task RecordNewJob(JobModel jobDetails, List <GeneratedMessage> generatedMessages, CancellationToken cancellationToken = default(CancellationToken))
{
if (!jobDetails.DcJobId.HasValue)
{
throw new InvalidOperationException("No DcJob Id found on the job.");
}
var isNewJob = await JobStorageService.StoreNewJob(jobDetails, CancellationToken.None);
Logger.LogDebug($"Finished storing new job: {jobDetails.Id} for dc job id: {jobDetails.DcJobId}. Now storing the job messages.");
await RecordJobInProgressMessages(jobDetails.DcJobId.Value, generatedMessages, cancellationToken).ConfigureAwait(false);
Logger.LogDebug($"Finished storing new job messages for job: {jobDetails.Id} for dc job id: {jobDetails.DcJobId}.");
if (isNewJob)
{
SendTelemetry(jobDetails);
}
Logger.LogInfo($"Finished saving the job info. DC Job Id: {jobDetails.DcJobId}");
}
private async Task CompleteDataLocks(long jobId, List <CompletedMessage> completedMessages,
List <InProgressMessage> inProgressMessages, CancellationToken cancellationToken)
{
var job = await JobStorageService.GetJob(jobId, cancellationToken).ConfigureAwait(false);
if (job == null)
{
return;
}
var dataLocksMessages = new[]
{
nameof(FunctionalSkillEarningFailedDataLockMatching),
nameof(PayableFunctionalSkillEarningEvent),
nameof(PayableEarningEvent),
nameof(EarningFailedDataLockMatching),
nameof(ProcessLearnerCommand),
nameof(Act1FunctionalSkillEarningsEvent),
nameof(ApprenticeshipContractType1EarningEvent),
nameof(EarningFailedDataLockMatching),
};
var inProgressDataLocks = inProgressMessages
.Where(inProgress =>
dataLocksMessages.Any(dlockType => inProgress.MessageName?.Contains(dlockType) ?? false))
.ToList();
if (!inProgressDataLocks.Any())
{
return;
}
if (!inProgressDataLocks.All(inProgress =>
completedMessages.Any(completed => completed.MessageId == inProgress.MessageId)))
{
return;
}
var completionTime = completedMessages
.Where(completed =>
inProgressDataLocks.Exists(inProgress => inProgress.MessageId == completed.MessageId))
.Max(completed => completed.CompletedTime);
await JobStorageService.SaveDataLocksCompletionTime(job.DcJobId.Value, completionTime, cancellationToken)
.ConfigureAwait(false);
var properties = new Dictionary <string, string>
{
{ TelemetryKeys.JobId, job.DcJobId.ToString() },
{ TelemetryKeys.JobType, job.JobType.ToString("G") },
{ TelemetryKeys.Ukprn, job.Ukprn?.ToString() ?? string.Empty },
{ TelemetryKeys.InternalJobId, job.Id.ToString() },
{ TelemetryKeys.CollectionPeriod, job.CollectionPeriod.ToString() },
{ TelemetryKeys.AcademicYear, job.AcademicYear.ToString() },
{ TelemetryKeys.Status, job.Status.ToString("G") }
};
var metrics = new Dictionary <string, double>
{
{ TelemetryKeys.Duration, (completionTime - job.StartTime).TotalMilliseconds },
{ "Learner Count", job.LearnerCount ?? 0 }
};
Telemetry.TrackEvent("Finished DataLocks", properties, metrics);
Logger.LogInfo(
$"Recorded DataLocks completion time for Job: {job.DcJobId}, completion time: {completionTime}");
}
public virtual async Task <bool> ManageStatus(long jobId, CancellationToken cancellationToken)
{
Logger.LogVerbose($"Now determining if job {jobId} has finished.");
var job = await JobStorageService.GetJob(jobId, cancellationToken).ConfigureAwait(false);
if (job != null)
{
if (await CheckSavedJobStatus(job, cancellationToken))
{
return(true);
}
if (await IsJobTimedOut(job, cancellationToken))
{
return(true);
}
}
var additionalJobChecksResult = await PerformAdditionalJobChecks(job, cancellationToken);
if (!additionalJobChecksResult.IsComplete)
{
return(false);
}
var inProgressMessages = await JobStorageService.GetInProgressMessages(jobId, cancellationToken).ConfigureAwait(false);
var completedItems = await GetCompletedMessages(jobId, inProgressMessages, cancellationToken).ConfigureAwait(false);
//TODO: make a little more elegant
Logger.LogDebug($"Inprogress count: {inProgressMessages.Count}, completed count: {completedItems.Count}. Job: {jobId}");
inProgressMessages.GroupBy(message => message.MessageName).ToList().ForEach(group => Logger.LogDebug($"Inprogress message type: {group.Key}, count: {group.Count()}. Job: {jobId}"));
if (!completedItems.Any())
{
Logger.LogVerbose($"Found no completed messages for job: {jobId}");
return(false);
}
cancellationToken.ThrowIfCancellationRequested();
await ManageMessageStatus(jobId, completedItems, inProgressMessages, cancellationToken).ConfigureAwait(false);
var currentJobStatus = await UpdateJobStatus(jobId, completedItems, cancellationToken).ConfigureAwait(false);
if (!inProgressMessages.TrueForAll(inProgress => completedItems.Any(item => item.MessageId == inProgress.MessageId)))
{
Logger.LogDebug($"Found {inProgressMessages.Count} in progress messages for job id: {jobId}. Cannot set status for job.");
return(false);
}
var jobStatus = additionalJobChecksResult.OverriddenJobStatus ?? (currentJobStatus.hasFailedMessages ? JobStatus.CompletedWithErrors : JobStatus.Completed);
var endTime =
(additionalJobChecksResult.completionTime.HasValue &&
additionalJobChecksResult.completionTime > currentJobStatus.endTime.Value)
? additionalJobChecksResult.completionTime.Value
: currentJobStatus.endTime.Value;
return(await CompleteJob(jobId, jobStatus, endTime, cancellationToken).ConfigureAwait(false));
}
