public async Task RecordNewJob(RecordEarningsJob earningsJobRequest, CancellationToken cancellationToken = default(CancellationToken))
{
logger.LogDebug($"Now recording new provider earnings job. Job Id: {earningsJobRequest.JobId}, Ukprn: {earningsJobRequest.Ukprn}.");
var jobDetails = new JobModel
{
JobType = JobType.EarningsJob,
StartTime = earningsJobRequest.StartTime,
CollectionPeriod = earningsJobRequest.CollectionPeriod,
AcademicYear = earningsJobRequest.CollectionYear,
Ukprn = earningsJobRequest.Ukprn,
DcJobId = earningsJobRequest.JobId,
IlrSubmissionTime = earningsJobRequest.IlrSubmissionTime,
Status = JobStatus.InProgress,
LearnerCount = earningsJobRequest.LearnerCount
};
var isNewJob = await jobStorageService.StoreNewJob(jobDetails, CancellationToken.None);
logger.LogDebug($"Finished storing new job: {jobDetails.Id} for dc job id: {jobDetails.DcJobId}, Ukprn: {earningsJobRequest.Ukprn}. Now storing the job messages.");
await RecordJobInProgressMessages(earningsJobRequest.JobId, earningsJobRequest.GeneratedMessages, cancellationToken).ConfigureAwait(false);
logger.LogDebug($"Finished storing new job messages for job: {jobDetails.Id} for dc job id: {jobDetails.DcJobId}, Ukprn: {earningsJobRequest.Ukprn}.");
if (isNewJob)
{
SendTelemetry(earningsJobRequest, jobDetails);
}
logger.LogInfo($"Finished saving the job info. DC Job Id: {earningsJobRequest.JobId}, Ukprn: {earningsJobRequest.Ukprn}.");
}
public async Task Stores_New_Jobs()
{
var jobStarted = new RecordEarningsJob
{
CollectionPeriod = 1,
CollectionYear = 1819,
JobId = 1,
Ukprn = 9999,
IlrSubmissionTime = DateTime.UtcNow.AddMinutes(-20),
StartTime = DateTimeOffset.UtcNow,
};
var service = mocker.Create <EarningsJobService>();
await service.RecordNewJob(jobStarted, default(CancellationToken));
mocker.Mock <IJobStorageService>()
.Verify(x => x.StoreNewJob(
It.Is <JobModel>(job =>
job.StartTime == jobStarted.StartTime &&
job.JobType == JobType.EarningsJob &&
job.Status == JobStatus.InProgress && job.DcJobId == jobStarted.JobId &&
job.CollectionPeriod == jobStarted.CollectionPeriod &&
job.AcademicYear == jobStarted.CollectionYear &&
job.IlrSubmissionTime == jobStarted.IlrSubmissionTime &&
job.Ukprn == jobStarted.Ukprn), It.IsAny <CancellationToken>()), Times.Once);
}
private void SendTelemetry(RecordEarningsJob startedEvent, JobModel jobDetails)
{
telemetry.TrackEvent("Started Job",
new Dictionary <string, string>
{
{ "JobType", JobType.EarningsJob.ToString("G") },
{ "Ukprn", startedEvent.Ukprn.ToString() },
{ "Id", jobDetails.Id.ToString() },
{ "ExternalJobId", startedEvent.JobId.ToString() },
{ "CollectionPeriod", startedEvent.CollectionPeriod.ToString() },
{ "CollectionYear", startedEvent.CollectionYear.ToString() }
},
new Dictionary <string, double> {
{ "LearnerCount", startedEvent.LearnerCount }
}
);
}
public async Task StartJob(long jobId, long ukprn, DateTime ilrSubmissionTime, short collectionYear, byte collectionPeriod, List <GeneratedMessage> generatedMessages, DateTimeOffset startTime)
{
logger.LogVerbose($"Sending request to record start of earnings job. Job Id: {jobId}, Ukprn: {ukprn}");
try
{
var batchSize = 1000; //TODO: this should come from config
List <GeneratedMessage> batch;
var providerEarningsEvent = new RecordEarningsJob
{
StartTime = startTime,
JobId = jobId,
Ukprn = ukprn,
IlrSubmissionTime = ilrSubmissionTime,
CollectionYear = collectionYear,
CollectionPeriod = collectionPeriod,
GeneratedMessages = generatedMessages.Take(batchSize).ToList(),
LearnerCount = generatedMessages.Count
};
var jobsEndpointName = config.GetSettingOrDefault("Monitoring_JobsService_EndpointName", "sfa-das-payments-monitoring-jobs");
var partitionedEndpointName = $"{jobsEndpointName}{partitionName.PartitionNameForJob(jobId, ukprn)}";
logger.LogVerbose($"Endpoint for RecordEarningsJob for Job Id {jobId} is `{partitionedEndpointName}`");
await messageSession.Send(partitionedEndpointName, providerEarningsEvent).ConfigureAwait(false);
var skip = batchSize;
while ((batch = generatedMessages.Skip(skip).Take(1000).ToList()).Count > 0)
{
skip += batchSize;
var providerEarningsAdditionalMessages = new RecordEarningsJobAdditionalMessages
{
JobId = jobId,
GeneratedMessages = batch,
};
await messageSession.Send(partitionedEndpointName, providerEarningsAdditionalMessages).ConfigureAwait(false);
}
logger.LogDebug($"Sent request(s) to record start of earnings job. Job Id: {jobId}, Ukprn: {ukprn}");
}
catch (Exception ex)
{
logger.LogError($"Failed to send the request to record the earnings job. Job: {jobId}, Error: {ex.Message}", ex);
throw;
}
}
public async Task RecordNewJob(RecordEarningsJob earningsJobRequest, CancellationToken cancellationToken)
{
Logger.LogDebug($"Now recording new provider earnings job. Job Id: {earningsJobRequest.JobId}, Ukprn: {earningsJobRequest.Ukprn}.");
var jobDetails = new JobModel
{
JobType = JobType.EarningsJob,
StartTime = earningsJobRequest.StartTime,
CollectionPeriod = earningsJobRequest.CollectionPeriod,
AcademicYear = earningsJobRequest.CollectionYear,
Ukprn = earningsJobRequest.Ukprn,
DcJobId = earningsJobRequest.JobId,
IlrSubmissionTime = earningsJobRequest.IlrSubmissionTime,
Status = JobStatus.InProgress,
LearnerCount = earningsJobRequest.LearnerCount
};
await RecordNewJob(jobDetails, earningsJobRequest.GeneratedMessages, cancellationToken)
.ConfigureAwait(false);
}
public async Task RecordEarningsJob(RecordEarningsJob message, CancellationToken cancellationToken)
{
try
{
using (var operation = telemetry.StartOperation("JobsService.RecordEarningsJob", message.CommandId.ToString()))
{
var stopwatch = Stopwatch.StartNew();
var service = lifetimeScope.Resolve <IEarningsJobService>();
await service.JobStarted(message, CancellationToken.None).ConfigureAwait(false);
telemetry.TrackDuration("JobsService.RecordEarningsJob", stopwatch.Elapsed);
telemetry.StopOperation(operation);
}
}
catch (Exception e)
{
logger.LogWarning($"Error recording earning job. Job: {message.JobId}, ukprn: {message.Ukprn}, Error: {e.Message}. {e}");
throw;
}
}
public async Task Stores_In_Progress_Messages()
{
var generatedMessageA = new GeneratedMessage
{
StartTime = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid(),
MessageName = "MessageA",
};
var generatedMessageB = new GeneratedMessage
{
StartTime = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid(),
MessageName = "MessageB",
};
var jobStarted = new RecordEarningsJob
{
CollectionPeriod = 1,
CollectionYear = 1819,
JobId = 1,
Ukprn = 9999,
IlrSubmissionTime = DateTime.UtcNow.AddMinutes(-20),
StartTime = DateTimeOffset.UtcNow,
GeneratedMessages = new List <GeneratedMessage> {
generatedMessageA, generatedMessageB
}
};
var service = mocker.Create <EarningsJobService>();
await service.RecordNewJob(jobStarted, default(CancellationToken));
mocker.Mock <IJobStorageService>()
.Verify(x => x.StoreInProgressMessages(It.Is <long>(jobId => jobId == jobStarted.JobId), It.Is <List <InProgressMessage> >(identifiers =>
identifiers.Count == 2 &&
identifiers.Exists(inProgress => inProgress.MessageId == generatedMessageA.MessageId) &&
identifiers.Exists(inProgress => inProgress.MessageId == generatedMessageB.MessageId)), It.IsAny <CancellationToken>()), Times.Once);
}
