public async Task <int> Process(int batchSize, CancellationToken cancellationToken)
{
logger.LogVerbose("Processing batch.");
var batch = await cache.GetPayments(batchSize, cancellationToken).ConfigureAwait(false);
if (batch.Count < 1)
{
logger.LogVerbose("No records found to process.");
return(0);
}
using (var scope = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionScopeAsyncFlowOption.Enabled))
{
try
{
foreach (var item in batch)
{
logger.LogVerbose($"Saving {typeof(T).Name} to table: {item.ToString()}");
await bulkWriter.Write(item, cancellationToken).ConfigureAwait(false);
}
await bulkWriter.Flush(cancellationToken).ConfigureAwait(false);
scope.Complete();
}
catch (Exception e)
{
logger.LogError($"Error performing bulk copy for model type: {typeof(T).Name}. Error: {e.Message}", e);
throw;
}
}
return(batch.Count);
}
public async Task AddProviderAdjustments(List <ProviderAdjustment> payments)
{
foreach (var providerAdjustment in payments)
{
await bulkWriter.Write(providerAdjustment, default(CancellationToken));
}
await bulkWriter.Flush(default(CancellationToken)).ConfigureAwait(false);
}
private void Write(Func <KeyValuePair <string, TLog>, TLog> selector)
{
lock (padlock)
{
var orderedLogs = logs
.OrderByDescending(DateTimeKey())
.Select(selector);
bulkWriter.Write(orderedLogs);
logs.Clear();
}
}
private async Task SaveEventPeriods(CancellationToken cancellationToken, DataLockStatusChanged dataLockStatusChangedEvent, int transactionTypeFlag, bool isError, string commitmentVersionId)
{
var collectionPeriod = dataLockStatusChangedEvent.CollectionPeriod.Period;
var dataLockEventPeriod = new LegacyDataLockEventPeriod
{
DataLockEventId = dataLockStatusChangedEvent.EventId,
TransactionTypesFlag = transactionTypeFlag,
CollectionPeriodYear = dataLockStatusChangedEvent.CollectionPeriod.AcademicYear,
CollectionPeriodName = $"{dataLockStatusChangedEvent.CollectionPeriod.AcademicYear}-{collectionPeriod:D2}",
CollectionPeriodMonth = (collectionPeriod < 6) ? collectionPeriod + 7 : collectionPeriod - 5,
IsPayable = !isError,
CommitmentVersion = commitmentVersionId
};
logger.LogVerbose($"Saving DataLockEventPeriod {dataLockEventPeriod.CollectionPeriodName} for legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}");
await dataLockEventPeriodWriter.Write(dataLockEventPeriod, cancellationToken).ConfigureAwait(false);
}
private async Task <LegacyDataLockEvent> SaveDataLockEvent(CancellationToken cancellationToken, DataLockStatusChanged dataLockStatusChangedEvent, EarningPeriod earningPeriod, long?apprenticeshipId)
{
var priceEpisode = dataLockStatusChangedEvent.PriceEpisodes.Single(e => e.Identifier == earningPeriod.PriceEpisodeIdentifier);
var hasTnp3 = priceEpisode.TotalNegotiatedPrice3.GetValueOrDefault(0) > 0;
var dataLockEvent = new LegacyDataLockEvent // commitment ID
{
AcademicYear = dataLockStatusChangedEvent.CollectionPeriod.AcademicYear.ToString(),
UKPRN = dataLockStatusChangedEvent.Ukprn,
DataLockEventId = dataLockStatusChangedEvent.EventId,
EventSource = 1, // submission
HasErrors = !(dataLockStatusChangedEvent is DataLockStatusChangedToPassed),
ULN = dataLockStatusChangedEvent.Learner.Uln,
Status = dataLockStatusChangedEvent is DataLockStatusChangedToPassed ? 3 : dataLockStatusChangedEvent is DataLockStatusChangedToFailed ? 1 : 2,
ProcessDateTime = DateTime.UtcNow,
LearnRefNumber = dataLockStatusChangedEvent.Learner.ReferenceNumber,
IlrFrameworkCode = dataLockStatusChangedEvent.LearningAim.FrameworkCode,
IlrPathwayCode = dataLockStatusChangedEvent.LearningAim.PathwayCode,
IlrProgrammeType = dataLockStatusChangedEvent.LearningAim.ProgrammeType,
IlrStandardCode = dataLockStatusChangedEvent.LearningAim.StandardCode,
SubmittedDateTime = dataLockStatusChangedEvent.IlrSubmissionDateTime,
PriceEpisodeIdentifier = earningPeriod.PriceEpisodeIdentifier,
CommitmentId = apprenticeshipId.GetValueOrDefault(0),
EmployerAccountId = earningPeriod.AccountId.GetValueOrDefault(0),
AimSeqNumber = dataLockStatusChangedEvent.LearningAim.SequenceNumber,
IlrPriceEffectiveFromDate = priceEpisode.EffectiveTotalNegotiatedPriceStartDate,
IlrPriceEffectiveToDate = priceEpisode.ActualEndDate.GetValueOrDefault(priceEpisode.PlannedEndDate),
IlrEndpointAssessorPrice = hasTnp3 ? priceEpisode.TotalNegotiatedPrice4 : priceEpisode.TotalNegotiatedPrice2,
IlrFileName = TrimUkprnFromIlrFileNameLimitToValidLength(dataLockStatusChangedEvent.IlrFileName),
IlrStartDate = priceEpisode.CourseStartDate,
IlrTrainingPrice = hasTnp3 ? priceEpisode.TotalNegotiatedPrice3 : priceEpisode.TotalNegotiatedPrice1,
};
logger.LogVerbose($"Saving legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}");
await dataLockEventWriter.Write(dataLockEvent, cancellationToken).ConfigureAwait(false);
return(dataLockEvent);
}
private async Task SaveCommitmentVersion(CancellationToken cancellationToken, DataLockStatusChanged dataLockStatusChangedEvent, long apprenticeshipId, long apprenticeshipPriceEpisodeId)
{
var apprenticeship = apprenticeshipCache[apprenticeshipId];
var apprenticeshipPriceEpisode = apprenticeship.ApprenticeshipPriceEpisodes.Single(e => e.Id == apprenticeshipPriceEpisodeId);
var commitmentVersion = new LegacyDataLockEventCommitmentVersion
{
DataLockEventId = dataLockStatusChangedEvent.EventId,
CommitmentVersion = $"{apprenticeship.Id}-{apprenticeshipPriceEpisode.Id}",
CommitmentEffectiveDate = apprenticeshipPriceEpisode.StartDate,
CommitmentFrameworkCode = apprenticeship.FrameworkCode,
CommitmentNegotiatedPrice = apprenticeshipPriceEpisode.Cost,
CommitmentPathwayCode = apprenticeship.PathwayCode,
CommitmentProgrammeType = apprenticeship.ProgrammeType,
CommitmentStandardCode = apprenticeship.StandardCode,
CommitmentStartDate = apprenticeship.EstimatedStartDate
};
logger.LogVerbose($"Saving DataLockEventCommitmentVersion {commitmentVersion.CommitmentVersion} for legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}");
await dataLockEventCommitmentVersionWriter.Write(commitmentVersion, cancellationToken).ConfigureAwait(false);
}
private async Task SaveErrorCodes(DataLockStatusChanged dataLockStatusChangedEvent, List <DataLockFailure> dataLockFailures, HashSet <DataLockErrorCode> writtenCodes, CancellationToken cancellationToken)
{
foreach (var dataLockFailure in dataLockFailures)
{
if (writtenCodes.Contains(dataLockFailure.DataLockError))
{
continue;
}
var dataLockEventError = new LegacyDataLockEventError
{
DataLockEventId = dataLockStatusChangedEvent.EventId,
SystemDescription = GetDataLockDescription(dataLockFailure.DataLockError),
ErrorCode = dataLockFailure.DataLockError.ToString()
};
logger.LogVerbose($"Saving DataLockEventError {dataLockFailure.DataLockError} for legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}");
await dataLockEventErrorWriter.Write(dataLockEventError, cancellationToken).ConfigureAwait(false);
writtenCodes.Add(dataLockFailure.DataLockError);
}
}
private async Task <double> ProcessFm36Global(JobContextMessage message, int collectionPeriod, FM36Global fm36Output, string ilrFileName, CancellationToken cancellationToken)
{
logger.LogVerbose("Now building commands.");
var startTime = DateTimeOffset.UtcNow;
var learners = fm36Output.Learners ?? new List <FM36Learner>();
var commands = learners
.Select(learner => Build(learner, message.JobId, message.SubmissionDateTimeUtc, short.Parse(fm36Output.Year), collectionPeriod, fm36Output.UKPRN, ilrFileName))
.ToList();
var jobStatusClient = jobClientFactory.Create();
var messageName = typeof(ProcessLearnerCommand).FullName;
logger.LogVerbose($"Now sending the start job command for job: {message.JobId}");
await jobStatusClient.StartJob(message.JobId, fm36Output.UKPRN, message.SubmissionDateTimeUtc, short.Parse(fm36Output.Year), (byte)collectionPeriod,
commands.Select(cmd => new GeneratedMessage {
StartTime = startTime, MessageId = cmd.CommandId, MessageName = messageName
}).ToList(), startTime);
logger.LogDebug($"Now sending the process learner commands for job: {message.JobId}");
var stopwatch = new Stopwatch();
stopwatch.Start();
var endpointInstance = await factory.GetEndpointInstance();
foreach (var learnerCommand in commands)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
logger.LogWarning($"Cancellation requested, will now stop sending learners for job: {message.JobId}");
return(stopwatch.ElapsedMilliseconds);
}
await endpointInstance.Send(learnerCommand).ConfigureAwait(false);
var aims = submittedLearnerAimBuilder.Build(learnerCommand);
await Task.WhenAll(aims.Select(aim => submittedAimWriter.Write(aim, cancellationToken))).ConfigureAwait(false);
logger.LogVerbose($"Successfully sent ProcessLearnerCommand JobId: {learnerCommand.JobId}, LearnRefNumber: {learnerCommand.Learner.LearnRefNumber}, SubmissionTime: {message.SubmissionDateTimeUtc}, Collection Year: {fm36Output.Year}, Collection period: {collectionPeriod}");
}
catch (Exception ex)
{
logger.LogError($"Error sending the command: ProcessLearnerCommand. Job Id: {message.JobId}, Ukprn: {fm36Output.UKPRN}, Error: {ex.Message}", ex);
throw;
}
}
await submittedAimWriter.Flush(cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
var duration = stopwatch.ElapsedMilliseconds;
telemetry.TrackEvent("Sent All ProcessLearnerCommand Messages",
new Dictionary <string, string>
{
{ TelemetryKeys.Count, fm36Output.Learners.Count.ToString() },
{ TelemetryKeys.CollectionPeriod, collectionPeriod.ToString() },
{ TelemetryKeys.AcademicYear, fm36Output.Year },
{ TelemetryKeys.JobId, message.JobId.ToString() },
{ TelemetryKeys.Ukprn, fm36Output.UKPRN.ToString() },
},
new Dictionary <string, double>
{
{ TelemetryKeys.Duration, duration },
{ TelemetryKeys.Count, fm36Output.Learners.Count },
});
logger.LogDebug($"Took {stopwatch.ElapsedMilliseconds}ms to send {commands.Count} Process Learner Commands for Job: {message.JobId}");
return(stopwatch.ElapsedMilliseconds);
}
private async Task SaveErrorCode(LegacyDataLockEventError error, PriceEpisodeStatusChange dataLockStatusChangedEvent, CancellationToken cancellationToken)
{
logger.LogVerbose($"Saving DataLockEventError {error} for legacy DataLockEvent {dataLockStatusChangedEvent.DataLock.DataLockEventId} for UKPRN {dataLockStatusChangedEvent.DataLock.UKPRN}");
await dataLockEventErrorWriter.Write(error, cancellationToken);
}
private async Task SaveDataLockEvent(CancellationToken cancellationToken, PriceEpisodeStatusChange priceEpisodeStatus)
{
logger.LogVerbose($"Saving legacy DataLockEvent {priceEpisodeStatus.DataLock.DataLockEventId} for UKPRN {priceEpisodeStatus.DataLock.UKPRN}");
await dataLockEventWriter.Write(priceEpisodeStatus.DataLock, cancellationToken);
}
private async Task SaveCommitmentVersion(LegacyDataLockEventCommitmentVersion commitment, PriceEpisodeStatusChange dataLockStatusChangedEvent, CancellationToken cancellationToken)
{
logger.LogVerbose($"Saving DataLockEventCommitmentVersion {commitment} for PriceEpisodeStatusChange {dataLockStatusChangedEvent.DataLock.DataLockEventId} for UKPRN {dataLockStatusChangedEvent.DataLock.UKPRN}");
await dataLockEventCommitmentVersionWriter.Write(commitment, cancellationToken).ConfigureAwait(false);
}
private async Task SaveEventPeriods(LegacyDataLockEventPeriod period, PriceEpisodeStatusChange dataLockStatusChangedEvent, CancellationToken cancellationToken)
{
logger.LogVerbose($"Saving DataLockEventPeriod {period.CollectionPeriodName} for legacy DataLockEvent {dataLockStatusChangedEvent.DataLock.DataLockEventId} for UKPRN {dataLockStatusChangedEvent.DataLock.UKPRN}");
await dataLockEventPeriodWriter.Write(period, cancellationToken).ConfigureAwait(false);
}
