public async Task <int> Process(int batchSize, CancellationToken cancellationToken) { logger.LogVerbose("Processing batch."); var batch = await cache.GetPayments(batchSize, cancellationToken).ConfigureAwait(false); if (batch.Count < 1) { logger.LogVerbose("No records found to process."); return(0); } using (var scope = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionScopeAsyncFlowOption.Enabled)) { try { foreach (var item in batch) { logger.LogVerbose($"Saving {typeof(T).Name} to table: {item.ToString()}"); await bulkWriter.Write(item, cancellationToken).ConfigureAwait(false); } await bulkWriter.Flush(cancellationToken).ConfigureAwait(false); scope.Complete(); } catch (Exception e) { logger.LogError($"Error performing bulk copy for model type: {typeof(T).Name}. Error: {e.Message}", e); throw; } } return(batch.Count); }
public async Task AddProviderAdjustments(List <ProviderAdjustment> payments) { foreach (var providerAdjustment in payments) { await bulkWriter.Write(providerAdjustment, default(CancellationToken)); } await bulkWriter.Flush(default(CancellationToken)).ConfigureAwait(false); }
private void Write(Func <KeyValuePair <string, TLog>, TLog> selector) { lock (padlock) { var orderedLogs = logs .OrderByDescending(DateTimeKey()) .Select(selector); bulkWriter.Write(orderedLogs); logs.Clear(); } }
private async Task SaveEventPeriods(CancellationToken cancellationToken, DataLockStatusChanged dataLockStatusChangedEvent, int transactionTypeFlag, bool isError, string commitmentVersionId) { var collectionPeriod = dataLockStatusChangedEvent.CollectionPeriod.Period; var dataLockEventPeriod = new LegacyDataLockEventPeriod { DataLockEventId = dataLockStatusChangedEvent.EventId, TransactionTypesFlag = transactionTypeFlag, CollectionPeriodYear = dataLockStatusChangedEvent.CollectionPeriod.AcademicYear, CollectionPeriodName = $"{dataLockStatusChangedEvent.CollectionPeriod.AcademicYear}-{collectionPeriod:D2}", CollectionPeriodMonth = (collectionPeriod < 6) ? collectionPeriod + 7 : collectionPeriod - 5, IsPayable = !isError, CommitmentVersion = commitmentVersionId }; logger.LogVerbose($"Saving DataLockEventPeriod {dataLockEventPeriod.CollectionPeriodName} for legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}"); await dataLockEventPeriodWriter.Write(dataLockEventPeriod, cancellationToken).ConfigureAwait(false); }
private async Task <LegacyDataLockEvent> SaveDataLockEvent(CancellationToken cancellationToken, DataLockStatusChanged dataLockStatusChangedEvent, EarningPeriod earningPeriod, long?apprenticeshipId) { var priceEpisode = dataLockStatusChangedEvent.PriceEpisodes.Single(e => e.Identifier == earningPeriod.PriceEpisodeIdentifier); var hasTnp3 = priceEpisode.TotalNegotiatedPrice3.GetValueOrDefault(0) > 0; var dataLockEvent = new LegacyDataLockEvent // commitment ID { AcademicYear = dataLockStatusChangedEvent.CollectionPeriod.AcademicYear.ToString(), UKPRN = dataLockStatusChangedEvent.Ukprn, DataLockEventId = dataLockStatusChangedEvent.EventId, EventSource = 1, // submission HasErrors = !(dataLockStatusChangedEvent is DataLockStatusChangedToPassed), ULN = dataLockStatusChangedEvent.Learner.Uln, Status = dataLockStatusChangedEvent is DataLockStatusChangedToPassed ? 3 : dataLockStatusChangedEvent is DataLockStatusChangedToFailed ? 1 : 2, ProcessDateTime = DateTime.UtcNow, LearnRefNumber = dataLockStatusChangedEvent.Learner.ReferenceNumber, IlrFrameworkCode = dataLockStatusChangedEvent.LearningAim.FrameworkCode, IlrPathwayCode = dataLockStatusChangedEvent.LearningAim.PathwayCode, IlrProgrammeType = dataLockStatusChangedEvent.LearningAim.ProgrammeType, IlrStandardCode = dataLockStatusChangedEvent.LearningAim.StandardCode, SubmittedDateTime = dataLockStatusChangedEvent.IlrSubmissionDateTime, PriceEpisodeIdentifier = earningPeriod.PriceEpisodeIdentifier, CommitmentId = apprenticeshipId.GetValueOrDefault(0), EmployerAccountId = earningPeriod.AccountId.GetValueOrDefault(0), AimSeqNumber = dataLockStatusChangedEvent.LearningAim.SequenceNumber, IlrPriceEffectiveFromDate = priceEpisode.EffectiveTotalNegotiatedPriceStartDate, IlrPriceEffectiveToDate = priceEpisode.ActualEndDate.GetValueOrDefault(priceEpisode.PlannedEndDate), IlrEndpointAssessorPrice = hasTnp3 ? priceEpisode.TotalNegotiatedPrice4 : priceEpisode.TotalNegotiatedPrice2, IlrFileName = TrimUkprnFromIlrFileNameLimitToValidLength(dataLockStatusChangedEvent.IlrFileName), IlrStartDate = priceEpisode.CourseStartDate, IlrTrainingPrice = hasTnp3 ? priceEpisode.TotalNegotiatedPrice3 : priceEpisode.TotalNegotiatedPrice1, }; logger.LogVerbose($"Saving legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}"); await dataLockEventWriter.Write(dataLockEvent, cancellationToken).ConfigureAwait(false); return(dataLockEvent); }
private async Task SaveCommitmentVersion(CancellationToken cancellationToken, DataLockStatusChanged dataLockStatusChangedEvent, long apprenticeshipId, long apprenticeshipPriceEpisodeId) { var apprenticeship = apprenticeshipCache[apprenticeshipId]; var apprenticeshipPriceEpisode = apprenticeship.ApprenticeshipPriceEpisodes.Single(e => e.Id == apprenticeshipPriceEpisodeId); var commitmentVersion = new LegacyDataLockEventCommitmentVersion { DataLockEventId = dataLockStatusChangedEvent.EventId, CommitmentVersion = $"{apprenticeship.Id}-{apprenticeshipPriceEpisode.Id}", CommitmentEffectiveDate = apprenticeshipPriceEpisode.StartDate, CommitmentFrameworkCode = apprenticeship.FrameworkCode, CommitmentNegotiatedPrice = apprenticeshipPriceEpisode.Cost, CommitmentPathwayCode = apprenticeship.PathwayCode, CommitmentProgrammeType = apprenticeship.ProgrammeType, CommitmentStandardCode = apprenticeship.StandardCode, CommitmentStartDate = apprenticeship.EstimatedStartDate }; logger.LogVerbose($"Saving DataLockEventCommitmentVersion {commitmentVersion.CommitmentVersion} for legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}"); await dataLockEventCommitmentVersionWriter.Write(commitmentVersion, cancellationToken).ConfigureAwait(false); }
private async Task SaveErrorCodes(DataLockStatusChanged dataLockStatusChangedEvent, List <DataLockFailure> dataLockFailures, HashSet <DataLockErrorCode> writtenCodes, CancellationToken cancellationToken) { foreach (var dataLockFailure in dataLockFailures) { if (writtenCodes.Contains(dataLockFailure.DataLockError)) { continue; } var dataLockEventError = new LegacyDataLockEventError { DataLockEventId = dataLockStatusChangedEvent.EventId, SystemDescription = GetDataLockDescription(dataLockFailure.DataLockError), ErrorCode = dataLockFailure.DataLockError.ToString() }; logger.LogVerbose($"Saving DataLockEventError {dataLockFailure.DataLockError} for legacy DataLockEvent {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}"); await dataLockEventErrorWriter.Write(dataLockEventError, cancellationToken).ConfigureAwait(false); writtenCodes.Add(dataLockFailure.DataLockError); } }
private async Task <double> ProcessFm36Global(JobContextMessage message, int collectionPeriod, FM36Global fm36Output, string ilrFileName, CancellationToken cancellationToken) { logger.LogVerbose("Now building commands."); var startTime = DateTimeOffset.UtcNow; var learners = fm36Output.Learners ?? new List <FM36Learner>(); var commands = learners .Select(learner => Build(learner, message.JobId, message.SubmissionDateTimeUtc, short.Parse(fm36Output.Year), collectionPeriod, fm36Output.UKPRN, ilrFileName)) .ToList(); var jobStatusClient = jobClientFactory.Create(); var messageName = typeof(ProcessLearnerCommand).FullName; logger.LogVerbose($"Now sending the start job command for job: {message.JobId}"); await jobStatusClient.StartJob(message.JobId, fm36Output.UKPRN, message.SubmissionDateTimeUtc, short.Parse(fm36Output.Year), (byte)collectionPeriod, commands.Select(cmd => new GeneratedMessage { StartTime = startTime, MessageId = cmd.CommandId, MessageName = messageName }).ToList(), startTime); logger.LogDebug($"Now sending the process learner commands for job: {message.JobId}"); var stopwatch = new Stopwatch(); stopwatch.Start(); var endpointInstance = await factory.GetEndpointInstance(); foreach (var learnerCommand in commands) { try { if (cancellationToken.IsCancellationRequested) { logger.LogWarning($"Cancellation requested, will now stop sending learners for job: {message.JobId}"); return(stopwatch.ElapsedMilliseconds); } await endpointInstance.Send(learnerCommand).ConfigureAwait(false); var aims = submittedLearnerAimBuilder.Build(learnerCommand); await Task.WhenAll(aims.Select(aim => submittedAimWriter.Write(aim, cancellationToken))).ConfigureAwait(false); logger.LogVerbose($"Successfully sent ProcessLearnerCommand JobId: {learnerCommand.JobId}, LearnRefNumber: {learnerCommand.Learner.LearnRefNumber}, SubmissionTime: {message.SubmissionDateTimeUtc}, Collection Year: {fm36Output.Year}, Collection period: {collectionPeriod}"); } catch (Exception ex) { logger.LogError($"Error sending the command: ProcessLearnerCommand. Job Id: {message.JobId}, Ukprn: {fm36Output.UKPRN}, Error: {ex.Message}", ex); throw; } } await submittedAimWriter.Flush(cancellationToken).ConfigureAwait(false); stopwatch.Stop(); var duration = stopwatch.ElapsedMilliseconds; telemetry.TrackEvent("Sent All ProcessLearnerCommand Messages", new Dictionary <string, string> { { TelemetryKeys.Count, fm36Output.Learners.Count.ToString() }, { TelemetryKeys.CollectionPeriod, collectionPeriod.ToString() }, { TelemetryKeys.AcademicYear, fm36Output.Year }, { TelemetryKeys.JobId, message.JobId.ToString() }, { TelemetryKeys.Ukprn, fm36Output.UKPRN.ToString() }, }, new Dictionary <string, double> { { TelemetryKeys.Duration, duration }, { TelemetryKeys.Count, fm36Output.Learners.Count }, }); logger.LogDebug($"Took {stopwatch.ElapsedMilliseconds}ms to send {commands.Count} Process Learner Commands for Job: {message.JobId}"); return(stopwatch.ElapsedMilliseconds); }
private async Task SaveErrorCode(LegacyDataLockEventError error, PriceEpisodeStatusChange dataLockStatusChangedEvent, CancellationToken cancellationToken) { logger.LogVerbose($"Saving DataLockEventError {error} for legacy DataLockEvent {dataLockStatusChangedEvent.DataLock.DataLockEventId} for UKPRN {dataLockStatusChangedEvent.DataLock.UKPRN}"); await dataLockEventErrorWriter.Write(error, cancellationToken); }
private async Task SaveDataLockEvent(CancellationToken cancellationToken, PriceEpisodeStatusChange priceEpisodeStatus) { logger.LogVerbose($"Saving legacy DataLockEvent {priceEpisodeStatus.DataLock.DataLockEventId} for UKPRN {priceEpisodeStatus.DataLock.UKPRN}"); await dataLockEventWriter.Write(priceEpisodeStatus.DataLock, cancellationToken); }
private async Task SaveCommitmentVersion(LegacyDataLockEventCommitmentVersion commitment, PriceEpisodeStatusChange dataLockStatusChangedEvent, CancellationToken cancellationToken) { logger.LogVerbose($"Saving DataLockEventCommitmentVersion {commitment} for PriceEpisodeStatusChange {dataLockStatusChangedEvent.DataLock.DataLockEventId} for UKPRN {dataLockStatusChangedEvent.DataLock.UKPRN}"); await dataLockEventCommitmentVersionWriter.Write(commitment, cancellationToken).ConfigureAwait(false); }
private async Task SaveEventPeriods(LegacyDataLockEventPeriod period, PriceEpisodeStatusChange dataLockStatusChangedEvent, CancellationToken cancellationToken) { logger.LogVerbose($"Saving DataLockEventPeriod {period.CollectionPeriodName} for legacy DataLockEvent {dataLockStatusChangedEvent.DataLock.DataLockEventId} for UKPRN {dataLockStatusChangedEvent.DataLock.UKPRN}"); await dataLockEventPeriodWriter.Write(period, cancellationToken).ConfigureAwait(false); }