public async Task <int> Process(int batchSize, CancellationToken cancellationToken)
{
logger.LogVerbose("Processing batch.");
var batch = await cache.GetPayments(batchSize, cancellationToken).ConfigureAwait(false);
if (batch.Count < 1)
{
logger.LogVerbose("No records found to process.");
return(0);
}
using (var scope = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionScopeAsyncFlowOption.Enabled))
{
try
{
foreach (var item in batch)
{
logger.LogVerbose($"Saving {typeof(T).Name} to table: {item.ToString()}");
await bulkWriter.Write(item, cancellationToken).ConfigureAwait(false);
}
await bulkWriter.Flush(cancellationToken).ConfigureAwait(false);
scope.Complete();
}
catch (Exception e)
{
logger.LogError($"Error performing bulk copy for model type: {typeof(T).Name}. Error: {e.Message}", e);
throw;
}
}
return(batch.Count);
}
public async Task <int> Process(int batchSize, CancellationToken cancellationToken)
{
logger.LogVerbose("Processing batch.");
var batch = await cache.GetPayments(batchSize, cancellationToken).ConfigureAwait(false);
if (batch.Count < 1)
{
logger.LogVerbose("No records found to process.");
return(0);
}
try
{
using (var scope = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionScopeAsyncFlowOption.Enabled))
{
foreach (var changeEvent in batch)
{
await SaveDataLockEvent(cancellationToken, changeEvent);
foreach (var period in changeEvent.Periods)
{
await SaveEventPeriods(period, changeEvent, cancellationToken);
}
foreach (var commitment in changeEvent.CommitmentVersions)
{
await SaveCommitmentVersion(commitment, changeEvent, cancellationToken);
}
foreach (var error in changeEvent.Errors)
{
await SaveErrorCode(error, changeEvent, cancellationToken);
}
logger.LogDebug(
$"Saved PriceEpisodeStatusChange event {changeEvent.DataLock.DataLockEventId} for UKPRN {changeEvent.DataLock.UKPRN}. " +
$"Commitment versions: {changeEvent.CommitmentVersions.Length}, " +
$"periods: {changeEvent.Periods.Length}, errors: {changeEvent.Errors.Length}");
}
await dataLockEventWriter.Flush(cancellationToken);
await dataLockEventCommitmentVersionWriter.Flush(cancellationToken);
await dataLockEventPeriodWriter.Flush(cancellationToken);
await dataLockEventErrorWriter.Flush(cancellationToken);
scope.Complete();
}
}
catch (Exception e)
{
logger.LogError($"Error saving batch of DataLockStatusChanged events. Error: {e.Message}", e);
throw;
}
return(batch.Count);
}
public async Task <int> Process(int batchSize, CancellationToken cancellationToken)
{
logger.LogVerbose("Processing batch.");
var batch = await cache.GetPayments(batchSize, cancellationToken).ConfigureAwait(false);
if (batch.Count < 1)
{
logger.LogVerbose("No records found to process.");
return(0);
}
try
{
await PopulateApprenticeshipCache(batch, cancellationToken).ConfigureAwait(false);
using (var scope = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionScopeAsyncFlowOption.Enabled))
{
foreach (var dataLockStatusChangedEvent in batch)
{
int savedEvents = 0, savedCommitmentVersions = 0, savedPeriods = 0, savedErrors = 0;
var isError = dataLockStatusChangedEvent is DataLockStatusChangedToFailed || dataLockStatusChangedEvent is DataLockFailureChanged;
var flatPeriodList = dataLockStatusChangedEvent.TransactionTypesAndPeriods.SelectMany(tp => tp.Value).ToList();
var writtenErrors = new HashSet <DataLockErrorCode>();
// there may be multiple commitment IDs when employer changes or separate errors against two employers
var apprenticeshipIds = flatPeriodList.Select(p => p.ApprenticeshipId)
.Concat(flatPeriodList.Where(p => p.DataLockFailures != null).SelectMany(p => p.DataLockFailures.Select(f => f.ApprenticeshipId)))
.Distinct()
.ToList();
foreach (var apprenticeshipId in apprenticeshipIds)
{
// v1 doesn't use delivery period, get one earning period for each price episode
var earningPeriodsByPriceEpisode = flatPeriodList
.GroupBy(p => p.PriceEpisodeIdentifier)
.Select(g => g.FirstOrDefault(p => p.ApprenticeshipId == apprenticeshipId || (p.DataLockFailures != null && p.DataLockFailures.Any(f => f.ApprenticeshipId == apprenticeshipId))))
.Where(g => g != null)
.ToList();
foreach (var earningPeriod in earningPeriodsByPriceEpisode)
{
// only records null apprenticeship when DLOCK 01 & 02
if (!apprenticeshipId.HasValue)
{
if (earningPeriod.DataLockFailures.Any(f => f.ApprenticeshipId.HasValue))
{
continue;
}
}
await SaveDataLockEvent(cancellationToken, dataLockStatusChangedEvent, earningPeriod, apprenticeshipId).ConfigureAwait(false);
savedEvents++;
if (apprenticeshipId.HasValue)
{
if (isError)
{
var writtenVersions = new HashSet <(long apprenticeshipId, long apprenticeshipPriceEpisodeId)>();
foreach (var dataLockFailure in earningPeriod.DataLockFailures.Where(f => f.ApprenticeshipId == apprenticeshipId))
{
foreach (var apprenticeshipPriceEpisodeId in dataLockFailure.ApprenticeshipPriceEpisodeIds)
{
// there are multiple errors recorded for the same apprenticeship episode
if (writtenVersions.Contains((apprenticeshipId.Value, apprenticeshipPriceEpisodeId)))
{
continue;
}
savedPeriods += await SaveCommitmentVersionAndPeriods(dataLockStatusChangedEvent, dataLockFailure.ApprenticeshipId.Value, apprenticeshipPriceEpisodeId, isError, cancellationToken);
savedCommitmentVersions++;
writtenVersions.Add((apprenticeshipId.Value, apprenticeshipPriceEpisodeId));
}
}
}
else
{
savedPeriods += await SaveCommitmentVersionAndPeriods(dataLockStatusChangedEvent, earningPeriod.ApprenticeshipId.Value, earningPeriod.ApprenticeshipPriceEpisodeId.Value, isError, cancellationToken);
savedCommitmentVersions++;
}
}
if (earningPeriod.DataLockFailures?.Count > 0)
{
await SaveErrorCodes(dataLockStatusChangedEvent, earningPeriod.DataLockFailures, writtenErrors, cancellationToken).ConfigureAwait(false);
savedErrors += earningPeriod.DataLockFailures.Count;
}
}
}
logger.LogDebug($"Saved DataLockStatusChanged event {dataLockStatusChangedEvent.EventId} for UKPRN {dataLockStatusChangedEvent.Ukprn}. Legacy events: {savedEvents}, commitment versions: {savedCommitmentVersions}, periods: {savedPeriods}, errors: {savedErrors}");
}
await dataLockEventWriter.Flush(cancellationToken).ConfigureAwait(false);
await dataLockEventCommitmentVersionWriter.Flush(cancellationToken).ConfigureAwait(false);
await dataLockEventPeriodWriter.Flush(cancellationToken).ConfigureAwait(false);
await dataLockEventErrorWriter.Flush(cancellationToken).ConfigureAwait(false);
scope.Complete();
}
}
catch (Exception e)
{
logger.LogError($"Error saving batch of DataLockStatusChanged events. Error: {e.Message}", e);
throw;
}
return(batch.Count);
}