public async Task <bool> Callback(CancellationToken cancellationToken)
{
_logger.LogInfo("Reporting callback invoked");
var reportZipFileKey = $"R{_reportServiceContext.ReturnPeriod:00}_{_reportServiceContext.Ukprn}_Reports.zip";
cancellationToken.ThrowIfCancellationRequested();
MemoryStream memoryStream = new MemoryStream();
var zipFileExists = await _streamableKeyValuePersistenceService.ContainsAsync(reportZipFileKey, cancellationToken);
if (zipFileExists)
{
if (_reportServiceContext.Tasks.Any(x => x.CaseInsensitiveEquals(ReportTaskNameConstants.TaskClearPeriodEndDASZip)))
{
await _streamableKeyValuePersistenceService.RemoveAsync(reportZipFileKey, cancellationToken);
}
else
{
await _streamableKeyValuePersistenceService.GetAsync(reportZipFileKey, memoryStream, cancellationToken);
}
}
using (memoryStream)
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Update, true))
{
await ExecuteTasks(_reportServiceContext, archive, cancellationToken);
}
await _streamableKeyValuePersistenceService.SaveAsync(reportZipFileKey, memoryStream, cancellationToken);
}
return(true);
}
public async Task <IMessage> GetIlrFile(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
await _getIlrLock.WaitAsync(cancellationToken);
try
{
if (_message != null)
{
return(_message);
}
cancellationToken.ThrowIfCancellationRequested();
string filename = reportServiceContext.Filename;
int ukPrn = reportServiceContext.Ukprn;
if (string.Equals(reportServiceContext.CollectionName, "ILR1819", StringComparison.OrdinalIgnoreCase))
{
using (MemoryStream ms = new MemoryStream())
{
await _storage.GetAsync(filename, ms, cancellationToken);
ms.Seek(0, SeekOrigin.Begin);
_message = _xmlSerializationService.Deserialize <Message>(ms);
}
}
else
{
DateTime submittedDate;
DateTime filePreparationDate;
using (var ilrContext = _ilrValidContextFactory())
{
submittedDate = ilrContext.Sources.SingleOrDefault(x => x.UKPRN == ukPrn)?.DateTime ?? _dateTimeProvider.ConvertUtcToUk(_dateTimeProvider.GetNowUtc());
filePreparationDate = ilrContext.CollectionDetails.SingleOrDefault(x => x.UKPRN == ukPrn)?.FilePreparationDate ?? _dateTimeProvider.ConvertUtcToUk(_dateTimeProvider.GetNowUtc());
}
_message = new Message
{
Header = new MessageHeader
{
Source = new MessageHeaderSource
{
UKPRN = ukPrn,
DateTime = submittedDate
},
CollectionDetails = new MessageHeaderCollectionDetails
{
FilePreparationDate = filePreparationDate
}
}
};
}
}
finally
{
_getIlrLock.Release();
}
return(_message);
}
public async Task <bool> Callback(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
_logger.LogInfo("Reporting callback invoked");
var reportZipFileKey = $"{reportServiceContext.Ukprn}_{reportServiceContext.JobId}_Reports.zip";
cancellationToken.ThrowIfCancellationRequested();
MemoryStream memoryStream = new MemoryStream();
var zipFileExists = await _streamableKeyValuePersistenceService.ContainsAsync(reportZipFileKey, cancellationToken);
if (zipFileExists)
{
await _streamableKeyValuePersistenceService.GetAsync(reportZipFileKey, memoryStream, cancellationToken);
}
using (memoryStream)
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Update, true))
{
await ExecuteTasks(reportServiceContext, archive, cancellationToken);
}
await _streamableKeyValuePersistenceService.SaveAsync(reportZipFileKey, memoryStream, cancellationToken);
}
return(true);
}
private async Task CopyFile(string inFile, string outFile, IStreamableKeyValuePersistenceService streamableKeyValuePersistenceService, CancellationToken cancellationToken)
{
using (var memoryStream = new MemoryStream())
{
await streamableKeyValuePersistenceService.GetAsync(inFile, memoryStream, cancellationToken);
memoryStream.Seek(0, SeekOrigin.Begin);
await streamableKeyValuePersistenceService.SaveAsync(outFile, memoryStream, cancellationToken);
}
}
public async Task <Stream> Provide(CancellationToken cancellationToken)
{
var startDateTime = DateTime.UtcNow;
var stopwatch = new Stopwatch();
stopwatch.Start();
MemoryStream memoryStream = new MemoryStream();
await _keyValuePersistenceService.GetAsync(_preValidationContext.Input, memoryStream, cancellationToken);
var processTimes = new StringBuilder();
processTimes.AppendLine($"Start Time : {startDateTime}");
processTimes.AppendLine($"Total Time : {(DateTime.UtcNow - startDateTime).TotalMilliseconds}");
_logger.LogDebug($"Blob download :{processTimes} ");
return(memoryStream);
}
public async Task <bool> Callback(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
_logger.LogInfo("Data Match Reporting callback invoked", jobIdOverride: reportServiceContext.JobId);
cancellationToken.ThrowIfCancellationRequested();
string reportZipFileKey;
if (reportServiceContext.IsIlrSubmission)
{
reportZipFileKey = $"{reportServiceContext.Ukprn}_{reportServiceContext.JobId}_Reports.zip";
}
else
{
reportZipFileKey = $"R{reportServiceContext.ReturnPeriod:00}_{reportServiceContext.Ukprn}_Reports.zip";
}
MemoryStream memoryStream = new MemoryStream();
var zipFileExists = await _streamableKeyValuePersistenceService.ContainsAsync(reportZipFileKey, cancellationToken);
if (zipFileExists)
{
await _streamableKeyValuePersistenceService.GetAsync(reportZipFileKey, memoryStream, cancellationToken);
}
using (memoryStream)
{
bool needZip;
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Update, true))
{
needZip = await ExecuteTasks(reportServiceContext, archive, cancellationToken);
}
if (needZip)
{
await _streamableKeyValuePersistenceService.SaveAsync(reportZipFileKey, memoryStream, cancellationToken);
}
}
return(true);
}
public async Task <IMessage> Provide(string fileName)
{
Message message = null;
try
{
using (MemoryStream ms = new MemoryStream())
{
await _blobStoragePersistenceService.GetAsync(fileName, ms).ConfigureAwait(false);
ms.Seek(0, SeekOrigin.Begin);
message = _xmlSerializationService.Deserialize <Message>(ms);
}
}
catch (Exception ex)
{
_logger.LogError($"Failed to retrieve and deserialise ILR file {fileName}", ex);
throw;
}
return(message);
}
private async Task AddZipContents(string zip, ZipArchive archive, IStreamableKeyValuePersistenceService streamableKeyValuePersistenceService, CancellationToken cancellationToken)
{
using (var memoryStream = new MemoryStream())
{
await streamableKeyValuePersistenceService.GetAsync(zip, memoryStream, cancellationToken);
memoryStream.Seek(0, SeekOrigin.Begin);
using (var archiveIn = new ZipArchive(memoryStream, ZipArchiveMode.Read, true))
{
foreach (ZipArchiveEntry zipArchiveEntry in archiveIn.Entries)
{
ZipArchiveEntry newEntry = archive.CreateEntry(zipArchiveEntry.Name);
using (Stream streamOut = newEntry.Open())
{
using (var streamIn = zipArchiveEntry.Open())
{
await streamIn.CopyToAsync(streamOut, cancellationToken);
}
}
}
}
}
}
public async Task <FM35Global> GetFM35Data(
IReportServiceContext reportServiceContext,
CancellationToken cancellationToken)
{
await _getDataLock.WaitAsync(cancellationToken);
try
{
if (_loadedDataAlready)
{
return(_fundingOutputs);
}
cancellationToken.ThrowIfCancellationRequested();
_loadedDataAlready = true;
int ukPrn = reportServiceContext.Ukprn;
if (string.Equals(reportServiceContext.CollectionName, "ILR1819", StringComparison.OrdinalIgnoreCase))
{
string fm35Filename = reportServiceContext.FundingFM35OutputKey;
_logger.LogWarning($"Reading {fm35Filename}; Storage is {_storage}; CancellationToken is {cancellationToken}");
if (await _storage.ContainsAsync(fm35Filename, cancellationToken))
{
_logger.LogWarning($"Available {fm35Filename}");
using (MemoryStream ms = new MemoryStream())
{
await _storage.GetAsync(fm35Filename, ms, cancellationToken);
_logger.LogWarning($"Deserialising {fm35Filename} with {ms.Length}");
_fundingOutputs = _jsonSerializationService.Deserialize <FM35Global>(ms);
}
}
_logger.LogWarning($"Finished {fm35Filename}");
}
else
{
FM35Global fm35Global = new FM35Global();
using (var ilrContext = _ilrRulebaseContextFactory())
{
var fm35GlobalDb = await ilrContext.FM35_globals.FirstOrDefaultAsync(x => x.UKPRN == ukPrn, cancellationToken);
FM35_LearningDelivery[] res = await ilrContext.FM35_LearningDeliveries.Where(x => x.UKPRN == ukPrn)
.Include(x => x.FM35_LearningDelivery_PeriodisedValues).ToArrayAsync(cancellationToken);
IGrouping <string, FM35_LearningDelivery>[] learners = res.GroupBy(x => x.LearnRefNumber).ToArray();
fm35Global.Learners = new List <FM35Learner>();
foreach (IGrouping <string, FM35_LearningDelivery> fm35LearningDeliveries in learners)
{
var learningDeliveryDto = new List <LearningDelivery>();
foreach (var ld in fm35LearningDeliveries)
{
var ldPeriodisedValues = ld.FM35_LearningDelivery_PeriodisedValues.Select(ldpv => new LearningDeliveryPeriodisedValue()
{
AttributeName = ldpv.AttributeName,
Period1 = ldpv.Period_1,
Period2 = ldpv.Period_2,
Period3 = ldpv.Period_3,
Period4 = ldpv.Period_4,
Period5 = ldpv.Period_5,
Period6 = ldpv.Period_6,
Period7 = ldpv.Period_7,
Period8 = ldpv.Period_8,
Period9 = ldpv.Period_9,
Period10 = ldpv.Period_10,
Period11 = ldpv.Period_11,
Period12 = ldpv.Period_12
}).ToList();
learningDeliveryDto.Add(new LearningDelivery()
{
AimSeqNumber = ld.AimSeqNumber,
LearningDeliveryPeriodisedValues = ldPeriodisedValues,
LearningDeliveryValue = new LearningDeliveryValue()
{
FundLine = ld.FundLine,
AchApplicDate = ld.AchApplicDate // todo: finish the entire LearningDeliveryValue here
}
});
}
FM35Learner learner = new FM35Learner()
{
LearnRefNumber = fm35LearningDeliveries.Key,
LearningDeliveries = learningDeliveryDto
};
fm35Global.Learners.Add(learner);
}
if (fm35GlobalDb != null)
{
fm35Global.LARSVersion = fm35GlobalDb.LARSVersion;
fm35Global.OrgVersion = fm35GlobalDb.OrgVersion;
fm35Global.PostcodeDisadvantageVersion = fm35GlobalDb.PostcodeDisadvantageVersion;
fm35Global.RulebaseVersion = fm35GlobalDb.RulebaseVersion;
fm35Global.UKPRN = fm35GlobalDb.UKPRN;
}
}
_fundingOutputs = fm35Global;
}
}
finally
{
_getDataLock.Release();
}
return(_fundingOutputs);
}
public async Task <Stream> Provide(CancellationToken cancellationToken)
{
var startDateTime = _dateTimeProvider.GetNowUtc();
var stopwatch = new Stopwatch();
stopwatch.Start();
MemoryStream outputStream = new MemoryStream();
try
{
using (var memoryStream = new MemoryStream())
{
await _streamableKeyValuePersistenceService.GetAsync(_preValidationContext.Input, memoryStream, cancellationToken);
using (ZipArchive archive = new ZipArchive(memoryStream))
{
List <ZipArchiveEntry> xmlFiles = archive.Entries.Where(x =>
x.Name.EndsWith(".xml", StringComparison.InvariantCultureIgnoreCase)).ToList();
if (xmlFiles.Count == 1)
{
ZipArchiveEntry zippedFile = xmlFiles.First();
using (Stream stream = zippedFile.Open())
{
await stream.CopyToAsync(outputStream, 81920, cancellationToken);
}
string xmlFileName = $"{ExtractUkrpn(_preValidationContext.Input)}/{zippedFile.Name}";
_preValidationContext.Input = xmlFileName;
await _streamableKeyValuePersistenceService.SaveAsync(
xmlFileName,
outputStream,
cancellationToken);
}
else
{
_logger.LogWarning(
$"Zip file contains either more than one file will or no xml file, returning empty stream: jobId: {_preValidationContext.JobId}, file name: {_preValidationContext.Input}");
}
}
}
}
catch (Exception ex)
{
_logger.LogError(
$"Failed to extract the zip file from storage: jobId: {_preValidationContext.JobId}, file name: {_preValidationContext.Input}",
ex);
}
stopwatch.Stop();
var processTimes = new StringBuilder();
processTimes.Append("Start Time: ");
processTimes.AppendLine(startDateTime.ToString(CultureInfo.InvariantCulture));
processTimes.Append("Total Time: ");
processTimes.AppendLine((DateTime.UtcNow - startDateTime).TotalMilliseconds.ToString(CultureInfo.InvariantCulture));
_logger.LogDebug($"Blob download: {processTimes}");
return(outputStream);
}
public async Task <ALBGlobal> GetAllbData(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
await _getDataLock.WaitAsync(cancellationToken);
try
{
if (_loadedDataAlready)
{
return(_fundingOutputs);
}
cancellationToken.ThrowIfCancellationRequested();
_loadedDataAlready = true;
int ukPrn = reportServiceContext.Ukprn;
_logger.LogWarning($"ReportServiceCollectionName {reportServiceContext.CollectionName};");
if (string.Equals(reportServiceContext.CollectionName, "ILR1819", StringComparison.OrdinalIgnoreCase))
{
string albFilename = reportServiceContext.FundingALBOutputKey;
_logger.LogWarning($"Reading {albFilename}; Storage is {_storage}; CancellationToken is {cancellationToken}");
using (MemoryStream ms = new MemoryStream())
{
await _storage.GetAsync(albFilename, ms, cancellationToken);
_fundingOutputs = _jsonSerializationService.Deserialize <ALBGlobal>(ms);
}
}
else
{
ALBGlobal albGlobal = new ALBGlobal();
using (var ilrContext = _ilrRulebaseContextFactory())
{
_logger.LogWarning($"AllbProviderService - Accessing Db;");
var albGlobalDb = await ilrContext.ALB_globals.FirstOrDefaultAsync(x => x.UKPRN == ukPrn, cancellationToken);
using (var ilrValidContext = _ilrValidContextFactory())
{
ALB_LearningDelivery[] res = await ilrContext.ALB_LearningDeliveries
.Where(x => x.UKPRN == ukPrn)
.Include(x => x.ALB_LearningDelivery_PeriodisedValues).ToArrayAsync(cancellationToken);
IGrouping <string, ALB_LearningDelivery>[] learners = res.GroupBy(x => x.LearnRefNumber).ToArray();
albGlobal.Learners = new System.Collections.Generic.List <ALBLearner>();
foreach (IGrouping <string, ALB_LearningDelivery> albLearningDeliveries in learners)
{
var learningDeliveryDto = new List <LearningDelivery>();
foreach (var ld in albLearningDeliveries)
{
var ldPeriodisedValues = ld.ALB_LearningDelivery_PeriodisedValues.Select(ldpv =>
new LearningDeliveryPeriodisedValue()
{
AttributeName = ldpv.AttributeName,
Period1 = ldpv.Period_1,
Period2 = ldpv.Period_2,
Period3 = ldpv.Period_3,
Period4 = ldpv.Period_4,
Period5 = ldpv.Period_5,
Period6 = ldpv.Period_6,
Period7 = ldpv.Period_7,
Period8 = ldpv.Period_8,
Period9 = ldpv.Period_9,
Period10 = ldpv.Period_10,
Period11 = ldpv.Period_11,
Period12 = ldpv.Period_12
}).ToList();
learningDeliveryDto.Add(new LearningDelivery()
{
AimSeqNumber = ld.AimSeqNumber,
LearningDeliveryPeriodisedValues = ldPeriodisedValues,
LearningDeliveryValue = new LearningDeliveryValue()
{
FundLine = ld.FundLine // todo: finish the entire LearningDeliveryValue here
}
});
}
ALBLearner learner = new ALBLearner()
{
LearnRefNumber = albLearningDeliveries.Key,
LearningDeliveries = learningDeliveryDto
};
albGlobal.Learners.Add(learner);
}
}
if (albGlobalDb != null)
{
albGlobal.LARSVersion = albGlobalDb.LARSVersion;
albGlobal.PostcodeAreaCostVersion = albGlobalDb.PostcodeAreaCostVersion;
albGlobal.RulebaseVersion = albGlobalDb.RulebaseVersion;
albGlobal.UKPRN = albGlobalDb.UKPRN;
}
}
_fundingOutputs = albGlobal;
}
}
finally
{
_getDataLock.Release();
}
return(_fundingOutputs);
}
