public async Task Merge(long jobId, string zip1, string zip2, IStreamableKeyValuePersistenceService streamableKeyValuePersistenceService, ILogger logger, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(zip1))
{
if (string.IsNullOrEmpty(zip2))
{
logger.LogWarning($"Cross loading can't find any reports for Job Id {jobId}");
return;
}
await CopyFile(zip2, GetNewFilename(zip2), streamableKeyValuePersistenceService, cancellationToken);
return;
}
if (string.IsNullOrEmpty(zip2))
{
await CopyFile(zip1, GetNewFilename(zip1), streamableKeyValuePersistenceService, cancellationToken);
return;
}
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
await AddZipContents(zip1, archive, streamableKeyValuePersistenceService, cancellationToken);
await AddZipContents(zip2, archive, streamableKeyValuePersistenceService, cancellationToken);
}
await streamableKeyValuePersistenceService.SaveAsync(GetNewFilename(zip1), memoryStream, cancellationToken);
}
}
public async Task <bool> Callback(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
_logger.LogInfo("Reporting callback invoked");
var reportZipFileKey = $"{reportServiceContext.Ukprn}_{reportServiceContext.JobId}_Reports.zip";
cancellationToken.ThrowIfCancellationRequested();
MemoryStream memoryStream = new MemoryStream();
var zipFileExists = await _streamableKeyValuePersistenceService.ContainsAsync(reportZipFileKey, cancellationToken);
if (zipFileExists)
{
await _streamableKeyValuePersistenceService.GetAsync(reportZipFileKey, memoryStream, cancellationToken);
}
using (memoryStream)
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Update, true))
{
await ExecuteTasks(reportServiceContext, archive, cancellationToken);
}
await _streamableKeyValuePersistenceService.SaveAsync(reportZipFileKey, memoryStream, cancellationToken);
}
return(true);
}
public async Task <bool> Callback(CancellationToken cancellationToken)
{
_logger.LogInfo("Reporting callback invoked");
var reportZipFileKey = $"R{_reportServiceContext.ReturnPeriod:00}_{_reportServiceContext.Ukprn}_Reports.zip";
cancellationToken.ThrowIfCancellationRequested();
MemoryStream memoryStream = new MemoryStream();
var zipFileExists = await _streamableKeyValuePersistenceService.ContainsAsync(reportZipFileKey, cancellationToken);
if (zipFileExists)
{
if (_reportServiceContext.Tasks.Any(x => x.CaseInsensitiveEquals(ReportTaskNameConstants.TaskClearPeriodEndDASZip)))
{
await _streamableKeyValuePersistenceService.RemoveAsync(reportZipFileKey, cancellationToken);
}
else
{
await _streamableKeyValuePersistenceService.GetAsync(reportZipFileKey, memoryStream, cancellationToken);
}
}
using (memoryStream)
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Update, true))
{
await ExecuteTasks(_reportServiceContext, archive, cancellationToken);
}
await _streamableKeyValuePersistenceService.SaveAsync(reportZipFileKey, memoryStream, cancellationToken);
}
return(true);
}
private async Task StoreIlrFile(int ukPrn, string ilrFileName, string ilrFile)
{
var byteArray = Encoding.UTF8.GetBytes(ilrFile);
var stream = new MemoryStream(byteArray);
var ilrStoragePathAndFileName = $"{ukPrn}/{ilrFileName}";
await storageService.SaveAsync(ilrStoragePathAndFileName, stream);
}
public override async Task GenerateReport(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
_logger.LogInfo($"In {ReportFileName} report.");
var externalFileName = GetFilename(reportServiceContext);
int collectionId = await _jobQueueManagerProviderService.GetCollectionIdAsync(
$"{ReportTaskNameConstants.IlrCollectionName}{reportServiceContext.CollectionYear}",
cancellationToken);
var providerReturns = (await _jobQueueManagerProviderService.GetReturnersAndPeriodsAsync(collectionId, reportServiceContext.ReturnPeriod, cancellationToken)).ToList();
var ilrFileDetails = await _ilrPeriodEndProviderService.GeFileDetailsSubmittedAsync(cancellationToken);
providerReturns = providerReturns.Select(x => new ProviderReturnPeriod
{
Ukprn = x.Ukprn,
ReturnPeriod = x.ReturnPeriod,
FileName = x.FileName.Replace(".ZIP", ".XML")
}).ToList();
var fileDetails = providerReturns.Select(x => Convert(x, ilrFileDetails)).Where(x => x != null).ToList();
List <OrganisationCollectionModel> expectedReturners = (await _jobQueueManagerProviderService
.GetExpectedReturnersUKPRNsAsync(
collectionId,
cancellationToken)).ToList();
IEnumerable <long> actualReturners = await _jobQueueManagerProviderService
.GetActualReturnersUKPRNsAsync(
collectionId,
reportServiceContext.ReturnPeriod,
cancellationToken);
var ukPrns = providerReturns.Select(x => x.Ukprn).Union(expectedReturners.Select(x => x.Ukprn)).ToList();
var orgDetails = await _orgProviderService.GetOrgDetailsDictionaryForUKPRNSAsync(ukPrns, cancellationToken);
IEnumerable <ProviderSubmissionModel> providerSubmissionsModel = _providerSubmissionsModelBuilder
.BuildModel(
fileDetails,
orgDetails,
expectedReturners,
actualReturners,
reportServiceContext.ILRPeriodsAdjustedTimes,
reportServiceContext.ReturnPeriod);
Workbook providerSubmissionWorkbook = GenerateWorkbook(
reportServiceContext.ReturnPeriod,
providerSubmissionsModel);
using (var ms = new MemoryStream())
{
providerSubmissionWorkbook.Save(ms, SaveFormat.Xlsx);
await _streamableKeyValuePersistenceService.SaveAsync($"{externalFileName}.xlsx", ms, cancellationToken);
}
}
private async Task CopyFile(string inFile, string outFile, IStreamableKeyValuePersistenceService streamableKeyValuePersistenceService, CancellationToken cancellationToken)
{
using (var memoryStream = new MemoryStream())
{
await streamableKeyValuePersistenceService.GetAsync(inFile, memoryStream, cancellationToken);
memoryStream.Seek(0, SeekOrigin.Begin);
await streamableKeyValuePersistenceService.SaveAsync(outFile, memoryStream, cancellationToken);
}
}
public override async Task GenerateReport(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
var externalFileName = $"R{reportServiceContext.ReturnPeriod:D2}_CollectionStats";
IEnumerable <CollectionStatsModel> collectionStatsInfo = (await _jobQueueManagerProviderService.GetCollectionStatsModels(
reportServiceContext.CollectionYear, reportServiceContext.ReturnPeriod, cancellationToken)).ToList();
string json = await GetJson(collectionStatsInfo, cancellationToken);
await _streamableKeyValuePersistenceService.SaveAsync($"{externalFileName}.json", json, cancellationToken);
}
public override async Task GenerateReport(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
_logger.LogInfo($"In {ReportFileName} report.");
List <long> ukprns = new List <long>();
string externalFileName = GetFilename(reportServiceContext);
int collectionId = await _jobQueueManagerProviderService.GetCollectionIdAsync(
$"{ReportTaskNameConstants.IlrCollectionName}{reportServiceContext.CollectionYear}",
cancellationToken);
var fileDetails = await _jobQueueManagerProviderService.GetFilePeriodInfoForCollection(collectionId, cancellationToken);
IEnumerable <DataQualityReturningProviders> dataQualityModels = await _ilrPeriodEndProviderService.GetReturningProvidersAsync(
reportServiceContext.CollectionYear,
reportServiceContext.ILRPeriodsAdjustedTimes,
fileDetails,
CancellationToken.None);
IEnumerable <RuleViolationsInfo> ruleViolations = await _ilrPeriodEndProviderService.GetTop20RuleViolationsAsync(CancellationToken.None);
IEnumerable <ProviderWithoutValidLearners> providersWithoutValidLearners = (await
_ilrPeriodEndProviderService.GetProvidersWithoutValidLearners(fileDetails, CancellationToken.None)).ToList();
IEnumerable <Top10ProvidersWithInvalidLearners> providersWithInvalidLearners = (await
_ilrPeriodEndProviderService.GetProvidersWithInvalidLearners(
reportServiceContext.CollectionYear,
reportServiceContext.ILRPeriodsAdjustedTimes,
fileDetails,
CancellationToken.None)).ToList();
ukprns.AddRange(providersWithoutValidLearners.Select(x => x.Ukprn));
ukprns.AddRange(providersWithInvalidLearners.Select(x => x.Ukprn));
IEnumerable <OrgModel> orgDetails = await _orgProviderService.GetOrgDetailsForUKPRNsAsync(ukprns.Distinct().ToList(), CancellationToken.None);
PopulateModelsWithOrgDetails(orgDetails, providersWithoutValidLearners, providersWithInvalidLearners);
Workbook dataQualityWorkbook = GenerateWorkbook(
reportServiceContext.ReturnPeriod,
dataQualityModels,
ruleViolations,
providersWithoutValidLearners,
providersWithInvalidLearners);
using (var ms = new MemoryStream())
{
dataQualityWorkbook.Save(ms, SaveFormat.Xlsx);
await _streamableKeyValuePersistenceService.SaveAsync($"{externalFileName}.xlsx", ms, cancellationToken);
}
}
public async Task FileLevelErrorReport(
SupplementaryDataWrapper wrapper,
SourceFileModel sourceFile,
CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
foreach (var validationReport in _validationReports)
{
await validationReport.GenerateReport(sourceFile, wrapper, archive, cancellationToken);
}
}
await _streamableKeyValuePersistenceService.SaveAsync($"{sourceFile.UKPRN}_{sourceFile.JobId}_Reports.zip", memoryStream, cancellationToken);
}
}
protected async Task <long> SubmitJob(string collectionName, IFormFile file)
{
long jobId;
var collection = await _collectionManagementService.GetCollectionAsync(Ukprn, collectionName);
if (collection == null || !collection.IsOpen)
{
Logger.LogWarning($"collection {collectionName} for ukprn : {Ukprn} is not open/available, but file is being uploaded");
throw new ArgumentOutOfRangeException(collectionName);
}
var period = await GetCurrentPeriodAsync(collectionName);
if (period == null)
{
Logger.LogWarning($"No active period for collection : {collectionName}");
period = await GetNextPeriodAsync(collectionName);
}
try
{
var fileName = $"{Ukprn}/{file.FileName}".ToUpper();
// push file to Storage
await _storageService.SaveAsync(fileName, file?.OpenReadStream());
// add to the queue
jobId = await _jobService.SubmitJob(new SubmissionMessageViewModel(_jobType, Ukprn)
{
FileName = fileName,
FileSizeBytes = file.Length,
SubmittedBy = User.Name(),
CollectionName = collectionName,
Period = period.PeriodNumber,
NotifyEmail = User.Email(),
StorageReference = _storageKeyValueConfig.ContainerName,
CollectionYear = collection.CollectionYear
});
}
catch (Exception ex)
{
Logger.LogError($"Error trying to subnmit ILR file with name : {file?.Name}", ex);
throw;
}
return(jobId);
}
public override async Task GenerateReport(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
var externalFileName = GetFilename(reportServiceContext);
var collectionName = $"ILR{reportServiceContext.CollectionYear}";
IEnumerable <DataExtractModel> summarisationInfo = (await _summarisationProviderService.GetSummarisedActualsForDataExtractReport(
collectionName,
new[] { reportServiceContext.CollectionReturnCodeApp, reportServiceContext.CollectionReturnCodeDC, reportServiceContext.CollectionReturnCodeESF },
cancellationToken)).ToList();
IEnumerable <string> organisationIds = summarisationInfo?.Select(x => x.OrganisationId).Distinct();
IEnumerable <DataExtractFcsInfo> fcsInfo = await _fcsProviderService.GetFCSForDataExtractReport(organisationIds, cancellationToken);
var dataExtractModel = _modelBuilder.BuildModel(summarisationInfo, fcsInfo);
string csv = await GetCsv(dataExtractModel, cancellationToken);
await _streamableKeyValuePersistenceService.SaveAsync($"{externalFileName}.csv", csv, cancellationToken);
}
public override async Task GenerateReport(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
string externalFileName = GetFilename(reportServiceContext);
IEnumerable <ActCountModel> models = await GenerateModelsAsync(cancellationToken);
using (MemoryStream ms = new MemoryStream())
{
using (var writer = new StreamWriter(ms, Encoding.UTF8, 1024, true))
{
using (var csv = new CsvWriter(writer))
{
csv.Configuration.RegisterClassMap <ActCountModelMapper>();
csv.WriteRecords(models);
}
}
await _streamableKeyValuePersistenceService.SaveAsync($"{externalFileName}.csv", ms, cancellationToken);
}
}
public async Task <bool> Callback(IReportServiceContext reportServiceContext, CancellationToken cancellationToken)
{
_logger.LogInfo("Data Match Reporting callback invoked", jobIdOverride: reportServiceContext.JobId);
cancellationToken.ThrowIfCancellationRequested();
string reportZipFileKey;
if (reportServiceContext.IsIlrSubmission)
{
reportZipFileKey = $"{reportServiceContext.Ukprn}_{reportServiceContext.JobId}_Reports.zip";
}
else
{
reportZipFileKey = $"R{reportServiceContext.ReturnPeriod:00}_{reportServiceContext.Ukprn}_Reports.zip";
}
MemoryStream memoryStream = new MemoryStream();
var zipFileExists = await _streamableKeyValuePersistenceService.ContainsAsync(reportZipFileKey, cancellationToken);
if (zipFileExists)
{
await _streamableKeyValuePersistenceService.GetAsync(reportZipFileKey, memoryStream, cancellationToken);
}
using (memoryStream)
{
bool needZip;
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Update, true))
{
needZip = await ExecuteTasks(reportServiceContext, archive, cancellationToken);
}
if (needZip)
{
await _streamableKeyValuePersistenceService.SaveAsync(reportZipFileKey, memoryStream, cancellationToken);
}
}
return(true);
}
public async Task GenerateReport(
SupplementaryDataWrapper supplementaryDataWrapper,
SourceFileModel sourceFile,
ZipArchive archive,
CancellationToken cancellationToken)
{
var ukPrn = Convert.ToInt32(sourceFile.UKPRN);
var sourceFiles = await _supplementaryDataService.GetImportFiles(sourceFile.UKPRN, cancellationToken);
var supplementaryData =
await _supplementaryDataService.GetSupplementaryData(sourceFiles, cancellationToken);
var ilrYearlyFileData = await _ilrService.GetIlrFileDetails(ukPrn, cancellationToken);
var fm70YearlyData = (await _ilrService.GetYearlyIlrData(ukPrn, cancellationToken)).ToList();
FundingSummaryHeaderModel fundingSummaryHeaderModel =
PopulateReportHeader(sourceFile, ilrYearlyFileData, ukPrn, cancellationToken);
var workbook = new Workbook();
workbook.Worksheets.Clear();
foreach (var file in sourceFiles)
{
var fundingYear = FileNameHelper.GetFundingYearFromFileName(file.FileName);
var thisYearsFm70Data = fm70YearlyData.Where(d => d.FundingYear == fundingYear);
var fundingSummaryModels = PopulateReportData(thisYearsFm70Data, supplementaryData[file.SourceFileId]).ToList();
ApplyFundingYearToEmptyFundingYears(fundingSummaryModels, fundingYear);
FundingSummaryFooterModel fundingSummaryFooterModel = PopulateReportFooter(cancellationToken);
FundingSummaryModel rowOfData = fundingSummaryModels.FirstOrDefault(x => x.DeliverableCode == "ST01" && x.YearlyValues.Any());
var yearAndDataLengthModels = new List <YearAndDataLengthModel>();
if (rowOfData != null)
{
int valCount = rowOfData.YearlyValues.Sum(x => x.Values.Length);
_reportWidth = valCount + rowOfData.Totals.Count + 2;
foreach (FundingSummaryReportYearlyValueModel fundingSummaryReportYearlyValueModel in
rowOfData.YearlyValues)
{
yearAndDataLengthModels.Add(new YearAndDataLengthModel(
fundingSummaryReportYearlyValueModel.FundingYear,
fundingSummaryReportYearlyValueModel.Values.Length));
}
}
_cachedHeaders = GetHeaderEntries(yearAndDataLengthModels);
_cellStyles = _excelStyleProvider.GetFundingSummaryStyles(workbook);
Worksheet sheet = workbook.Worksheets.Add(file.ConRefNumber);
workbook = GetWorkbookReport(workbook, sheet, fundingSummaryHeaderModel, fundingSummaryModels, fundingSummaryFooterModel);
ApplyAdditionalFormatting(workbook, rowOfData);
}
string externalFileName = GetExternalFilename(sourceFile.UKPRN, sourceFile.JobId ?? 0, sourceFile.SuppliedDate ?? DateTime.MinValue);
string fileName = GetFilename(sourceFile.UKPRN, sourceFile.JobId ?? 0, sourceFile.SuppliedDate ?? DateTime.MinValue);
using (var ms = new MemoryStream())
{
workbook.Save(ms, SaveFormat.Xlsx);
await _storage.SaveAsync($"{externalFileName}.xlsx", ms, cancellationToken);
await WriteZipEntry(archive, $"{fileName}.xlsx", ms, cancellationToken);
}
}
public async Task <Stream> Provide(CancellationToken cancellationToken)
{
var startDateTime = _dateTimeProvider.GetNowUtc();
var stopwatch = new Stopwatch();
stopwatch.Start();
MemoryStream outputStream = new MemoryStream();
try
{
using (var memoryStream = new MemoryStream())
{
await _streamableKeyValuePersistenceService.GetAsync(_preValidationContext.Input, memoryStream, cancellationToken);
using (ZipArchive archive = new ZipArchive(memoryStream))
{
List <ZipArchiveEntry> xmlFiles = archive.Entries.Where(x =>
x.Name.EndsWith(".xml", StringComparison.InvariantCultureIgnoreCase)).ToList();
if (xmlFiles.Count == 1)
{
ZipArchiveEntry zippedFile = xmlFiles.First();
using (Stream stream = zippedFile.Open())
{
await stream.CopyToAsync(outputStream, 81920, cancellationToken);
}
string xmlFileName = $"{ExtractUkrpn(_preValidationContext.Input)}/{zippedFile.Name}";
_preValidationContext.Input = xmlFileName;
await _streamableKeyValuePersistenceService.SaveAsync(
xmlFileName,
outputStream,
cancellationToken);
}
else
{
_logger.LogWarning(
$"Zip file contains either more than one file will or no xml file, returning empty stream: jobId: {_preValidationContext.JobId}, file name: {_preValidationContext.Input}");
}
}
}
}
catch (Exception ex)
{
_logger.LogError(
$"Failed to extract the zip file from storage: jobId: {_preValidationContext.JobId}, file name: {_preValidationContext.Input}",
ex);
}
stopwatch.Stop();
var processTimes = new StringBuilder();
processTimes.Append("Start Time: ");
processTimes.AppendLine(startDateTime.ToString(CultureInfo.InvariantCulture));
processTimes.Append("Total Time: ");
processTimes.AppendLine((DateTime.UtcNow - startDateTime).TotalMilliseconds.ToString(CultureInfo.InvariantCulture));
_logger.LogDebug($"Blob download: {processTimes}");
return(outputStream);
}
