/// <summary>
/// 从csv文件里面读取对象
/// </summary>
/// <typeparam name="T">读取对象的类型</typeparam>
/// <returns>对象集合的迭代器</returns>
public IEnumerable <T> ReadObjects <T>() where T : class, new()
{
CsvFilePath.ShouldExistSuchFile();
IsFirstLineHeader.ShouldEqual(true);
return(ReadObjects <T>(CsvFilePath, Separators, Encoding));
}
public override async Task Process(Message message)
{
string specificationId = message.GetUserProperty <string>("specification-id");
string specificationName = message.GetUserProperty <string>("specification-name");
if (specificationId == null)
{
string error = "Specification id missing";
_logger.Error(error);
throw new NonRetriableException(error);
}
IEnumerable <string> jobDefinitions = new List <string>
{
JobConstants.DefinitionNames.CreateInstructAllocationJob
};
IEnumerable <string> jobTypesRunning = await GetJobTypes(specificationId, jobDefinitions);
if (!jobTypesRunning.IsNullOrEmpty())
{
string errorMessage = string.Join(Environment.NewLine, jobTypesRunning.Select(_ => $"{_} is still running"));
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
string temporaryFilePath = new CsvFilePath(_fileSystemCacheSettings.Path, specificationId);
EnsureFileIsNew(temporaryFilePath);
bool outputHeaders = true;
ApiResponse <SpecificationSummary> specificationSummaryResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(specificationId));
if (specificationSummaryResponse?.Content == null)
{
string errorMessage = $"Specification: {specificationId} not found";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
SpecificationSummary specificationSummary = specificationSummaryResponse.Content;
IEnumerable <TemplateMappingItem> allMappings = new TemplateMappingItem[0];
foreach (Reference reference in specificationSummary.FundingStreams)
{
ApiResponse <TemplateMapping> templateMapping = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specificationId, reference.Id));
if (templateMapping?.Content == null)
{
continue;
}
allMappings = allMappings.Concat(templateMapping.Content.TemplateMappingItems);
}
if (allMappings.Any(_ => _.CalculationId == null))
{
string errorMessage = $"Specification: {specificationId} has missing calculations in template mapping";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.ProviderResultsBatchProcessing(specificationId,
providerResults =>
{
IEnumerable <ExpandoObject> csvRows = _resultsToCsvRowsTransformation.TransformProviderResultsIntoCsvRows(providerResults, allMappings.ToDictionary(_ => _.CalculationId));
string csv = _csvUtils.AsCsv(csvRows, outputHeaders);
_fileSystemAccess.Append(temporaryFilePath, csv)
.GetAwaiter()
.GetResult();
outputHeaders = false;
return(Task.CompletedTask);
}, BatchSize)
);
ICloudBlob blob = _blobClient.GetBlockBlobReference($"calculation-results-{specificationId}.csv");
blob.Properties.ContentDisposition = $"attachment; filename={GetPrettyFileName(specificationName)}";
await using Stream csvFileStream = _fileSystemAccess.OpenRead(temporaryFilePath);
await _blobClientPolicy.ExecuteAsync(() => UploadBlob(blob, csvFileStream, GetMetadata(specificationId, specificationName)));
}
/// <summary>
/// 依次将每行分隔, 返回字符串数组
/// </summary>
/// <returns></returns>
public IEnumerable <string[]> ReadStrings()
{
CsvFilePath.ShouldExistSuchFile();
return(ReadStrings(CsvFilePath, Separators, IsFirstLineHeader, Encoding));
}
/// <summary>
/// 将每行分隔后, 返回字符串数组和行号
/// </summary>
public IEnumerable <(string[] values, int lineNumber)> ReadStringsWithLineNumber()
{
CsvFilePath.ShouldExistSuchFile();
return(ReadStringsWithLineNumber(CsvFilePath, Separators, IsFirstLineHeader, Encoding));
}
/// <summary>
/// 读取文件标题行
/// </summary>
/// <returns>Csv标题</returns>
public CsvHeader ReadHeader()
{
CsvFilePath.ShouldExistSuchFile();
return(ReadHeader(CsvFilePath, Separators, Encoding));
}
