public async Task Merge_ShouldGetTheNewAndUpdatedData(string latestBlobFileName, string blobFileNameToMerge, string resultsFile, string definitionFileName)
{
DatasetDefinition datasetDefinition = GetDatasetDefinitionByName(definitionFileName);
await using Stream latestDatasetStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{latestBlobFileName}");
Mock <ICloudBlob> latestFileBlob = new Mock <ICloudBlob>();
_blobClient.Setup(_ => _
.GetBlobReferenceFromServerAsync(latestBlobFileName))
.ReturnsAsync(latestFileBlob.Object);
_blobClient.Setup(_ => _
.DownloadToStreamAsync(latestFileBlob.Object))
.ReturnsAsync(latestDatasetStream);
await using Stream fileToMergeDatasetStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{blobFileNameToMerge}");
Mock <ICloudBlob> fileToMergeBlob = new Mock <ICloudBlob>();
_blobClient.Setup(_ => _
.GetBlobReferenceFromServerAsync(blobFileNameToMerge))
.ReturnsAsync(fileToMergeBlob.Object);
_blobClient.Setup(_ => _
.DownloadToStreamAsync(fileToMergeBlob.Object))
.ReturnsAsync(fileToMergeDatasetStream);
await using MemoryStream uploadedStream = new MemoryStream();
fileToMergeBlob.Setup(_ => _.UploadFromStreamAsync(It.IsAny <Stream>()))
.Callback <Stream>(_ =>
{
_?.Seek(0, SeekOrigin.Begin);
// ReSharper disable once AccessToDisposedClosure
_?.CopyTo(uploadedStream);
});
DatasetDataMergeResult result = await _service.Merge(datasetDefinition, latestBlobFileName, blobFileNameToMerge);
result.TablesMergeResults.Count().Should().Be(1);
await using Stream expectedResultStream = File.OpenRead($"TestItems{Path.DirectorySeparatorChar}{resultsFile}");
using ExcelPackage expected = new ExcelPackage(expectedResultStream);
using ExcelPackage actual = new ExcelPackage(uploadedStream);
ExcelWorksheet expectedWorksheet = expected.Workbook.Worksheets[1];
ExcelWorksheet actualWorksheet = expected.Workbook.Worksheets[1];
for (int i = 1; i <= expectedWorksheet.Dimension.Rows; i++)
{
for (int j = 1; j <= expectedWorksheet.Dimension.Columns; j++)
{
actualWorksheet.Cells[i, j].Value.Should().Be(expectedWorksheet.Cells[i, j].Value);
}
}
}
public async Task <DatasetDataMergeResult> Merge(DatasetDefinition datasetDefinition,
string latestBlobFileName,
string blobFileNameToMerge)
{
DatasetDataMergeResult result = new DatasetDataMergeResult();
bool success;
string errorMessage;
List <TableLoadResult> latestTableLoadResults;
List <TableLoadResult> tableLoadResultsToMerge;
(success, errorMessage, latestTableLoadResults) = await ReadExcelDatasetData(datasetDefinition, latestBlobFileName);
if (!success)
{
result.ErrorMessage = errorMessage;
_logger.Error(errorMessage);
return(result);
}
(success, errorMessage, tableLoadResultsToMerge) = await ReadExcelDatasetData(datasetDefinition, blobFileNameToMerge);
if (!success)
{
result.ErrorMessage = errorMessage;
_logger.Error(errorMessage);
return(result);
}
foreach (TableLoadResult latestTableLoadResult in latestTableLoadResults)
{
TableLoadResult tableLoadResultToMerge = tableLoadResultsToMerge.FirstOrDefault(x => x.TableDefinition?.Name == latestTableLoadResult.TableDefinition.Name);
if (tableLoadResultToMerge == null || !tableLoadResultToMerge.Rows.Any())
{
result.TablesMergeResults.Add(new DatasetDataTableMergeResult
{
TableDefinitionName = latestTableLoadResult.TableDefinition.Name
});
}
else
{
// Merge updates latestTableLoadResult with tableLoadResultToMerge data
result.TablesMergeResults.Add(Merge(latestTableLoadResult, tableLoadResultToMerge));
}
}
if (result.HasChanges)
{
// NOTE: If any new / updated rows after merge (rows merged into latest (previous version) dataset), then the merge file will be replaced with latest merge data.
byte[] excelAsBytes = _excelDatasetWriter.Write(datasetDefinition, latestTableLoadResults);
ICloudBlob blob = await _blobClient.GetBlobReferenceFromServerAsync(blobFileNameToMerge);
try
{
await using MemoryStream memoryStream = new MemoryStream(excelAsBytes);
await _blobClientPolicy.ExecuteAsync(() => blob.UploadFromStreamAsync(memoryStream));
}
catch (Exception ex)
{
result.ErrorMessage = $"Failed to upload {datasetDefinition.Name} to blob storage after merge.";
_logger.Error(ex, result.ErrorMessage);
}
}
return(result);
}
