public async Task GivenSqlBulkImporter_WhenImportDataWithUnExceptedExceptionInErrorLogUpload_ThenChannelShouldBeCompleteAndExceptionShouldThrow()
{
Channel <ImportResource> inputs = Channel.CreateUnbounded <ImportResource>();
await inputs.Writer.WriteAsync(new ImportResource(0, 0, "Error message"));
inputs.Writer.Complete();
ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>();
ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>();
IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>();
List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>();
IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >();
operationsConfiguration.Value.Returns(new OperationsConfiguration());
SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance);
List <string> errorLogs = new List <string>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
importErrorStore.UploadErrorsAsync(Arg.Any <string[]>(), Arg.Any <CancellationToken>())
.Returns((_) => throw new InvalidOperationException());
(Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None);
await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
{
// Do nothing...
}
await Assert.ThrowsAsync <InvalidOperationException>(() => importTask);
}
public SqlResourceBulkImporter(
ISqlImportOperation sqlImportOperation,
ISqlBulkCopyDataWrapperFactory sqlBulkCopyDataWrapperFactory,
IImportErrorSerializer importErrorSerializer,
List <TableBulkCopyDataGenerator> generators,
IOptions <OperationsConfiguration> operationsConfig,
ILogger <SqlResourceBulkImporter> logger)
{
EnsureArg.IsNotNull(sqlImportOperation, nameof(sqlImportOperation));
EnsureArg.IsNotNull(sqlBulkCopyDataWrapperFactory, nameof(sqlBulkCopyDataWrapperFactory));
EnsureArg.IsNotNull(importErrorSerializer, nameof(importErrorSerializer));
EnsureArg.IsNotNull(generators, nameof(generators));
EnsureArg.IsNotNull(operationsConfig, nameof(operationsConfig));
EnsureArg.IsNotNull(logger, nameof(logger));
_sqlImportOperation = sqlImportOperation;
_sqlBulkCopyDataWrapperFactory = sqlBulkCopyDataWrapperFactory;
_importErrorSerializer = importErrorSerializer;
_generators = generators;
_importTaskConfiguration = operationsConfig.Value.Import;
_logger = logger;
}
private static async Task VerifyBulkImporterBehaviourAsync(Channel <ImportResource> inputs, long expectedSucceedCount, long expectedFailedCount, long expectedEndIndex, int maxResourceCountInBatch, int checkpointBatchCount, int maxConcurrentCount)
{
DataTable table1 = new DataTable();
DataTable table2 = new DataTable();
DataTable dupTable = new DataTable();
List <SqlBulkCopyDataWrapper> importedResources = new List <SqlBulkCopyDataWrapper>();
ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>();
testFhirDataBulkOperation
.When(t => t.BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>()))
.Do(call =>
{
DataTable table = (DataTable)call[0];
if (table.TableName.Equals("Table1"))
{
table1.Merge(table);
}
else if (table.TableName.Equals("Table2"))
{
table2.Merge(table);
}
else if (table.TableName.Equals("Dup"))
{
dupTable.Merge(table);
}
});
testFhirDataBulkOperation
.BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>())
.Returns(call =>
{
IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0];
importedResources.AddRange(resources);
return(resources);
});
IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>();
ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>();
dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>())
.Returns((callInfo) =>
{
ImportResource resource = (ImportResource)callInfo[0];
return(new SqlBulkCopyDataWrapper()
{
ResourceSurrogateId = resource.Id,
});
});
List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>()
{
new TestDataGenerator("Table1", 1),
new TestDataGenerator("Table2", 2),
new TestDupDataGenerator("Dup"),
};
IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >();
OperationsConfiguration operationsConfig = new OperationsConfiguration();
operationsConfig.Import.SqlBatchSizeForImportResourceOperation = maxResourceCountInBatch;
operationsConfig.Import.SqlMaxImportOperationConcurrentCount = maxConcurrentCount;
operationsConfig.Import.SqlImportBatchSizeForCheckpoint = checkpointBatchCount;
operationsConfiguration.Value.Returns(operationsConfig);
SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance);
List <string> errorLogs = new List <string>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
importErrorStore.When(t => t.UploadErrorsAsync(Arg.Any <string[]>(), Arg.Any <CancellationToken>()))
.Do(call =>
{
string[] errors = (string[])call[0];
errorLogs.AddRange(errors);
});
(Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None);
ImportProcessingProgress finalProgress = new ImportProcessingProgress();
await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
{
Assert.True(finalProgress.CurrentIndex <= progress.CurrentIndex);
finalProgress = progress;
}
await importTask;
Assert.Equal(expectedSucceedCount, finalProgress.SucceedImportCount);
Assert.Equal(expectedFailedCount, finalProgress.FailedImportCount);
Assert.Equal(expectedEndIndex, finalProgress.CurrentIndex);
Assert.Equal(expectedSucceedCount, importedResources.Count);
Assert.Equal(expectedSucceedCount, table1.Rows.Count);
Assert.Equal(expectedSucceedCount * 2, table2.Rows.Count);
Assert.Equal(expectedSucceedCount, dupTable.Rows.Count);
Assert.Equal(expectedFailedCount, errorLogs.Count);
}
public async Task GivenSqlBulkImporter_WhenImportDataWithUnExceptedExceptionInBulkOpertation_ThenChannelShouldBeCompleteAndExceptionShouldThrow()
{
Channel <ImportResource> inputs = Channel.CreateUnbounded <ImportResource>();
await inputs.Writer.WriteAsync(new ImportResource(0, 0, default(ResourceWrapper)));
inputs.Writer.Complete();
ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>();
testFhirDataBulkOperation
.BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>())
.Returns((callInfo) =>
{
throw new InvalidOperationException();
});
testFhirDataBulkOperation
.BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>())
.Returns(call =>
{
IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0];
return(resources);
});
IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>();
ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>();
dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>())
.Returns((callInfo) =>
{
ImportResource resource = (ImportResource)callInfo[0];
return(new SqlBulkCopyDataWrapper()
{
ResourceSurrogateId = resource.Id,
});
});
List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>()
{
new TestDataGenerator("Table1", 1),
new TestDataGenerator("Table2", 2),
};
IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >();
operationsConfiguration.Value.Returns(new OperationsConfiguration());
SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance);
List <string> errorLogs = new List <string>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
(Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None);
await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
{
// Do nothing...
}
await Assert.ThrowsAsync <InvalidOperationException>(() => importTask);
}
public SqlResourceBulkImporter(
ISqlImportOperation sqlImportOperation,
ISqlBulkCopyDataWrapperFactory sqlBulkCopyDataWrapperFactory,
IImportErrorSerializer importErrorSerializer,
CompartmentAssignmentTableBulkCopyDataGenerator compartmentAssignmentTableBulkCopyDataGenerator,
ResourceWriteClaimTableBulkCopyDataGenerator resourceWriteClaimTableBulkCopyDataGenerator,
DateTimeSearchParamsTableBulkCopyDataGenerator dateTimeSearchParamsTableBulkCopyDataGenerator,
NumberSearchParamsTableBulkCopyDataGenerator numberSearchParamsTableBulkCopyDataGenerator,
QuantitySearchParamsTableBulkCopyDataGenerator quantitySearchParamsTableBulkCopyDataGenerator,
ReferenceSearchParamsTableBulkCopyDataGenerator referenceSearchParamsTableBulkCopyDataGenerator,
ReferenceTokenCompositeSearchParamsTableBulkCopyDataGenerator referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator,
StringSearchParamsTableBulkCopyDataGenerator stringSearchParamsTableBulkCopyDataGenerator,
TokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator,
TokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator,
TokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator,
TokenSearchParamsTableBulkCopyDataGenerator tokenSearchParamsTableBulkCopyDataGenerator,
TokenStringCompositeSearchParamsTableBulkCopyDataGenerator tokenStringCompositeSearchParamsTableBulkCopyDataGenerator,
TokenTextSearchParamsTableBulkCopyDataGenerator tokenTextSearchParamsTableBulkCopyDataGenerator,
TokenTokenCompositeSearchParamsTableBulkCopyDataGenerator tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator,
UriSearchParamsTableBulkCopyDataGenerator uriSearchParamsTableBulkCopyDataGenerator,
IOptions <OperationsConfiguration> operationsConfig,
ILogger <SqlResourceBulkImporter> logger)
{
EnsureArg.IsNotNull(sqlImportOperation, nameof(sqlImportOperation));
EnsureArg.IsNotNull(sqlBulkCopyDataWrapperFactory, nameof(sqlBulkCopyDataWrapperFactory));
EnsureArg.IsNotNull(importErrorSerializer, nameof(importErrorSerializer));
EnsureArg.IsNotNull(compartmentAssignmentTableBulkCopyDataGenerator, nameof(compartmentAssignmentTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(resourceWriteClaimTableBulkCopyDataGenerator, nameof(resourceWriteClaimTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(dateTimeSearchParamsTableBulkCopyDataGenerator, nameof(dateTimeSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(numberSearchParamsTableBulkCopyDataGenerator, nameof(numberSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(quantitySearchParamsTableBulkCopyDataGenerator, nameof(quantitySearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(referenceSearchParamsTableBulkCopyDataGenerator, nameof(referenceSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator, nameof(referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(stringSearchParamsTableBulkCopyDataGenerator, nameof(stringSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(tokenSearchParamsTableBulkCopyDataGenerator, nameof(tokenSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(tokenStringCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenStringCompositeSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(tokenTextSearchParamsTableBulkCopyDataGenerator, nameof(tokenTextSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(uriSearchParamsTableBulkCopyDataGenerator, nameof(uriSearchParamsTableBulkCopyDataGenerator));
EnsureArg.IsNotNull(operationsConfig, nameof(operationsConfig));
EnsureArg.IsNotNull(logger, nameof(logger));
_sqlImportOperation = sqlImportOperation;
_sqlBulkCopyDataWrapperFactory = sqlBulkCopyDataWrapperFactory;
_importErrorSerializer = importErrorSerializer;
_generators.Add(compartmentAssignmentTableBulkCopyDataGenerator);
_generators.Add(resourceWriteClaimTableBulkCopyDataGenerator);
_generators.Add(dateTimeSearchParamsTableBulkCopyDataGenerator);
_generators.Add(numberSearchParamsTableBulkCopyDataGenerator);
_generators.Add(quantitySearchParamsTableBulkCopyDataGenerator);
_generators.Add(referenceSearchParamsTableBulkCopyDataGenerator);
_generators.Add(referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator);
_generators.Add(stringSearchParamsTableBulkCopyDataGenerator);
_generators.Add(tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator);
_generators.Add(tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator);
_generators.Add(tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator);
_generators.Add(tokenSearchParamsTableBulkCopyDataGenerator);
_generators.Add(tokenStringCompositeSearchParamsTableBulkCopyDataGenerator);
_generators.Add(tokenTextSearchParamsTableBulkCopyDataGenerator);
_generators.Add(tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator);
_generators.Add(uriSearchParamsTableBulkCopyDataGenerator);
_importTaskConfiguration = operationsConfig.Value.Import;
_logger = logger;
}
