public async Task GivenSqlBulkImporter_WhenImportDataWithUnExceptedExceptionInErrorLogUpload_ThenChannelShouldBeCompleteAndExceptionShouldThrow() { Channel <ImportResource> inputs = Channel.CreateUnbounded <ImportResource>(); await inputs.Writer.WriteAsync(new ImportResource(0, 0, "Error message")); inputs.Writer.Complete(); ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>(); ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>(); IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>(); List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>(); IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >(); operationsConfiguration.Value.Returns(new OperationsConfiguration()); SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance); List <string> errorLogs = new List <string>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); importErrorStore.UploadErrorsAsync(Arg.Any <string[]>(), Arg.Any <CancellationToken>()) .Returns((_) => throw new InvalidOperationException()); (Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None); await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync()) { // Do nothing... } await Assert.ThrowsAsync <InvalidOperationException>(() => importTask); }
public async Task GivenResourceLoader_WhenLoadResourcesWithDifferentResourceType_ThenResourcesWithDifferentTypeShouldBeSkipped() { string errorMessage = "Resource type not match."; using MemoryStream stream = new MemoryStream(); using StreamWriter writer = new StreamWriter(stream); await writer.WriteLineAsync("test"); await writer.FlushAsync(); stream.Position = 0; IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream); integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty); IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>(); importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>()) .Returns(callInfo => { ImportResource importResource = new ImportResource(null); return(importResource); }); IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>(); serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>()) .Returns(callInfo => { Exception ex = (Exception)callInfo[1]; return(ex.Message); }); Func <long, long> idGenerator = (i) => i; ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance); (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, "DummyType", idGenerator, CancellationToken.None); int errorCount = 0; await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync()) { Assert.Equal(errorMessage, resource.ImportError); ++errorCount; } await importTask; Assert.Equal(1, errorCount); }
public ImportResourceLoader( IIntegrationDataStoreClient integrationDataStoreClient, IImportResourceParser importResourceParser, IImportErrorSerializer importErrorSerializer, ILogger <ImportResourceLoader> logger) { EnsureArg.IsNotNull(integrationDataStoreClient, nameof(integrationDataStoreClient)); EnsureArg.IsNotNull(importResourceParser, nameof(importResourceParser)); EnsureArg.IsNotNull(importErrorSerializer, nameof(importErrorSerializer)); EnsureArg.IsNotNull(logger, nameof(logger)); _integrationDataStoreClient = integrationDataStoreClient; _importResourceParser = importResourceParser; _importErrorSerializer = importErrorSerializer; _logger = logger; }
public SqlResourceBulkImporter( ISqlImportOperation sqlImportOperation, ISqlBulkCopyDataWrapperFactory sqlBulkCopyDataWrapperFactory, IImportErrorSerializer importErrorSerializer, List <TableBulkCopyDataGenerator> generators, IOptions <OperationsConfiguration> operationsConfig, ILogger <SqlResourceBulkImporter> logger) { EnsureArg.IsNotNull(sqlImportOperation, nameof(sqlImportOperation)); EnsureArg.IsNotNull(sqlBulkCopyDataWrapperFactory, nameof(sqlBulkCopyDataWrapperFactory)); EnsureArg.IsNotNull(importErrorSerializer, nameof(importErrorSerializer)); EnsureArg.IsNotNull(generators, nameof(generators)); EnsureArg.IsNotNull(operationsConfig, nameof(operationsConfig)); EnsureArg.IsNotNull(logger, nameof(logger)); _sqlImportOperation = sqlImportOperation; _sqlBulkCopyDataWrapperFactory = sqlBulkCopyDataWrapperFactory; _importErrorSerializer = importErrorSerializer; _generators = generators; _importTaskConfiguration = operationsConfig.Value.Import; _logger = logger; }
private async Task VerifyResourceLoaderAsync(int resourcCount, int batchSize, long startIndex) { long startId = 1; List <string> inputStrings = new List <string>(); using MemoryStream stream = new MemoryStream(); using StreamWriter writer = new StreamWriter(stream); for (int i = 0; i < resourcCount; ++i) { string content = (i + startId).ToString(); inputStrings.Add(content); await writer.WriteLineAsync(content); } await writer.FlushAsync(); stream.Position = 0; IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream); integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty); IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>(); importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>()) .Returns(callInfo => { long surrogatedId = (long)callInfo[0]; long index = (long)callInfo[1]; string content = (string)callInfo[2]; ResourceWrapper resourceWrapper = new ResourceWrapper( content, "0", "Dummy", new RawResource(content, Fhir.Core.Models.FhirResourceFormat.Json, true), new ResourceRequest("POST"), DateTimeOffset.UtcNow, false, null, null, null, "SearchParam"); return(new ImportResource(surrogatedId, index, resourceWrapper)); }); IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>(); Func <long, long> idGenerator = (i) => startId + i; ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance); loader.MaxBatchSize = batchSize; (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", startIndex, null, idGenerator, CancellationToken.None); long currentIndex = startIndex; await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync()) { string content = idGenerator(currentIndex++).ToString(); Assert.Equal(content, resource.Resource.ResourceId); } await importTask; Assert.Equal(resourcCount, currentIndex); }
public async Task GivenResourceLoader_WhenCancelLoadTask_ThenDataLoadTaskShouldBeCanceled() { string errorMessage = "error"; using MemoryStream stream = new MemoryStream(); using StreamWriter writer = new StreamWriter(stream); await writer.WriteLineAsync("test"); await writer.WriteLineAsync("test"); await writer.WriteLineAsync("test"); await writer.WriteLineAsync("test"); await writer.FlushAsync(); stream.Position = 0; AutoResetEvent resetEvent1 = new AutoResetEvent(false); ManualResetEvent resetEvent2 = new ManualResetEvent(false); IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream); integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty); IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>(); importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>()) .Returns(callInfo => { resetEvent1.Set(); resetEvent2.WaitOne(); throw new InvalidCastException(errorMessage); }); IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>(); serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>()) .Returns(callInfo => { Exception ex = (Exception)callInfo[1]; return(ex.Message); }); Func <long, long> idGenerator = (i) => i; ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, null, idGenerator, cancellationTokenSource.Token); resetEvent1.WaitOne(); cancellationTokenSource.Cancel(); resetEvent2.Set(); await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync()) { // do nothing. } try { await importTask; throw new InvalidOperationException(); } catch (TaskCanceledException) { // Expected error } catch (OperationCanceledException) { // Expected error } }
private static async Task VerifyBulkImporterBehaviourAsync(Channel <ImportResource> inputs, long expectedSucceedCount, long expectedFailedCount, long expectedEndIndex, int maxResourceCountInBatch, int checkpointBatchCount, int maxConcurrentCount) { DataTable table1 = new DataTable(); DataTable table2 = new DataTable(); DataTable dupTable = new DataTable(); List <SqlBulkCopyDataWrapper> importedResources = new List <SqlBulkCopyDataWrapper>(); ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>(); testFhirDataBulkOperation .When(t => t.BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>())) .Do(call => { DataTable table = (DataTable)call[0]; if (table.TableName.Equals("Table1")) { table1.Merge(table); } else if (table.TableName.Equals("Table2")) { table2.Merge(table); } else if (table.TableName.Equals("Dup")) { dupTable.Merge(table); } }); testFhirDataBulkOperation .BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>()) .Returns(call => { IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0]; importedResources.AddRange(resources); return(resources); }); IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>(); ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>(); dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>()) .Returns((callInfo) => { ImportResource resource = (ImportResource)callInfo[0]; return(new SqlBulkCopyDataWrapper() { ResourceSurrogateId = resource.Id, }); }); List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>() { new TestDataGenerator("Table1", 1), new TestDataGenerator("Table2", 2), new TestDupDataGenerator("Dup"), }; IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >(); OperationsConfiguration operationsConfig = new OperationsConfiguration(); operationsConfig.Import.SqlBatchSizeForImportResourceOperation = maxResourceCountInBatch; operationsConfig.Import.SqlMaxImportOperationConcurrentCount = maxConcurrentCount; operationsConfig.Import.SqlImportBatchSizeForCheckpoint = checkpointBatchCount; operationsConfiguration.Value.Returns(operationsConfig); SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance); List <string> errorLogs = new List <string>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); importErrorStore.When(t => t.UploadErrorsAsync(Arg.Any <string[]>(), Arg.Any <CancellationToken>())) .Do(call => { string[] errors = (string[])call[0]; errorLogs.AddRange(errors); }); (Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None); ImportProcessingProgress finalProgress = new ImportProcessingProgress(); await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync()) { Assert.True(finalProgress.CurrentIndex <= progress.CurrentIndex); finalProgress = progress; } await importTask; Assert.Equal(expectedSucceedCount, finalProgress.SucceedImportCount); Assert.Equal(expectedFailedCount, finalProgress.FailedImportCount); Assert.Equal(expectedEndIndex, finalProgress.CurrentIndex); Assert.Equal(expectedSucceedCount, importedResources.Count); Assert.Equal(expectedSucceedCount, table1.Rows.Count); Assert.Equal(expectedSucceedCount * 2, table2.Rows.Count); Assert.Equal(expectedSucceedCount, dupTable.Rows.Count); Assert.Equal(expectedFailedCount, errorLogs.Count); }
public async Task GivenSqlBulkImporter_WhenImportDataWithUnExceptedExceptionInBulkOpertation_ThenChannelShouldBeCompleteAndExceptionShouldThrow() { Channel <ImportResource> inputs = Channel.CreateUnbounded <ImportResource>(); await inputs.Writer.WriteAsync(new ImportResource(0, 0, default(ResourceWrapper))); inputs.Writer.Complete(); ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>(); testFhirDataBulkOperation .BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>()) .Returns((callInfo) => { throw new InvalidOperationException(); }); testFhirDataBulkOperation .BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>()) .Returns(call => { IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0]; return(resources); }); IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>(); ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>(); dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>()) .Returns((callInfo) => { ImportResource resource = (ImportResource)callInfo[0]; return(new SqlBulkCopyDataWrapper() { ResourceSurrogateId = resource.Id, }); }); List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>() { new TestDataGenerator("Table1", 1), new TestDataGenerator("Table2", 2), }; IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >(); operationsConfiguration.Value.Returns(new OperationsConfiguration()); SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance); List <string> errorLogs = new List <string>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); (Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None); await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync()) { // Do nothing... } await Assert.ThrowsAsync <InvalidOperationException>(() => importTask); }
public SqlResourceBulkImporter( ISqlImportOperation sqlImportOperation, ISqlBulkCopyDataWrapperFactory sqlBulkCopyDataWrapperFactory, IImportErrorSerializer importErrorSerializer, CompartmentAssignmentTableBulkCopyDataGenerator compartmentAssignmentTableBulkCopyDataGenerator, ResourceWriteClaimTableBulkCopyDataGenerator resourceWriteClaimTableBulkCopyDataGenerator, DateTimeSearchParamsTableBulkCopyDataGenerator dateTimeSearchParamsTableBulkCopyDataGenerator, NumberSearchParamsTableBulkCopyDataGenerator numberSearchParamsTableBulkCopyDataGenerator, QuantitySearchParamsTableBulkCopyDataGenerator quantitySearchParamsTableBulkCopyDataGenerator, ReferenceSearchParamsTableBulkCopyDataGenerator referenceSearchParamsTableBulkCopyDataGenerator, ReferenceTokenCompositeSearchParamsTableBulkCopyDataGenerator referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator, StringSearchParamsTableBulkCopyDataGenerator stringSearchParamsTableBulkCopyDataGenerator, TokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator, TokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator, TokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator, TokenSearchParamsTableBulkCopyDataGenerator tokenSearchParamsTableBulkCopyDataGenerator, TokenStringCompositeSearchParamsTableBulkCopyDataGenerator tokenStringCompositeSearchParamsTableBulkCopyDataGenerator, TokenTextSearchParamsTableBulkCopyDataGenerator tokenTextSearchParamsTableBulkCopyDataGenerator, TokenTokenCompositeSearchParamsTableBulkCopyDataGenerator tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator, UriSearchParamsTableBulkCopyDataGenerator uriSearchParamsTableBulkCopyDataGenerator, IOptions <OperationsConfiguration> operationsConfig, ILogger <SqlResourceBulkImporter> logger) { EnsureArg.IsNotNull(sqlImportOperation, nameof(sqlImportOperation)); EnsureArg.IsNotNull(sqlBulkCopyDataWrapperFactory, nameof(sqlBulkCopyDataWrapperFactory)); EnsureArg.IsNotNull(importErrorSerializer, nameof(importErrorSerializer)); EnsureArg.IsNotNull(compartmentAssignmentTableBulkCopyDataGenerator, nameof(compartmentAssignmentTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(resourceWriteClaimTableBulkCopyDataGenerator, nameof(resourceWriteClaimTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(dateTimeSearchParamsTableBulkCopyDataGenerator, nameof(dateTimeSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(numberSearchParamsTableBulkCopyDataGenerator, nameof(numberSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(quantitySearchParamsTableBulkCopyDataGenerator, nameof(quantitySearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(referenceSearchParamsTableBulkCopyDataGenerator, nameof(referenceSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator, nameof(referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(stringSearchParamsTableBulkCopyDataGenerator, nameof(stringSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(tokenSearchParamsTableBulkCopyDataGenerator, nameof(tokenSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(tokenStringCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenStringCompositeSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(tokenTextSearchParamsTableBulkCopyDataGenerator, nameof(tokenTextSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator, nameof(tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(uriSearchParamsTableBulkCopyDataGenerator, nameof(uriSearchParamsTableBulkCopyDataGenerator)); EnsureArg.IsNotNull(operationsConfig, nameof(operationsConfig)); EnsureArg.IsNotNull(logger, nameof(logger)); _sqlImportOperation = sqlImportOperation; _sqlBulkCopyDataWrapperFactory = sqlBulkCopyDataWrapperFactory; _importErrorSerializer = importErrorSerializer; _generators.Add(compartmentAssignmentTableBulkCopyDataGenerator); _generators.Add(resourceWriteClaimTableBulkCopyDataGenerator); _generators.Add(dateTimeSearchParamsTableBulkCopyDataGenerator); _generators.Add(numberSearchParamsTableBulkCopyDataGenerator); _generators.Add(quantitySearchParamsTableBulkCopyDataGenerator); _generators.Add(referenceSearchParamsTableBulkCopyDataGenerator); _generators.Add(referenceTokenCompositeSearchParamsTableBulkCopyDataGenerator); _generators.Add(stringSearchParamsTableBulkCopyDataGenerator); _generators.Add(tokenDateTimeCompositeSearchParamsTableBulkCopyDataGenerator); _generators.Add(tokenNumberNumberCompositeSearchParamsTableBulkCopyDataGenerator); _generators.Add(tokenQuantityCompositeSearchParamsTableBulkCopyDataGenerator); _generators.Add(tokenSearchParamsTableBulkCopyDataGenerator); _generators.Add(tokenStringCompositeSearchParamsTableBulkCopyDataGenerator); _generators.Add(tokenTextSearchParamsTableBulkCopyDataGenerator); _generators.Add(tokenTokenCompositeSearchParamsTableBulkCopyDataGenerator); _generators.Add(uriSearchParamsTableBulkCopyDataGenerator); _importTaskConfiguration = operationsConfig.Value.Import; _logger = logger; }