public async Task GivenResourceLoader_WhenLoadResourcesWithDifferentResourceType_ThenResourcesWithDifferentTypeShouldBeSkipped() { string errorMessage = "Resource type not match."; using MemoryStream stream = new MemoryStream(); using StreamWriter writer = new StreamWriter(stream); await writer.WriteLineAsync("test"); await writer.FlushAsync(); stream.Position = 0; IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>(); integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream); integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty); IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>(); importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>()) .Returns(callInfo => { ImportResource importResource = new ImportResource(null); return(importResource); }); IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>(); serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>()) .Returns(callInfo => { Exception ex = (Exception)callInfo[1]; return(ex.Message); }); Func <long, long> idGenerator = (i) => i; ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance); (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, "DummyType", idGenerator, CancellationToken.None); int errorCount = 0; await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync()) { Assert.Equal(errorMessage, resource.ImportError); ++errorCount; } await importTask; Assert.Equal(1, errorCount); }
public SqlBulkCopyDataWrapper CreateSqlBulkCopyDataWrapper(ImportResource resource) { var resourceMetadata = new ResourceMetadata( resource.Resource.CompartmentIndices, resource.Resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)), resource.Resource.LastModifiedClaims); short resourceTypeId = _model.GetResourceTypeId(resource.Resource.ResourceTypeName); resource.CompressedStream.Seek(0, 0); return(new SqlBulkCopyDataWrapper() { Metadata = resourceMetadata, ResourceTypeId = resourceTypeId, Resource = resource.Resource, ResourceSurrogateId = resource.Id, Index = resource.Index, BulkImportResource = resource.ExtractBulkImportResourceTypeV1Row(resourceTypeId), }); }
internal static bool ContainsError(this ImportResource importResource) { return(!string.IsNullOrEmpty(importResource.ImportError)); }
internal static BulkImportResourceTypeV1Row ExtractBulkImportResourceTypeV1Row(this ImportResource importResource, short resourceTypeId) { return(new BulkImportResourceTypeV1Row(resourceTypeId, importResource.Resource.ResourceId, 0, false, importResource.Id, false, "POST", importResource.CompressedStream, true, importResource.Resource.SearchParameterHash)); }
private static async Task VerifyBulkImporterBehaviourAsync(Channel <ImportResource> inputs, long expectedSucceedCount, long expectedFailedCount, long expectedEndIndex, int maxResourceCountInBatch, int checkpointBatchCount, int maxConcurrentCount) { DataTable table1 = new DataTable(); DataTable table2 = new DataTable(); DataTable dupTable = new DataTable(); List <SqlBulkCopyDataWrapper> importedResources = new List <SqlBulkCopyDataWrapper>(); ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>(); testFhirDataBulkOperation .When(t => t.BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>())) .Do(call => { DataTable table = (DataTable)call[0]; if (table.TableName.Equals("Table1")) { table1.Merge(table); } else if (table.TableName.Equals("Table2")) { table2.Merge(table); } else if (table.TableName.Equals("Dup")) { dupTable.Merge(table); } }); testFhirDataBulkOperation .BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>()) .Returns(call => { IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0]; importedResources.AddRange(resources); return(resources); }); IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>(); ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>(); dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>()) .Returns((callInfo) => { ImportResource resource = (ImportResource)callInfo[0]; return(new SqlBulkCopyDataWrapper() { ResourceSurrogateId = resource.Id, }); }); List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>() { new TestDataGenerator("Table1", 1), new TestDataGenerator("Table2", 2), new TestDupDataGenerator("Dup"), }; IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >(); OperationsConfiguration operationsConfig = new OperationsConfiguration(); operationsConfig.Import.SqlBatchSizeForImportResourceOperation = maxResourceCountInBatch; operationsConfig.Import.SqlMaxImportOperationConcurrentCount = maxConcurrentCount; operationsConfig.Import.SqlImportBatchSizeForCheckpoint = checkpointBatchCount; operationsConfiguration.Value.Returns(operationsConfig); SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance); List <string> errorLogs = new List <string>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); importErrorStore.When(t => t.UploadErrorsAsync(Arg.Any <string[]>(), Arg.Any <CancellationToken>())) .Do(call => { string[] errors = (string[])call[0]; errorLogs.AddRange(errors); }); (Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None); ImportProcessingProgress finalProgress = new ImportProcessingProgress(); await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync()) { Assert.True(finalProgress.CurrentIndex <= progress.CurrentIndex); finalProgress = progress; } await importTask; Assert.Equal(expectedSucceedCount, finalProgress.SucceedImportCount); Assert.Equal(expectedFailedCount, finalProgress.FailedImportCount); Assert.Equal(expectedEndIndex, finalProgress.CurrentIndex); Assert.Equal(expectedSucceedCount, importedResources.Count); Assert.Equal(expectedSucceedCount, table1.Rows.Count); Assert.Equal(expectedSucceedCount * 2, table2.Rows.Count); Assert.Equal(expectedSucceedCount, dupTable.Rows.Count); Assert.Equal(expectedFailedCount, errorLogs.Count); }
public async Task GivenSqlBulkImporter_WhenImportDataWithUnExceptedExceptionInBulkOpertation_ThenChannelShouldBeCompleteAndExceptionShouldThrow() { Channel <ImportResource> inputs = Channel.CreateUnbounded <ImportResource>(); await inputs.Writer.WriteAsync(new ImportResource(0, 0, default(ResourceWrapper))); inputs.Writer.Complete(); ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>(); testFhirDataBulkOperation .BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>()) .Returns((callInfo) => { throw new InvalidOperationException(); }); testFhirDataBulkOperation .BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>()) .Returns(call => { IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0]; return(resources); }); IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>(); ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>(); dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>()) .Returns((callInfo) => { ImportResource resource = (ImportResource)callInfo[0]; return(new SqlBulkCopyDataWrapper() { ResourceSurrogateId = resource.Id, }); }); List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>() { new TestDataGenerator("Table1", 1), new TestDataGenerator("Table2", 2), }; IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >(); operationsConfiguration.Value.Returns(new OperationsConfiguration()); SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance); List <string> errorLogs = new List <string>(); IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>(); (Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None); await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync()) { // Do nothing... } await Assert.ThrowsAsync <InvalidOperationException>(() => importTask); }