public async Task GivenResourceLoader_WhenLoadResourcesWithDifferentResourceType_ThenResourcesWithDifferentTypeShouldBeSkipped()
{
string errorMessage = "Resource type not match.";
using MemoryStream stream = new MemoryStream();
using StreamWriter writer = new StreamWriter(stream);
await writer.WriteLineAsync("test");
await writer.FlushAsync();
stream.Position = 0;
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream);
integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty);
IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>();
importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>())
.Returns(callInfo =>
{
ImportResource importResource = new ImportResource(null);
return(importResource);
});
IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>();
serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>())
.Returns(callInfo =>
{
Exception ex = (Exception)callInfo[1];
return(ex.Message);
});
Func <long, long> idGenerator = (i) => i;
ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance);
(Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, "DummyType", idGenerator, CancellationToken.None);
int errorCount = 0;
await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync())
{
Assert.Equal(errorMessage, resource.ImportError);
++errorCount;
}
await importTask;
Assert.Equal(1, errorCount);
}
public SqlBulkCopyDataWrapper CreateSqlBulkCopyDataWrapper(ImportResource resource)
{
var resourceMetadata = new ResourceMetadata(
resource.Resource.CompartmentIndices,
resource.Resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
resource.Resource.LastModifiedClaims);
short resourceTypeId = _model.GetResourceTypeId(resource.Resource.ResourceTypeName);
resource.CompressedStream.Seek(0, 0);
return(new SqlBulkCopyDataWrapper()
{
Metadata = resourceMetadata,
ResourceTypeId = resourceTypeId,
Resource = resource.Resource,
ResourceSurrogateId = resource.Id,
Index = resource.Index,
BulkImportResource = resource.ExtractBulkImportResourceTypeV1Row(resourceTypeId),
});
}
internal static bool ContainsError(this ImportResource importResource)
{
return(!string.IsNullOrEmpty(importResource.ImportError));
}
internal static BulkImportResourceTypeV1Row ExtractBulkImportResourceTypeV1Row(this ImportResource importResource, short resourceTypeId)
{
return(new BulkImportResourceTypeV1Row(resourceTypeId, importResource.Resource.ResourceId, 0, false, importResource.Id, false, "POST", importResource.CompressedStream, true, importResource.Resource.SearchParameterHash));
}
private static async Task VerifyBulkImporterBehaviourAsync(Channel <ImportResource> inputs, long expectedSucceedCount, long expectedFailedCount, long expectedEndIndex, int maxResourceCountInBatch, int checkpointBatchCount, int maxConcurrentCount)
{
DataTable table1 = new DataTable();
DataTable table2 = new DataTable();
DataTable dupTable = new DataTable();
List <SqlBulkCopyDataWrapper> importedResources = new List <SqlBulkCopyDataWrapper>();
ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>();
testFhirDataBulkOperation
.When(t => t.BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>()))
.Do(call =>
{
DataTable table = (DataTable)call[0];
if (table.TableName.Equals("Table1"))
{
table1.Merge(table);
}
else if (table.TableName.Equals("Table2"))
{
table2.Merge(table);
}
else if (table.TableName.Equals("Dup"))
{
dupTable.Merge(table);
}
});
testFhirDataBulkOperation
.BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>())
.Returns(call =>
{
IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0];
importedResources.AddRange(resources);
return(resources);
});
IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>();
ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>();
dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>())
.Returns((callInfo) =>
{
ImportResource resource = (ImportResource)callInfo[0];
return(new SqlBulkCopyDataWrapper()
{
ResourceSurrogateId = resource.Id,
});
});
List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>()
{
new TestDataGenerator("Table1", 1),
new TestDataGenerator("Table2", 2),
new TestDupDataGenerator("Dup"),
};
IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >();
OperationsConfiguration operationsConfig = new OperationsConfiguration();
operationsConfig.Import.SqlBatchSizeForImportResourceOperation = maxResourceCountInBatch;
operationsConfig.Import.SqlMaxImportOperationConcurrentCount = maxConcurrentCount;
operationsConfig.Import.SqlImportBatchSizeForCheckpoint = checkpointBatchCount;
operationsConfiguration.Value.Returns(operationsConfig);
SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance);
List <string> errorLogs = new List <string>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
importErrorStore.When(t => t.UploadErrorsAsync(Arg.Any <string[]>(), Arg.Any <CancellationToken>()))
.Do(call =>
{
string[] errors = (string[])call[0];
errorLogs.AddRange(errors);
});
(Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None);
ImportProcessingProgress finalProgress = new ImportProcessingProgress();
await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
{
Assert.True(finalProgress.CurrentIndex <= progress.CurrentIndex);
finalProgress = progress;
}
await importTask;
Assert.Equal(expectedSucceedCount, finalProgress.SucceedImportCount);
Assert.Equal(expectedFailedCount, finalProgress.FailedImportCount);
Assert.Equal(expectedEndIndex, finalProgress.CurrentIndex);
Assert.Equal(expectedSucceedCount, importedResources.Count);
Assert.Equal(expectedSucceedCount, table1.Rows.Count);
Assert.Equal(expectedSucceedCount * 2, table2.Rows.Count);
Assert.Equal(expectedSucceedCount, dupTable.Rows.Count);
Assert.Equal(expectedFailedCount, errorLogs.Count);
}
public async Task GivenSqlBulkImporter_WhenImportDataWithUnExceptedExceptionInBulkOpertation_ThenChannelShouldBeCompleteAndExceptionShouldThrow()
{
Channel <ImportResource> inputs = Channel.CreateUnbounded <ImportResource>();
await inputs.Writer.WriteAsync(new ImportResource(0, 0, default(ResourceWrapper)));
inputs.Writer.Complete();
ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>();
testFhirDataBulkOperation
.BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>())
.Returns((callInfo) =>
{
throw new InvalidOperationException();
});
testFhirDataBulkOperation
.BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>())
.Returns(call =>
{
IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0];
return(resources);
});
IImportErrorSerializer errorSerializer = Substitute.For <IImportErrorSerializer>();
ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>();
dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>())
.Returns((callInfo) =>
{
ImportResource resource = (ImportResource)callInfo[0];
return(new SqlBulkCopyDataWrapper()
{
ResourceSurrogateId = resource.Id,
});
});
List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>()
{
new TestDataGenerator("Table1", 1),
new TestDataGenerator("Table2", 2),
};
IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >();
operationsConfiguration.Value.Returns(new OperationsConfiguration());
SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance);
List <string> errorLogs = new List <string>();
IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();
(Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None);
await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
{
// Do nothing...
}
await Assert.ThrowsAsync <InvalidOperationException>(() => importTask);
}
