public async Task GivenResourceLoader_WhenLoadResourcesWithDifferentResourceType_ThenResourcesWithDifferentTypeShouldBeSkipped()
{
string errorMessage = "Resource type not match.";
using MemoryStream stream = new MemoryStream();
using StreamWriter writer = new StreamWriter(stream);
await writer.WriteLineAsync("test");
await writer.FlushAsync();
stream.Position = 0;
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream);
integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty);
IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>();
importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>())
.Returns(callInfo =>
{
ImportResource importResource = new ImportResource(null);
return(importResource);
});
IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>();
serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>())
.Returns(callInfo =>
{
Exception ex = (Exception)callInfo[1];
return(ex.Message);
});
Func <long, long> idGenerator = (i) => i;
ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance);
(Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, "DummyType", idGenerator, CancellationToken.None);
int errorCount = 0;
await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync())
{
Assert.Equal(errorMessage, resource.ImportError);
++errorCount;
}
await importTask;
Assert.Equal(1, errorCount);
}
private async Task VerifyResourceLoaderAsync(int resourcCount, int batchSize, long startIndex)
{
long startId = 1;
List <string> inputStrings = new List <string>();
using MemoryStream stream = new MemoryStream();
using StreamWriter writer = new StreamWriter(stream);
for (int i = 0; i < resourcCount; ++i)
{
string content = (i + startId).ToString();
inputStrings.Add(content);
await writer.WriteLineAsync(content);
}
await writer.FlushAsync();
stream.Position = 0;
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream);
integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty);
IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>();
importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>())
.Returns(callInfo =>
{
long surrogatedId = (long)callInfo[0];
long index = (long)callInfo[1];
string content = (string)callInfo[2];
ResourceWrapper resourceWrapper = new ResourceWrapper(
content,
"0",
"Dummy",
new RawResource(content, Fhir.Core.Models.FhirResourceFormat.Json, true),
new ResourceRequest("POST"),
DateTimeOffset.UtcNow,
false,
null,
null,
null,
"SearchParam");
return(new ImportResource(surrogatedId, index, resourceWrapper));
});
IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>();
Func <long, long> idGenerator = (i) => startId + i;
ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance);
loader.MaxBatchSize = batchSize;
(Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", startIndex, null, idGenerator, CancellationToken.None);
long currentIndex = startIndex;
await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync())
{
string content = idGenerator(currentIndex++).ToString();
Assert.Equal(content, resource.Resource.ResourceId);
}
await importTask;
Assert.Equal(resourcCount, currentIndex);
}
public async Task GivenResourceLoader_WhenCancelLoadTask_ThenDataLoadTaskShouldBeCanceled()
{
string errorMessage = "error";
using MemoryStream stream = new MemoryStream();
using StreamWriter writer = new StreamWriter(stream);
await writer.WriteLineAsync("test");
await writer.WriteLineAsync("test");
await writer.WriteLineAsync("test");
await writer.WriteLineAsync("test");
await writer.FlushAsync();
stream.Position = 0;
AutoResetEvent resetEvent1 = new AutoResetEvent(false);
ManualResetEvent resetEvent2 = new ManualResetEvent(false);
IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();
integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream);
integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty);
IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>();
importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>())
.Returns(callInfo =>
{
resetEvent1.Set();
resetEvent2.WaitOne();
throw new InvalidCastException(errorMessage);
});
IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>();
serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>())
.Returns(callInfo =>
{
Exception ex = (Exception)callInfo[1];
return(ex.Message);
});
Func <long, long> idGenerator = (i) => i;
ImportResourceLoader loader = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance);
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
(Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, null, idGenerator, cancellationTokenSource.Token);
resetEvent1.WaitOne();
cancellationTokenSource.Cancel();
resetEvent2.Set();
await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync())
{
// do nothing.
}
try
{
await importTask;
throw new InvalidOperationException();
}
catch (TaskCanceledException)
{
// Expected error
}
catch (OperationCanceledException)
{
// Expected error
}
}
private async Task LoadResourcesInternalAsync(Channel <ImportResource> outputChannel, string resourceLocation, long startIndex, string resourceType, Func <long, long> sequenceIdGenerator, CancellationToken cancellationToken)
{
string leaseId = null;
try
{
_logger.LogInformation("Start to load resource from store.");
// Try to acquire lease to block change on the blob.
leaseId = await _integrationDataStoreClient.TryAcquireLeaseAsync(new Uri(resourceLocation), Guid.NewGuid().ToString("N"), cancellationToken);
using Stream inputDataStream = _integrationDataStoreClient.DownloadResource(new Uri(resourceLocation), 0, cancellationToken);
using StreamReader inputDataReader = new StreamReader(inputDataStream);
string content = null;
long currentIndex = 0;
List <(string content, long index)> buffer = new List <(string content, long index)>();
Queue <Task <IEnumerable <ImportResource> > > processingTasks = new Queue <Task <IEnumerable <ImportResource> > >();
while (!string.IsNullOrEmpty(content = await inputDataReader.ReadLineAsync()))
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException();
}
// TODO: improve to load from offset in file
if (currentIndex < startIndex)
{
currentIndex++;
continue;
}
buffer.Add((content, currentIndex));
currentIndex++;
if (buffer.Count < MaxBatchSize)
{
continue;
}
while (processingTasks.Count >= MaxConcurrentCount)
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException();
}
IEnumerable <ImportResource> importResources = await processingTasks.Dequeue();
foreach (ImportResource importResource in importResources)
{
await outputChannel.Writer.WriteAsync(importResource, cancellationToken);
}
}
processingTasks.Enqueue(ParseImportRawContentAsync(resourceType, buffer.ToArray(), sequenceIdGenerator));
buffer.Clear();
}
processingTasks.Enqueue(ParseImportRawContentAsync(resourceType, buffer.ToArray(), sequenceIdGenerator));
while (processingTasks.Count > 0)
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException();
}
IEnumerable <ImportResource> importResources = await processingTasks.Dequeue();
foreach (ImportResource importResource in importResources)
{
await outputChannel.Writer.WriteAsync(importResource, cancellationToken);
}
}
_logger.LogInformation($"{currentIndex} lines loaded.");
}
finally
{
outputChannel.Writer.Complete();
if (!string.IsNullOrEmpty(leaseId))
{
await _integrationDataStoreClient.TryReleaseLeaseAsync(new Uri(resourceLocation), leaseId, cancellationToken);
}
_logger.LogInformation("Load resource from store complete.");
}
}