Exemplo n.º 1
0
        public async Task GivenResourceLoader_WhenLoadResourcesWithDifferentResourceType_ThenResourcesWithDifferentTypeShouldBeSkipped()
        {
            string errorMessage = "Resource type not match.";

            using MemoryStream stream = new MemoryStream();
            using StreamWriter writer = new StreamWriter(stream);
            await writer.WriteLineAsync("test");

            await writer.FlushAsync();

            stream.Position = 0;

            IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();

            integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream);
            integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty);

            IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>();

            importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>())
            .Returns(callInfo =>
            {
                ImportResource importResource = new ImportResource(null);
                return(importResource);
            });

            IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>();

            serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>())
            .Returns(callInfo =>
            {
                Exception ex = (Exception)callInfo[1];
                return(ex.Message);
            });

            Func <long, long>    idGenerator = (i) => i;
            ImportResourceLoader loader      = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance);

            (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, "DummyType", idGenerator, CancellationToken.None);

            int errorCount = 0;

            await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync())
            {
                Assert.Equal(errorMessage, resource.ImportError);
                ++errorCount;
            }

            await importTask;

            Assert.Equal(1, errorCount);
        }
Exemplo n.º 2
0
        private async Task VerifyResourceLoaderAsync(int resourcCount, int batchSize, long startIndex)
        {
            long          startId      = 1;
            List <string> inputStrings = new List <string>();

            using MemoryStream stream = new MemoryStream();
            using StreamWriter writer = new StreamWriter(stream);
            for (int i = 0; i < resourcCount; ++i)
            {
                string content = (i + startId).ToString();
                inputStrings.Add(content);
                await writer.WriteLineAsync(content);
            }

            await writer.FlushAsync();

            stream.Position = 0;

            IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();

            integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream);
            integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty);

            IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>();

            importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>())
            .Returns(callInfo =>
            {
                long surrogatedId = (long)callInfo[0];
                long index        = (long)callInfo[1];
                string content    = (string)callInfo[2];
                ResourceWrapper resourceWrapper = new ResourceWrapper(
                    content,
                    "0",
                    "Dummy",
                    new RawResource(content, Fhir.Core.Models.FhirResourceFormat.Json, true),
                    new ResourceRequest("POST"),
                    DateTimeOffset.UtcNow,
                    false,
                    null,
                    null,
                    null,
                    "SearchParam");
                return(new ImportResource(surrogatedId, index, resourceWrapper));
            });

            IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>();

            Func <long, long>    idGenerator = (i) => startId + i;
            ImportResourceLoader loader      = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance);

            loader.MaxBatchSize = batchSize;

            (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", startIndex, null, idGenerator, CancellationToken.None);

            long currentIndex = startIndex;

            await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync())
            {
                string content = idGenerator(currentIndex++).ToString();
                Assert.Equal(content, resource.Resource.ResourceId);
            }

            await importTask;

            Assert.Equal(resourcCount, currentIndex);
        }
Exemplo n.º 3
0
        public async Task GivenResourceLoader_WhenCancelLoadTask_ThenDataLoadTaskShouldBeCanceled()
        {
            string errorMessage = "error";

            using MemoryStream stream = new MemoryStream();
            using StreamWriter writer = new StreamWriter(stream);
            await writer.WriteLineAsync("test");

            await writer.WriteLineAsync("test");

            await writer.WriteLineAsync("test");

            await writer.WriteLineAsync("test");

            await writer.FlushAsync();

            stream.Position = 0;

            AutoResetEvent   resetEvent1 = new AutoResetEvent(false);
            ManualResetEvent resetEvent2 = new ManualResetEvent(false);

            IIntegrationDataStoreClient integrationDataStoreClient = Substitute.For <IIntegrationDataStoreClient>();

            integrationDataStoreClient.DownloadResource(Arg.Any <Uri>(), Arg.Any <long>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(stream);
            integrationDataStoreClient.TryAcquireLeaseAsync(Arg.Any <Uri>(), Arg.Any <string>(), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(string.Empty);

            IImportResourceParser importResourceParser = Substitute.For <IImportResourceParser>();

            importResourceParser.Parse(Arg.Any <long>(), Arg.Any <long>(), Arg.Any <string>())
            .Returns(callInfo =>
            {
                resetEvent1.Set();
                resetEvent2.WaitOne();

                throw new InvalidCastException(errorMessage);
            });

            IImportErrorSerializer serializer = Substitute.For <IImportErrorSerializer>();

            serializer.Serialize(Arg.Any <long>(), Arg.Any <Exception>())
            .Returns(callInfo =>
            {
                Exception ex = (Exception)callInfo[1];
                return(ex.Message);
            });

            Func <long, long>    idGenerator = (i) => i;
            ImportResourceLoader loader      = new ImportResourceLoader(integrationDataStoreClient, importResourceParser, serializer, NullLogger <ImportResourceLoader> .Instance);

            CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();

            (Channel <ImportResource> outputChannel, Task importTask) = loader.LoadResources("http://dummy", 0, null, idGenerator, cancellationTokenSource.Token);

            resetEvent1.WaitOne();
            cancellationTokenSource.Cancel();
            resetEvent2.Set();

            await foreach (ImportResource resource in outputChannel.Reader.ReadAllAsync())
            {
                // do nothing.
            }

            try
            {
                await importTask;
                throw new InvalidOperationException();
            }
            catch (TaskCanceledException)
            {
                // Expected error
            }
            catch (OperationCanceledException)
            {
                // Expected error
            }
        }
Exemplo n.º 4
0
        private async Task LoadResourcesInternalAsync(Channel <ImportResource> outputChannel, string resourceLocation, long startIndex, string resourceType, Func <long, long> sequenceIdGenerator, CancellationToken cancellationToken)
        {
            string leaseId = null;

            try
            {
                _logger.LogInformation("Start to load resource from store.");

                // Try to acquire lease to block change on the blob.
                leaseId = await _integrationDataStoreClient.TryAcquireLeaseAsync(new Uri(resourceLocation), Guid.NewGuid().ToString("N"), cancellationToken);

                using Stream inputDataStream       = _integrationDataStoreClient.DownloadResource(new Uri(resourceLocation), 0, cancellationToken);
                using StreamReader inputDataReader = new StreamReader(inputDataStream);

                string content      = null;
                long   currentIndex = 0;
                List <(string content, long index)>           buffer          = new List <(string content, long index)>();
                Queue <Task <IEnumerable <ImportResource> > > processingTasks = new Queue <Task <IEnumerable <ImportResource> > >();

                while (!string.IsNullOrEmpty(content = await inputDataReader.ReadLineAsync()))
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        throw new OperationCanceledException();
                    }

                    // TODO: improve to load from offset in file
                    if (currentIndex < startIndex)
                    {
                        currentIndex++;
                        continue;
                    }

                    buffer.Add((content, currentIndex));
                    currentIndex++;

                    if (buffer.Count < MaxBatchSize)
                    {
                        continue;
                    }

                    while (processingTasks.Count >= MaxConcurrentCount)
                    {
                        if (cancellationToken.IsCancellationRequested)
                        {
                            throw new OperationCanceledException();
                        }

                        IEnumerable <ImportResource> importResources = await processingTasks.Dequeue();

                        foreach (ImportResource importResource in importResources)
                        {
                            await outputChannel.Writer.WriteAsync(importResource, cancellationToken);
                        }
                    }

                    processingTasks.Enqueue(ParseImportRawContentAsync(resourceType, buffer.ToArray(), sequenceIdGenerator));
                    buffer.Clear();
                }

                processingTasks.Enqueue(ParseImportRawContentAsync(resourceType, buffer.ToArray(), sequenceIdGenerator));
                while (processingTasks.Count > 0)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        throw new OperationCanceledException();
                    }

                    IEnumerable <ImportResource> importResources = await processingTasks.Dequeue();

                    foreach (ImportResource importResource in importResources)
                    {
                        await outputChannel.Writer.WriteAsync(importResource, cancellationToken);
                    }
                }

                _logger.LogInformation($"{currentIndex} lines loaded.");
            }
            finally
            {
                outputChannel.Writer.Complete();

                if (!string.IsNullOrEmpty(leaseId))
                {
                    await _integrationDataStoreClient.TryReleaseLeaseAsync(new Uri(resourceLocation), leaseId, cancellationToken);
                }

                _logger.LogInformation("Load resource from store complete.");
            }
        }