public async Task GivenImportTaskInput_WhenExceptionThrowForLoad_ThenRetriableExceptionShouldBeThrow()
        {
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                long startIndex = (long)callInfo[1];
                Func <long, long> idGenerator            = (Func <long, long>)callInfo[3];
                Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();

                Task loadTask = Task.Run(() =>
                {
                    try
                    {
                        throw new InvalidOperationException();
                    }
                    finally
                    {
                        resourceChannel.Writer.Complete();
                    }
                });

                return(resourceChannel, loadTask);
            });

            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
        }
        public async Task GivenImportTaskInput_WhenExceptionThrowForImport_ThenContextShouldBeUpdatedBeforeFailure()
        {
            long currentIndex = 100;
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                long startIndex = (long)callInfo[1];
                Func <long, long> idGenerator            = (Func <long, long>)callInfo[3];
                Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
                resourceChannel.Writer.Complete();

                return(resourceChannel, Task.CompletedTask);
            });

            importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();

                Task loadTask = Task.Run(async() =>
                {
                    try
                    {
                        ImportProcessingProgress progress = new ImportProcessingProgress();
                        progress.CurrentIndex             = currentIndex;

                        await progressChannel.Writer.WriteAsync(progress);
                        throw new InvalidOperationException();
                    }
                    finally
                    {
                        progressChannel.Writer.Complete();
                    }
                });

                return(progressChannel, loadTask);
            });

            string context = null;

            contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                context = (string)callInfo[0];

                return(Task.CompletedTask);
            });

            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());

            ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);

            Assert.Equal(progressForContext.CurrentIndex, currentIndex);
        }
        private static async Task VerifyCommonImportTaskAsync(ImportProcessingTaskInputData inputData, ImportProcessingProgress progress)
        {
            long startIndexFromProgress   = progress.CurrentIndex;
            long succeedCountFromProgress = progress.SucceedImportCount;
            long failedCountFromProgress  = progress.FailedImportCount;

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            long cleanStart = -1;
            long cleanEnd   = -1;

            importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                var inputData        = (ImportProcessingTaskInputData)callInfo[0];
                var progress         = (ImportProcessingProgress)callInfo[1];
                long beginSequenceId = inputData.BeginSequenceId;
                long endSequenceId   = inputData.EndSequenceId;
                long endIndex        = progress.CurrentIndex;

                cleanStart = beginSequenceId + endIndex;
                cleanEnd   = endSequenceId;

                return(Task.CompletedTask);
            });

            loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                long startIndex = (long)callInfo[1];
                Func <long, long> idGenerator            = (Func <long, long>)callInfo[3];
                Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();

                Task loadTask = Task.Run(async() =>
                {
                    ResourceWrapper resourceWrapper = new ResourceWrapper(
                        Guid.NewGuid().ToString(),
                        "0",
                        "Dummy",
                        new RawResource(Guid.NewGuid().ToString(), Fhir.Core.Models.FhirResourceFormat.Json, true),
                        new ResourceRequest("POST"),
                        DateTimeOffset.UtcNow,
                        false,
                        null,
                        null,
                        null,
                        "SearchParam");

                    await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex), startIndex, resourceWrapper));
                    await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex + 1), startIndex + 1, "Error"));
                    resourceChannel.Writer.Complete();
                });

                return(resourceChannel, loadTask);
            });

            importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                Channel <ImportResource> resourceChannel           = (Channel <ImportResource>)callInfo[0];
                Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();

                Task loadTask = Task.Run(async() =>
                {
                    ImportProcessingProgress progress = new ImportProcessingProgress();
                    await foreach (ImportResource resource in resourceChannel.Reader.ReadAllAsync())
                    {
                        if (string.IsNullOrEmpty(resource.ImportError))
                        {
                            progress.SucceedImportCount++;
                        }
                        else
                        {
                            progress.FailedImportCount++;
                        }

                        progress.CurrentIndex = resource.Index + 1;
                    }

                    await progressChannel.Writer.WriteAsync(progress);
                    progressChannel.Writer.Complete();
                });

                return(progressChannel, loadTask);
            });

            string context = null;

            contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                context = (string)callInfo[0];

                return(Task.CompletedTask);
            });

            progress.NeedCleanData = true;
            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            TaskResultData taskResult = await task.ExecuteAsync();

            Assert.Equal(TaskResult.Success, taskResult.Result);
            ImportProcessingTaskResult result = JsonConvert.DeserializeObject <ImportProcessingTaskResult>(taskResult.ResultData);

            Assert.Equal(1 + failedCountFromProgress, result.FailedCount);
            Assert.Equal(1 + succeedCountFromProgress, result.SucceedCount);

            ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);

            Assert.Equal(progressForContext.SucceedImportCount, result.SucceedCount);
            Assert.Equal(progressForContext.FailedImportCount, result.FailedCount);
            Assert.Equal(startIndexFromProgress + 2, progressForContext.CurrentIndex);

            Assert.Equal(startIndexFromProgress, cleanStart);
            Assert.Equal(inputData.EndSequenceId, cleanEnd);
        }
        public async Task <TaskResultData> ExecuteAsync()
        {
            var fhirRequestContext = new FhirRequestContext(
                method: "Import",
                uriString: _inputData.UriString,
                baseUriString: _inputData.BaseUriString,
                correlationId: _inputData.TaskId,
                requestHeaders: new Dictionary <string, StringValues>(),
                responseHeaders: new Dictionary <string, StringValues>())
            {
                IsBackgroundTask = true,
            };

            _contextAccessor.RequestContext = fhirRequestContext;

            CancellationToken cancellationToken = _cancellationTokenSource.Token;

            long succeedImportCount = _importProgress.SucceedImportCount;
            long failedImportCount  = _importProgress.FailedImportCount;

            ImportProcessingTaskResult result = new ImportProcessingTaskResult();

            result.ResourceType = _inputData.ResourceType;

            try
            {
                if (cancellationToken.IsCancellationRequested)
                {
                    throw new OperationCanceledException();
                }

                Func <long, long> sequenceIdGenerator = (index) => _inputData.BeginSequenceId + index;

                // Clean resources before import start
                await _resourceBulkImporter.CleanResourceAsync(_inputData, _importProgress, cancellationToken);

                _importProgress.NeedCleanData = true;
                await _contextUpdater.UpdateContextAsync(JsonConvert.SerializeObject(_importProgress), cancellationToken);

                // Initialize error store
                IImportErrorStore importErrorStore = await _importErrorStoreFactory.InitializeAsync(GetErrorFileName(), cancellationToken);

                result.ErrorLogLocation = importErrorStore.ErrorFileLocation;

                // Load and parse resource from bulk resource
                (Channel <ImportResource> importResourceChannel, Task loadTask) = _importResourceLoader.LoadResources(_inputData.ResourceLocation, _importProgress.CurrentIndex, _inputData.ResourceType, sequenceIdGenerator, cancellationToken);

                // Import to data store
                (Channel <ImportProcessingProgress> progressChannel, Task importTask) = _resourceBulkImporter.Import(importResourceChannel, importErrorStore, cancellationToken);

                // Update progress for checkpoints
                await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        throw new OperationCanceledException("Import task is canceled by user.");
                    }

                    _importProgress.SucceedImportCount = progress.SucceedImportCount + succeedImportCount;
                    _importProgress.FailedImportCount  = progress.FailedImportCount + failedImportCount;
                    _importProgress.CurrentIndex       = progress.CurrentIndex;
                    result.SucceedCount = _importProgress.SucceedImportCount;
                    result.FailedCount  = _importProgress.FailedImportCount;

                    _logger.LogInformation("Import task progress: {0}", JsonConvert.SerializeObject(_importProgress));

                    try
                    {
                        await _contextUpdater.UpdateContextAsync(JsonConvert.SerializeObject(_importProgress), cancellationToken);
                    }
                    catch (Exception ex)
                    {
                        // ignore exception for progresss update
                        _logger.LogInformation(ex, "Failed to update context.");
                    }
                }

                // Pop up exception during load & import
                // Put import task before load task for resource channel full and blocking issue.
                try
                {
                    await importTask;
                }
                catch (TaskCanceledException)
                {
                    throw;
                }
                catch (OperationCanceledException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex, "Failed to import data.");
                    throw new RetriableTaskException("Failed to import data.", ex);
                }

                try
                {
                    await loadTask;
                }
                catch (TaskCanceledException)
                {
                    throw;
                }
                catch (OperationCanceledException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex, "Failed to load data.");
                    throw new RetriableTaskException("Failed to load data", ex);
                }

                return(new TaskResultData(TaskResult.Success, JsonConvert.SerializeObject(result)));
            }
            catch (TaskCanceledException canceledEx)
            {
                _logger.LogInformation(canceledEx, "Data processing task is canceled.");

                await CleanResourceForFailureAsync(canceledEx);

                return(new TaskResultData(TaskResult.Canceled, JsonConvert.SerializeObject(result)));
            }
            catch (OperationCanceledException canceledEx)
            {
                _logger.LogInformation(canceledEx, "Data processing task is canceled.");

                await CleanResourceForFailureAsync(canceledEx);

                return(new TaskResultData(TaskResult.Canceled, JsonConvert.SerializeObject(result)));
            }
            catch (RetriableTaskException retriableEx)
            {
                _logger.LogInformation(retriableEx, "Error in data processing task.");

                await CleanResourceForFailureAsync(retriableEx);

                throw;
            }
            catch (Exception ex)
            {
                _logger.LogInformation(ex, "Critical error in data processing task.");

                await CleanResourceForFailureAsync(ex);

                throw new RetriableTaskException(ex.Message);
            }
            finally
            {
                if (!_cancellationTokenSource.IsCancellationRequested)
                {
                    _cancellationTokenSource.Cancel();
                }
            }
        }