public async Task GivenImportTaskInput_WhenExceptionThrowForCleanData_ThenRetriableExceptionShouldBeThrow()
        {
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                throw new InvalidOperationException();
            });

            progress.NeedCleanData = true;
            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
        }
        public async Task GivenImportTaskInput_WhenStartFromClean_ThenAllResoruceShouldBeImported()
        {
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            await VerifyCommonImportTaskAsync(inputData, progress);
        }
        public async Task GivenImportTaskInput_WhenOperationWasCancelledExceptionThrow_ThenTaskShouldBeCanceled()
        {
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                throw new OperationCanceledException();
            });

            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            TaskResultData result = await task.ExecuteAsync();

            Assert.Equal(TaskResult.Canceled, result.Result);
        }
        public async Task GivenImportTaskInput_WhenStartFromMiddle_ThenAllResoruceShouldBeImported()
        {
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            progress.SucceedImportCount = 3;
            progress.FailedImportCount  = 1;
            progress.CurrentIndex       = 4;

            await VerifyCommonImportTaskAsync(inputData, progress);
        }
        public async Task GivenImportTaskInput_WhenExceptionThrowForLoad_ThenRetriableExceptionShouldBeThrow()
        {
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                long startIndex = (long)callInfo[1];
                Func <long, long> idGenerator            = (Func <long, long>)callInfo[3];
                Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();

                Task loadTask = Task.Run(() =>
                {
                    try
                    {
                        throw new InvalidOperationException();
                    }
                    finally
                    {
                        resourceChannel.Writer.Complete();
                    }
                });

                return(resourceChannel, loadTask);
            });

            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());
        }
        private async Task <Task <ImportProcessingProgress> > EnqueueTaskAsync(Queue <Task <ImportProcessingProgress> > importTasks, Func <Task <ImportProcessingProgress> > newTaskFactory, Channel <ImportProcessingProgress> progressChannel)
        {
            while (importTasks.Count >= _importTaskConfiguration.SqlMaxImportOperationConcurrentCount)
            {
                ImportProcessingProgress progress = await importTasks.Dequeue();

                if (progress != null)
                {
                    await progressChannel.Writer.WriteAsync(progress);
                }
            }

            Task <ImportProcessingProgress> newTask = newTaskFactory();

            importTasks.Enqueue(newTask);

            return(newTask);
        }
Beispiel #7
0
        public ITask Create(TaskInfo taskInfo)
        {
            EnsureArg.IsNotNull(taskInfo, nameof(taskInfo));

            if (taskInfo.TaskTypeId == ImportProcessingTask.ImportProcessingTaskId)
            {
                IContextUpdater contextUpdater               = _contextUpdaterFactory.CreateContextUpdater(taskInfo.TaskId, taskInfo.RunId);
                ImportProcessingTaskInputData inputData      = JsonConvert.DeserializeObject <ImportProcessingTaskInputData>(taskInfo.InputData);
                ImportProcessingProgress      importProgress = string.IsNullOrEmpty(taskInfo.Context) ? new ImportProcessingProgress() : JsonConvert.DeserializeObject <ImportProcessingProgress>(taskInfo.Context);
                return(new ImportProcessingTask(
                           inputData,
                           importProgress,
                           _importResourceLoader,
                           _resourceBulkImporter,
                           _importErrorStoreFactory,
                           contextUpdater,
                           _contextAccessor,
                           _loggerFactory));
            }

            if (taskInfo.TaskTypeId == ImportOrchestratorTask.ImportOrchestratorTaskId)
            {
                IContextUpdater contextUpdater            = _contextUpdaterFactory.CreateContextUpdater(taskInfo.TaskId, taskInfo.RunId);
                ImportOrchestratorTaskInputData inputData = JsonConvert.DeserializeObject <ImportOrchestratorTaskInputData>(taskInfo.InputData);
                ImportOrchestratorTaskContext   orchestratorTaskProgress = string.IsNullOrEmpty(taskInfo.Context) ? new ImportOrchestratorTaskContext() : JsonConvert.DeserializeObject <ImportOrchestratorTaskContext>(taskInfo.Context);

                return(new ImportOrchestratorTask(
                           _mediator,
                           inputData,
                           orchestratorTaskProgress,
                           _taskmanager,
                           _sequenceIdGenerator,
                           contextUpdater,
                           _contextAccessor,
                           _importOrchestratorTaskDataStoreOperation,
                           _integrationDataStoreClient,
                           _loggerFactory));
            }

            return(null);
        }
        private async Task <ImportProcessingProgress> UploadImportErrorsAsync(IImportErrorStore importErrorStore, long succeedCount, long failedCount, string[] importErrors, long lastIndex, CancellationToken cancellationToken)
        {
            try
            {
                await importErrorStore.UploadErrorsAsync(importErrors, cancellationToken);
            }
            catch (Exception ex)
            {
                _logger.LogInformation(ex, "Failed to upload error logs.");
                throw;
            }

            ImportProcessingProgress progress = new ImportProcessingProgress();

            progress.SucceedImportCount = succeedCount;
            progress.FailedImportCount  = failedCount;
            progress.CurrentIndex       = lastIndex + 1;

            // Return progress for checkpoint progress
            return(progress);
        }
        public async Task GivenImportTaskInput_WhenExceptionThrowForImport_ThenContextShouldBeUpdatedBeforeFailure()
        {
            long currentIndex = 100;
            ImportProcessingTaskInputData inputData = GetInputData();
            ImportProcessingProgress      progress  = new ImportProcessingProgress();

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                long startIndex = (long)callInfo[1];
                Func <long, long> idGenerator            = (Func <long, long>)callInfo[3];
                Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();
                resourceChannel.Writer.Complete();

                return(resourceChannel, Task.CompletedTask);
            });

            importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();

                Task loadTask = Task.Run(async() =>
                {
                    try
                    {
                        ImportProcessingProgress progress = new ImportProcessingProgress();
                        progress.CurrentIndex             = currentIndex;

                        await progressChannel.Writer.WriteAsync(progress);
                        throw new InvalidOperationException();
                    }
                    finally
                    {
                        progressChannel.Writer.Complete();
                    }
                });

                return(progressChannel, loadTask);
            });

            string context = null;

            contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                context = (string)callInfo[0];

                return(Task.CompletedTask);
            });

            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            await Assert.ThrowsAsync <RetriableTaskException>(() => task.ExecuteAsync());

            ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);

            Assert.Equal(progressForContext.CurrentIndex, currentIndex);
        }
        private static async Task VerifyCommonImportTaskAsync(ImportProcessingTaskInputData inputData, ImportProcessingProgress progress)
        {
            long startIndexFromProgress   = progress.CurrentIndex;
            long succeedCountFromProgress = progress.SucceedImportCount;
            long failedCountFromProgress  = progress.FailedImportCount;

            IImportResourceLoader    loader                              = Substitute.For <IImportResourceLoader>();
            IResourceBulkImporter    importer                            = Substitute.For <IResourceBulkImporter>();
            IImportErrorStore        importErrorStore                    = Substitute.For <IImportErrorStore>();
            IImportErrorStoreFactory importErrorStoreFactory             = Substitute.For <IImportErrorStoreFactory>();
            IContextUpdater          contextUpdater                      = Substitute.For <IContextUpdater>();
            RequestContextAccessor <IFhirRequestContext> contextAccessor = Substitute.For <RequestContextAccessor <IFhirRequestContext> >();
            ILoggerFactory loggerFactory = new NullLoggerFactory();

            long cleanStart = -1;
            long cleanEnd   = -1;

            importer.CleanResourceAsync(Arg.Any <ImportProcessingTaskInputData>(), Arg.Any <ImportProcessingProgress>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                var inputData        = (ImportProcessingTaskInputData)callInfo[0];
                var progress         = (ImportProcessingProgress)callInfo[1];
                long beginSequenceId = inputData.BeginSequenceId;
                long endSequenceId   = inputData.EndSequenceId;
                long endIndex        = progress.CurrentIndex;

                cleanStart = beginSequenceId + endIndex;
                cleanEnd   = endSequenceId;

                return(Task.CompletedTask);
            });

            loader.LoadResources(Arg.Any <string>(), Arg.Any <long>(), Arg.Any <string>(), Arg.Any <Func <long, long> >(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                long startIndex = (long)callInfo[1];
                Func <long, long> idGenerator            = (Func <long, long>)callInfo[3];
                Channel <ImportResource> resourceChannel = Channel.CreateUnbounded <ImportResource>();

                Task loadTask = Task.Run(async() =>
                {
                    ResourceWrapper resourceWrapper = new ResourceWrapper(
                        Guid.NewGuid().ToString(),
                        "0",
                        "Dummy",
                        new RawResource(Guid.NewGuid().ToString(), Fhir.Core.Models.FhirResourceFormat.Json, true),
                        new ResourceRequest("POST"),
                        DateTimeOffset.UtcNow,
                        false,
                        null,
                        null,
                        null,
                        "SearchParam");

                    await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex), startIndex, resourceWrapper));
                    await resourceChannel.Writer.WriteAsync(new ImportResource(idGenerator(startIndex + 1), startIndex + 1, "Error"));
                    resourceChannel.Writer.Complete();
                });

                return(resourceChannel, loadTask);
            });

            importer.Import(Arg.Any <Channel <ImportResource> >(), Arg.Any <IImportErrorStore>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                Channel <ImportResource> resourceChannel           = (Channel <ImportResource>)callInfo[0];
                Channel <ImportProcessingProgress> progressChannel = Channel.CreateUnbounded <ImportProcessingProgress>();

                Task loadTask = Task.Run(async() =>
                {
                    ImportProcessingProgress progress = new ImportProcessingProgress();
                    await foreach (ImportResource resource in resourceChannel.Reader.ReadAllAsync())
                    {
                        if (string.IsNullOrEmpty(resource.ImportError))
                        {
                            progress.SucceedImportCount++;
                        }
                        else
                        {
                            progress.FailedImportCount++;
                        }

                        progress.CurrentIndex = resource.Index + 1;
                    }

                    await progressChannel.Writer.WriteAsync(progress);
                    progressChannel.Writer.Complete();
                });

                return(progressChannel, loadTask);
            });

            string context = null;

            contextUpdater.UpdateContextAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
            .Returns(callInfo =>
            {
                context = (string)callInfo[0];

                return(Task.CompletedTask);
            });

            progress.NeedCleanData = true;
            ImportProcessingTask task = new ImportProcessingTask(
                inputData,
                progress,
                loader,
                importer,
                importErrorStoreFactory,
                contextUpdater,
                contextAccessor,
                loggerFactory);

            TaskResultData taskResult = await task.ExecuteAsync();

            Assert.Equal(TaskResult.Success, taskResult.Result);
            ImportProcessingTaskResult result = JsonConvert.DeserializeObject <ImportProcessingTaskResult>(taskResult.ResultData);

            Assert.Equal(1 + failedCountFromProgress, result.FailedCount);
            Assert.Equal(1 + succeedCountFromProgress, result.SucceedCount);

            ImportProcessingProgress progressForContext = JsonConvert.DeserializeObject <ImportProcessingProgress>(context);

            Assert.Equal(progressForContext.SucceedImportCount, result.SucceedCount);
            Assert.Equal(progressForContext.FailedImportCount, result.FailedCount);
            Assert.Equal(startIndexFromProgress + 2, progressForContext.CurrentIndex);

            Assert.Equal(startIndexFromProgress, cleanStart);
            Assert.Equal(inputData.EndSequenceId, cleanEnd);
        }
        private static async Task VerifyBulkImporterBehaviourAsync(Channel <ImportResource> inputs, long expectedSucceedCount, long expectedFailedCount, long expectedEndIndex, int maxResourceCountInBatch, int checkpointBatchCount, int maxConcurrentCount)
        {
            DataTable table1   = new DataTable();
            DataTable table2   = new DataTable();
            DataTable dupTable = new DataTable();
            List <SqlBulkCopyDataWrapper> importedResources = new List <SqlBulkCopyDataWrapper>();

            ISqlImportOperation testFhirDataBulkOperation = Substitute.For <ISqlImportOperation>();

            testFhirDataBulkOperation
            .When(t => t.BulkCopyDataAsync(Arg.Any <DataTable>(), Arg.Any <CancellationToken>()))
            .Do(call =>
            {
                DataTable table = (DataTable)call[0];
                if (table.TableName.Equals("Table1"))
                {
                    table1.Merge(table);
                }
                else if (table.TableName.Equals("Table2"))
                {
                    table2.Merge(table);
                }
                else if (table.TableName.Equals("Dup"))
                {
                    dupTable.Merge(table);
                }
            });
            testFhirDataBulkOperation
            .BulkMergeResourceAsync(Arg.Any <IEnumerable <SqlBulkCopyDataWrapper> >(), Arg.Any <CancellationToken>())
            .Returns(call =>
            {
                IEnumerable <SqlBulkCopyDataWrapper> resources = (IEnumerable <SqlBulkCopyDataWrapper>)call[0];
                importedResources.AddRange(resources);

                return(resources);
            });

            IImportErrorSerializer         errorSerializer    = Substitute.For <IImportErrorSerializer>();
            ISqlBulkCopyDataWrapperFactory dataWrapperFactory = Substitute.For <ISqlBulkCopyDataWrapperFactory>();

            dataWrapperFactory.CreateSqlBulkCopyDataWrapper(Arg.Any <ImportResource>())
            .Returns((callInfo) =>
            {
                ImportResource resource = (ImportResource)callInfo[0];
                return(new SqlBulkCopyDataWrapper()
                {
                    ResourceSurrogateId = resource.Id,
                });
            });

            List <TableBulkCopyDataGenerator> generators = new List <TableBulkCopyDataGenerator>()
            {
                new TestDataGenerator("Table1", 1),
                new TestDataGenerator("Table2", 2),
                new TestDupDataGenerator("Dup"),
            };

            IOptions <OperationsConfiguration> operationsConfiguration = Substitute.For <IOptions <OperationsConfiguration> >();
            OperationsConfiguration            operationsConfig        = new OperationsConfiguration();

            operationsConfig.Import.SqlBatchSizeForImportResourceOperation = maxResourceCountInBatch;
            operationsConfig.Import.SqlMaxImportOperationConcurrentCount   = maxConcurrentCount;
            operationsConfig.Import.SqlImportBatchSizeForCheckpoint        = checkpointBatchCount;
            operationsConfiguration.Value.Returns(operationsConfig);

            SqlResourceBulkImporter importer = new SqlResourceBulkImporter(testFhirDataBulkOperation, dataWrapperFactory, errorSerializer, generators, operationsConfiguration, NullLogger <SqlResourceBulkImporter> .Instance);

            List <string>     errorLogs        = new List <string>();
            IImportErrorStore importErrorStore = Substitute.For <IImportErrorStore>();

            importErrorStore.When(t => t.UploadErrorsAsync(Arg.Any <string[]>(), Arg.Any <CancellationToken>()))
            .Do(call =>
            {
                string[] errors = (string[])call[0];
                errorLogs.AddRange(errors);
            });
            (Channel <ImportProcessingProgress> progressChannel, Task importTask) = importer.Import(inputs, importErrorStore, CancellationToken.None);
            ImportProcessingProgress finalProgress = new ImportProcessingProgress();

            await foreach (ImportProcessingProgress progress in progressChannel.Reader.ReadAllAsync())
            {
                Assert.True(finalProgress.CurrentIndex <= progress.CurrentIndex);
                finalProgress = progress;
            }

            await importTask;

            Assert.Equal(expectedSucceedCount, finalProgress.SucceedImportCount);
            Assert.Equal(expectedFailedCount, finalProgress.FailedImportCount);
            Assert.Equal(expectedEndIndex, finalProgress.CurrentIndex);

            Assert.Equal(expectedSucceedCount, importedResources.Count);
            Assert.Equal(expectedSucceedCount, table1.Rows.Count);
            Assert.Equal(expectedSucceedCount * 2, table2.Rows.Count);
            Assert.Equal(expectedSucceedCount, dupTable.Rows.Count);
            Assert.Equal(expectedFailedCount, errorLogs.Count);
        }
        private async Task ImportInternalAsync(Channel <ImportResource> inputChannel, Channel <ImportProcessingProgress> outputChannel, IImportErrorStore importErrorStore, CancellationToken cancellationToken)
        {
            try
            {
                _logger.LogInformation("Start to import data to SQL data store.");

                Task <ImportProcessingProgress> checkpointTask = Task.FromResult <ImportProcessingProgress>(null);

                long succeedCount        = 0;
                long failedCount         = 0;
                long?lastCheckpointIndex = null;
                long currentIndex        = -1;
                Dictionary <string, DataTable> resourceParamsBuffer = new Dictionary <string, DataTable>();
                List <string> importErrorBuffer = new List <string>();
                Queue <Task <ImportProcessingProgress> > importTasks = new Queue <Task <ImportProcessingProgress> >();

                List <ImportResource> resourceBuffer = new List <ImportResource>();
                await _sqlBulkCopyDataWrapperFactory.EnsureInitializedAsync();

                await foreach (ImportResource resource in inputChannel.Reader.ReadAllAsync(cancellationToken))
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        throw new OperationCanceledException();
                    }

                    lastCheckpointIndex = lastCheckpointIndex ?? resource.Index - 1;
                    currentIndex        = resource.Index;

                    resourceBuffer.Add(resource);
                    if (resourceBuffer.Count < _importTaskConfiguration.SqlBatchSizeForImportResourceOperation)
                    {
                        continue;
                    }

                    try
                    {
                        // Handle resources in buffer
                        IEnumerable <ImportResource>         resourcesWithError = resourceBuffer.Where(r => r.ContainsError());
                        IEnumerable <SqlBulkCopyDataWrapper> inputResources     = resourceBuffer.Where(r => !r.ContainsError()).Select(r => _sqlBulkCopyDataWrapperFactory.CreateSqlBulkCopyDataWrapper(r));
                        IEnumerable <SqlBulkCopyDataWrapper> mergedResources    = await _sqlImportOperation.BulkMergeResourceAsync(inputResources, cancellationToken);

                        IEnumerable <SqlBulkCopyDataWrapper> duplicateResourcesNotMerged = inputResources.Except(mergedResources);

                        importErrorBuffer.AddRange(resourcesWithError.Select(r => r.ImportError));
                        FillResourceParamsBuffer(mergedResources, resourceParamsBuffer);
                        AppendDuplicatedResouceErrorToBuffer(duplicateResourcesNotMerged, importErrorBuffer);

                        succeedCount += mergedResources.Count();
                        failedCount  += resourcesWithError.Count() + duplicateResourcesNotMerged.Count();
                    }
                    finally
                    {
                        foreach (ImportResource importResource in resourceBuffer)
                        {
                            importResource?.CompressedStream?.Dispose();
                        }

                        resourceBuffer.Clear();
                    }

                    bool shouldCreateCheckpoint = resource.Index - lastCheckpointIndex >= _importTaskConfiguration.SqlImportBatchSizeForCheckpoint;
                    if (shouldCreateCheckpoint)
                    {
                        // Create checkpoint for all tables not empty
                        string[] tableNameNeedImport = resourceParamsBuffer.Where(r => r.Value.Rows.Count > 0).Select(r => r.Key).ToArray();

                        foreach (string tableName in tableNameNeedImport)
                        {
                            DataTable dataTable = resourceParamsBuffer[tableName];
                            resourceParamsBuffer.Remove(tableName);
                            await EnqueueTaskAsync(importTasks, () => ImportDataTableAsync(dataTable, cancellationToken), outputChannel);
                        }

                        // wait previous checkpoint task complete
                        await checkpointTask;

                        // upload error logs for import errors
                        string[] importErrors = importErrorBuffer.ToArray();
                        importErrorBuffer.Clear();
                        lastCheckpointIndex = resource.Index;
                        checkpointTask      = await EnqueueTaskAsync(importTasks, () => UploadImportErrorsAsync(importErrorStore, succeedCount, failedCount, importErrors, currentIndex, cancellationToken), outputChannel);
                    }
                    else
                    {
                        // import table >= MaxResourceCountInBatch
                        string[] tableNameNeedImport =
                            resourceParamsBuffer.Where(r => r.Value.Rows.Count >= _importTaskConfiguration.SqlBatchSizeForImportParamsOperation).Select(r => r.Key).ToArray();

                        foreach (string tableName in tableNameNeedImport)
                        {
                            DataTable dataTable = resourceParamsBuffer[tableName];
                            resourceParamsBuffer.Remove(tableName);
                            await EnqueueTaskAsync(importTasks, () => ImportDataTableAsync(dataTable, cancellationToken), outputChannel);
                        }
                    }
                }

                try
                {
                    // Handle resources in buffer
                    IEnumerable <ImportResource>         resourcesWithError = resourceBuffer.Where(r => r.ContainsError());
                    IEnumerable <SqlBulkCopyDataWrapper> inputResources     = resourceBuffer.Where(r => !r.ContainsError()).Select(r => _sqlBulkCopyDataWrapperFactory.CreateSqlBulkCopyDataWrapper(r));
                    IEnumerable <SqlBulkCopyDataWrapper> mergedResources    = await _sqlImportOperation.BulkMergeResourceAsync(inputResources, cancellationToken);

                    IEnumerable <SqlBulkCopyDataWrapper> duplicateResourcesNotMerged = inputResources.Except(mergedResources);
                    importErrorBuffer.AddRange(resourcesWithError.Select(r => r.ImportError));

                    FillResourceParamsBuffer(mergedResources, resourceParamsBuffer);

                    AppendDuplicatedResouceErrorToBuffer(duplicateResourcesNotMerged, importErrorBuffer);
                    succeedCount += mergedResources.Count();
                    failedCount  += resourcesWithError.Count() + duplicateResourcesNotMerged.Count();
                }
                finally
                {
                    foreach (ImportResource importResource in resourceBuffer)
                    {
                        importResource?.CompressedStream?.Dispose();
                    }

                    resourceBuffer.Clear();
                }

                // Import all remain tables
                string[] allTablesNotNull = resourceParamsBuffer.Where(r => r.Value.Rows.Count > 0).Select(r => r.Key).ToArray();
                foreach (string tableName in allTablesNotNull)
                {
                    DataTable dataTable = resourceParamsBuffer[tableName];
                    await EnqueueTaskAsync(importTasks, () => ImportDataTableAsync(dataTable, cancellationToken), outputChannel);
                }

                // Wait all table import task complete
                while (importTasks.Count > 0)
                {
                    await importTasks.Dequeue();
                }

                // Upload remain error logs
                ImportProcessingProgress progress = await UploadImportErrorsAsync(importErrorStore, succeedCount, failedCount, importErrorBuffer.ToArray(), currentIndex, cancellationToken);

                await outputChannel.Writer.WriteAsync(progress, cancellationToken);
            }
            finally
            {
                outputChannel.Writer.Complete();
                _logger.LogInformation("Import data to SQL data store complete.");
            }
        }
        public async Task CleanResourceAsync(ImportProcessingTaskInputData inputData, ImportProcessingProgress progress, CancellationToken cancellationToken)
        {
            if (!progress.NeedCleanData)
            {
                // Skip clean data step for first run.
                return;
            }

            long beginSequenceId = inputData.BeginSequenceId;
            long endSequenceId   = inputData.EndSequenceId;
            long endIndex        = progress.CurrentIndex;

            try
            {
                await _sqlBulkCopyDataWrapperFactory.EnsureInitializedAsync();

                await _sqlImportOperation.CleanBatchResourceAsync(inputData.ResourceType, beginSequenceId + endIndex, endSequenceId, cancellationToken);
            }
            catch (OperationCanceledException)
            {
                throw;
            }
            catch (Exception ex)
            {
                _logger.LogInformation(ex, "Failed to clean batch resource.");
                throw new RetriableTaskException("Failed to clean resource before import task start.", ex);
            }
        }