public static ImportRun CsvImport(ImportConfig config, string csvData)
        {
            ImportSettings settings = new ImportSettings
            {
                FileToken      = "whatever",
                ImportConfigId = config.Id
            };


            IFileRepository fileRepository = MockFileRepository.FromText(csvData);

            using (var scope = Factory.Current
                               .BeginLifetimeScope(builder => builder.RegisterInstance(fileRepository).Named <IFileRepository>(FileRepositoryModule.TemporaryFileRepositoryName)))
            {
                Assert.That(scope.ResolveKeyed <IFileRepository>(FileRepositoryModule.TemporaryFileRepositoryName), Is.InstanceOf <MockFileRepository>( ));

                // Create import run
                var       si          = ( SpreadsheetImporter )scope.Resolve <ISpreadsheetImporter>( );
                ImportRun importRun   = si.CreateImportRunEntity(config, settings);
                long      importRunId = importRun.Id;

                IImportRunWorker worker = scope.Resolve <IImportRunWorker>( );

                worker.StartImport(importRunId);
                return(importRun);
            }
        }
        /// <summary>
        /// Read an import status out of an import run entity.
        /// </summary>
        /// <param name="importRun">The import run entity.</param>
        /// <returns>The import status.</returns>
        internal static ImportStatus GetImportStatus(ImportRun importRun)
        {
            if (importRun == null)
            {
                throw new ArgumentNullException(nameof(importRun));
            }

            var runStatus = importRun.ImportRunStatus_Enum;

            if (runStatus == null)
            {
                return(ImportStatus.Failed); // assert false
            }
            switch (runStatus.Value)
            {
            case WorkflowRunState_Enumeration.WorkflowRunQueued:
            case WorkflowRunState_Enumeration.WorkflowRunStarted:
                return(ImportStatus.InProgress);

            case WorkflowRunState_Enumeration.WorkflowRunCompleted:
                return(ImportStatus.Success);

            case WorkflowRunState_Enumeration.WorkflowRunFailed:
                return(ImportStatus.Failed);

            case WorkflowRunState_Enumeration.WorkflowRunCancelled:        // will do for now
                return(ImportStatus.Cancelled);

            case WorkflowRunState_Enumeration.WorkflowRunPaused:
            default:
                throw new InvalidOperationException( );
            }
        }
        public static ImportRun RunTest(EntityType entityType, string fileName, ImportFormat importFormat, string sheetName = null)
        {
            string fileToken;

            using (Stream stream = SheetTestHelper.GetStream(fileName))
            {
                fileToken = FileRepositoryHelper.AddTemporaryFile(stream);
            }


            EntityType   type         = entityType;
            ImportConfig importConfig = CreateImportConfig(type, importFormat, sheetName);

            ImportRun importRun = CreateImportRun(importConfig, fileToken);

            ISpreadsheetInspector inspector = Factory.Current.Resolve <ISpreadsheetInspector>( );
            SpreadsheetInfo       info      = inspector.GetSpreadsheetInfo(fileToken, importFormat);
            SampleTable           sample    = inspector.GetSampleTable(fileToken, importFormat, sheetName, 1, 2, null);

            AddAllFields(importConfig, sample);

            // Run import
            IImportRunWorker worker = Factory.Current.Resolve <IImportRunWorker>( );

            worker.StartImport(importRun.Id);

            return(importRun);
        }
Exemple #4
0
        private async Task GetShows(ImportRun run, HttpClient httpClient, CancellationToken stoppingToken)
        {
            var keepGoing = true;

            while (keepGoing)
            {
                List <Show> shows;

                // Wait for 200 ms to not flood the api
                await Task.Delay(200, stoppingToken);

                // Call the next page of shows on tvMaze
                using (var response = await httpClient.GetAsync($"http://api.tvmaze.com/shows?page={run.EndPageNum}"))
                {
                    if (!response.IsSuccessStatusCode)
                    {
                        // reached the last page
                        keepGoing = false;
                        run.EndPageNum--;
                        break;
                    }

                    string apiResponse = await response.Content.ReadAsStringAsync();

                    shows = JsonConvert.DeserializeObject <List <Show> >(apiResponse);

                    foreach (var show in shows)
                    {
                        await ProcessShow(show, httpClient, stoppingToken);
                    }

                    run.EndPageNum++;
                }
            }
        }
Exemple #5
0
        public IEntityRepository GetMockEntityRepository(long runId, ImportRun importRun)
        {
            var entityRepository = new Mock <IEntityRepository>( );

            entityRepository.Setup(r => r.Get <ImportRun>(runId)).Returns(importRun);
            return(entityRepository.Object);
        }
        /// <summary>
        ///     Get Import status of the importing task.
        /// </summary>
        /// <param name="importRunId">Import run ID.</param>
        /// <returns>Returns the import status info.</returns>
        public ImportResultInfo GetImportStatus(long importRunId)
        {
            ImportRun importRun = GetImportRun(importRunId);

            ImportResultInfo result = GetImportResultInfo(importRun);

            return(result);
        }
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="importRun">A the import run entity that will be monitored, which must be provided by an automatically updating value.</param>
        public ImportRunCancellationWatcher([NotNull] ImportRun importRun)
        {
            if (importRun == null)
            {
                throw new ArgumentNullException(nameof(importRun));
            }

            _importRun = importRun;
        }
        public void Test_CompletedWorks( )
        {
            ImportRun            importRun = new ImportRun( );
            ICancellationWatcher watcher   = new ImportRunCancellationWatcher(importRun);

            Assert.That(watcher.IsCancellationRequested, Is.False);
            importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunStarted;
            Assert.That(watcher.IsCancellationRequested, Is.False);
            importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunCompleted;
            Assert.That(watcher.IsCancellationRequested, Is.False);
        }
        private static ImportRun CreateImportRun(ImportConfig importConfig, string fileToken)
        {
            SpreadsheetImporter importer       = ( SpreadsheetImporter )Factory.Current.Resolve <ISpreadsheetImporter>( );
            ImportSettings      importSettings = new ImportSettings
            {
                FileToken      = fileToken,
                ImportConfigId = importConfig.Id,
                TimeZoneName   = ""
            };
            ImportRun importRun = importer.CreateImportRunEntity(importConfig, importSettings);

            return(importRun);
        }
        /// <summary>
        /// Extracts general progress/result information from an import run.
        /// </summary>
        /// <param name="importRun">The import run entity.</param>
        /// <returns>Result object.</returns>
        private ImportResultInfo GetImportResultInfo(ImportRun importRun)
        {
            ImportResultInfo result = new ImportResultInfo
            {
                ImportStatus     = ImportHelpers.GetImportStatus(importRun),
                ImportMessages   = importRun.ImportMessages,
                RecordsTotal     = importRun.ImportRecordsTotal ?? 0,
                RecordsSucceeded = importRun.ImportRecordsSucceeded ?? 0,
                RecordsFailed    = importRun.ImportRecordsFailed ?? 0
            };

            return(result);
        }
        public void Test_ReportOk_Single( )
        {
            ImportRun         importRun = new ImportRun( );
            ImportRunReporter reporter  = new ImportRunReporter(importRun);

            reporter.ReportOk( );
            reporter.ReportOk(2);
            reporter.Flush( );

            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(0));
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(3));
            Assert.That(importRun.ImportMessages, Is.Null);
        }
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="importRun">A Writable reference to the import run entity that represents this import.</param>
        public ImportRunReporter([NotNull] ImportRun importRun)
        {
            if (importRun == null)
            {
                throw new ArgumentNullException(nameof(importRun));
            }
            if (importRun.IsReadOnly)
            {
                throw new ArgumentException("Import run entity is read-only.", nameof(importRun));
            }

            _importRun = importRun;
        }
        public void Test_ReportError(string location, string expectedMessage)
        {
            ImportRun         importRun = new ImportRun( );
            ImportRunReporter reporter  = new ImportRunReporter(importRun);

            Mock <IObjectReader> mockObject = new Mock <IObjectReader>( );

            mockObject.Setup(obj => obj.GetLocation( )).Returns(location);
            reporter.ReportError(mockObject.Object, "Message");
            reporter.Flush( );

            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(1));
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(0));
            Assert.That(importRun.ImportMessages, Is.EqualTo(expectedMessage));
        }
        /// <summary>
        ///     Creates an importRun entity - does not save it.
        /// </summary>
        /// <param name="importConfig">The import configuration.</param>
        /// <param name="importSettings">Settings passed in for the current run.</param>
        /// <returns>Returns the ID of the import run.</returns>
        internal ImportRun CreateImportRunEntity(ImportConfig importConfig, ImportSettings importSettings)
        {
            // Create a new import run
            ImportRun importRun = _entityRepository.Create <ImportRun>( );

            importRun.ImportRunStatus_Enum   = WorkflowRunState_Enumeration.WorkflowRunQueued;
            importRun.ImportConfigUsed       = importConfig;
            importRun.ImportFileId           = importSettings.FileToken;
            importRun.ImportFileName         = importSettings.FileName;
            importRun.ImportTimeZone         = importSettings.TimeZoneName;
            importRun.ImportRecordsSucceeded = 0;
            importRun.ImportRecordsFailed    = 0;
            importRun.ImportTestRun          = importSettings.TestRun;
            return(importRun);
        }
Exemple #15
0
        public void ImportDataFromExcel_TestBooleanConstraints_Valid( )
        {
            EntityType type      = ImportTestHelper.CreateTestBooleanEntityType( );
            ImportRun  importRun = ImportTestHelper.RunTest(type, "Boolean.csv", ImportFormat.CSV);

            //check for the import status
            Assert.That(importRun.ImportRunStatus_Enum, Is.EqualTo(WorkflowRunState_Enumeration.WorkflowRunCompleted), importRun.ImportMessages);
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(13));
            Assert.That(importRun.ImportRecordsTotal, Is.EqualTo(14));
            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(1));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 15: Value for 'Boolean_False' was formatted incorrectly"));

            var instances = Entity.GetInstancesOfType(type, false);

            Assert.That(instances, Has.Count.EqualTo(13));
        }
Exemple #16
0
        /// <summary>
        ///     Creates an importRun entity - does not save it.
        /// </summary>
        /// <param name="config">The import configuration.</param>
        /// <param name="message"></param>
        /// <returns>Returns the ID of the import run.</returns>
        private void CreateFailedImportRunEntity(ScheduledImportConfig config, string message)
        {
            var importConfig = config.SicImportConfig;

            // Create a new import run
            ImportRun importRun = Entity.Create <ImportRun>();

            importRun.ImportFileName       = config.SicUrl ?? string.Empty;
            importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunFailed;
            importRun.ImportConfigUsed     = importConfig;
            importRun.ImportMessages       = message;
            importRun.ImportRunStarted     = DateTime.UtcNow;
            importRun.ImportRunFinished    = DateTime.UtcNow;

            importRun.Save();
        }
Exemple #17
0
        public AutoMock GetMock( )
        {
            ImportRun importRun = GetMockRun(MockToken);

            AutoMock mock = AutoMock.GetStrict();

            mock.Provide(importRun);
            mock.Provide(GetMockFileRepository(MockToken));
            mock.Provide(GetMockEntityRepository(MockRunId, importRun));

            // OK, that's enough mocking..
            mock.Provide(Factory.Current.Resolve <IReaderToEntityAdapterProvider>());
            mock.Provide(Factory.Current.Resolve <Func <ImportFormat, IDataFileReaderService> >( ));
            mock.Provide(Factory.Current.Resolve <RecordImporter.Factory>( ));
            mock.Provide(Factory.Current.Resolve <IEntitySaver>( ));
            return(mock);
        }
        /// <summary>
        ///     Begin task to import spreadsheet data.
        /// </summary>
        /// <param name="importSettings">The settings of the import.</param>
        /// <returns>Returns the ID of the import run.</returns>
        public long StartImport(ImportSettings importSettings)
        {
            // Validate
            if (importSettings == null)
            {
                throw new ArgumentNullException(nameof(importSettings));
            }
            if (string.IsNullOrEmpty(importSettings.FileToken))
            {
                throw new ArgumentException("importSettings.FileToken");
            }

            // Load the config
            ImportConfig importConfig = SecurityBypassContext.ElevateIf(
                importSettings.SuppressSecurityCheckOnImportConfig,
                () => _entityRepository.Get <ImportConfig>(importSettings.ImportConfigId));

            if (importConfig == null)
            {
                throw new ArgumentException("importSettings.ImportConfigId");
            }

            // Create a new import run
            ImportRun importRun = CreateImportRunEntity(importConfig, importSettings);

            SecurityBypassContext.Elevate(importRun.Save);

            long importRunId = importRun.Id;

            try
            {
                _asyncRunner.Start(() => _importRunWorker.StartImport(importRunId));
            }
            catch
            {
                // Async operation failed to start
                // (This is not reached if the import itself fails)
                importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunFailed;
                importRun.ImportMessages       = "Failed to start importer.";
                SecurityBypassContext.Elevate(importRun.Save);
                throw;
            }

            return(importRun.Id);
        }
        /// <summary>
        ///     Cancel Import operation.
        /// </summary>
        /// <param name="importRunId">Import run ID.</param>
        /// <returns>Returns the import status info.</returns>
        public ImportResultInfo CancelImportOperation(long importRunId)
        {
            ImportRun importRun = GetImportRun(importRunId).AsWritable <ImportRun>( );

            // Check status - only cancel if still running
            ImportStatus importStatus = ImportHelpers.GetImportStatus(importRun);

            if (importStatus == ImportStatus.InProgress)
            {
                // Cancel
                importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunCancelled;
                importRun.Save( );
            }

            ImportResultInfo result = GetImportResultInfo(importRun);

            return(result);
        }
        public void Test_MessageReordering( )
        {
            ImportRun         importRun = new ImportRun( );
            ImportRunReporter reporter  = new ImportRunReporter(importRun);

            Mock <IObjectReader> mockObject  = new Mock <IObjectReader>( );
            Mock <IObjectReader> mockObject2 = new Mock <IObjectReader>( );

            mockObject.Setup(obj => obj.GetLocation( )).Returns(() => "Line 10");
            mockObject2.Setup(obj => obj.GetLocation( )).Returns(() => "Line 2");
            reporter.ReportError(mockObject.Object, "Message");
            reporter.ReportError(mockObject2.Object, "Message");
            reporter.ReportError(mockObject.Object, "Message");
            reporter.ReportError(mockObject2.Object, "Message");
            reporter.Flush( );

            Assert.That(importRun.ImportMessages, Is.EqualTo("Line 2: Message\r\nLine 2: Message\r\nLine 10: Message\r\nLine 10: Message\r\n"));
        }
Exemple #21
0
        public void ImportDataFromCSV_TestNumericConstraints_Valid( )
        {
            EntityType type      = ImportTestHelper.CreateTestNumericEntityType( );
            ImportRun  importRun = ImportTestHelper.RunTest(type, "Numeric.csv", ImportFormat.CSV);

            //check for the import status
            Assert.That(importRun.ImportRunStatus_Enum, Is.EqualTo(WorkflowRunState_Enumeration.WorkflowRunCompleted), importRun.ImportMessages);
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(7));
            Assert.That(importRun.ImportRecordsTotal, Is.EqualTo(10));
            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(3));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 2: 'Numeric' value must not be less than 10"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 7: 'Numeric' value is required"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 11: 'Numeric' value must not be greater than 99"));

            var instances = Entity.GetInstancesOfType(type, false);

            Assert.That(instances, Has.Count.EqualTo(7));
        }
        public void Test_Calls_DontAccumulateForSameRecord( )
        {
            ImportRun         importRun = new ImportRun( );
            ImportRunReporter reporter  = new ImportRunReporter(importRun);

            Mock <IObjectReader> mockObject = new Mock <IObjectReader>( );

            mockObject.Setup(obj => obj.GetLocation( )).Returns(() => "Line 1");
            reporter.ReportError(mockObject.Object, "Message");
            reporter.ReportError(mockObject.Object, "Message");
            reporter.ReportOk( );
            reporter.Flush( );
            reporter.ReportOk(2);
            reporter.Flush( );

            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(1));
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(3));
            Assert.That(importRun.ImportMessages, Is.EqualTo("Line 1: Message\r\nLine 1: Message\r\n"));
        }
Exemple #23
0
        public void ImportDataFromCSV_TestStringConstraints_Valid( )
        {
            EntityType type      = ImportTestHelper.CreateTestStringEntityType( );
            ImportRun  importRun = ImportTestHelper.RunTest(type, "String.csv", ImportFormat.CSV);

            //check for the import status
            Assert.That(importRun.ImportRunStatus_Enum, Is.EqualTo(WorkflowRunState_Enumeration.WorkflowRunCompleted), importRun.ImportMessages);
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(5));
            Assert.That(importRun.ImportRecordsTotal, Is.EqualTo(9));
            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(4));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 2: 'State' length must not be less than 3"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 7: 'City' value is required."));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 8: 'City' value is required."));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 3: 'String1' length must not be greater than 20"));     // this should say line 4, but there's a problem in the .Net TextFieldParser

            var instances = Entity.GetInstancesOfType(type, false);

            Assert.That(instances, Has.Count.EqualTo(5));
        }
        public void Test_PostCreate_OK( )
        {
            string uri = string.Format("upload/{0}/{1}/{2}?key={3}", TenantName, ApiAddress, EndpointAddress, ApiKey);

            using (var request = new PlatformHttpRequest(uri, PlatformHttpMethod.Post, null, true))
            {
                Stream testFile = ImportSpreadsheetControllerTests.GetStream( );
                request.PopulateBodyStream(testFile, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");

                var response = request.GetResponse( );

                // check that it worked (200)
                Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
            }

            // Check run started
            using (new TenantAdministratorContext(TenantName))
            {
                //Thread.Sleep( 1000 );
                ImportRun run = importConfig.ImportRuns.SingleOrDefault( );
                Assert.That(run, Is.Not.Null);

                // Poll progress
                int count = 25;
                while (run.ImportRunStatus_Enum == WorkflowRunState_Enumeration.WorkflowRunStarted || run.ImportRunStatus_Enum == WorkflowRunState_Enumeration.WorkflowRunQueued)
                {
                    Thread.Sleep(100);

                    run = Entity.Get <ImportRun>(run.Id);
                    Assert.That(run, Is.Not.Null);
                    if (count-- == 0)
                    {
                        throw new Exception("Timed out: " + run.ImportRunStatus_Enum);
                    }
                }
                Assert.That(run.ImportRunStatus_Enum, Is.EqualTo(WorkflowRunState_Enumeration.WorkflowRunCompleted), run.ImportMessages);
                Assert.That(run.ImportRecordsSucceeded, Is.EqualTo(7));

                // Check instances
                var instances = Entity.GetInstancesOfType(type.Id).ToList( );
                Assert.That(instances, Has.Count.EqualTo(7));
            }
        }
        public void ImportDataFromExcel_TestDecimalConstraints_Valid( )
        {
            EntityType type      = ImportTestHelper.CreateTestDecimalEntityType( );
            ImportRun  importRun = ImportTestHelper.RunTest(type, ExcelFile, ImportFormat.Excel, "Decimal");

            //check for the import status
            Assert.That(importRun.ImportRunStatus_Enum, Is.EqualTo(WorkflowRunState_Enumeration.WorkflowRunCompleted), importRun.ImportMessages);
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(5), importRun.ImportMessages);
            Assert.That(importRun.ImportRecordsTotal, Is.EqualTo(9));
            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(4));
            Assert.That(importRun.ImportMessages, Contains.Substring("Row 3: 'Decimal' value must not be less than 10"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Row 6: 'Decimal' value must not be greater than 100"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Row 7: 'Decimal' value is required"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Row 8: 'Decimal' value must not be less than 10"));

            var instances = Entity.GetInstancesOfType(type, false);

            Assert.That(instances, Has.Count.EqualTo(5));
        }
Exemple #26
0
        public void ImportDataFromCSV_TestDateConstraints_Valid( )
        {
            EntityType type      = ImportTestHelper.CreateTestDateEntityType( );
            ImportRun  importRun = ImportTestHelper.RunTest(type, "DateTime.csv", ImportFormat.CSV);

            //check for the import status
            Assert.That(importRun.ImportRunStatus_Enum, Is.EqualTo(WorkflowRunState_Enumeration.WorkflowRunCompleted), importRun.ImportMessages);
            Assert.That(importRun.ImportRecordsSucceeded, Is.EqualTo(5));
            Assert.That(importRun.ImportRecordsTotal, Is.EqualTo(10));
            Assert.That(importRun.ImportRecordsFailed, Is.EqualTo(5));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 3: 'Entered Date' value must not be less than 1/06/2012"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 6: 'Entered Date' value must not be greater than 30/06/2012"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 7: 'Entered Date' value is required."));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 8: 'Entered Date' value must not be less than 1/06/2012"));
            Assert.That(importRun.ImportMessages, Contains.Substring("Line 9: 'Entered Date' value must not be greater than 30/06/2012"));

            var instances = Entity.GetInstancesOfType(type, false);

            Assert.That(instances, Has.Count.EqualTo(5));
        }
Exemple #27
0
        public ImportRun GetMockRun(string token)
        {
            EntityType type = new EntityType();

            ApiResourceMapping mapping = new ApiResourceMapping();

            mapping.MappedType             = type;
            mapping.MappingSourceReference = "Test1";
            mapping.ImportHeadingRow       = 3;
            mapping.ImportDataRow          = 4;

            ImportConfig importConfig = new ImportConfig( );

            importConfig.ImportFileType_Enum = ImportFileTypeEnum_Enumeration.ImportFileTypeExcel;
            importConfig.ImportConfigMapping = mapping;

            ImportRun importRun = new ImportRun( );

            importRun.ImportConfigUsed     = importConfig;
            importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunQueued;
            importRun.ImportFileId         = token;
            return(importRun);
        }
Exemple #28
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            while (!stoppingToken.IsCancellationRequested)
            {
                _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);

                var nextPageNumber = await GetLastPageNumber();

                var run = new ImportRun {
                    StartTime    = DateTime.UtcNow,
                    StartPageNum = nextPageNumber,
                    EndPageNum   = nextPageNumber + 1
                };

                await _importRunRepository.AddAsync(run);

                using (var httpClient = new HttpClient())
                {
                    await GetShows(run, httpClient, stoppingToken);
                }

                await Task.Delay(1000, stoppingToken);
            }
        }
        /// <summary>
        /// Open up a records reader to read the contents of the file.
        /// </summary>
        /// <remarks>
        /// Caller closes stream.
        /// </remarks>
        private IObjectsReader GetRecordsReader(ImportRun importRun, ImportConfig importConfig)
        {
            // Get settings
            DataFileReaderSettings settings = CreateReaderSettings(importConfig);

            // Get the timezone
            if (!string.IsNullOrEmpty(importRun.ImportTimeZone))
            {
                settings.TimeZoneInfo = TimeZoneHelper.GetTimeZoneInfo(importRun.ImportTimeZone);
            }

            // Get file reader
            IDataFileReaderService fileReader = _readerActivator(settings.ImportFormat);

            // Open stream
            string fileUploadId = importRun.ImportFileId;

            if (string.IsNullOrEmpty(fileUploadId))
            {
                throw new Exception("File handle not set");
            }

            Stream fileStream;

            try
            {
                fileStream = FileRepository.Get(fileUploadId);
            }
            catch (Exception ex)
            {
                throw new Exception("Could not retrieve file. " + ex.Message, ex);
            }
            IObjectsReader recordsReader = fileReader.OpenDataFile(fileStream, settings);

            return(recordsReader);
        }
        /// <summary>
        /// Start processing an import run.
        /// </summary>
        /// <param name="importRunId">ID of the import run to process.</param>
        public void StartImport(long importRunId)
        {
            using (new SecurityBypassContext( ))
            {
                ImportRun importRun = _entityRepository.Get <ImportRun>(importRunId);
                if (importRun == null)
                {
                    throw new ArgumentException(nameof(importRunId));
                }

                importRun = importRun.AsWritable <ImportRun>( );

                // Verify that import run is ready to run.
                if (importRun.ImportRunStatus_Enum != WorkflowRunState_Enumeration.WorkflowRunQueued)
                {
                    throw new Exception("Import run is not marked as queued, or has already been started.");
                }

                // Mark it as started
                importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunStarted;
                importRun.ImportRunStarted     = DateTime.UtcNow;
                importRun.Save( );

                // Trap errors
                try
                {
                    string verb = "imported";
                    if (importRun.ImportTestRun == true)
                    {
                        verb = "verified";
                        importRun.ImportMessages += "This is a test import. No records are being saved.\r\n";
                    }

                    StartImportSafe(importRun);

                    if (importRun.ImportRecordsTotal == 0 || importRun.ImportRecordsTotal == null)
                    {
                        throw new Exception($"No records were found to be {verb}.");
                    }
                    if (importRun.ImportRecordsSucceeded == 0)
                    {
                        throw new Exception($"No records were successfully {verb}.");
                    }

                    // Mark run as completed
                    if (importRun.ImportRunStatus_Enum != WorkflowRunState_Enumeration.WorkflowRunCancelled)
                    {
                        importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunCompleted;
                    }
                }
                catch (Exception ex)
                {
                    importRun.ImportRunStatus_Enum = WorkflowRunState_Enumeration.WorkflowRunFailed;
                    importRun.ImportMessages       = "Failed: " + ex.Message + "\r\n" + importRun.ImportMessages; // TODO : Something better than this
                }
                finally
                {
                    importRun.ImportRunFinished = DateTime.UtcNow;
                    importRun.Save( );
                }
            }
        }