/// <summary> /// The run elastic search pipeline. /// </summary> /// <param name="container"> /// The container. /// </param> /// <param name="config"> /// The config. /// </param> /// <param name="cancellationToken"> /// The cancellation token /// </param> public void RunElasticSearchPipeline( IUnityContainer container, IJob config, CancellationToken cancellationToken) { var stopwatch = new Stopwatch(); stopwatch.Start(); var pipelineRunner = new PipelineRunner(container, cancellationToken); pipelineRunner.RunElasticSearchPipeline(config); stopwatch.Stop(); var timeElapsed = stopwatch.Elapsed.ToString(@"hh\:mm\:ss"); var threadText = config.Config.UseMultipleThreads ? "multiple threads" : "single thread"; container.Resolve <ILogger>().Verbose($"Finished in {timeElapsed} using {threadText}"); }
public void TestElasticSearchPipelineSingleThreaded() { var sql = @"SELECT CustomerNM ,CustomerID , AliasPatientID , GenderNormDSC ,RaceNormDSC ,MaritalStatusNormDSC FROM CAFEEDW.SharedClinicalUnion.ElasticsearchInputPatient where CustomerID = 4"; var job = new Job { Config = new QueryConfig { ConnectionString = "foo", LocalSaveFolder = Path.GetTempPath(), TopLevelKeyColumn = "AliasPatientID", Url = "http://foo", UploadToElasticSearch = false, EntitiesPerUploadFile = 1 }, Data = new JobData { MyDataSources = new List <DataSource> { new DataSource { Sql = sql } } } }; var mockRepository = new MockRepository(MockBehavior.Strict); var mockDatabusSqlReader = mockRepository.Create <IDatabusSqlReader>(); mockDatabusSqlReader.Setup( service => service.ReadDataFromQueryAsync( It.IsAny <IDataSource>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <ILogger>(), It.IsAny <string>())).ReturnsAsync( new ReadSqlDataResult { ColumnList = new List <ColumnInfo> { new ColumnInfo { Name = "CustomerNM", ElasticSearchType = ElasticSearchTypes.keyword.ToString() }, new ColumnInfo { Name = "CustomerID", ElasticSearchType = ElasticSearchTypes.integer.ToString() }, new ColumnInfo { Name = "AliasPatientID", ElasticSearchType = ElasticSearchTypes.integer.ToString() }, new ColumnInfo { Name = "GenderNormDSC", ElasticSearchType = ElasticSearchTypes.keyword.ToString() }, new ColumnInfo { Name = "RaceNormDSC", ElasticSearchType = ElasticSearchTypes.keyword.ToString() }, new ColumnInfo { Name = "MaritalStatusNormDSC", ElasticSearchType = ElasticSearchTypes.keyword.ToString() } }, Data = new Dictionary <string, List <object[]> > { { "ElasticsearchInputPatient", new List <object[]> { new object[] { "name", 1, 2, "M", "White", "Married" } } } } }); var mockFileUploaderFactory = mockRepository.Create <IElasticSearchUploaderFactory>(); var mockFileUploader = mockRepository.Create <IElasticSearchUploader>(); mockFileUploaderFactory .Setup(service => service.Create(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <List <string> >(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())) .Returns(mockFileUploader.Object); mockFileUploader.Setup( service => service.SendStreamToHosts( It.IsAny <string>(), It.IsAny <int>(), It.IsAny <Stream>(), It.IsAny <bool>(), It.IsAny <bool>())).Returns(Task.CompletedTask); mockFileUploader .Setup(service => service.StartUploadAsync()) .Returns(Task.CompletedTask); ILogger logger = new LoggerConfiguration() .MinimumLevel.Verbose() .WriteTo.Debug() .CreateLogger(); using (var progressMonitor = new ProgressMonitor(new TestConsoleProgressLogger())) { using (var cancellationTokenSource = new CancellationTokenSource()) { var container = new UnityContainer(); container.RegisterInstance <IProgressMonitor>(progressMonitor); container.RegisterInstance(mockDatabusSqlReader.Object); container.RegisterInstance(mockFileUploaderFactory.Object); container.RegisterInstance(logger); container.RegisterType <IPipelineExecutorFactory, SingleThreadedPipelineExecutorFactory>(); var pipelineRunner = new PipelineRunner(container, cancellationTokenSource.Token); try { pipelineRunner.RunElasticSearchPipeline(job); } catch (OperationCanceledException e) { Console.WriteLine(e.ToString()); throw; } catch (AggregateException e) { Console.WriteLine(e.Flatten().ToString()); throw; } catch (Exception e) { Console.WriteLine(e); throw; } } } }