/// <summary>
/// The run elastic search pipeline.
/// </summary>
/// <param name="container">
/// The container.
/// </param>
/// <param name="config">
/// The config.
/// </param>
/// <param name="cancellationToken">
/// The cancellation token
/// </param>
public void RunElasticSearchPipeline(
IUnityContainer container,
IJob config,
CancellationToken cancellationToken)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var pipelineRunner = new PipelineRunner(container, cancellationToken);
pipelineRunner.RunElasticSearchPipeline(config);
stopwatch.Stop();
var timeElapsed = stopwatch.Elapsed.ToString(@"hh\:mm\:ss");
var threadText = config.Config.UseMultipleThreads ? "multiple threads" : "single thread";
container.Resolve <ILogger>().Verbose($"Finished in {timeElapsed} using {threadText}");
}
public void TestElasticSearchPipelineSingleThreaded()
{
var sql = @"SELECT
CustomerNM
,CustomerID
, AliasPatientID
, GenderNormDSC
,RaceNormDSC
,MaritalStatusNormDSC
FROM CAFEEDW.SharedClinicalUnion.ElasticsearchInputPatient where CustomerID = 4";
var job = new Job
{
Config = new QueryConfig
{
ConnectionString = "foo",
LocalSaveFolder = Path.GetTempPath(),
TopLevelKeyColumn = "AliasPatientID",
Url = "http://foo",
UploadToElasticSearch = false,
EntitiesPerUploadFile = 1
},
Data = new JobData
{
MyDataSources = new List <DataSource>
{
new DataSource
{
Sql = sql
}
}
}
};
var mockRepository = new MockRepository(MockBehavior.Strict);
var mockDatabusSqlReader = mockRepository.Create <IDatabusSqlReader>();
mockDatabusSqlReader.Setup(
service => service.ReadDataFromQueryAsync(
It.IsAny <IDataSource>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <ILogger>(),
It.IsAny <string>())).ReturnsAsync(
new ReadSqlDataResult
{
ColumnList = new List <ColumnInfo>
{
new ColumnInfo {
Name = "CustomerNM", ElasticSearchType = ElasticSearchTypes.keyword.ToString()
},
new ColumnInfo {
Name = "CustomerID", ElasticSearchType = ElasticSearchTypes.integer.ToString()
},
new ColumnInfo {
Name = "AliasPatientID", ElasticSearchType = ElasticSearchTypes.integer.ToString()
},
new ColumnInfo {
Name = "GenderNormDSC", ElasticSearchType = ElasticSearchTypes.keyword.ToString()
},
new ColumnInfo {
Name = "RaceNormDSC", ElasticSearchType = ElasticSearchTypes.keyword.ToString()
},
new ColumnInfo {
Name = "MaritalStatusNormDSC", ElasticSearchType = ElasticSearchTypes.keyword.ToString()
}
},
Data = new Dictionary <string, List <object[]> >
{
{
"ElasticsearchInputPatient",
new List <object[]> {
new object[] { "name", 1, 2, "M", "White", "Married" }
}
}
}
});
var mockFileUploaderFactory = mockRepository.Create <IElasticSearchUploaderFactory>();
var mockFileUploader = mockRepository.Create <IElasticSearchUploader>();
mockFileUploaderFactory
.Setup(service => service.Create(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <List <string> >(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(mockFileUploader.Object);
mockFileUploader.Setup(
service => service.SendStreamToHosts(
It.IsAny <string>(),
It.IsAny <int>(),
It.IsAny <Stream>(),
It.IsAny <bool>(),
It.IsAny <bool>())).Returns(Task.CompletedTask);
mockFileUploader
.Setup(service => service.StartUploadAsync())
.Returns(Task.CompletedTask);
ILogger logger = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.Debug()
.CreateLogger();
using (var progressMonitor = new ProgressMonitor(new TestConsoleProgressLogger()))
{
using (var cancellationTokenSource = new CancellationTokenSource())
{
var container = new UnityContainer();
container.RegisterInstance <IProgressMonitor>(progressMonitor);
container.RegisterInstance(mockDatabusSqlReader.Object);
container.RegisterInstance(mockFileUploaderFactory.Object);
container.RegisterInstance(logger);
container.RegisterType <IPipelineExecutorFactory, SingleThreadedPipelineExecutorFactory>();
var pipelineRunner = new PipelineRunner(container, cancellationTokenSource.Token);
try
{
pipelineRunner.RunElasticSearchPipeline(job);
}
catch (OperationCanceledException e)
{
Console.WriteLine(e.ToString());
throw;
}
catch (AggregateException e)
{
Console.WriteLine(e.Flatten().ToString());
throw;
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
}
}