/// <inheritdoc />
/// <summary>
/// Initializes a new instance of the <see cref="T:FileUploadPipelineStep.FileUploadPipelineStep" /> class.
/// </summary>
/// <param name="jobConfig">
/// The queue context.
/// </param>
/// <param name="elasticSearchUploader">
/// The elasticSearchUploader
/// </param>
/// <param name="logger">
/// The logger.
/// </param>
/// <param name="queueManager">
/// The queue Manager.
/// </param>
/// <param name="progressMonitor"></param>
/// <param name="cancellationToken"></param>
public FileUploadPipelineStep(
IJobConfig jobConfig,
IElasticSearchUploader elasticSearchUploader,
ILogger logger,
IQueueManager queueManager,
IProgressMonitor progressMonitor,
CancellationToken cancellationToken)
: base(jobConfig, logger, queueManager, progressMonitor, cancellationToken)
{
this.elasticSearchUploader = elasticSearchUploader;
}
/// <inheritdoc />
/// <summary>
/// Initializes a new instance of the <see cref="T:MappingUploadPipelineStep.MappingUploadPipelineStep" /> class.
/// </summary>
/// <param name="jobConfig">
/// The queue context.
/// </param>
/// <param name="logger">
/// The logger.
/// </param>
/// <param name="elasticSearchUploader"></param>
/// <param name="queueManager"></param>
/// <param name="progressMonitor"></param>
/// <param name="cancellationToken"></param>
public MappingUploadPipelineStep(
IJobConfig jobConfig,
ILogger logger,
IElasticSearchUploader elasticSearchUploader,
IQueueManager queueManager,
IProgressMonitor progressMonitor,
CancellationToken cancellationToken)
: base(jobConfig, logger, queueManager, progressMonitor, cancellationToken)
{
this.elasticSearchUploader = elasticSearchUploader ?? throw new ArgumentNullException(nameof(elasticSearchUploader));
this.folder = Path.Combine(this.Config.LocalSaveFolder, $"{this.UniqueId}-{LoggerName}");
if (this.Config.WriteDetailedTemporaryFilesToDisk)
{
Directory.CreateDirectory(this.folder);
}
}
/// <inheritdoc />
public SqlGetSchemaPipelineStep(
IJobConfig jobConfig,
ILogger logger,
IElasticSearchUploader elasticSearchUploader,
IQueueManager queueManager,
IProgressMonitor progressMonitor,
ISchemaLoader schemaLoader,
IDetailedTemporaryFileWriter fileWriter,
CancellationToken cancellationToken)
: base(jobConfig, logger, queueManager, progressMonitor, cancellationToken)
{
this.elasticSearchUploader = elasticSearchUploader ?? throw new ArgumentNullException(nameof(elasticSearchUploader));
this.schemaLoader = schemaLoader ?? throw new ArgumentNullException(nameof(schemaLoader));
this.fileWriter = fileWriter ?? throw new ArgumentNullException(nameof(fileWriter));
this.folder = Path.Combine(this.Config.LocalSaveFolder, $"{this.UniqueId}-{LoggerName}");
this.fileWriter.CreateDirectory(this.folder);
}
/// <summary>
/// The run pipeline.
/// </summary>
/// <param name="job">
/// The job.
/// </param>
public void RunElasticSearchPipeline(IJob job)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
if (job.Config == null)
{
throw new ArgumentNullException(nameof(job.Config));
}
var config = job.Config;
this.InitContainer(config);
if (config.WriteTemporaryFilesToDisk)
{
this.container.Resolve <IFileWriter>().DeleteDirectory(config.LocalSaveFolder);
}
var stopwatch = new Stopwatch();
stopwatch.Start();
int loadNumber = 0;
// add sequence number to every load
foreach (var load in job.Data.DataSources)
{
load.SequenceNumber = ++loadNumber;
}
// add job to the first queue
var sqlJobQueue = this.container.Resolve <IQueueManager>()
.CreateInputQueue <SqlJobQueueItem>(++this.stepNumber);
sqlJobQueue.Add(new SqlJobQueueItem
{
Job = job
});
sqlJobQueue.CompleteAdding();
var processors = new List <PipelineStepInfo>();
if (config.DropAndReloadIndex)
{
processors.AddRange(
new List <PipelineStepInfo>
{
new PipelineStepInfo {
Type = typeof(SqlGetSchemaPipelineStep), Count = 1
},
new PipelineStepInfo {
Type = typeof(SaveSchemaPipelineStep), Count = 1
},
new PipelineStepInfo
{
Type = config.UploadToElasticSearch
? typeof(MappingUploadPipelineStep)
: typeof(DummyMappingUploadPipelineStep),
Count = 1
}
});
}
processors.AddRange(
new List <PipelineStepInfo>
{
new PipelineStepInfo {
Type = typeof(SqlJobPipelineStep), Count = 1
},
new PipelineStepInfo {
Type = typeof(SqlBatchPipelineStep), Count = 1
},
new PipelineStepInfo {
Type = typeof(SqlImportPipelineStep), Count = 15
},
new PipelineStepInfo {
Type = typeof(ConvertDatabaseRowToJsonPipelineStep), Count = 5
},
new PipelineStepInfo {
Type = typeof(JsonDocumentMergerPipelineStep), Count = 1
},
new PipelineStepInfo {
Type = typeof(CreateBatchItemsPipelineStep), Count = 1
},
new PipelineStepInfo {
Type = typeof(SaveBatchPipelineStep), Count = 1
}
});
if (config.WriteTemporaryFilesToDisk)
{
processors.Add(new PipelineStepInfo
{
Type = typeof(FileSavePipelineStep),
Count = 1
});
}
if (config.UploadToElasticSearch)
{
processors.Add(new PipelineStepInfo
{
Type = typeof(FileUploadPipelineStep),
Count = 1
});
}
var elasticSearchUploaderFactory = this.container.Resolve <IElasticSearchUploaderFactory>();
IElasticSearchUploader elasticSearchUploader = elasticSearchUploaderFactory.Create(
config.ElasticSearchUserName,
config.ElasticSearchPassword,
false,
config.Urls,
config.Index,
config.Alias,
config.EntityType);
this.container.RegisterInstance(elasticSearchUploader);
var pipelineExecutorFactory = this.container.Resolve <IPipelineExecutorFactory>();
var pipelineExecutor = pipelineExecutorFactory.Create(this.container, this.cancellationTokenSource);
pipelineExecutor.RunPipelineTasks(config, processors, TimeoutInMilliseconds);
var stopwatchElapsed = stopwatch.Elapsed;
stopwatch.Stop();
Console.WriteLine(stopwatchElapsed);
}
