/// <inheritdoc /> /// <summary> /// Initializes a new instance of the <see cref="T:FileUploadPipelineStep.FileUploadPipelineStep" /> class. /// </summary> /// <param name="jobConfig"> /// The queue context. /// </param> /// <param name="elasticSearchUploader"> /// The elasticSearchUploader /// </param> /// <param name="logger"> /// The logger. /// </param> /// <param name="queueManager"> /// The queue Manager. /// </param> /// <param name="progressMonitor"></param> /// <param name="cancellationToken"></param> public FileUploadPipelineStep( IJobConfig jobConfig, IElasticSearchUploader elasticSearchUploader, ILogger logger, IQueueManager queueManager, IProgressMonitor progressMonitor, CancellationToken cancellationToken) : base(jobConfig, logger, queueManager, progressMonitor, cancellationToken) { this.elasticSearchUploader = elasticSearchUploader; }
/// <inheritdoc /> /// <summary> /// Initializes a new instance of the <see cref="T:MappingUploadPipelineStep.MappingUploadPipelineStep" /> class. /// </summary> /// <param name="jobConfig"> /// The queue context. /// </param> /// <param name="logger"> /// The logger. /// </param> /// <param name="elasticSearchUploader"></param> /// <param name="queueManager"></param> /// <param name="progressMonitor"></param> /// <param name="cancellationToken"></param> public MappingUploadPipelineStep( IJobConfig jobConfig, ILogger logger, IElasticSearchUploader elasticSearchUploader, IQueueManager queueManager, IProgressMonitor progressMonitor, CancellationToken cancellationToken) : base(jobConfig, logger, queueManager, progressMonitor, cancellationToken) { this.elasticSearchUploader = elasticSearchUploader ?? throw new ArgumentNullException(nameof(elasticSearchUploader)); this.folder = Path.Combine(this.Config.LocalSaveFolder, $"{this.UniqueId}-{LoggerName}"); if (this.Config.WriteDetailedTemporaryFilesToDisk) { Directory.CreateDirectory(this.folder); } }
/// <inheritdoc /> public SqlGetSchemaPipelineStep( IJobConfig jobConfig, ILogger logger, IElasticSearchUploader elasticSearchUploader, IQueueManager queueManager, IProgressMonitor progressMonitor, ISchemaLoader schemaLoader, IDetailedTemporaryFileWriter fileWriter, CancellationToken cancellationToken) : base(jobConfig, logger, queueManager, progressMonitor, cancellationToken) { this.elasticSearchUploader = elasticSearchUploader ?? throw new ArgumentNullException(nameof(elasticSearchUploader)); this.schemaLoader = schemaLoader ?? throw new ArgumentNullException(nameof(schemaLoader)); this.fileWriter = fileWriter ?? throw new ArgumentNullException(nameof(fileWriter)); this.folder = Path.Combine(this.Config.LocalSaveFolder, $"{this.UniqueId}-{LoggerName}"); this.fileWriter.CreateDirectory(this.folder); }
/// <summary> /// The run pipeline. /// </summary> /// <param name="job"> /// The job. /// </param> public void RunElasticSearchPipeline(IJob job) { if (job == null) { throw new ArgumentNullException(nameof(job)); } if (job.Config == null) { throw new ArgumentNullException(nameof(job.Config)); } var config = job.Config; this.InitContainer(config); if (config.WriteTemporaryFilesToDisk) { this.container.Resolve <IFileWriter>().DeleteDirectory(config.LocalSaveFolder); } var stopwatch = new Stopwatch(); stopwatch.Start(); int loadNumber = 0; // add sequence number to every load foreach (var load in job.Data.DataSources) { load.SequenceNumber = ++loadNumber; } // add job to the first queue var sqlJobQueue = this.container.Resolve <IQueueManager>() .CreateInputQueue <SqlJobQueueItem>(++this.stepNumber); sqlJobQueue.Add(new SqlJobQueueItem { Job = job }); sqlJobQueue.CompleteAdding(); var processors = new List <PipelineStepInfo>(); if (config.DropAndReloadIndex) { processors.AddRange( new List <PipelineStepInfo> { new PipelineStepInfo { Type = typeof(SqlGetSchemaPipelineStep), Count = 1 }, new PipelineStepInfo { Type = typeof(SaveSchemaPipelineStep), Count = 1 }, new PipelineStepInfo { Type = config.UploadToElasticSearch ? typeof(MappingUploadPipelineStep) : typeof(DummyMappingUploadPipelineStep), Count = 1 } }); } processors.AddRange( new List <PipelineStepInfo> { new PipelineStepInfo { Type = typeof(SqlJobPipelineStep), Count = 1 }, new PipelineStepInfo { Type = typeof(SqlBatchPipelineStep), Count = 1 }, new PipelineStepInfo { Type = typeof(SqlImportPipelineStep), Count = 15 }, new PipelineStepInfo { Type = typeof(ConvertDatabaseRowToJsonPipelineStep), Count = 5 }, new PipelineStepInfo { Type = typeof(JsonDocumentMergerPipelineStep), Count = 1 }, new PipelineStepInfo { Type = typeof(CreateBatchItemsPipelineStep), Count = 1 }, new PipelineStepInfo { Type = typeof(SaveBatchPipelineStep), Count = 1 } }); if (config.WriteTemporaryFilesToDisk) { processors.Add(new PipelineStepInfo { Type = typeof(FileSavePipelineStep), Count = 1 }); } if (config.UploadToElasticSearch) { processors.Add(new PipelineStepInfo { Type = typeof(FileUploadPipelineStep), Count = 1 }); } var elasticSearchUploaderFactory = this.container.Resolve <IElasticSearchUploaderFactory>(); IElasticSearchUploader elasticSearchUploader = elasticSearchUploaderFactory.Create( config.ElasticSearchUserName, config.ElasticSearchPassword, false, config.Urls, config.Index, config.Alias, config.EntityType); this.container.RegisterInstance(elasticSearchUploader); var pipelineExecutorFactory = this.container.Resolve <IPipelineExecutorFactory>(); var pipelineExecutor = pipelineExecutorFactory.Create(this.container, this.cancellationTokenSource); pipelineExecutor.RunPipelineTasks(config, processors, TimeoutInMilliseconds); var stopwatchElapsed = stopwatch.Elapsed; stopwatch.Stop(); Console.WriteLine(stopwatchElapsed); }