public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { var logger = pipelineContext.PipelineBatchContext.Logger; if (CanProcess(pipelineStep, pipelineContext)) { var identifierValue = GetIdentifierValue(pipelineStep, pipelineContext); if (string.IsNullOrWhiteSpace(identifierValue)) { logger.Error("Pipeline step processing will abort because no identifier value was resolved. (pipeline step: {0})", (object)pipelineStep.Name); } else { var readJsonObjectsSettings = pipelineStep.GetPlugin <ReadJsonObjectsSettings>(); var endpoint = pipelineStep.GetEndpointSettings().EndpointFrom; JObject jObject = ReadJsonData(endpoint, pipelineContext, readJsonObjectsSettings.Api, identifierValue); var resolvedObject = ExtractObject(readJsonObjectsSettings, logger, jObject); SaveResolvedObject(pipelineStep, pipelineContext, resolvedObject); } } }
private IItemModelRepository GetItemModelRepository(PipelineStep pipelineStep) { var endpointSettings = pipelineStep.GetEndpointSettings(); var endpointTo = endpointSettings?.EndpointTo; var repositorySettings = endpointTo?.GetItemModelRepositorySettings(); return(repositorySettings?.ItemModelRepository); }
private IItemModelRepository GetItemModelRepository(PipelineStep pipelineStep, PipelineContext pipelineContext) { var endpointsSettings = pipelineStep.GetEndpointSettings(); if (endpointsSettings == null) { return(null); } var endpoint = endpointsSettings.EndpointTo; var repositorySettings = endpoint?.GetItemModelRepositorySettings(); return(repositorySettings?.ItemModelRepository); }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var logger = pipelineContext.PipelineBatchContext.Logger; var endpointSettings = pipelineStep.GetEndpointSettings(); var endpointTo = endpointSettings.EndpointTo; var indexSettings = endpointTo.GetPlugin <IndexSettings>(); var indexName = indexSettings.IndexName; logger.Debug("Index Name: " + indexName); var dataSet = pipelineContext.GetPlugin <IterableDataSettings>(); var baseIndexables = dataSet.Data.Cast <IIndexable>(); var totalIndexables = baseIndexables.Count(); logger.Info("Indexable Items found: " + totalIndexables); if (totalIndexables <= 0) { logger.Debug("No indexable items found, quitting step."); return; } try { var indexHandle = ContentSearchManager.GetIndex(indexName); indexHandle.Initialize(); var flatCrawler = new IterableDatasetCrawler <IIndexable>(baseIndexables); using (var updateContext = indexHandle.CreateUpdateContext()) { logger.Debug("Crawler Initialize..."); flatCrawler.Initialize(indexHandle); logger.Debug("Index Rebuild..."); indexHandle.Rebuild(IndexingOptions.ForcedIndexing); logger.Debug("Crawler Rebuild..."); flatCrawler.RebuildFromRoot(updateContext, IndexingOptions.Default); updateContext.Commit(); logger.Debug("Rebuild Complete!"); } } catch (Exception ex) { logger.Error(ex.Message); } }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { var logger = pipelineContext.PipelineBatchContext.Logger; if (!this.CanProcess(pipelineStep, pipelineContext)) { logger.Error("Pipeline step processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", pipelineStep.Name); } else { EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); if (endpointSettings == null) { logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName); } else { ReadResourceDataSettings readDataSettings = pipelineStep.GetReadResourceDataSettings(); if (readDataSettings == null) { this.Log(logger.Error, pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", string.Format("plugin: {0}", typeof(ReadResourceDataSettings).FullName)); } else { Endpoint endpointFrom = endpointSettings.EndpointFrom; if (endpointFrom == null) { logger.Error( "Pipeline step processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(EndpointSettings).FullName, "EndpointFrom"); } else if (!this.IsEndpointValid(endpointFrom, pipelineStep, pipelineContext)) { logger.Error("Pipeline step processing will abort because the endpoint to read from is not valid. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name); } else { logger.Info("Pipeline step reading data. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName); var dataRead = Task.Run <bool>(async() => await this.ReadData(endpointFrom, pipelineStep, pipelineContext)).Result; logger.Info("Pipeline context has iterable data? {0} (pipeline step: {1}, plugin: {2})", pipelineContext.HasIterableDataSettings(), pipelineStep.Name, typeof(EndpointSettings).FullName); } } } } }
public override bool CanProcess(PipelineStep pipelineStep, PipelineContext pipelineContext) { var canProcess = false; var logger = pipelineContext.PipelineBatchContext.Logger; if (base.CanProcess(pipelineStep, pipelineContext)) { var endpointSettings = pipelineStep.GetEndpointSettings(); var endpointFrom = endpointSettings.EndpointFrom; if (endpointFrom == null) { logger.Error("Pipeline processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(EndpointSettings).FullName, "EndpointFrom"); } else if (IsEndpointValid(endpointFrom, pipelineStep, pipelineContext)) { var synchronizationSettings = pipelineContext.GetSynchronizationSettings(); if (synchronizationSettings.Source == null) { logger.Error("Pipeline processing will abort because the pipeline context has no source assigned. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(SynchronizationSettings).FullName, "Source"); } else { var jsonServiceSettings = endpointFrom.GetPlugin <JsonServiceEndpointSettings>(); if (jsonServiceSettings.Host == null || jsonServiceSettings.Protocol == null) { logger.Error("No 'Host' or 'Protocol' is specified on the endpoint. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name); } else { var readJsonObjectsSettings = pipelineStep.GetPlugin <ReadJsonObjectsSettings>(); if (readJsonObjectsSettings.Api == null) { logger.Error("No 'Api' is specified on the reader. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name); } else { canProcess = true; } } } } } if (!canProcess) { logger.Error("Pipeline processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", pipelineStep.Name); pipelineContext.CriticalError = true; } return(canProcess); }
protected override void ProcessPipelineStep(PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); if (endpointSettings == null) { logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName); } else { Endpoint endpointFrom = endpointSettings.EndpointFrom; ReadData(endpointFrom, pipelineStep, pipelineContext, logger); } }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var logger = pipelineContext.PipelineBatchContext.Logger; EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); var rssFeedSettings = endpointSettings.EndpointFrom.GetPlugin <RssFeedSettings>(); if (rssFeedSettings != null) { try { //Read rss feed and make them into an indexable item var indexableArticles = new List <RssIndexable>(); string url = rssFeedSettings.RssFeedUrl; XmlReader reader = XmlReader.Create(url); SyndicationFeed feed = SyndicationFeed.Load(reader); reader.Close(); foreach (SyndicationItem item in feed.Items) { string publishDate = item.PublishDate.ToString(); string subject = item.Title.Text; string summary = item.Summary.Text; string categories = string.Join("/", item.Categories?.Select(c => c.Name)); string authors = string.Join(", ", item.Authors?.Select(a => a.Name)); string baseUri = item.BaseUri?.AbsoluteUri; var rssIndexable = new RssIndexable(publishDate, subject, summary, authors, categories, baseUri); indexableArticles.Add(rssIndexable); } //Stuff indexables into pipeline batch var dataSettings = new IterableDataSettings(indexableArticles); pipelineContext.Plugins.Add(dataSettings); } catch (Exception ex) { logger.Error("RSS Processor Failed: " + ex.Message); } } }
protected override ITroubleshooterResult Troubleshoot(PipelineStep pipelineStep, TroubleshooterContext context) { EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); if (endpointSettings == null) { return(TroubleshooterResult.FailResult(string.Format("Pipeline step is missing a plugin. \n Plugin: {0}", typeof(EndpointSettings).FullName))); } Endpoint endpointFrom = endpointSettings.EndpointFrom; if (endpointFrom == null) { return(TroubleshooterResult.FailResult(string.Format("Pipeline step is missing a value. \n Property: {0}", Templates.BaseEndpointPipelineStep.FieldNames.EndpointFrom))); } return(Task.Run(async() => await this.TroubleshootReadData(endpointFrom, pipelineStep, context)).Result); }
protected override void ProcessPipelineStep(PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); if (endpointSettings == null) { logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName); } else { ReadResourceDataSettings readDataSettings = pipelineStep.GetReadResourceDataSettings(); if (readDataSettings == null) { this.Log(logger.Error, pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", string.Format("plugin: {0}", typeof(ReadResourceDataSettings).FullName)); } else { Endpoint endpointFrom = endpointSettings.EndpointFrom; if (endpointFrom == null) { logger.Error( "Pipeline step processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(EndpointSettings).FullName, Templates.BaseEndpointPipelineStep.FieldNames.EndpointFrom); } else if (!this.IsEndpointValid(endpointFrom, pipelineStep, pipelineContext, logger)) { logger.Error("Pipeline step processing will abort because the endpoint to read from is not valid. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name); } else { logger.Info("Pipeline step reading data. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName); var dataRead = Task.Run <bool>(async() => await this.ReadData(endpointFrom, pipelineStep, pipelineContext, logger)).Result; logger.Info("Pipeline context has data? {0} (pipeline step: {1}, plugin: {2})", pipelineContext.HasIterableDataSettings(), pipelineStep.Name, typeof(EndpointSettings).FullName); } } } }