Example #1
0
        public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            var logger = pipelineContext.PipelineBatchContext.Logger;

            if (CanProcess(pipelineStep, pipelineContext))
            {
                var identifierValue = GetIdentifierValue(pipelineStep, pipelineContext);
                if (string.IsNullOrWhiteSpace(identifierValue))
                {
                    logger.Error("Pipeline step processing will abort because no identifier value was resolved. (pipeline step: {0})", (object)pipelineStep.Name);
                }
                else
                {
                    var readJsonObjectsSettings = pipelineStep.GetPlugin <ReadJsonObjectsSettings>();
                    var endpoint = pipelineStep.GetEndpointSettings().EndpointFrom;

                    JObject jObject = ReadJsonData(endpoint, pipelineContext, readJsonObjectsSettings.Api, identifierValue);

                    var resolvedObject = ExtractObject(readJsonObjectsSettings, logger, jObject);



                    SaveResolvedObject(pipelineStep, pipelineContext, resolvedObject);
                }
            }
        }
        private IItemModelRepository GetItemModelRepository(PipelineStep pipelineStep)
        {
            var endpointSettings   = pipelineStep.GetEndpointSettings();
            var endpointTo         = endpointSettings?.EndpointTo;
            var repositorySettings = endpointTo?.GetItemModelRepositorySettings();

            return(repositorySettings?.ItemModelRepository);
        }
        private IItemModelRepository GetItemModelRepository(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            var endpointsSettings = pipelineStep.GetEndpointSettings();

            if (endpointsSettings == null)
            {
                return(null);
            }
            var endpoint           = endpointsSettings.EndpointTo;
            var repositorySettings = endpoint?.GetItemModelRepositorySettings();

            return(repositorySettings?.ItemModelRepository);
        }
Example #4
0
        public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            if (pipelineStep == null)
            {
                throw new ArgumentNullException(nameof(pipelineStep));
            }
            if (pipelineContext == null)
            {
                throw new ArgumentNullException(nameof(pipelineContext));
            }
            var logger = pipelineContext.PipelineBatchContext.Logger;

            var endpointSettings = pipelineStep.GetEndpointSettings();
            var endpointTo       = endpointSettings.EndpointTo;
            var indexSettings    = endpointTo.GetPlugin <IndexSettings>();
            var indexName        = indexSettings.IndexName;

            logger.Debug("Index Name: " + indexName);
            var dataSet         = pipelineContext.GetPlugin <IterableDataSettings>();
            var baseIndexables  = dataSet.Data.Cast <IIndexable>();
            var totalIndexables = baseIndexables.Count();

            logger.Info("Indexable Items found: " + totalIndexables);
            if (totalIndexables <= 0)
            {
                logger.Debug("No indexable items found, quitting step.");
                return;
            }
            try
            {
                var indexHandle = ContentSearchManager.GetIndex(indexName);
                indexHandle.Initialize();
                var flatCrawler = new IterableDatasetCrawler <IIndexable>(baseIndexables);
                using (var updateContext = indexHandle.CreateUpdateContext())
                {
                    logger.Debug("Crawler Initialize...");
                    flatCrawler.Initialize(indexHandle);
                    logger.Debug("Index Rebuild...");
                    indexHandle.Rebuild(IndexingOptions.ForcedIndexing);
                    logger.Debug("Crawler Rebuild...");
                    flatCrawler.RebuildFromRoot(updateContext, IndexingOptions.Default);
                    updateContext.Commit();
                    logger.Debug("Rebuild Complete!");
                }
            }
            catch (Exception ex)
            {
                logger.Error(ex.Message);
            }
        }
        public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            var logger = pipelineContext.PipelineBatchContext.Logger;

            if (!this.CanProcess(pipelineStep, pipelineContext))
            {
                logger.Error("Pipeline step processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", pipelineStep.Name);
            }
            else
            {
                EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();
                if (endpointSettings == null)
                {
                    logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name,
                                 typeof(EndpointSettings).FullName);
                }
                else
                {
                    ReadResourceDataSettings readDataSettings = pipelineStep.GetReadResourceDataSettings();
                    if (readDataSettings == null)
                    {
                        this.Log(logger.Error, pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", string.Format("plugin: {0}", typeof(ReadResourceDataSettings).FullName));
                    }
                    else
                    {
                        Endpoint endpointFrom = endpointSettings.EndpointFrom;
                        if (endpointFrom == null)
                        {
                            logger.Error(
                                "Pipeline step processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})",
                                pipelineStep.Name, typeof(EndpointSettings).FullName, "EndpointFrom");
                        }
                        else if (!this.IsEndpointValid(endpointFrom, pipelineStep, pipelineContext))
                        {
                            logger.Error("Pipeline step processing will abort because the endpoint to read from is not valid. (pipeline step: {0}, endpoint: {1})",
                                         pipelineStep.Name,
                                         endpointFrom.Name);
                        }
                        else
                        {
                            logger.Info("Pipeline step reading data. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName);

                            var dataRead = Task.Run <bool>(async() => await this.ReadData(endpointFrom, pipelineStep, pipelineContext)).Result;

                            logger.Info("Pipeline context has iterable data? {0} (pipeline step: {1}, plugin: {2})", pipelineContext.HasIterableDataSettings(), pipelineStep.Name, typeof(EndpointSettings).FullName);
                        }
                    }
                }
            }
        }
Example #6
0
        public override bool CanProcess(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            var canProcess = false;
            var logger     = pipelineContext.PipelineBatchContext.Logger;

            if (base.CanProcess(pipelineStep, pipelineContext))
            {
                var endpointSettings = pipelineStep.GetEndpointSettings();
                var endpointFrom     = endpointSettings.EndpointFrom;
                if (endpointFrom == null)
                {
                    logger.Error("Pipeline processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(EndpointSettings).FullName, "EndpointFrom");
                }
                else if (IsEndpointValid(endpointFrom, pipelineStep, pipelineContext))
                {
                    var synchronizationSettings = pipelineContext.GetSynchronizationSettings();
                    if (synchronizationSettings.Source == null)
                    {
                        logger.Error("Pipeline processing will abort because the pipeline context has no source assigned. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(SynchronizationSettings).FullName, "Source");
                    }
                    else
                    {
                        var jsonServiceSettings = endpointFrom.GetPlugin <JsonServiceEndpointSettings>();
                        if (jsonServiceSettings.Host == null || jsonServiceSettings.Protocol == null)
                        {
                            logger.Error("No 'Host' or 'Protocol' is specified on the endpoint. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name);
                        }
                        else
                        {
                            var readJsonObjectsSettings = pipelineStep.GetPlugin <ReadJsonObjectsSettings>();
                            if (readJsonObjectsSettings.Api == null)
                            {
                                logger.Error("No 'Api' is specified on the reader. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name);
                            }
                            else
                            {
                                canProcess = true;
                            }
                        }
                    }
                }
            }
            if (!canProcess)
            {
                logger.Error("Pipeline processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", pipelineStep.Name);
                pipelineContext.CriticalError = true;
            }
            return(canProcess);
        }
Example #7
0
        protected override void ProcessPipelineStep(PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
        {
            EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();

            if (endpointSettings == null)
            {
                logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name,
                             typeof(EndpointSettings).FullName);
            }
            else
            {
                Endpoint endpointFrom = endpointSettings.EndpointFrom;
                ReadData(endpointFrom, pipelineStep, pipelineContext, logger);
            }
        }
        public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            if (pipelineStep == null)
            {
                throw new ArgumentNullException(nameof(pipelineStep));
            }
            if (pipelineContext == null)
            {
                throw new ArgumentNullException(nameof(pipelineContext));
            }
            var logger = pipelineContext.PipelineBatchContext.Logger;

            EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();

            var rssFeedSettings = endpointSettings.EndpointFrom.GetPlugin <RssFeedSettings>();

            if (rssFeedSettings != null)
            {
                try
                {
                    //Read rss feed and make them into an indexable item
                    var             indexableArticles = new List <RssIndexable>();
                    string          url    = rssFeedSettings.RssFeedUrl;
                    XmlReader       reader = XmlReader.Create(url);
                    SyndicationFeed feed   = SyndicationFeed.Load(reader);
                    reader.Close();
                    foreach (SyndicationItem item in feed.Items)
                    {
                        string publishDate  = item.PublishDate.ToString();
                        string subject      = item.Title.Text;
                        string summary      = item.Summary.Text;
                        string categories   = string.Join("/", item.Categories?.Select(c => c.Name));
                        string authors      = string.Join(", ", item.Authors?.Select(a => a.Name));
                        string baseUri      = item.BaseUri?.AbsoluteUri;
                        var    rssIndexable = new RssIndexable(publishDate, subject, summary, authors, categories, baseUri);
                        indexableArticles.Add(rssIndexable);
                    }
                    //Stuff indexables into pipeline batch
                    var dataSettings = new IterableDataSettings(indexableArticles);
                    pipelineContext.Plugins.Add(dataSettings);
                }
                catch (Exception ex)
                {
                    logger.Error("RSS Processor Failed: " + ex.Message);
                }
            }
        }
Example #9
0
        protected override ITroubleshooterResult Troubleshoot(PipelineStep pipelineStep, TroubleshooterContext context)
        {
            EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();

            if (endpointSettings == null)
            {
                return(TroubleshooterResult.FailResult(string.Format("Pipeline step is missing a plugin. \n Plugin: {0}", typeof(EndpointSettings).FullName)));
            }

            Endpoint endpointFrom = endpointSettings.EndpointFrom;

            if (endpointFrom == null)
            {
                return(TroubleshooterResult.FailResult(string.Format("Pipeline step is missing a value. \n Property: {0}", Templates.BaseEndpointPipelineStep.FieldNames.EndpointFrom)));
            }

            return(Task.Run(async() => await this.TroubleshootReadData(endpointFrom, pipelineStep, context)).Result);
        }
Example #10
0
        protected override void ProcessPipelineStep(PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
        {
            EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();

            if (endpointSettings == null)
            {
                logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name,
                             typeof(EndpointSettings).FullName);
            }
            else
            {
                ReadResourceDataSettings readDataSettings = pipelineStep.GetReadResourceDataSettings();
                if (readDataSettings == null)
                {
                    this.Log(logger.Error, pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", string.Format("plugin: {0}", typeof(ReadResourceDataSettings).FullName));
                }
                else
                {
                    Endpoint endpointFrom = endpointSettings.EndpointFrom;
                    if (endpointFrom == null)
                    {
                        logger.Error(
                            "Pipeline step processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})",
                            pipelineStep.Name, typeof(EndpointSettings).FullName, Templates.BaseEndpointPipelineStep.FieldNames.EndpointFrom);
                    }
                    else if (!this.IsEndpointValid(endpointFrom, pipelineStep, pipelineContext, logger))
                    {
                        logger.Error("Pipeline step processing will abort because the endpoint to read from is not valid. (pipeline step: {0}, endpoint: {1})",
                                     pipelineStep.Name,
                                     endpointFrom.Name);
                    }
                    else
                    {
                        logger.Info("Pipeline step reading data. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName);

                        var dataRead = Task.Run <bool>(async() => await this.ReadData(endpointFrom, pipelineStep, pipelineContext, logger)).Result;

                        logger.Info("Pipeline context has data? {0} (pipeline step: {1}, plugin: {2})", pipelineContext.HasIterableDataSettings(), pipelineStep.Name, typeof(EndpointSettings).FullName);
                    }
                }
            }
        }