public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
{
var logger = pipelineContext.PipelineBatchContext.Logger;
if (CanProcess(pipelineStep, pipelineContext))
{
var identifierValue = GetIdentifierValue(pipelineStep, pipelineContext);
if (string.IsNullOrWhiteSpace(identifierValue))
{
logger.Error("Pipeline step processing will abort because no identifier value was resolved. (pipeline step: {0})", (object)pipelineStep.Name);
}
else
{
var readJsonObjectsSettings = pipelineStep.GetPlugin <ReadJsonObjectsSettings>();
var endpoint = pipelineStep.GetEndpointSettings().EndpointFrom;
JObject jObject = ReadJsonData(endpoint, pipelineContext, readJsonObjectsSettings.Api, identifierValue);
var resolvedObject = ExtractObject(readJsonObjectsSettings, logger, jObject);
SaveResolvedObject(pipelineStep, pipelineContext, resolvedObject);
}
}
}
private IItemModelRepository GetItemModelRepository(PipelineStep pipelineStep)
{
var endpointSettings = pipelineStep.GetEndpointSettings();
var endpointTo = endpointSettings?.EndpointTo;
var repositorySettings = endpointTo?.GetItemModelRepositorySettings();
return(repositorySettings?.ItemModelRepository);
}
private IItemModelRepository GetItemModelRepository(PipelineStep pipelineStep, PipelineContext pipelineContext)
{
var endpointsSettings = pipelineStep.GetEndpointSettings();
if (endpointsSettings == null)
{
return(null);
}
var endpoint = endpointsSettings.EndpointTo;
var repositorySettings = endpoint?.GetItemModelRepositorySettings();
return(repositorySettings?.ItemModelRepository);
}
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
{
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
var logger = pipelineContext.PipelineBatchContext.Logger;
var endpointSettings = pipelineStep.GetEndpointSettings();
var endpointTo = endpointSettings.EndpointTo;
var indexSettings = endpointTo.GetPlugin <IndexSettings>();
var indexName = indexSettings.IndexName;
logger.Debug("Index Name: " + indexName);
var dataSet = pipelineContext.GetPlugin <IterableDataSettings>();
var baseIndexables = dataSet.Data.Cast <IIndexable>();
var totalIndexables = baseIndexables.Count();
logger.Info("Indexable Items found: " + totalIndexables);
if (totalIndexables <= 0)
{
logger.Debug("No indexable items found, quitting step.");
return;
}
try
{
var indexHandle = ContentSearchManager.GetIndex(indexName);
indexHandle.Initialize();
var flatCrawler = new IterableDatasetCrawler <IIndexable>(baseIndexables);
using (var updateContext = indexHandle.CreateUpdateContext())
{
logger.Debug("Crawler Initialize...");
flatCrawler.Initialize(indexHandle);
logger.Debug("Index Rebuild...");
indexHandle.Rebuild(IndexingOptions.ForcedIndexing);
logger.Debug("Crawler Rebuild...");
flatCrawler.RebuildFromRoot(updateContext, IndexingOptions.Default);
updateContext.Commit();
logger.Debug("Rebuild Complete!");
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
}
}
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
{
var logger = pipelineContext.PipelineBatchContext.Logger;
if (!this.CanProcess(pipelineStep, pipelineContext))
{
logger.Error("Pipeline step processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", pipelineStep.Name);
}
else
{
EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();
if (endpointSettings == null)
{
logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name,
typeof(EndpointSettings).FullName);
}
else
{
ReadResourceDataSettings readDataSettings = pipelineStep.GetReadResourceDataSettings();
if (readDataSettings == null)
{
this.Log(logger.Error, pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", string.Format("plugin: {0}", typeof(ReadResourceDataSettings).FullName));
}
else
{
Endpoint endpointFrom = endpointSettings.EndpointFrom;
if (endpointFrom == null)
{
logger.Error(
"Pipeline step processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})",
pipelineStep.Name, typeof(EndpointSettings).FullName, "EndpointFrom");
}
else if (!this.IsEndpointValid(endpointFrom, pipelineStep, pipelineContext))
{
logger.Error("Pipeline step processing will abort because the endpoint to read from is not valid. (pipeline step: {0}, endpoint: {1})",
pipelineStep.Name,
endpointFrom.Name);
}
else
{
logger.Info("Pipeline step reading data. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName);
var dataRead = Task.Run <bool>(async() => await this.ReadData(endpointFrom, pipelineStep, pipelineContext)).Result;
logger.Info("Pipeline context has iterable data? {0} (pipeline step: {1}, plugin: {2})", pipelineContext.HasIterableDataSettings(), pipelineStep.Name, typeof(EndpointSettings).FullName);
}
}
}
}
}
public override bool CanProcess(PipelineStep pipelineStep, PipelineContext pipelineContext)
{
var canProcess = false;
var logger = pipelineContext.PipelineBatchContext.Logger;
if (base.CanProcess(pipelineStep, pipelineContext))
{
var endpointSettings = pipelineStep.GetEndpointSettings();
var endpointFrom = endpointSettings.EndpointFrom;
if (endpointFrom == null)
{
logger.Error("Pipeline processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(EndpointSettings).FullName, "EndpointFrom");
}
else if (IsEndpointValid(endpointFrom, pipelineStep, pipelineContext))
{
var synchronizationSettings = pipelineContext.GetSynchronizationSettings();
if (synchronizationSettings.Source == null)
{
logger.Error("Pipeline processing will abort because the pipeline context has no source assigned. (pipeline step: {0}, plugin: {1}, property: {2})", pipelineStep.Name, typeof(SynchronizationSettings).FullName, "Source");
}
else
{
var jsonServiceSettings = endpointFrom.GetPlugin <JsonServiceEndpointSettings>();
if (jsonServiceSettings.Host == null || jsonServiceSettings.Protocol == null)
{
logger.Error("No 'Host' or 'Protocol' is specified on the endpoint. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name);
}
else
{
var readJsonObjectsSettings = pipelineStep.GetPlugin <ReadJsonObjectsSettings>();
if (readJsonObjectsSettings.Api == null)
{
logger.Error("No 'Api' is specified on the reader. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpointFrom.Name);
}
else
{
canProcess = true;
}
}
}
}
}
if (!canProcess)
{
logger.Error("Pipeline processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", pipelineStep.Name);
pipelineContext.CriticalError = true;
}
return(canProcess);
}
protected override void ProcessPipelineStep(PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
{
EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();
if (endpointSettings == null)
{
logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name,
typeof(EndpointSettings).FullName);
}
else
{
Endpoint endpointFrom = endpointSettings.EndpointFrom;
ReadData(endpointFrom, pipelineStep, pipelineContext, logger);
}
}
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
{
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
var logger = pipelineContext.PipelineBatchContext.Logger;
EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();
var rssFeedSettings = endpointSettings.EndpointFrom.GetPlugin <RssFeedSettings>();
if (rssFeedSettings != null)
{
try
{
//Read rss feed and make them into an indexable item
var indexableArticles = new List <RssIndexable>();
string url = rssFeedSettings.RssFeedUrl;
XmlReader reader = XmlReader.Create(url);
SyndicationFeed feed = SyndicationFeed.Load(reader);
reader.Close();
foreach (SyndicationItem item in feed.Items)
{
string publishDate = item.PublishDate.ToString();
string subject = item.Title.Text;
string summary = item.Summary.Text;
string categories = string.Join("/", item.Categories?.Select(c => c.Name));
string authors = string.Join(", ", item.Authors?.Select(a => a.Name));
string baseUri = item.BaseUri?.AbsoluteUri;
var rssIndexable = new RssIndexable(publishDate, subject, summary, authors, categories, baseUri);
indexableArticles.Add(rssIndexable);
}
//Stuff indexables into pipeline batch
var dataSettings = new IterableDataSettings(indexableArticles);
pipelineContext.Plugins.Add(dataSettings);
}
catch (Exception ex)
{
logger.Error("RSS Processor Failed: " + ex.Message);
}
}
}
protected override ITroubleshooterResult Troubleshoot(PipelineStep pipelineStep, TroubleshooterContext context)
{
EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();
if (endpointSettings == null)
{
return(TroubleshooterResult.FailResult(string.Format("Pipeline step is missing a plugin. \n Plugin: {0}", typeof(EndpointSettings).FullName)));
}
Endpoint endpointFrom = endpointSettings.EndpointFrom;
if (endpointFrom == null)
{
return(TroubleshooterResult.FailResult(string.Format("Pipeline step is missing a value. \n Property: {0}", Templates.BaseEndpointPipelineStep.FieldNames.EndpointFrom)));
}
return(Task.Run(async() => await this.TroubleshootReadData(endpointFrom, pipelineStep, context)).Result);
}
protected override void ProcessPipelineStep(PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
{
EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings();
if (endpointSettings == null)
{
logger.Error("Pipeline step processing will abort because the pipeline step is missing a plugin. (pipeline step: {0}, plugin: {1})", pipelineStep.Name,
typeof(EndpointSettings).FullName);
}
else
{
ReadResourceDataSettings readDataSettings = pipelineStep.GetReadResourceDataSettings();
if (readDataSettings == null)
{
this.Log(logger.Error, pipelineContext, "Pipeline step processing will abort because the pipeline step is missing a plugin.", string.Format("plugin: {0}", typeof(ReadResourceDataSettings).FullName));
}
else
{
Endpoint endpointFrom = endpointSettings.EndpointFrom;
if (endpointFrom == null)
{
logger.Error(
"Pipeline step processing will abort because the pipeline step is missing an endpoint to read from. (pipeline step: {0}, plugin: {1}, property: {2})",
pipelineStep.Name, typeof(EndpointSettings).FullName, Templates.BaseEndpointPipelineStep.FieldNames.EndpointFrom);
}
else if (!this.IsEndpointValid(endpointFrom, pipelineStep, pipelineContext, logger))
{
logger.Error("Pipeline step processing will abort because the endpoint to read from is not valid. (pipeline step: {0}, endpoint: {1})",
pipelineStep.Name,
endpointFrom.Name);
}
else
{
logger.Info("Pipeline step reading data. (pipeline step: {0}, plugin: {1})", pipelineStep.Name, typeof(EndpointSettings).FullName);
var dataRead = Task.Run <bool>(async() => await this.ReadData(endpointFrom, pipelineStep, pipelineContext, logger)).Result;
logger.Info("Pipeline context has data? {0} (pipeline step: {1}, plugin: {2})", pipelineContext.HasIterableDataSettings(), pipelineStep.Name, typeof(EndpointSettings).FullName);
}
}
}
}