protected override void ReadData(Sitecore.DataExchange.Models.Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { // do some null checks Assert.ArgumentNotNull(endpoint, nameof(endpoint)); Assert.ArgumentNotNull(pipelineStep, nameof(pipelineStep)); Assert.ArgumentNotNull(pipelineContext, nameof(pipelineContext)); Assert.ArgumentNotNull(logger, nameof(logger)); // get the configuration values for the endpoint var endpointSettings = endpoint.GetPlugin <JsonApiSettings>(); if (endpointSettings == null) { return; } if (string.IsNullOrEmpty(endpointSettings.ApiUrl)) { logger.Error($"No API URL is specified on the endpoint. (pipeline step: { pipelineStep.Name }, endpoint: { endpoint.Name }"); } // get the configuration values for the processor step var pipelineStepSettings = pipelineStep.GetPlugin <ReadPaginatedJsonApiStepSettings>(); if (pipelineStepSettings == null) { return; } logger.Debug("Executing pipeline step: ", $"MaxCount: { pipelineStepSettings.MaxCount }", $"ResultsPerPage: { pipelineStepSettings.ResultsPerPage }", $"Page: { pipelineStepSettings.Page }", $"Offset: { pipelineStepSettings.Offset }"); // execute the API to retrieve the data var uri = new UriBuilder(endpointSettings.ApiUrl); var query = HttpUtility.ParseQueryString(uri.Query); query["per_page"] = pipelineStepSettings.ResultsPerPage.ToString(); uri.Query = query.ToString(); var data = new List <JObject>(); logger.Debug($"Executing API call { uri.Uri.AbsoluteUri }. "); var batch = GetDataAsync(uri.Uri.AbsoluteUri).GetAwaiter().GetResult().ToList(); var page = pipelineStepSettings.Page; while (batch.Any() && (pipelineStepSettings.MaxCount < 0 || pipelineStepSettings.MaxCount > 0 && data.Count < pipelineStepSettings.MaxCount)) { data.AddRange(batch); page++; query["page"] = page.ToString(); uri.Query = query.ToString(); logger.Debug($"Executing API call { uri.Uri.AbsoluteUri }. "); batch = GetDataAsync(uri.Uri.AbsoluteUri).GetAwaiter().GetResult().ToList(); logger.Debug($"Retrieved { batch.Count } items."); } // add this data as a plugin to the pipeline context var dataSettings = new IterableDataSettings(data); pipelineContext.AddPlugin(dataSettings); }
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } try { var settings = endpoint.GetWordpressSettings(); if (settings == null) { logger.Error("Empty WordPress settings"); return; } List <Category> categories = _wordpressService.Read <Category>(settings.CategoriesUrl, logger); var categoriesData = new IterableDataSettings(categories); pipelineContext.AddPlugin(categoriesData); } catch (Exception ex) { logger.Error($"Error in ReadCategoriesStepProcessor: {ex.InnerException}"); pipelineContext.CriticalError = true; } }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException("endpoint"); } if (pipelineStep == null) { throw new ArgumentNullException("pipelinestep"); } if (pipelineContext == null) { throw new ArgumentNullException("pipelineContext"); } var logger = pipelineContext.PipelineBatchContext.Logger; // //get the file path from the plugin on the endpoint var settings = endpoint.GetDatabaseSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.ConnectionString)) { logger.Error( "No connection string is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } var querySettings = pipelineStep.GetPlugin <QuerySettings>(); if (string.IsNullOrWhiteSpace(querySettings.Query)) { logger.Error( "No query has been configured" + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } SynchronizationSettings synchronizationSettings = pipelineContext.GetSynchronizationSettings(); Dictionary <string, string> source = null; if (synchronizationSettings != null) { source = synchronizationSettings.Source as Dictionary <string, string>; } ////add the data that was read from the file to a plugin var dataSettings = new IterableDataSettings(GetEnumerable(settings.ConnectionType, settings.ConnectionString, querySettings.Query, source, logger)); pipelineContext.Plugins.Add(dataSettings); }
private void RedditFeed(PipelineContext pipelineContext, string blogpath) { var reddit = new Reddit(); //var subreddit = reddit.(blogpath); var subreddit = reddit.GetSubreddit(blogpath); var dataSettings = new IterableDataSettings(subreddit.New.Take(25)); pipelineContext.AddPlugin(dataSettings); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } // //get the file path from the plugin on the endpoint var settings = endpoint.GetTextFileSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.Path)) { logger.Error( "No path is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } if (!File.Exists(settings.Path)) { logger.Error( "The path specified on the endpoint does not exist. " + "(pipeline step: {0}, endpoint: {1}, path: {2})", pipelineStep.Name, endpoint.Name, settings.Path); return; } // //add the data that was read from the file to a plugin var data = this.GetIterableData(settings); var dataSettings = new IterableDataSettings(data); // //add the plugin to the pipeline context pipelineContext.AddPlugin(dataSettings); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException("endpoint"); } if (pipelineStep == null) { throw new ArgumentNullException("pipelineStep"); } if (pipelineContext == null) { throw new ArgumentNullException("pipelineContext"); } var logger = pipelineContext.PipelineBatchContext.Logger; // //get the file path from the plugin on the endpoint var settings = endpoint.GetFileSystemSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.Path)) { logger.Error( "No path is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } //not a file or directory if (!System.IO.File.Exists(settings.Path) && !Directory.Exists(settings.Path)) { logger.Error( "The path specified on the endpoint does not exist. " + "(pipeline step: {0}, endpoint: {1}, path: {2})", pipelineStep.Name, endpoint.Name, settings.Path); return; } var dataSettings = new IterableDataSettings(GetEnumerable(settings, logger)); logger.Info( "path: {0} was processed.. (pipeline step: {1}, endpoint: {2})", settings.Path, pipelineStep.Name, endpoint.Name); // //add the plugin to the pipeline context pipelineContext.Plugins.Add(dataSettings); }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { var logger = pipelineContext.PipelineBatchContext.Logger; if (!this.CanProcess(pipelineStep, pipelineContext)) { logger.Error("Pipeline step processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", (object)pipelineStep.Name); return; } var settings = pipelineStep.GetPlugin <GroupIterableDataSettings>(); if (settings == null) { logger.Error("Cannot access ClearFacetCollectionSettings. (pipeline step: {0})", (object)pipelineStep.Name); return; } IterableDataSettings iterableDataSettings = pipelineContext.GetIterableDataSettings(); if (iterableDataSettings == null || iterableDataSettings.Data == null) { return; } var groupedData = new GroupedDataSettings(); foreach (object element in iterableDataSettings.Data) { var record = element as Dictionary <string, string>; if (record != null) { if (record.ContainsKey(settings.GroupFieldKey)) { var key = record[settings.GroupFieldKey]; if (!groupedData.Data.ContainsKey(key)) { groupedData.Data.Add(key, new List <Dictionary <string, string> >()); } groupedData.Data[key].Add(record); } } } pipelineContext.Plugins.Add(groupedData); if (settings.RemoveIterableSettingsPlugin) { // This should be checked if another pipeline step is going to add settings. pipelineContext.Plugins.Remove(iterableDataSettings); } }
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { var endpointSettings = endpoint.GetPlugin <FileSettings>(); if (string.IsNullOrWhiteSpace(endpointSettings.FilePath)) { logger.Error("File name should not be empty"); return; } var jsonSettings = pipelineStep.GetPlugin <JsonSettings>(); if ((jsonSettings == null) || (string.IsNullOrWhiteSpace(jsonSettings.JPath))) { logger.Error("Cannot read JToken settings"); return; } IEnumerable jarray; using (StreamReader file = System.IO.File.OpenText(endpointSettings.FilePath)) using (JsonTextReader reader = new JsonTextReader(file)) { JToken o2 = JToken.ReadFrom(reader); logger.Info("File was read"); JToken token = o2.SelectToken(jsonSettings.JPath); if (!(token is JArray)) { logger.Error("Element specified is not a JSon Array"); return; } jarray = (JArray)token; } if (!(jarray is IEnumerable)) { logger.Error("Trying to read object, that is not an IEnumerable"); return; } logger.Info("Data were read from file"); var dataSettings = new IterableDataSettings() { Data = jarray }; pipelineContext.AddPlugin(dataSettings); }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var logger = pipelineContext.PipelineBatchContext.Logger; EndpointSettings endpointSettings = pipelineStep.GetEndpointSettings(); var rssFeedSettings = endpointSettings.EndpointFrom.GetPlugin <RssFeedSettings>(); if (rssFeedSettings != null) { try { //Read rss feed and make them into an indexable item var indexableArticles = new List <RssIndexable>(); string url = rssFeedSettings.RssFeedUrl; XmlReader reader = XmlReader.Create(url); SyndicationFeed feed = SyndicationFeed.Load(reader); reader.Close(); foreach (SyndicationItem item in feed.Items) { string publishDate = item.PublishDate.ToString(); string subject = item.Title.Text; string summary = item.Summary.Text; string categories = string.Join("/", item.Categories?.Select(c => c.Name)); string authors = string.Join(", ", item.Authors?.Select(a => a.Name)); string baseUri = item.BaseUri?.AbsoluteUri; var rssIndexable = new RssIndexable(publishDate, subject, summary, authors, categories, baseUri); indexableArticles.Add(rssIndexable); } //Stuff indexables into pipeline batch var dataSettings = new IterableDataSettings(indexableArticles); pipelineContext.Plugins.Add(dataSettings); } catch (Exception ex) { logger.Error("RSS Processor Failed: " + ex.Message); } } }
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } var formDataProvider = (IFormDataProvider)Sitecore.DependencyInjection.ServiceLocator.ServiceProvider.GetService(typeof(IFormDataProvider)); if (formDataProvider == null) { throw new Exception("IFormDataProvider is missing"); } var settings = endpoint.GetFormsSettings(); if (settings == null) { return; } var readDataSettings = pipelineStep.GetPlugin <ReadDataSettings>(); if (readDataSettings == null || readDataSettings.FormID == Guid.Empty) { logger.Error("Form is not selected for Read Form Entries Step Processor"); return; } var data = this.GetIterableData(settings, readDataSettings, formDataProvider); var dataSettings = new IterableDataSettings(data); pipelineContext.AddPlugin(dataSettings); }
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { var logger = pipelineContext.PipelineBatchContext.Logger; var readJsonObjectsSettings = pipelineStep.GetPlugin <ReadJsonObjectsSettings>(); //read data JToken jToken = ReadJsonData(endpoint, pipelineContext, readJsonObjectsSettings.Api); //extract array JArray result = ExtractArray(readJsonObjectsSettings, logger, jToken); logger.Info("{0} json objects were read from endpoint. (pipeline step: {1}, endpoint: {2})", result?.Count ?? 0, pipelineStep.Name, endpoint.Name); //add the data that was read to a plugin var dataSettings = new IterableDataSettings(result); //add the plugin to the pipeline context pipelineContext.Plugins.Add(dataSettings); }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { ILogger logger = pipelineContext.PipelineBatchContext.Logger; if (!this.CanProcess(pipelineStep, pipelineContext)) { logger.Error("Pipeline step processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", (object)pipelineStep.Name); } else { PipelinesSettings pipelinesSettings = pipelineStep.GetPipelinesSettings(); if (pipelinesSettings == null || !pipelinesSettings.Pipelines.Any <Pipeline>()) { logger.Error("Pipeline step processing will abort because the pipeline step has no sub-pipelines assigned. (pipeline step: {0})", (object)pipelineStep.Name); } else { IterableDataSettings iterableDataSettings = pipelineContext.GetIterableDataSettings(); if (iterableDataSettings == null || iterableDataSettings.Data == null) { return; } int num = 0; try { var childRecordSettings = new ChildRecordSettings(); foreach (object element in iterableDataSettings.Data) { if (!pipelineContext.PipelineBatchContext.Stopped) { PipelineContext pipelineContext1 = new PipelineContext(pipelineContext.PipelineBatchContext); SynchronizationSettings synchronizationSettings = this.ResolveSynchronizationSettingsAndSetElement(pipelineStep, pipelineContext, element); //instatiate Target for filling. synchronizationSettings.Target = new Dictionary <string, string>(); pipelineContext1.Plugins.Add((IPlugin)synchronizationSettings); ParentPipelineContextSettings pipelineContextSettings = new ParentPipelineContextSettings() { ParentPipelineContext = pipelineContext }; pipelineContext1.Plugins.Add((IPlugin)pipelineContextSettings); this.ProcessPipelines(pipelineStep, pipelinesSettings.Pipelines, pipelineContext1); //Now Let's get the results var record = pipelineContext1.GetPlugin <SynchronizationSettings>().Target as Dictionary <string, string>; if (record != null) { childRecordSettings.Records.Add(record); } } num++; } //Add to the context so we can do something with this collection pipelineContext.Plugins.Add(childRecordSettings); logger.Info("{0} elements were iterated. (pipeline: {1}, pipeline step: {2})", (object)num, (object)pipelineContext.CurrentPipeline.Name, (object)pipelineContext.CurrentPipelineStep.Name, (object)pipelineContext); } catch (Exception ex) { logger.Error(ex.Message); logger.Error(ex.StackTrace); pipelineContext.CriticalError = true; } } } }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { ILogger logger = pipelineContext.PipelineBatchContext.Logger; if (!this.CanProcess(pipelineStep, pipelineContext)) { logger.Error("Pipeline step processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", (object)pipelineStep.Name); } else { PipelinesSettings pipelinesSettings = pipelineStep.GetPipelinesSettings(); if (pipelinesSettings == null || !pipelinesSettings.Pipelines.Any <Pipeline>()) { logger.Error("Pipeline step processing will abort because the pipeline step has no sub-pipelines assigned. (pipeline step: {0})", (object)pipelineStep.Name); } else { IterableDataSettings iterableDataSettings = pipelineContext.GetIterableDataSettings(); if (iterableDataSettings == null || iterableDataSettings.Data == null) { return; } int num = 0; try { int maxThreads = 2; int.TryParse(Sitecore.Configuration.Settings.GetSetting("SF.DEF.General.MaxThreads"), out maxThreads); //only set min threads if it's configured in the envrionment. int minThreads = 2; if (int.TryParse(Sitecore.Configuration.Settings.GetSetting("SF.DEF.General.MionThreads"), out minThreads)) { System.Threading.ThreadPool.SetMinThreads(minThreads, minThreads); } logger.Info("Starting Async Processor with {0} threads", maxThreads); Parallel.ForEach(iterableDataSettings.Data.Cast <object>(), new ParallelOptions { MaxDegreeOfParallelism = maxThreads }, (element) => { if (!pipelineContext.PipelineBatchContext.Stopped) { PipelineContext pipelineContext1 = new PipelineContext(pipelineContext.PipelineBatchContext); SynchronizationSettings synchronizationSettings = this.ResolveSynchronizationSettingsAndSetElement(pipelineStep, pipelineContext, element); pipelineContext1.Plugins.Add((IPlugin)synchronizationSettings); ParentPipelineContextSettings pipelineContextSettings = new ParentPipelineContextSettings() { ParentPipelineContext = pipelineContext }; pipelineContext1.Plugins.Add((IPlugin)pipelineContextSettings); this.ProcessPipelines(pipelineStep, pipelinesSettings.Pipelines, pipelineContext1); } num++; }); logger.Info("{0} elements were iterated. (pipeline: {1}, pipeline step: {2})", (object)num, (object)pipelineContext.CurrentPipeline.Name, (object)pipelineContext.CurrentPipelineStep.Name, (object)pipelineContext); } catch (Exception ex) { logger.Error(ex.Message); logger.Error(ex.StackTrace); pipelineContext.CriticalError = true; } } } }
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException(); } if (pipelineStep == null) { throw new ArgumentNullException(); } if (pipelineContext == null) { throw new ArgumentNullException(); } var logger = pipelineContext.PipelineBatchContext.Logger; // //get the file path from the plugin on the endpoint var settings = endpoint.GetXMLFileSettings(); if (settings == null) { logger.Error("No xml file settings are specified on the endpoint. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } if (string.IsNullOrWhiteSpace(settings.Path)) { logger.Error("No path is specified on the endpoint. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } // //if the path is relative, the base directory is used to build an //absolute path so that when this code runs on the Sitecore server, //relative paths will be based on the webroot var path = settings.Path; if (!Path.IsPathRooted(path)) { path = string.Format("{0}{1}", AppDomain.CurrentDomain.BaseDirectory, path); } // if (!File.Exists(path)) { logger.Error("The path specified on the endpoint does not exist. (pipeline step: {0}, endpoint: {1}, path: {2})", pipelineStep.Name, endpoint.Name, path); return; } // //read the file, one line at a time List <string[]> result = new List <string[]>(); List <XElement> XMLChildNodes = new List <XElement>(); var ChildNodes = settings.ChildNodes.Split(',').ToList(); foreach (string childnode in ChildNodes) { XMLChildNodes.Add(new XElement(childnode)); } foreach (XElement level1Element in XElement.Load(settings.Path).Elements(settings.RootNode)) { //result.AppendLine(level1Element.Attribute("name").Value); foreach (XElement level2Element in XMLChildNodes) { var arr = new string[] { level2Element.Value }; result.Add(arr); } } //add the data that was read from the file to a plugin var dataSettings = new IterableDataSettings(result); logger.Info("{0} rows were read from the file. (pipeline step: {1}, endpoint: {2})", result.Count, pipelineStep.Name, endpoint.Name); // //add the plugin to the pipeline context pipelineContext.Plugins.Add(dataSettings); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var logger = pipelineContext.PipelineBatchContext.Logger; // //get the file path from the plugin on the endpoint var settings = endpoint.GetXMLSystemSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.XMLPath)) { logger.Error( "No path is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } // //if the path is relative, the base directory is used to build an //absolute path so that when this code runs on the Sitecore server, //relative paths will be based on the webroot var path = settings.XMLPath; Uri uriResult; bool result = Uri.TryCreate(path, UriKind.Absolute, out uriResult) && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); if (!result) { if (!Path.IsPathRooted(path)) { path = string.Format("{0}{1}", AppDomain.CurrentDomain.BaseDirectory, path); } // if (!File.Exists(path)) { logger.Error( "The path specified on the endpoint does not exist. " + "(pipeline step: {0}, endpoint: {1}, path: {2})", pipelineStep.Name, endpoint.Name, path); return; } } var lines = new List <string[]>(); XmlDocument document = new XmlDocument(); document.Load(path); XmlNodeList xmlNodeList = document.GetElementsByTagName(settings.XMLNodeName); XmlNode[] nodeArray = xmlNodeList.Cast <XmlNode>().ToArray(); for (int i = 0; i < xmlNodeList.Count; i++) { List <string> strs = new List <string>(); for (int j = 0; j < xmlNodeList[i].ChildNodes.Count; j++) { strs.Add(xmlNodeList[i].ChildNodes[j].InnerText); } lines.Add(strs.ToArray()); } // //add the data that was read from the xml file to a plugin var dataSettings = new IterableDataSettings(nodeArray); logger.Info( "{0} rows were read from the file. (pipeline step: {1}, endpoint: {2})", lines.Count, pipelineStep.Name, endpoint.Name); // //add the plugin to the pipeline context pipelineContext.Plugins.Add(dataSettings); }
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } var settings = endpoint.GetServiceBusSettings(); if (settings == null) { logger.Error("Unable to read data, there are no settings set in the ednpoint."); return; } // Check the endpoint settings if (settings.MaxNumberOfMessages <= 0 || settings.MaxNumberOfMessages > MaximumNumberOfMessages) { logger.Error($"Invalid settings. {nameof(settings.MaxNumberOfMessages)} have to be between 0 and {MaximumNumberOfMessages}. (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})"); return; } if (settings.BatchSize <= 0 || settings.BatchSize > MaximumServiceBusMessagesBatchSize) { logger.Error($"Invalid settings. {nameof(settings.BatchSize)} have to be between 0 and {MaximumServiceBusMessagesBatchSize}. (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})"); return; } if (settings.BatchSize > settings.MaxNumberOfMessages) { logger.Error($"Invalid settings. {nameof(settings.BatchSize)} ({settings.BatchSize}) have to be <= {nameof(settings.MaxNumberOfMessages)} ({settings.MaxNumberOfMessages}). (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})"); return; } // Create the ServiceBus Reciever ServiceBusReceiver messageReceiver; try { var serviceBusService = new ServiceBusService(); messageReceiver = new ServiceBusReceiver(MessagingFactory.CreateFromConnectionString( settings.ConnectionString), settings.Topic, serviceBusService.GetSubscriptionName(settings.Market, settings.Sender), logger); } catch (Exception exception) { logger.Error($"Unable to connect with ServiceBus ({exception.Message}). Please run the Troubleshooter to propper set the connection first."); return; } // read messages from the ServiceBus (in batches, up to the maximum number of messages) var dataList = RecieveMessages(settings, messageReceiver, logger).ToList(); if (!dataList.Any()) { logger.Info("No data read."); return; } logger.Debug($"{dataList.Count} messages read from ServiceBus."); var dataSettings = new IterableDataSettings(dataList); // //add the plugin to the pipeline context pipelineContext.AddPlugin(dataSettings); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException("Endpoint"); } if (pipelineStep == null) { throw new ArgumentNullException("PipelineStep"); } if (pipelineContext == null) { throw new ArgumentNullException("PipelineContext"); } var logger = pipelineContext.PipelineBatchContext.Logger; //get the rss feed url from the plugin on the endpoint var settings = endpoint.GetRssSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.FeedUrl)) { logger.Error( "No feed url is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } var lines = new List <string[]>(); //todo, fetch the RSS feed and then put into the context. try { var rssReader = XmlReader.Create(settings.FeedUrl); var feed = SyndicationFeed.Load(rssReader); //add the data that was read from the file to a plugin var dataSettings = new IterableDataSettings(GetFeedEnumerable(feed)); logger.Info( "{0} was loaded. (pipeline step: {1}, endpoint: {2})", settings.FeedUrl, pipelineStep.Name, endpoint.Name); // //add the plugin to the pipeline context pipelineContext.Plugins.Add(dataSettings); } catch (Exception ex) { logger.Error( "Unable to download and parse RSS feed" + "(pipeline step: {0}, endpoint: {1}, feedUrl: {2}, error: {3})", pipelineStep.Name, endpoint.Name, settings.FeedUrl, ex); } }
protected async Task <bool> ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var logger = pipelineContext.PipelineBatchContext.Logger; var repositorySettings = Context.GetPlugin <RepositorySettings>(); if (repositorySettings == null) { logger.Error("No repository settings plugin is specified on the context (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (repositorySettings.Client == null) { logger.Error("No client is specified on the repository settings (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } var applicationEndpointSettings = endpoint.GetApplicationEndpointSettings(); var applicationSettings = (ApplicationSettings)applicationEndpointSettings?.Application?.RefreshPlugin.Invoke(); if (applicationSettings == null) { logger.Error("No application is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(applicationSettings.BaseUrl)) { logger.Error("No Base Url is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(applicationSettings.AccessToken)) { logger.Warn("No access token is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); //return false; } var resourceSettings = pipelineStep.GetResourceSettings(); if (resourceSettings == null) { logger.Error("No resource is specified on the pipeline step (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(resourceSettings.Url)) { logger.Error("No url is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(resourceSettings.Method)) { logger.Error("No method is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } var readDataSettings = pipelineStep.GetReadResourceDataSettings(); if (readDataSettings == null || string.IsNullOrWhiteSpace(readDataSettings.PathExpression)) { logger.Error("No path expression is specified on the pipeline step. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } var iterableData = new JArray(); bool hasMore; do { hasMore = false; var response = await repositorySettings.Client.SendAsync(applicationSettings, resourceSettings); var json = await response.Content.ReadAsStringAsync(); var jObject = JsonConvert.DeserializeObject <JObject>(json); if (jObject == null) { logger.Debug("No data returned from request. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); } else { var jArray = (JArray)jObject.SelectToken(readDataSettings.PathExpression, false); if (jArray == null) { logger.Debug("No data returned from path expression. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); } else if (jArray.Count == 0) { logger.Info("No items returned from request. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); } else { logger.Info("{0} rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", jArray.Count, pipelineStep.Name, endpoint.Name); iterableData.Merge(jArray); if (resourceSettings.Paging != null) { if (!string.IsNullOrEmpty(resourceSettings.Paging.NextTokenPathExpression)) { var nextToken = jObject.SelectToken(resourceSettings.Paging.NextTokenPathExpression, false); resourceSettings.Paging.NextToken = nextToken?.Value <string>(); hasMore = !string.IsNullOrEmpty(nextToken?.Value <string>()); } else { var pageToken = jObject.SelectToken(resourceSettings.Paging.CurrentPagePathExpression, false); var pageSizeToken = jObject.SelectToken(resourceSettings.Paging.PageSizePathExpression, false); var totalCountToken = jObject.SelectToken(resourceSettings.Paging.TotalCountPathExpression, false); var page = pageToken?.Value <int?>() ?? 0; var pageSize = pageSizeToken?.Value <int?>() ?? resourceSettings.Paging.PageSize; var totalCount = totalCountToken?.Value <int?>() ?? int.MinValue; var maxCount = resourceSettings.Paging.MaximumCount; resourceSettings.Paging.Page = page + 1; resourceSettings.Paging.PageSize = pageSize; resourceSettings.Paging.TotalCount = totalCount; hasMore = page * pageSize > 0 && page * pageSize < totalCount && page * pageSize < maxCount; } } } } } while (resourceSettings.Paging != null && hasMore); logger.Info("{0} total rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", iterableData.Count, pipelineStep.Name, endpoint.Name); var dataSettings = new IterableDataSettings(iterableData); pipelineContext.Plugins.Add(dataSettings); return(true); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var logger = pipelineContext.PipelineBatchContext.Logger; // //get the file path from the plugin on the endpoint var settings = endpoint.GetDropboxSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.ApplicationName)) { logger.Error( "No application name is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } if (string.IsNullOrWhiteSpace(settings.AccessToken)) { logger.Error( "No access token name is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } if (string.IsNullOrWhiteSpace(settings.RootPath)) { logger.Error( "No root path is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } var dropboxRepository = new DropBoxRepository(); var dropboxFiles = dropboxRepository.ReadAll(settings); // //add the data that was read from the file to a plugin var dataSettings = new IterableDataSettings(dropboxFiles); logger.Info( "{0} rows were read from the file. (pipeline step: {1}, endpoint: {2})", dropboxFiles.Count(), pipelineStep.Name, endpoint.Name); SitecoreItemUtilities sitecoreItemUtility = new SitecoreItemUtilities() { IsItemNameValid = (string x) => ItemUtil.IsItemNameValid(x), ProposeValidItemName = (string x) => ItemUtil.ProposeValidItemName(x) }; Context.Plugins.Add(sitecoreItemUtility); //add the plugin to the pipeline context pipelineContext.Plugins.Add(dataSettings); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } // //get the file path from the plugin on the endpoint var settings = endpoint.GetPlugin <JsonApiEndpointSettings>(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.EndpointURL)) { logger.Error( "No endpoint URL is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } Uri validatedUri = null; if (!Uri.TryCreate(settings.EndpointURL, UriKind.RelativeOrAbsolute, out validatedUri)) { logger.Error( "The endpoint URL is not a valid URL structure. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } // //add the data that was read from the file to a plugin var data = this.GetIterableData(settings, logger); //foreach(var d in data) //{ // logger.Info($"ReadFromJsonApiStepProcessor.GetIterableData {d["id"]}"); //} var dataSettings = new IterableDataSettings(data); // //add the plugin to the pipeline context pipelineContext.AddPlugin(dataSettings); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var logger = pipelineContext.PipelineBatchContext.Logger; // //get the file path from the plugin on the endpoint var settings = endpoint.GetXmlFileSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.Path)) { logger.Error( "No path is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } List <string> xmlFilePath = Directory.GetFiles(settings.Path, "*.xml", SearchOption.AllDirectories).ToList(); if (xmlFilePath == null || xmlFilePath.Count() <= 0) { logger.Error( "The path specified on the endpoint does not have any xml file. " + "(pipeline step: {0}, endpoint: {1}, path: {2})", pipelineStep.Name, endpoint.Name, settings.Path); return; } var newsNodes = new List <XmlNode>(); XmlNamespaceManager xmlNameSpaceMgr; xmlNameSpaceMgr = new XmlNamespaceManager(new XmlDocument().NameTable); xmlNameSpaceMgr.AddNamespace("jcr", "http://www.jcp.org/jcr/1.0"); foreach (string path in xmlFilePath) { XmlDocument xdoc = new XmlDocument(); xdoc.Load(File.OpenRead(path)); var catNodes = xdoc.SelectNodes(settings.XmlRoot, xmlNameSpaceMgr); XmlAttribute attr = xdoc.CreateAttribute("filePath"); attr.Value = path; string itemName = Path.GetFileName(Path.GetDirectoryName(path)); itemName = itemName.Trim('-'); itemName = itemName.Trim(' '); //^[\w\*\$][\w\s\-\$] * (\(\d{ 1,}\)){ 0,1}$ XmlAttribute attrItemName = xdoc.CreateAttribute("itemName"); attrItemName.Value = itemName.Trim(); foreach (XmlNode node in catNodes) { node.Attributes.SetNamedItem(attr); node.Attributes.SetNamedItem(attrItemName); newsNodes.Add(node); } } var dataSettings = new IterableDataSettings(newsNodes); pipelineContext.Plugins.Add(dataSettings); }
protected override void ReadData( Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext) { if (endpoint == null) { throw new ArgumentNullException("endpoint"); } if (pipelineStep == null) { throw new ArgumentNullException("pipelinestep"); } if (pipelineContext == null) { throw new ArgumentNullException("pipelineContext"); } var logger = pipelineContext.PipelineBatchContext.Logger; // //get the file path from the plugin on the endpoint var settings = endpoint.GetExcelSettings(); if (settings == null) { return; } if (string.IsNullOrWhiteSpace(settings.FileLocation)) { logger.Error( "No File Location is specified on the endpoint. " + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } var excelSettings = pipelineStep.GetPlugin <ReadExcelTabSettings>(); if (string.IsNullOrWhiteSpace(excelSettings.Sheet)) { logger.Error( "No tab has been configured" + "(pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return; } SynchronizationSettings synchronizationSettings = pipelineContext.GetSynchronizationSettings(); Dictionary <string, string> source = null; if (synchronizationSettings != null) { source = synchronizationSettings.Source as Dictionary <string, string>; } try { ////add the data that was read from the file to a plugin var dataSettings = new IterableDataSettings(GetEnumerable(settings.FileLocation, excelSettings.Sheet, excelSettings.FirstRowHasColumnNames, logger)); pipelineContext.Plugins.Add(dataSettings); } catch (Exception ex) { Sitecore.Diagnostics.Log.Error($"ReadExcelTabStepProcessor: error occurred.pipelineStep: {pipelineStep.Name}, endpoint: {endpoint.Name}", ex); return; } }
protected async Task <bool> ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger) { if (endpoint == null) { throw new ArgumentNullException(nameof(endpoint)); } if (pipelineStep == null) { throw new ArgumentNullException(nameof(pipelineStep)); } if (pipelineContext == null) { throw new ArgumentNullException(nameof(pipelineContext)); } var repositorySettings = Context.GetPlugin <RepositorySettings>(); if (repositorySettings == null) { logger.Error("No repository settings plugin is specified on the context (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (repositorySettings.Client == null) { logger.Error("No client is specified on the repository settings (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } var applicationEndpointSettings = endpoint.GetApplicationEndpointSettings(); var applicationSettings = (ApplicationSettings)applicationEndpointSettings?.Application?.RefreshPlugin.Invoke(); if (applicationSettings == null) { logger.Error("No application is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(applicationSettings.BaseUrl)) { logger.Error("No Base Url is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(applicationSettings.AccessToken)) { logger.Warn("No access token is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); //return false; } var resourceSettings = pipelineStep.GetResourceSettings(); if (resourceSettings == null) { logger.Error("No resource is specified on the pipeline step (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(resourceSettings.Url)) { logger.Error("No url is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } if (string.IsNullOrWhiteSpace(resourceSettings.Method)) { logger.Error("No method is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } var readDataSettings = pipelineStep.GetReadResourceDataSettings(); if (readDataSettings == null || string.IsNullOrWhiteSpace(readDataSettings.PathExpression)) { logger.Error("No path expression is specified on the pipeline step. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); return(false); } var iterableData = new JArray(); string json = null; try { bool hasMore; do { hasMore = false; var response = await repositorySettings.Client.SendAsync(applicationSettings, resourceSettings); logger.Debug($"Data read from {response.RequestMessage.RequestUri} (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})"); json = await response.Content.ReadAsStringAsync(); var jObject = JsonConvert.DeserializeObject <JObject>(json); if (jObject == null) { logger.Debug("No data returned from request. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); } else { var jArray = (JArray)jObject.SelectToken(readDataSettings.PathExpression, false); if (jArray == null) { logger.Debug("No data returned from path expression. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name); } else { logger.Info("{0} rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", jArray.Count, pipelineStep.Name, endpoint.Name); iterableData.Merge(jArray); if (resourceSettings.Paging != null) { if (!string.IsNullOrEmpty(resourceSettings.Paging.NextTokenPathExpression)) { var nextToken = jObject.SelectToken(resourceSettings.Paging.NextTokenPathExpression, false); hasMore = !string.IsNullOrEmpty(nextToken?.Value <string>()); } else { var pageToken = jObject.SelectToken(resourceSettings.Paging.CurrentPagePathExpression, false); var pageSizeToken = jObject.SelectToken(resourceSettings.Paging.PageSizePathExpression, false); var totalCountToken = jObject.SelectToken(resourceSettings.Paging.TotalCountPathExpression, false); var page = pageToken?.Value <int?>() ?? 0; var pageSize = pageSizeToken?.Value <int?>() ?? resourceSettings.Paging.PageSize; var totalCount = totalCountToken?.Value <int?>() ?? int.MinValue; hasMore = page * pageSize > 0 && page * pageSize < totalCount; } } } } } while (resourceSettings.Paging != null && hasMore); } catch (Exception ex) { string truncatedResponse = json; if (truncatedResponse?.Length > 200) { truncatedResponse = $"{truncatedResponse.Substring(0, 200)}..."; } logger.Error($"An exception was thrown reading data: {ex.Message}. Enable DEBUG to see the first 200 characters of the response. (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})"); logger.Debug($"First 200 characters of the response: \"{truncatedResponse}\"."); throw; } logger.Info("{0} total rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", iterableData.Count, pipelineStep.Name, endpoint.Name); var dataSettings = new IterableDataSettings(iterableData); pipelineContext.AddPlugins(dataSettings); return(true); }
public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext) { ILogger logger = pipelineContext.PipelineBatchContext.Logger; if (!this.CanProcess(pipelineStep, pipelineContext)) { logger.Error("Pipeline step processing will abort because the pipeline step cannot be processed. (pipeline step: {0})", (object)pipelineStep.Name); } else { PipelinesSettings pipelinesSettings = pipelineStep.GetPipelinesSettings(); if (pipelinesSettings == null || !pipelinesSettings.Pipelines.Any <Pipeline>()) { logger.Error("Pipeline step processing will abort because the pipeline step has no sub-pipelines assigned. (pipeline step: {0})", (object)pipelineStep.Name); } else { IterableDataSettings iterableDataSettings = pipelineContext.GetIterableDataSettings(); if (iterableDataSettings == null || iterableDataSettings.Data == null) { return; } int num = 0; try { List <Task> tasks = new List <Task>(); foreach (object element in iterableDataSettings.Data) { Task task = Task.Factory.StartNew(() => { if (!pipelineContext.PipelineBatchContext.Stopped) { PipelineContext pipelineContext1 = new PipelineContext(pipelineContext.PipelineBatchContext); SynchronizationSettings synchronizationSettings = this.ResolveSynchronizationSettingsAndSetElement(pipelineStep, pipelineContext, element); pipelineContext1.Plugins.Add((IPlugin)synchronizationSettings); ParentPipelineContextSettings pipelineContextSettings = new ParentPipelineContextSettings() { ParentPipelineContext = pipelineContext }; pipelineContext1.Plugins.Add((IPlugin)pipelineContextSettings); this.ProcessPipelines(pipelineStep, pipelinesSettings.Pipelines, pipelineContext1); } }); num++; } Task.WaitAll(tasks.ToArray()); logger.Info("{0} elements were iterated. (pipeline: {1}, pipeline step: {2})", (object)num, (object)pipelineContext.CurrentPipeline.Name, (object)pipelineContext.CurrentPipelineStep.Name, (object)pipelineContext); } catch (Exception ex) { logger.Error(ex.Message); logger.Error(ex.StackTrace); pipelineContext.CriticalError = true; } } } }