protected override void ReadData(Sitecore.DataExchange.Models.Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
{
// do some null checks
Assert.ArgumentNotNull(endpoint, nameof(endpoint));
Assert.ArgumentNotNull(pipelineStep, nameof(pipelineStep));
Assert.ArgumentNotNull(pipelineContext, nameof(pipelineContext));
Assert.ArgumentNotNull(logger, nameof(logger));
// get the configuration values for the endpoint
var endpointSettings = endpoint.GetPlugin <JsonApiSettings>();
if (endpointSettings == null)
{
return;
}
if (string.IsNullOrEmpty(endpointSettings.ApiUrl))
{
logger.Error($"No API URL is specified on the endpoint. (pipeline step: { pipelineStep.Name }, endpoint: { endpoint.Name }");
}
// get the configuration values for the processor step
var pipelineStepSettings = pipelineStep.GetPlugin <ReadPaginatedJsonApiStepSettings>();
if (pipelineStepSettings == null)
{
return;
}
logger.Debug("Executing pipeline step: ", $"MaxCount: { pipelineStepSettings.MaxCount }", $"ResultsPerPage: { pipelineStepSettings.ResultsPerPage }", $"Page: { pipelineStepSettings.Page }", $"Offset: { pipelineStepSettings.Offset }");
// execute the API to retrieve the data
var uri = new UriBuilder(endpointSettings.ApiUrl);
var query = HttpUtility.ParseQueryString(uri.Query);
query["per_page"] = pipelineStepSettings.ResultsPerPage.ToString();
uri.Query = query.ToString();
var data = new List <JObject>();
logger.Debug($"Executing API call { uri.Uri.AbsoluteUri }. ");
var batch = GetDataAsync(uri.Uri.AbsoluteUri).GetAwaiter().GetResult().ToList();
var page = pipelineStepSettings.Page;
while (batch.Any() && (pipelineStepSettings.MaxCount < 0 || pipelineStepSettings.MaxCount > 0 && data.Count < pipelineStepSettings.MaxCount))
{
data.AddRange(batch);
page++;
query["page"] = page.ToString();
uri.Query = query.ToString();
logger.Debug($"Executing API call { uri.Uri.AbsoluteUri }. ");
batch = GetDataAsync(uri.Uri.AbsoluteUri).GetAwaiter().GetResult().ToList();
logger.Debug($"Retrieved { batch.Count } items.");
}
// add this data as a plugin to the pipeline context
var dataSettings = new IterableDataSettings(data);
pipelineContext.AddPlugin(dataSettings);
}
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext,
ILogger logger)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
try
{
var settings = endpoint.GetWordpressSettings();
if (settings == null)
{
logger.Error("Empty WordPress settings");
return;
}
List <Category> categories = _wordpressService.Read <Category>(settings.CategoriesUrl, logger);
var categoriesData = new IterableDataSettings(categories);
pipelineContext.AddPlugin(categoriesData);
}
catch (Exception ex)
{
logger.Error($"Error in ReadCategoriesStepProcessor: {ex.InnerException}");
pipelineContext.CriticalError = true;
}
}
private void RedditFeed(PipelineContext pipelineContext, string blogpath)
{
var reddit = new Reddit();
//var subreddit = reddit.(blogpath);
var subreddit = reddit.GetSubreddit(blogpath);
var dataSettings = new IterableDataSettings(subreddit.New.Take(25));
pipelineContext.AddPlugin(dataSettings);
}
protected override void ReadData(
Endpoint endpoint,
PipelineStep pipelineStep,
PipelineContext pipelineContext,
ILogger logger)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
//
//get the file path from the plugin on the endpoint
var settings = endpoint.GetTextFileSettings();
if (settings == null)
{
return;
}
if (string.IsNullOrWhiteSpace(settings.Path))
{
logger.Error(
"No path is specified on the endpoint. " +
"(pipeline step: {0}, endpoint: {1})",
pipelineStep.Name, endpoint.Name);
return;
}
if (!File.Exists(settings.Path))
{
logger.Error(
"The path specified on the endpoint does not exist. " +
"(pipeline step: {0}, endpoint: {1}, path: {2})",
pipelineStep.Name, endpoint.Name, settings.Path);
return;
}
//
//add the data that was read from the file to a plugin
var data = this.GetIterableData(settings);
var dataSettings = new IterableDataSettings(data);
//
//add the plugin to the pipeline context
pipelineContext.AddPlugin(dataSettings);
}
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext,
ILogger logger)
{
var endpointSettings = endpoint.GetPlugin <FileSettings>();
if (string.IsNullOrWhiteSpace(endpointSettings.FilePath))
{
logger.Error("File name should not be empty");
return;
}
var jsonSettings = pipelineStep.GetPlugin <JsonSettings>();
if ((jsonSettings == null) || (string.IsNullOrWhiteSpace(jsonSettings.JPath)))
{
logger.Error("Cannot read JToken settings");
return;
}
IEnumerable jarray;
using (StreamReader file = System.IO.File.OpenText(endpointSettings.FilePath))
using (JsonTextReader reader = new JsonTextReader(file))
{
JToken o2 = JToken.ReadFrom(reader);
logger.Info("File was read");
JToken token = o2.SelectToken(jsonSettings.JPath);
if (!(token is JArray))
{
logger.Error("Element specified is not a JSon Array");
return;
}
jarray = (JArray)token;
}
if (!(jarray is IEnumerable))
{
logger.Error("Trying to read object, that is not an IEnumerable");
return;
}
logger.Info("Data were read from file");
var dataSettings = new IterableDataSettings()
{
Data = jarray
};
pipelineContext.AddPlugin(dataSettings);
}
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
var formDataProvider = (IFormDataProvider)Sitecore.DependencyInjection.ServiceLocator.ServiceProvider.GetService(typeof(IFormDataProvider));
if (formDataProvider == null)
{
throw new Exception("IFormDataProvider is missing");
}
var settings = endpoint.GetFormsSettings();
if (settings == null)
{
return;
}
var readDataSettings = pipelineStep.GetPlugin <ReadDataSettings>();
if (readDataSettings == null || readDataSettings.FormID == Guid.Empty)
{
logger.Error("Form is not selected for Read Form Entries Step Processor");
return;
}
var data = this.GetIterableData(settings, readDataSettings, formDataProvider);
var dataSettings = new IterableDataSettings(data);
pipelineContext.AddPlugin(dataSettings);
}
protected override void ReadData(
Endpoint endpoint,
PipelineStep pipelineStep,
PipelineContext pipelineContext,
ILogger logger)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
//
//get the file path from the plugin on the endpoint
var settings = endpoint.GetPlugin <JsonApiEndpointSettings>();
if (settings == null)
{
return;
}
if (string.IsNullOrWhiteSpace(settings.EndpointURL))
{
logger.Error(
"No endpoint URL is specified on the endpoint. " +
"(pipeline step: {0}, endpoint: {1})",
pipelineStep.Name, endpoint.Name);
return;
}
Uri validatedUri = null;
if (!Uri.TryCreate(settings.EndpointURL, UriKind.RelativeOrAbsolute, out validatedUri))
{
logger.Error(
"The endpoint URL is not a valid URL structure. " +
"(pipeline step: {0}, endpoint: {1})",
pipelineStep.Name, endpoint.Name);
return;
}
//
//add the data that was read from the file to a plugin
var data = this.GetIterableData(settings, logger);
//foreach(var d in data)
//{
// logger.Info($"ReadFromJsonApiStepProcessor.GetIterableData {d["id"]}");
//}
var dataSettings = new IterableDataSettings(data);
//
//add the plugin to the pipeline context
pipelineContext.AddPlugin(dataSettings);
}
protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
var settings = endpoint.GetServiceBusSettings();
if (settings == null)
{
logger.Error("Unable to read data, there are no settings set in the ednpoint.");
return;
}
// Check the endpoint settings
if (settings.MaxNumberOfMessages <= 0 || settings.MaxNumberOfMessages > MaximumNumberOfMessages)
{
logger.Error($"Invalid settings. {nameof(settings.MaxNumberOfMessages)} have to be between 0 and {MaximumNumberOfMessages}. (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})");
return;
}
if (settings.BatchSize <= 0 || settings.BatchSize > MaximumServiceBusMessagesBatchSize)
{
logger.Error($"Invalid settings. {nameof(settings.BatchSize)} have to be between 0 and {MaximumServiceBusMessagesBatchSize}. (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})");
return;
}
if (settings.BatchSize > settings.MaxNumberOfMessages)
{
logger.Error($"Invalid settings. {nameof(settings.BatchSize)} ({settings.BatchSize}) have to be <= {nameof(settings.MaxNumberOfMessages)} ({settings.MaxNumberOfMessages}). (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})");
return;
}
// Create the ServiceBus Reciever
ServiceBusReceiver messageReceiver;
try
{
var serviceBusService = new ServiceBusService();
messageReceiver = new ServiceBusReceiver(MessagingFactory.CreateFromConnectionString(
settings.ConnectionString),
settings.Topic,
serviceBusService.GetSubscriptionName(settings.Market, settings.Sender),
logger);
}
catch (Exception exception)
{
logger.Error($"Unable to connect with ServiceBus ({exception.Message}). Please run the Troubleshooter to propper set the connection first.");
return;
}
// read messages from the ServiceBus (in batches, up to the maximum number of messages)
var dataList = RecieveMessages(settings, messageReceiver, logger).ToList();
if (!dataList.Any())
{
logger.Info("No data read.");
return;
}
logger.Debug($"{dataList.Count} messages read from ServiceBus.");
var dataSettings = new IterableDataSettings(dataList);
//
//add the plugin to the pipeline context
pipelineContext.AddPlugin(dataSettings);
}
