protected async Task <bool> ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
var repositorySettings = Context.GetPlugin <RepositorySettings>();
if (repositorySettings == null)
{
logger.Error("No repository settings plugin is specified on the context (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (repositorySettings.Client == null)
{
logger.Error("No client is specified on the repository settings (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
var applicationEndpointSettings = endpoint.GetApplicationEndpointSettings();
var applicationSettings = (ApplicationSettings)applicationEndpointSettings?.Application?.RefreshPlugin.Invoke();
if (applicationSettings == null)
{
logger.Error("No application is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(applicationSettings.BaseUrl))
{
logger.Error("No Base Url is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(applicationSettings.AccessToken))
{
logger.Warn("No access token is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
//return false;
}
var resourceSettings = pipelineStep.GetResourceSettings();
if (resourceSettings == null)
{
logger.Error("No resource is specified on the pipeline step (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(resourceSettings.Url))
{
logger.Error("No url is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(resourceSettings.Method))
{
logger.Error("No method is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
var readDataSettings = pipelineStep.GetReadResourceDataSettings();
if (readDataSettings == null || string.IsNullOrWhiteSpace(readDataSettings.PathExpression))
{
logger.Error("No path expression is specified on the pipeline step. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
var iterableData = new JArray();
string json = null;
try
{
bool hasMore;
do
{
hasMore = false;
var response = await repositorySettings.Client.SendAsync(applicationSettings, resourceSettings);
logger.Debug($"Data read from {response.RequestMessage.RequestUri} (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})");
json = await response.Content.ReadAsStringAsync();
var jObject = JsonConvert.DeserializeObject <JObject>(json);
if (jObject == null)
{
logger.Debug("No data returned from request. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
}
else
{
var jArray = (JArray)jObject.SelectToken(readDataSettings.PathExpression, false);
if (jArray == null)
{
logger.Debug("No data returned from path expression. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
}
else
{
logger.Info("{0} rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", jArray.Count, pipelineStep.Name, endpoint.Name);
iterableData.Merge(jArray);
if (resourceSettings.Paging != null)
{
if (!string.IsNullOrEmpty(resourceSettings.Paging.NextTokenPathExpression))
{
var nextToken = jObject.SelectToken(resourceSettings.Paging.NextTokenPathExpression, false);
hasMore = !string.IsNullOrEmpty(nextToken?.Value <string>());
}
else
{
var pageToken = jObject.SelectToken(resourceSettings.Paging.CurrentPagePathExpression, false);
var pageSizeToken = jObject.SelectToken(resourceSettings.Paging.PageSizePathExpression, false);
var totalCountToken = jObject.SelectToken(resourceSettings.Paging.TotalCountPathExpression, false);
var page = pageToken?.Value <int?>() ?? 0;
var pageSize = pageSizeToken?.Value <int?>() ?? resourceSettings.Paging.PageSize;
var totalCount = totalCountToken?.Value <int?>() ?? int.MinValue;
hasMore = page * pageSize > 0 &&
page * pageSize < totalCount;
}
}
}
}
} while (resourceSettings.Paging != null && hasMore);
}
catch (Exception ex)
{
string truncatedResponse = json;
if (truncatedResponse?.Length > 200)
{
truncatedResponse = $"{truncatedResponse.Substring(0, 200)}...";
}
logger.Error($"An exception was thrown reading data: {ex.Message}. Enable DEBUG to see the first 200 characters of the response. (pipeline step: {pipelineStep.Name}, endpoint: {endpoint.Name})");
logger.Debug($"First 200 characters of the response: \"{truncatedResponse}\".");
throw;
}
logger.Info("{0} total rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", iterableData.Count, pipelineStep.Name, endpoint.Name);
var dataSettings = new IterableDataSettings(iterableData);
pipelineContext.AddPlugins(dataSettings);
return(true);
}
protected async Task <bool> ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext, ILogger logger)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
if (pipelineStep == null)
{
throw new ArgumentNullException(nameof(pipelineStep));
}
if (pipelineContext == null)
{
throw new ArgumentNullException(nameof(pipelineContext));
}
var repositorySettings = Context.GetPlugin <RepositorySettings>();
if (repositorySettings == null)
{
logger.Error("No repository settings plugin is specified on the context (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (repositorySettings.Client == null)
{
logger.Error("No client is specified on the repository settings (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
var applicationEndpointSettings = endpoint.GetApplicationEndpointSettings();
var applicationSettings = (ApplicationSettings)applicationEndpointSettings?.Application?.RefreshPlugin.Invoke();
if (applicationSettings == null)
{
logger.Error("No application is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(applicationSettings.BaseUrl))
{
logger.Error("No Base Url is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(applicationSettings.AccessToken))
{
logger.Warn("No access token is specified on the endpoint (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
//return false;
}
var resourceSettings = pipelineStep.GetResourceSettings();
if (resourceSettings == null)
{
logger.Error("No resource is specified on the pipeline step (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(resourceSettings.Url))
{
logger.Error("No url is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
if (string.IsNullOrWhiteSpace(resourceSettings.Method))
{
logger.Error("No method is specified on the resource (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
var readDataSettings = pipelineStep.GetReadResourceDataSettings();
if (readDataSettings == null || string.IsNullOrWhiteSpace(readDataSettings.PathExpression))
{
logger.Error("No path expression is specified on the pipeline step. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
return(false);
}
var iterableData = new JArray();
bool hasMore;
do
{
hasMore = false;
var response = await repositorySettings.Client.SendAsync(applicationSettings, resourceSettings);
var json = await response.Content.ReadAsStringAsync();
var jObject = JsonConvert.DeserializeObject <JObject>(json);
if (jObject == null)
{
logger.Debug("No data returned from request. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
}
else
{
var jArray = (JArray)jObject.SelectToken(readDataSettings.PathExpression, false);
if (jArray == null)
{
logger.Debug("No data returned from path expression. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
}
else if (jArray.Count == 0)
{
logger.Info("No items returned from request. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
}
else
{
logger.Info("{0} rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", jArray.Count, pipelineStep.Name, endpoint.Name);
iterableData.Merge(jArray);
if (resourceSettings.Paging != null)
{
if (!string.IsNullOrEmpty(resourceSettings.Paging.NextTokenPathExpression))
{
var nextToken = jObject.SelectToken(resourceSettings.Paging.NextTokenPathExpression, false);
resourceSettings.Paging.NextToken = nextToken?.Value <string>();
hasMore = !string.IsNullOrEmpty(nextToken?.Value <string>());
}
else
{
var pageToken = jObject.SelectToken(resourceSettings.Paging.CurrentPagePathExpression, false);
var pageSizeToken = jObject.SelectToken(resourceSettings.Paging.PageSizePathExpression, false);
var totalCountToken = jObject.SelectToken(resourceSettings.Paging.TotalCountPathExpression, false);
var page = pageToken?.Value <int?>() ?? 0;
var pageSize = pageSizeToken?.Value <int?>() ?? resourceSettings.Paging.PageSize;
var totalCount = totalCountToken?.Value <int?>() ?? int.MinValue;
var maxCount = resourceSettings.Paging.MaximumCount;
resourceSettings.Paging.Page = page + 1;
resourceSettings.Paging.PageSize = pageSize;
resourceSettings.Paging.TotalCount = totalCount;
hasMore = page * pageSize > 0 &&
page * pageSize < totalCount &&
page * pageSize < maxCount;
}
}
}
}
} while (resourceSettings.Paging != null && hasMore);
logger.Info("{0} total rows were read from endpoint. (pipeline step: {1}, endpoint: {2})", iterableData.Count, pipelineStep.Name, endpoint.Name);
var dataSettings = new IterableDataSettings(iterableData);
pipelineContext.AddPlugins(dataSettings);
return(true);
}
