public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource)
{
yield return(Utils.Uri.QueryString.SetParameter(
value,
dependencyDataSource.Resolve(ParameterName),
dependencyDataSource.Resolve(Value)
));
}
private static IList <ResourceLink> ReadEntryLinksSection(XmlReader reader, WebsiteConfig config, WebsiteJob job)
{
var result = new List <ResourceLink>();
while (!(reader.Name == "entry" && reader.NodeType == XmlNodeType.EndElement) && reader.Read())
{
if (!reader.IsStartElement())
{
continue;
}
switch (reader.Name)
{
case "link":
var extractionLink = ReadExtractionLinkSection(reader, config);
var linkExtractedItems = new CollectionDictionary <string, StringWithDependencies>();
foreach (var extractionItem in extractionLink.PredefinedExtractionItems.Values)
{
linkExtractedItems.AddValue(extractionItem.Name, extractionItem.Value);
}
var dependencyDataSource = new DependencyDataSource(
linkExtractedItems
.Where(pred => pred.Value.Any(value => !value.RequiresResolve))
.ToCollectionDictionary(
pred => pred.Key,
pred => pred.Value
.Where(value => !value.RequiresResolve)
.Select(sel => sel.FormatString)
),
config.PredefinedValues,
job.PredefinedValues
);
// NOTE: These are entry links, so they can't have any location to extract items from, only constant values
var extractedLink = new AutoDetectLink(
dependencyDataSource.Resolve(extractionLink.Value),
extractionLink.HttpMethod,
dependencyDataSource.Resolve(extractionLink.Parameters),
dependencyDataSource.Resolve(extractionLink.Headers),
extractionLink.ExtractLinks,
extractionLink.ExtractData,
config,
job,
dependencyDataSource.Resolve(linkExtractedItems),
(job as CrawlingBase ?? config)?.InitializationDocumentLink
);
result.Add(extractedLink);
break;
default:
throw new ArgumentException("Unrecognized element", reader.Name);
}
}
return(result);
}
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource)
{
if (
(EqualTo == null || value == dependencyDataSource.Resolve(EqualTo)) &&
(NotEqualTo == null || value != dependencyDataSource.Resolve(NotEqualTo))
)
{
yield return(value);
}
}
public override IEnumerable <string> Execute(IEnumerable <string> values, DependencyDataSource dependencyDataSource)
{
if (!values.Any())
{
yield return(dependencyDataSource.Resolve(ReplacementValue));
}
}
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource)
{
if (value != null)
{
var uri = new Uri(value);
yield return(Utils.Uri.QueryString.GetParameter(uri.Query, dependencyDataSource.Resolve(ParameterName)));
}
}
public override void Resolve(DependencyDataSource dependencyDataSource)
{
base.Resolve(dependencyDataSource);
PreExtractedItems = PreExtractedItemsWithDependencies.ToCollectionDictionary(
pred => pred.Key,
pred => pred.Value.Select(value => dependencyDataSource.Resolve(value))
);
}
public override IEnumerable <string> Execute(CsvDocument value, DependencyDataSource dependencyDataSource)
{
var resultBuilder = new StringBuilder();
using (var stringWriter = new StringWriter(resultBuilder))
{
using (var writer = new CsvWriter(stringWriter, new Configuration
{
Delimiter = _outputDelimiter
}))
{
// Names could repeat to support alternative column calculation methods
foreach (var column in _columnsOrder)
{
writer.WriteField(column);
}
foreach (var row in value.Rows)
{
writer.NextRecord();
foreach (var columnTransitionName in _columnsOrder)
{
string resultColumnValue = null;
foreach (var columnTransitionOption in _columnTransitions[columnTransitionName])
{
string sourceColumnValue = null;
if (columnTransitionOption.SourceName != null && value.Columns.Contains(columnTransitionOption.SourceName))
{
var sourceColumnIndex = value.Columns.IndexOf(columnTransitionOption.SourceName);
sourceColumnValue = row[sourceColumnIndex] as string;
foreach (var postProcessor in columnTransitionOption.PostProcessors)
{
sourceColumnValue = postProcessor.Execute(sourceColumnValue, dependencyDataSource).SingleOrDefault();
}
}
resultColumnValue = sourceColumnValue ?? dependencyDataSource.Resolve(columnTransitionOption.Value);
if (resultColumnValue != null)
{
break;
}
}
if (resultColumnValue != null)
{
writer.WriteField(resultColumnValue);
}
}
}
}
}
yield return(resultBuilder.ToString());
}
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource)
{
var options = RegexOptions.Compiled;
if (!CaseSensitive)
{
options |= RegexOptions.IgnoreCase;
}
var regex = new Regex(RegexPattern, options);
yield return(regex.Replace(value, dependencyDataSource.Resolve(ReplaceValue)));
}
public virtual void Resolve(DependencyDataSource dependencyDataSource)
{
Url = dependencyDataSource.Resolve(UrlWithDependencies);
Parameters = ParametersWithDependencies.ToDictionary(pred => pred.Key, pred => dependencyDataSource.Resolve(pred.Value));
Headers = HeadersWithDependencies.ToDictionary(pred => pred.Key, pred => dependencyDataSource.Resolve(pred.Value));
}
public override IEnumerable <string> Execute(decimal?value, DependencyDataSource dependencyDataSource)
{
var addendumDecimal = decimal.Parse(dependencyDataSource.Resolve(AddendumString), CultureInfo.InvariantCulture);
yield return(ToString(value ?? 0 + addendumDecimal));
}
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource)
{
return(Enumerable.Repeat(value, int.Parse(dependencyDataSource.Resolve(NumberString))));
}
public override IEnumerable <string> Execute(decimal?value, DependencyDataSource dependencyDataSource)
{
var multiplierDecimal = decimal.Parse(dependencyDataSource.Resolve(MultiplierString), CultureInfo.InvariantCulture);
yield return(ToString(value * multiplierDecimal));
}
public override IEnumerable <string> Execute(IEnumerable <string> values, DependencyDataSource dependencyDataSource)
{
yield return(string.Join(dependencyDataSource.Resolve(Separator), values));
}
public override IEnumerable <string> Execute(IEnumerable <string> values, DependencyDataSource dependencyDataSource)
{
return(values.Take(int.Parse(dependencyDataSource.Resolve(NumberString))));
}
