public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource) { yield return(Utils.Uri.QueryString.SetParameter( value, dependencyDataSource.Resolve(ParameterName), dependencyDataSource.Resolve(Value) )); }
private static IList <ResourceLink> ReadEntryLinksSection(XmlReader reader, WebsiteConfig config, WebsiteJob job) { var result = new List <ResourceLink>(); while (!(reader.Name == "entry" && reader.NodeType == XmlNodeType.EndElement) && reader.Read()) { if (!reader.IsStartElement()) { continue; } switch (reader.Name) { case "link": var extractionLink = ReadExtractionLinkSection(reader, config); var linkExtractedItems = new CollectionDictionary <string, StringWithDependencies>(); foreach (var extractionItem in extractionLink.PredefinedExtractionItems.Values) { linkExtractedItems.AddValue(extractionItem.Name, extractionItem.Value); } var dependencyDataSource = new DependencyDataSource( linkExtractedItems .Where(pred => pred.Value.Any(value => !value.RequiresResolve)) .ToCollectionDictionary( pred => pred.Key, pred => pred.Value .Where(value => !value.RequiresResolve) .Select(sel => sel.FormatString) ), config.PredefinedValues, job.PredefinedValues ); // NOTE: These are entry links, so they can't have any location to extract items from, only constant values var extractedLink = new AutoDetectLink( dependencyDataSource.Resolve(extractionLink.Value), extractionLink.HttpMethod, dependencyDataSource.Resolve(extractionLink.Parameters), dependencyDataSource.Resolve(extractionLink.Headers), extractionLink.ExtractLinks, extractionLink.ExtractData, config, job, dependencyDataSource.Resolve(linkExtractedItems), (job as CrawlingBase ?? config)?.InitializationDocumentLink ); result.Add(extractedLink); break; default: throw new ArgumentException("Unrecognized element", reader.Name); } } return(result); }
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource) { if ( (EqualTo == null || value == dependencyDataSource.Resolve(EqualTo)) && (NotEqualTo == null || value != dependencyDataSource.Resolve(NotEqualTo)) ) { yield return(value); } }
public override IEnumerable <string> Execute(IEnumerable <string> values, DependencyDataSource dependencyDataSource) { if (!values.Any()) { yield return(dependencyDataSource.Resolve(ReplacementValue)); } }
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource) { if (value != null) { var uri = new Uri(value); yield return(Utils.Uri.QueryString.GetParameter(uri.Query, dependencyDataSource.Resolve(ParameterName))); } }
public override void Resolve(DependencyDataSource dependencyDataSource) { base.Resolve(dependencyDataSource); PreExtractedItems = PreExtractedItemsWithDependencies.ToCollectionDictionary( pred => pred.Key, pred => pred.Value.Select(value => dependencyDataSource.Resolve(value)) ); }
public override IEnumerable <string> Execute(CsvDocument value, DependencyDataSource dependencyDataSource) { var resultBuilder = new StringBuilder(); using (var stringWriter = new StringWriter(resultBuilder)) { using (var writer = new CsvWriter(stringWriter, new Configuration { Delimiter = _outputDelimiter })) { // Names could repeat to support alternative column calculation methods foreach (var column in _columnsOrder) { writer.WriteField(column); } foreach (var row in value.Rows) { writer.NextRecord(); foreach (var columnTransitionName in _columnsOrder) { string resultColumnValue = null; foreach (var columnTransitionOption in _columnTransitions[columnTransitionName]) { string sourceColumnValue = null; if (columnTransitionOption.SourceName != null && value.Columns.Contains(columnTransitionOption.SourceName)) { var sourceColumnIndex = value.Columns.IndexOf(columnTransitionOption.SourceName); sourceColumnValue = row[sourceColumnIndex] as string; foreach (var postProcessor in columnTransitionOption.PostProcessors) { sourceColumnValue = postProcessor.Execute(sourceColumnValue, dependencyDataSource).SingleOrDefault(); } } resultColumnValue = sourceColumnValue ?? dependencyDataSource.Resolve(columnTransitionOption.Value); if (resultColumnValue != null) { break; } } if (resultColumnValue != null) { writer.WriteField(resultColumnValue); } } } } } yield return(resultBuilder.ToString()); }
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource) { var options = RegexOptions.Compiled; if (!CaseSensitive) { options |= RegexOptions.IgnoreCase; } var regex = new Regex(RegexPattern, options); yield return(regex.Replace(value, dependencyDataSource.Resolve(ReplaceValue))); }
public virtual void Resolve(DependencyDataSource dependencyDataSource) { Url = dependencyDataSource.Resolve(UrlWithDependencies); Parameters = ParametersWithDependencies.ToDictionary(pred => pred.Key, pred => dependencyDataSource.Resolve(pred.Value)); Headers = HeadersWithDependencies.ToDictionary(pred => pred.Key, pred => dependencyDataSource.Resolve(pred.Value)); }
public override IEnumerable <string> Execute(decimal?value, DependencyDataSource dependencyDataSource) { var addendumDecimal = decimal.Parse(dependencyDataSource.Resolve(AddendumString), CultureInfo.InvariantCulture); yield return(ToString(value ?? 0 + addendumDecimal)); }
public override IEnumerable <string> Execute(string value, DependencyDataSource dependencyDataSource) { return(Enumerable.Repeat(value, int.Parse(dependencyDataSource.Resolve(NumberString)))); }
public override IEnumerable <string> Execute(decimal?value, DependencyDataSource dependencyDataSource) { var multiplierDecimal = decimal.Parse(dependencyDataSource.Resolve(MultiplierString), CultureInfo.InvariantCulture); yield return(ToString(value * multiplierDecimal)); }
public override IEnumerable <string> Execute(IEnumerable <string> values, DependencyDataSource dependencyDataSource) { yield return(string.Join(dependencyDataSource.Resolve(Separator), values)); }
public override IEnumerable <string> Execute(IEnumerable <string> values, DependencyDataSource dependencyDataSource) { return(values.Take(int.Parse(dependencyDataSource.Resolve(NumberString)))); }