public List <ScrapyResult> Process(ScrapyResponse response, ScrapySelector selector) { var results = new List <ScrapyResult>(); if (selector.IsSingle == true) { var value = response.BodyContent; if (!string.IsNullOrEmpty(selector.Query)) { var nodes = _htmlParser.Parse(response.BodyContent).QuerySelectorAll(selector.Query); value = nodes.Length > 0 ? nodes[0].OuterHtml : ""; } var result = GetScrapyResult(selector.Name, value, response.HttpHeaders, selector); results.Add(result); } else { var nodes = _htmlParser.Parse(response.BodyContent).QuerySelectorAll(selector.Query); for (int index = 0; index < nodes.Length; index++) { var node = nodes[index]; results.Add(GetScrapyResult($"{selector.Name}[{index}]", node.OuterHtml, response.HttpHeaders, selector)); } } return(results); }
public ScrapyResult Process(ScrapyResponse response, ScrapySelector selector) { var results = GetResults(response, selector); var parameters = GetParameters(results, selector); return(new ScrapyResult() { SubResults = results, Parameters = parameters, Name = selector.Name }); }
public List <ScrapyResult> Process(ScrapyResponse response, ScrapySelector selector) { var results = new List <ScrapyResult>(); foreach (var header in response.HttpHeaders) { results.Add(new ScrapyResult() { Name = header.Name, Value = header.Value, ResultType = selector.ResultType }); } return(results); }
private List <ScrapyResult> GetResults(ScrapyResponse response, ScrapySelector selector) { // get source Processor var processor = _scrapySourceProcessorBuilder.Processors[selector.SourceType]; var results = processor.Process(response, selector).ToList(); if (selector.SubSelectors.Count() > 0) { foreach (var result in results) { foreach (var subSelector in selector.SubSelectors) { var subResults = GetResults(result.ProcessedResponse, subSelector); result.SubResults.AddRange(subResults); } } } return(results); }
public List <ScrapyResult> Process(ScrapyResponse content, ScrapySelector selector) { throw new NotImplementedException(); }