public virtual void ScoreResults()
{
var items = SearchResults.Where(x => x.Uri != null).GroupBy(x => x.Uri).Select(x => x.First()).ToList();
Parallel.ForEach(items, (a) =>
{
var html = HtmlDocumentCollector.GetDocument(a.Uri.ToString()).Result;
var doc = HtmlDocumentCollector.GetHtmlDocument(html).Result;
var content = doc.DocumentNode.InnerText;
ScoredKeywords.Select(x => x.Key).ToList().ForEach(kw =>
{
if (content.Contains(kw))
{
a.KeywordScore += ScoredKeywords[kw];
}
});
});
}
public void CollectSearches()
{
SearchResults = new List <SearchResult>();
SearchUrls.ForEach(x =>
{
var content = HtmlDocumentCollector.GetDocument(x).Result;
var doc = HtmlDocumentCollector.GetHtmlDocument(content).Result;
Searcher.DocumentNode = doc.DocumentNode;
var nodes = Searcher.GetSearchResultNodes().Result;
nodes.ForEach(node =>
{
var item = Searcher.GetSearchResult(node).Result;
if (item != null && item.Uri != null & !ExcludeUrls.Contains(item.Uri.ToString()))
{
SearchResults.Add(item);
}
});
});
}
