private void DoExtractContent(CrawlData crawlData)
{
var text = _indexStrips.Aggregate(crawlData.OriginalContent, (current, strip) => StripText(current, strip.Key, strip.Value));
var html = new HtmlDocument(text);
crawlData.Title = html.FindTagsByName("title").Select(t => t.InnerText).FirstOrDefault();
var metaData = new CaseInvariantNameValueCollection();
foreach (var keyValue in html.FindTagsByName("meta").Select(GetKeyValue).Where(keyValue => keyValue.HasValue))
{
if (metaData.HasKey(keyValue.Value.Key))
throw new ApplicationException("Duplicate meta tags:" + keyValue.Value.Key);
metaData.Append(keyValue.Value.Key, keyValue.Value.Value);
}
crawlData.MetaData = metaData.ToDictionary();
crawlData.FilteredContent = html.ExtractContent();
}