private void DoExtractContent(CrawlData crawlData) { var text = _indexStrips.Aggregate(crawlData.OriginalContent, (current, strip) => StripText(current, strip.Key, strip.Value)); var html = new HtmlDocument(text); crawlData.Title = html.FindTagsByName("title").Select(t => t.InnerText).FirstOrDefault(); var metaData = new CaseInvariantNameValueCollection(); foreach (var keyValue in html.FindTagsByName("meta").Select(GetKeyValue).Where(keyValue => keyValue.HasValue)) { if (metaData.HasKey(keyValue.Value.Key)) throw new ApplicationException("Duplicate meta tags:" + keyValue.Value.Key); metaData.Append(keyValue.Value.Key, keyValue.Value.Value); } crawlData.MetaData = metaData.ToDictionary(); crawlData.FilteredContent = html.ExtractContent(); }