private static string ExtractCss(HtmlDocument doc)
{
var styleTags = doc.FindTagsByName("style", "link").ToArray();
var css = new StringBuilder();
foreach (var tag in styleTags)
{
if (tag.IsType("style"))
css.AppendLine(tag.ChildElements.ToSeparatedString(" "));
else if (tag.IsType("link") && (string.Compare(tag.Attributes["type"], "text/css", true) == 0 || string.Compare(tag.Attributes["rel"], "stylesheet") == 0))
{
var src = tag.Attributes["href"];
if (string.IsNullOrEmpty(src))
continue;
var uri = new Uri(src, UriKind.RelativeOrAbsolute);
if (!uri.IsAbsoluteUri && HttpContext.Current != null && HttpContext.Current.Request != null)
{
uri = new Uri(HttpContext.Current.Request.Url, uri);
}
if (!uri.IsAbsoluteUri)
continue;
var client = new WebClient();
css.Append(client.DownloadString(uri));
}
}
foreach (var tag in styleTags)
{
tag.Remove();
}
return css.ToString();
}
private void DoExtractContent(CrawlData crawlData)
{
var text = _indexStrips.Aggregate(crawlData.OriginalContent, (current, strip) => StripText(current, strip.Key, strip.Value));
var html = new HtmlDocument(text);
crawlData.Title = html.FindTagsByName("title").Select(t => t.InnerText).FirstOrDefault();
var metaData = new CaseInvariantNameValueCollection();
foreach (var keyValue in html.FindTagsByName("meta").Select(GetKeyValue).Where(keyValue => keyValue.HasValue))
{
if (metaData.HasKey(keyValue.Value.Key))
throw new ApplicationException("Duplicate meta tags:" + keyValue.Value.Key);
metaData.Append(keyValue.Value.Key, keyValue.Value.Value);
}
crawlData.MetaData = metaData.ToDictionary();
crawlData.FilteredContent = html.ExtractContent();
}
private void DoExtractLinks(CrawlData data)
{
var text = _followStrips.Aggregate(data.OriginalContent, (current, strip) => StripText(current, strip.Key, strip.Value));
var html = new HtmlDocument(text);
data.Links = html.FindTagsByName("a").Select(a => a.Attributes["href"]).ToArray();
}
private static HtmlTag ExtractBodyTag(HtmlDocument htmlDocument)
{
return htmlDocument.FindTagsByName("body").FirstOrDefault();
}