public IList<ParsedContent> ParseRaw(string rawContent, FollowLinksOptions linkOpts)
{
ParsedContent parsed = new ParsedContent();
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(rawContent);
ReadMeta(doc, ref parsed);
foreach (string invalidNode in new string[] { "script", "style", "link", "object", "embed", "title" }) {
foreach (HtmlNode script in new List<HtmlNode>(doc.DocumentNode.Descendants(invalidNode)))
script.Remove();
}
HtmlNode body = doc.DocumentNode.SelectSingleNode("/html/body");
if (body == null)
return new ParsedContent[] { parsed };
// extract links to foolow
if (true) {
HtmlNodeCollection links = body.SelectNodes("//a");
if (links != null) {
foreach (HtmlNode a in links) {
parsed.Links.Add(new ParsedLink(a));
}
}
}
// this is plain page, extract and index as HTML
parsed.PlainContent = body.InnerText.Trim();
parsed.LinkOpts = linkOpts;
parsed.LinkOpts.CurrentDepth++;
return new ParsedContent[] { parsed };
}
public IList<ParsedContent> ParseRaw(string rawContent, FollowLinksOptions linkOpts)
{
// This is RSS or Sitemap
XmlDocument xmlDoc = new XmlDocument();
try {
xmlDoc.LoadXml(rawContent);
} catch (Exception ex) {
Logger.Error("Invalid XML!", ex);
Logger.Debug(rawContent);
return new List<ParsedContent>();
}
// check type
var parsed = new ParsedContent();
if (xmlDoc.DocumentElement.Name == "urlset") {
// this is a sitemap
XmlNamespaceManager mgr = new XmlNamespaceManager(xmlDoc.NameTable);
mgr.AddNamespace("ns", "http://www.sitemaps.org/schemas/sitemap/0.9");
foreach (XmlElement xmlUrl in xmlDoc.DocumentElement.SelectNodes("//ns:url", mgr)) {
parsed.Links.Add(new ParsedLink() {
Url = xmlUrl["loc"].InnerText.Trim()
});
}
} else if (xmlDoc.DocumentElement.Name == "rss") {
try { parsed.Title = xmlDoc.DocumentElement["channel"]["title"].InnerText; } catch { }
try { parsed.Description = xmlDoc.DocumentElement["channel"]["description"].InnerText; } catch { }
try { parsed.Author = xmlDoc.DocumentElement["channel"]["managingEditor"].InnerText; } catch { }
try { parsed.Metadata["link"] = xmlDoc.DocumentElement["channel"]["link"].InnerText; } catch { }
foreach (XmlElement xmlUrl in xmlDoc.DocumentElement["channel"].SelectNodes("item")) {
var link = new ParsedLink();
link.Url = xmlUrl["link"].InnerText.Trim();
try { link.Title = xmlUrl["title"].InnerText.Trim(); } catch { }
try { link.Description = xmlUrl["description"].InnerText.Trim(); } catch { }
parsed.Links.Add(link);
}
}
parsed.LinkOpts = new FollowLinksOptions();
parsed.LinkOpts.Follow = true;
parsed.LinkOpts.CurrentDepth = 1;
return new ParsedContent[] { parsed };
}
void ReadMeta(HtmlDocument doc, ref ParsedContent parsed)
{
// first, normalize values for meta name attribute
if (doc.DocumentNode.SelectNodes("/html/head/meta") != null) {
foreach (HtmlNode xmlMeta in doc.DocumentNode.SelectNodes("/html/head/meta")) {
xmlMeta.Attributes["name"].Value = xmlMeta.Attributes["name"].Value.ToLower();
}
}
if (string.IsNullOrEmpty(parsed.Title)) {
try { parsed.Title = doc.DocumentNode.SelectSingleNode("/html/head/title").InnerText.Trim(); } catch { }
}
if (string.IsNullOrEmpty(parsed.Description)) {
try { parsed.Description = doc.DocumentNode.SelectSingleNode("/html/head/meta[@name='description']").Attributes["content"].Value.Trim(); } catch { }
}
}