public static void PrintToFile(this SpiderTreeNode node, string fileName) { using (StreamWriter sw = new StreamWriter(fileName)) { node.Print(sw); } }
public static void FillDirectoryTree(SpiderTreeNode node, bool recursive) { node.Print(Console.Out); if (node.Name.Contains(".Level_") || node.Name.Contains(".mage-tab.")) { return; } var content = WebUtils.DownloadHtml(node.Uri); //Console.WriteLine(content); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(content); foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]")) { HtmlAttribute att = link.Attributes["href"]; var curlink = att.Value; if (curlink.StartsWith("/") || !curlink.EndsWith("/") || curlink.Equals("lost+found/")) { continue; } string cururi; if (node.Uri.EndsWith("/")) { cururi = node.Uri + curlink; } else { cururi = node.Uri + "/" + curlink; } var curname = curlink.TrimEnd('/'); var lastModified = link.NextSibling.InnerText.Trim(); lastModified = lastModified.Substring(0, lastModified.Length - 1).Trim(); node.Nodes.Add(new SpiderTreeNode() { Name = curname, Uri = cururi, LastModified = lastModified, Depth = node.Depth + 1 }); } if (recursive) { foreach (var subnode in node.Nodes) { FillDirectoryTree(subnode, true); } } }
public static void FillDirectoryTree(SpiderTreeNode node, bool recursive) { node.Print(Console.Out); if (node.Name.Contains(".Level_") || node.Name.Contains(".mage-tab.")) { return; } var content = WebUtils.DownloadHtml(node.Uri); //Console.WriteLine(content); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(content); foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]")) { HtmlAttribute att = link.Attributes["href"]; var curlink = att.Value; if (curlink.StartsWith("/") || !curlink.EndsWith("/") || curlink.Equals("lost+found/")) { continue; } string cururi; if (node.Uri.EndsWith("/")) { cururi = node.Uri + curlink; } else { cururi = node.Uri + "/" + curlink; } var curname = curlink.TrimEnd('/'); var lastModified = link.NextSibling.InnerText.Trim(); lastModified = lastModified.Substring(0, lastModified.Length - 1).Trim(); node.Nodes.Add(new SpiderTreeNode() { Name = curname, Uri = cururi, LastModified = lastModified, Depth = node.Depth + 1 }); } if (recursive) { foreach (var subnode in node.Nodes) { FillDirectoryTree(subnode, true); } } }