public static void PrintToFile(this SpiderTreeNode node, string fileName)
{
using (StreamWriter sw = new StreamWriter(fileName))
{
node.Print(sw);
}
}
public static void FillDirectoryTree(SpiderTreeNode node, bool recursive)
{
node.Print(Console.Out);
if (node.Name.Contains(".Level_") || node.Name.Contains(".mage-tab."))
{
return;
}
var content = WebUtils.DownloadHtml(node.Uri);
//Console.WriteLine(content);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(content);
foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
{
HtmlAttribute att = link.Attributes["href"];
var curlink = att.Value;
if (curlink.StartsWith("/") || !curlink.EndsWith("/") || curlink.Equals("lost+found/"))
{
continue;
}
string cururi;
if (node.Uri.EndsWith("/"))
{
cururi = node.Uri + curlink;
}
else
{
cururi = node.Uri + "/" + curlink;
}
var curname = curlink.TrimEnd('/');
var lastModified = link.NextSibling.InnerText.Trim();
lastModified = lastModified.Substring(0, lastModified.Length - 1).Trim();
node.Nodes.Add(new SpiderTreeNode()
{
Name = curname,
Uri = cururi,
LastModified = lastModified,
Depth = node.Depth + 1
});
}
if (recursive)
{
foreach (var subnode in node.Nodes)
{
FillDirectoryTree(subnode, true);
}
}
}
public static void FillDirectoryTree(SpiderTreeNode node, bool recursive)
{
node.Print(Console.Out);
if (node.Name.Contains(".Level_") || node.Name.Contains(".mage-tab."))
{
return;
}
var content = WebUtils.DownloadHtml(node.Uri);
//Console.WriteLine(content);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(content);
foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
{
HtmlAttribute att = link.Attributes["href"];
var curlink = att.Value;
if (curlink.StartsWith("/") || !curlink.EndsWith("/") || curlink.Equals("lost+found/"))
{
continue;
}
string cururi;
if (node.Uri.EndsWith("/"))
{
cururi = node.Uri + curlink;
}
else
{
cururi = node.Uri + "/" + curlink;
}
var curname = curlink.TrimEnd('/');
var lastModified = link.NextSibling.InnerText.Trim();
lastModified = lastModified.Substring(0, lastModified.Length - 1).Trim();
node.Nodes.Add(new SpiderTreeNode()
{
Name = curname,
Uri = cururi,
LastModified = lastModified,
Depth = node.Depth + 1
});
}
if (recursive)
{
foreach (var subnode in node.Nodes)
{
FillDirectoryTree(subnode, true);
}
}
}
