public void Download()
{
Thread.Sleep(SpiderController.IdleTime());
string filename = Path.GetFileName(this.uri.LocalPath);
UriBuilder uri = new UriBuilder(this.uri.AbsoluteUri);
string path = SpiderController.DownloadFolder + Regex.Replace(Path.GetDirectoryName(uri.Path), "/", "\\");
if (!path.EndsWith("\\"))
{
path += "\\";
}
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
using (WebClient client = new WebClient())
{
client.DownloadFileAsync(this.uri, path + filename);
Log.DownloadedFile(this.uri.AbsoluteUri);
}
}
private void LoadNextURL()
{
while (this.URLQueue.Count > 0)
{
if (threadManager.ThreadList.Count >= SpiderController.MaxThreads)
{
break;
}
Url url = new Url();
lock (this.URLQueue)
{
if (this.URLQueue.Count > 0)
{
url = this.URLQueue.Dequeue();
}
}
if (SpiderController.ShouldContinue(url.depth))
{
Thread.Sleep(SpiderController.IdleTime());
threadManager.LaunchThread(FetchNewPage, url);
}
}
threadManager.KillThread();
}