public void CrawlWorkerMethod()
{
Uri currentUri;
CrawlerListBoxItem item = new CrawlerListBoxItem();
lock (UriItems)
{
if (UriItems.Count <= 0)
{
_semaphore.Release();
Console.WriteLine("CrawlWorker ran out of Uri's!");
return;
}
currentUri = UriItems[0];
item.Name = currentUri.AbsolutePath;
try
{
Application.Current.Dispatcher.Invoke((Action)(() =>
{
lock (TodoItems)
{
TodoItems.Add(item);
listBoxTodo.ScrollIntoView(item);
UriItems.RemoveAt(0);
}
}));
}
catch (Exception e)
{
}
}
string destinationString = _rootUri.MakeRelativeUri(currentUri).ToString();
DownloadFile(currentUri, destinationString, item);
if (_terminate)
{
Console.WriteLine("CrawlWorkerMethod() Caught terminate signal");
_semaphore.Release();
return;
}
if (_UriItemsPopulated == false) //first crawl, UriItems populated (hopefully)
{
_UriItemsPopulated = true;
_semaphore.Release(THREADPOOL_SIZE);
}
else
{
_semaphore.Release();
}
}
private void DownloadFile(Uri sourceUri, string destinationString, CrawlerListBoxItem item)
{
try
{
long iFileSize = 0;
int iBufferSize = 1;
iBufferSize *= 1000;
int lastSlash = destinationString.LastIndexOf("/") + 1;
string relativeDirectoryString = destinationString.Substring(0, lastSlash);
if (relativeDirectoryString.Length > 0)
{
Directory.CreateDirectory(relativeDirectoryString);
}
FileStream saveFileStream;
saveFileStream = new FileStream(destinationString,
FileMode.Create, FileAccess.Write,
FileShare.ReadWrite);
System.Net.HttpWebRequest hwRq;
System.Net.HttpWebResponse hwRes;
hwRq = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(sourceUri);
//Console.WriteLine(sourceUri);
Stream smRespStream;
hwRes = (System.Net.HttpWebResponse)hwRq.GetResponse();
smRespStream = hwRes.GetResponseStream();
iFileSize = hwRes.ContentLength;
int iByteSize;
int iDownloadedByteSize = 0;
byte[] downBuffer = new byte[iBufferSize];
while ((iByteSize = smRespStream.Read(downBuffer, 0, downBuffer.Length)) > 0)
{
saveFileStream.Write(downBuffer, 0, iByteSize);
iDownloadedByteSize += iByteSize;
item.Progress = ((0.0 + iDownloadedByteSize) / iFileSize);
_re.WaitOne();
if (_terminate)
{
Console.WriteLine("DownloadFile() Caught terminate signal");
saveFileStream.Close();
return;
}
//Thread.Sleep(50); // Debugging, slow things down
}
saveFileStream.Close();
item.Name = item.Name + " | Done";
doneList.Add(sourceUri);
Scraper.Scrape(sourceUri, _rootUri, File.ReadAllText(destinationString), UriItems, doneList);
}
catch (Exception e)
{
Console.WriteLine("DownloadFile() Caught Exception: " + e.ToString());
}
}
