public void CrawlWorkerMethod() { Uri currentUri; CrawlerListBoxItem item = new CrawlerListBoxItem(); lock (UriItems) { if (UriItems.Count <= 0) { _semaphore.Release(); Console.WriteLine("CrawlWorker ran out of Uri's!"); return; } currentUri = UriItems[0]; item.Name = currentUri.AbsolutePath; try { Application.Current.Dispatcher.Invoke((Action)(() => { lock (TodoItems) { TodoItems.Add(item); listBoxTodo.ScrollIntoView(item); UriItems.RemoveAt(0); } })); } catch (Exception e) { } } string destinationString = _rootUri.MakeRelativeUri(currentUri).ToString(); DownloadFile(currentUri, destinationString, item); if (_terminate) { Console.WriteLine("CrawlWorkerMethod() Caught terminate signal"); _semaphore.Release(); return; } if (_UriItemsPopulated == false) //first crawl, UriItems populated (hopefully) { _UriItemsPopulated = true; _semaphore.Release(THREADPOOL_SIZE); } else { _semaphore.Release(); } }
private void DownloadFile(Uri sourceUri, string destinationString, CrawlerListBoxItem item) { try { long iFileSize = 0; int iBufferSize = 1; iBufferSize *= 1000; int lastSlash = destinationString.LastIndexOf("/") + 1; string relativeDirectoryString = destinationString.Substring(0, lastSlash); if (relativeDirectoryString.Length > 0) { Directory.CreateDirectory(relativeDirectoryString); } FileStream saveFileStream; saveFileStream = new FileStream(destinationString, FileMode.Create, FileAccess.Write, FileShare.ReadWrite); System.Net.HttpWebRequest hwRq; System.Net.HttpWebResponse hwRes; hwRq = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(sourceUri); //Console.WriteLine(sourceUri); Stream smRespStream; hwRes = (System.Net.HttpWebResponse)hwRq.GetResponse(); smRespStream = hwRes.GetResponseStream(); iFileSize = hwRes.ContentLength; int iByteSize; int iDownloadedByteSize = 0; byte[] downBuffer = new byte[iBufferSize]; while ((iByteSize = smRespStream.Read(downBuffer, 0, downBuffer.Length)) > 0) { saveFileStream.Write(downBuffer, 0, iByteSize); iDownloadedByteSize += iByteSize; item.Progress = ((0.0 + iDownloadedByteSize) / iFileSize); _re.WaitOne(); if (_terminate) { Console.WriteLine("DownloadFile() Caught terminate signal"); saveFileStream.Close(); return; } //Thread.Sleep(50); // Debugging, slow things down } saveFileStream.Close(); item.Name = item.Name + " | Done"; doneList.Add(sourceUri); Scraper.Scrape(sourceUri, _rootUri, File.ReadAllText(destinationString), UriItems, doneList); } catch (Exception e) { Console.WriteLine("DownloadFile() Caught Exception: " + e.ToString()); } }