/// <summary> /// Sets up PageManager threads and begin search /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void Work(object sender, DoWorkEventArgs e) { _working = true; if (Thread.CurrentThread.Name == null) Thread.CurrentThread.Name = "Search Manager: " + _searchCriteria.SearchText; System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); // Set up the page managers, each running on an async thread _pageManagers = new PageManager[_threadCount]; for (int i = 0; i < _pageManagers.Count(); i++) { // Keep track of the highest page we've // attempted to scrape (page numbers are not zero-based) _pageNumber = i+1; // PageManagers internally fire off an async worker which each // call the ResultReturned / WorkerFinished event handlers. _pageManagers[i] = new PageManager( _pageNumber, _searchCriteria, Scraper.LoadSearchPage, // inject method for testing here ResultReturned, WorkerFinished); _pageManagers[i].RunWorkerAsync(); } while (_working) { // User can cancel a search through the UI. if (CancellationPending) { HaltAllOngoingWork(); } } stopwatch.Stop(); string msg = "Search time : {0} ms" + Environment.NewLine; msg = string.Format(msg, stopwatch.ElapsedMilliseconds); Debug.WriteLine(msg); }
/// <summary> /// Called by PageManagers when their task is complete. /// If there is still work to be done, replaces the finished /// pageManager with a new one (creates a new thread) /// Exits if all work is complete. /// </summary> /// <param name="obj">sender object</param> /// <param name="args">Result<AmazonItem></param> public void WorkerFinished(object obj, RunWorkerCompletedEventArgs args) { // See if any of the workers are reporting that they're out // of results. bool outOfResults = _pageManagers.Any(i => i.WorkStatus == PageManager.Status.NoResults); // If so, don't deploy another thread. if (outOfResults) { string msg = "PageManager reporting no more results;"; msg += " not deploying another thread."; Debug.WriteLine(msg); return; } // Or if there are no threads that are marked "Working", // we are done if (IsWorkFinished()) { HaltAllOngoingWork(); return; } if (args == null) { return; } if (args.Error != null) { Debug.WriteLine(args.Error.Message); return; } if (args.Result == null || args.Result.GetType() != typeof(PageManager)) { return; } // If this PageManager is done but we haven't hit our // target number of results, we should spawn a new thread PageManager finished = (PageManager)args.Result; // Get the index of the PageManager whose search page number // matches the one that just finished (we're going to replace // it with a new PageManager) int index = _pageManagers.ToList().FindIndex(i => i.PageNumber == finished.PageNumber); // Increment the variable that tracks the // highest page number we've searched so far // TODO: since page number is shared state, there is a // slight chance that two PageManagers might hit this // code _pageNumber += 1; // Start searching a new page PageManager newPageManager = new PageManager( _pageNumber, _searchCriteria, Scraper.LoadSearchPage, // inject method for testing here ResultReturned, WorkerFinished); _pageManagers[index].Dispose(); // get rid of old one _pageManagers[index] = newPageManager; _pageManagers[index].RunWorkerAsync(); }
/// <summary> /// Called by PageManagers when their task is complete. /// If there is still work to be done, replaces the finished /// pageManager with a new one (creates a new thread) /// Exits if all work is complete. /// </summary> /// <param name="obj">sender object</param> /// <param name="args">Result<AmazonItem></param> public void WorkerFinished(object obj, RunWorkerCompletedEventArgs args) { // See if any of the workers are reporting that they're out // of results. bool outOfResults = _pageManagers.Any(i => i.WorkStatus == PageManager.Status.NoResults); // If so, don't deploy another thread. if (outOfResults) { string msg = "PageManager reporting no more results;"; msg += " not deploying another thread."; Debug.WriteLine(msg); return; } // Or if there are no threads that are marked "Working", // we are done if (IsWorkFinished()) { HaltAllOngoingWork(); return; } if (args == null) return; if (args.Error != null) { Debug.WriteLine(args.Error.Message); return; } if (args.Result == null || args.Result.GetType() != typeof(PageManager)) return; // If this PageManager is done but we haven't hit our // target number of results, we should spawn a new thread PageManager finished = (PageManager)args.Result; // Get the index of the PageManager whose search page number // matches the one that just finished (we're going to replace // it with a new PageManager) int index =_pageManagers.ToList().FindIndex(i => i.PageNumber == finished.PageNumber); // Increment the variable that tracks the // highest page number we've searched so far // TODO: since page number is shared state, there is a // slight chance that two PageManagers might hit this // code _pageNumber += 1; // Start searching a new page PageManager newPageManager = new PageManager( _pageNumber, _searchCriteria, Scraper.LoadSearchPage, // inject method for testing here ResultReturned, WorkerFinished); _pageManagers[index].Dispose(); // get rid of old one _pageManagers[index] = newPageManager; _pageManagers[index].RunWorkerAsync(); }