private async void FetchResults(IScrapable <IMetacriticData> item) { List <UrlResponsePair> urlResponsePairs = item.Scrape(); var tasks = urlResponsePairs.Where(p => !string.IsNullOrEmpty(p.Response)). Select(pairs => Task.Run(() => item.Parse(pairs))); RequestTrackerItem tItem; lock (m_requestTrackerLock) { tItem = m_requestTracker.FirstOrDefault(i => i.RequestId == item.RequestId); if (!EqualityComparer <RequestTrackerItem> .Default.Equals(tItem, default(RequestTrackerItem))) { m_requestTracker.Remove(tItem); } } if (!EqualityComparer <RequestTrackerItem> .Default.Equals(tItem, default(RequestTrackerItem))) { try { IMetacriticData[] htmlResp = await Task.WhenAll(tasks); if (htmlResp != null && htmlResp.Length > 0) { PublishResult(tItem.RequestId, htmlResp); } else { throw new Errors.EmptyResponseException("No matching data found."); } } catch (EmptyResponseException) { Logger.Info("No response received"); Error[] error = new Error[1]; Error err = new Error("No matching item found!"); error[0] = err; PublishResult(tItem.RequestId, error); RequestTrackerItem reqItem = m_requestTracker.FirstOrDefault(r => r.RequestId == tItem.RequestId); m_requestTracker.Remove(reqItem); } catch (Exception) { var exceptions = tasks.Where(t => t.Exception != null).Select(t => t.Exception); Logger.Error("Encountered exception while parsing. Exceptions: "); foreach (Exception ex in exceptions) { Logger.Error("-- {0}", ex.ToString()); } } } else { Logger.Warn("Item not found in request tracker"); } }
private void DataFetchThreadProc() { while (m_isRunning) { IScrapable <IMetacriticData> item = m_dataFetchQueue.Dequeue(); if (item != null) { FetchResults(item); } else { m_dataFetchQueue.WaitOnEmpty(10000); } Thread.Sleep(10); } }