private async void FetchResults(IScrapable <IMetacriticData> item)
{
List <UrlResponsePair> urlResponsePairs = item.Scrape();
var tasks = urlResponsePairs.Where(p => !string.IsNullOrEmpty(p.Response)).
Select(pairs => Task.Run(() => item.Parse(pairs)));
RequestTrackerItem tItem;
lock (m_requestTrackerLock)
{
tItem = m_requestTracker.FirstOrDefault(i => i.RequestId == item.RequestId);
if (!EqualityComparer <RequestTrackerItem> .Default.Equals(tItem, default(RequestTrackerItem)))
{
m_requestTracker.Remove(tItem);
}
}
if (!EqualityComparer <RequestTrackerItem> .Default.Equals(tItem, default(RequestTrackerItem)))
{
try
{
IMetacriticData[] htmlResp = await Task.WhenAll(tasks);
if (htmlResp != null && htmlResp.Length > 0)
{
PublishResult(tItem.RequestId, htmlResp);
}
else
{
throw new Errors.EmptyResponseException("No matching data found.");
}
}
catch (EmptyResponseException)
{
Logger.Info("No response received");
Error[] error = new Error[1];
Error err = new Error("No matching item found!");
error[0] = err;
PublishResult(tItem.RequestId, error);
RequestTrackerItem reqItem = m_requestTracker.FirstOrDefault(r => r.RequestId == tItem.RequestId);
m_requestTracker.Remove(reqItem);
}
catch (Exception)
{
var exceptions = tasks.Where(t => t.Exception != null).Select(t => t.Exception);
Logger.Error("Encountered exception while parsing. Exceptions: ");
foreach (Exception ex in exceptions)
{
Logger.Error("-- {0}", ex.ToString());
}
}
}
else
{
Logger.Warn("Item not found in request tracker");
}
}
private void DataFetchThreadProc()
{
while (m_isRunning)
{
IScrapable <IMetacriticData> item = m_dataFetchQueue.Dequeue();
if (item != null)
{
FetchResults(item);
}
else
{
m_dataFetchQueue.WaitOnEmpty(10000);
}
Thread.Sleep(10);
}
}
