private void ExceptionHandlerOnDownloadData(Exception ex, ScraperDataWrapper item, string runningId, int retryCount)
{
var response = ScrapperMapper.ToResponse(item);
response.Exception = ex;
if (ex is WebException)
{
if (retryCount < MaxRetryCount)
{
Thread.Sleep(2000);
GetDataFromWebServer(runningId, item, retryCount + 1);
}
else
{
if (item.OnThrownException != null)
{
item.OnThrownException?.Invoke(response);
RemoveItemFromRunningCollection(item, runningId);
}
}
}
else
{
if (item.OnThrownException != null)
{
item.OnThrownException?.Invoke(response);
RemoveItemFromRunningCollection(item, runningId);
}
}
}
private void GetDataFromWebServer(string runningId, ScraperDataWrapper item, int retryCount = 0)
{
try
{
//Create response object
var response = ScrapperMapper.ToResponse(item);
//Raise on dequeue event
item.OnDequeue?.Invoke(response);
switch (item.ScraperType)
{
case ScraperType.String:
ProcessAsHtml(runningId, item, response, retryCount);
return;
case ScraperType.Binary:
ProcessAsBinary(runningId, item, response, retryCount);
return;
default:
throw new Exception("ScraperType " + item.ScraperType + " not valid");
}
}
catch (Exception ex)
{
ExceptionHandlerOnDownloadData(ex, item, runningId, retryCount);
}
}
private void RemoveItemFromRunningCollection(ScraperData item, string key, int retryCount = 0)
{
string dummyValue;
var response = ScrapperMapper.ToResponse(item);
if (!Running.TryRemove(key, out dummyValue))
{
if (retryCount < MaxRetryCount)
{
RemoveItemFromRunningCollection(item, key, retryCount + 1);
}
else
{
response.Exception = new Exception("The scraper data response cannot be deleted from running collection.");
item.OnThrownException?.Invoke(response);
}
}
}
