private void ProcessAsBinary(string runningId, ScraperDataWrapper wrapper, ScraperDataResponse response, int retryCount)
{
//Get HTML from web server
using (var webClient = new WebClient())
{
//If proxy was configured, it attach to webclient instance
if (wrapper.Proxy != null)
{
webClient.Proxy = wrapper.Proxy;
}
webClient.DownloadDataCompleted += (sender, args) =>
{
if (args.Error != null)
{
ExceptionHandlerOnDownloadData(args.Error, wrapper, runningId, retryCount);
return;
}
//Delete from running
RemoveItemFromRunningCollection(wrapper, runningId);
//Set response
response.Response = args.Result;
//Raise on data arrived event
wrapper.OnDataArrived?.Invoke(response);
//Raise a signal to notify that process was end
CheckIfProcessIsFinished();
};
webClient.DownloadDataAsync(wrapper.Uri);
}
}
private void ExceptionHandlerOnDownloadData(Exception ex, ScraperDataWrapper item, string runningId, int retryCount)
{
var response = ScrapperMapper.ToResponse(item);
response.Exception = ex;
if (ex is WebException)
{
if (retryCount < MaxRetryCount)
{
Thread.Sleep(2000);
GetDataFromWebServer(runningId, item, retryCount + 1);
}
else
{
if (item.OnThrownException != null)
{
item.OnThrownException?.Invoke(response);
RemoveItemFromRunningCollection(item, runningId);
}
}
}
else
{
if (item.OnThrownException != null)
{
item.OnThrownException?.Invoke(response);
RemoveItemFromRunningCollection(item, runningId);
}
}
}
private void GetDataFromWebServer(string runningId, ScraperDataWrapper item, int retryCount = 0)
{
try
{
//Create response object
var response = ScrapperMapper.ToResponse(item);
//Raise on dequeue event
item.OnDequeue?.Invoke(response);
switch (item.ScraperType)
{
case ScraperType.String:
ProcessAsHtml(runningId, item, response, retryCount);
return;
case ScraperType.Binary:
ProcessAsBinary(runningId, item, response, retryCount);
return;
default:
throw new Exception("ScraperType " + item.ScraperType + " not valid");
}
}
catch (Exception ex)
{
ExceptionHandlerOnDownloadData(ex, item, runningId, retryCount);
}
}
