private void ProcessAsBinary(string runningId, ScraperDataWrapper wrapper, ScraperDataResponse response, int retryCount)
{
//Get HTML from web server
using (var webClient = new WebClient())
{
//If proxy was configured, it attach to webclient instance
if (wrapper.Proxy != null)
{
webClient.Proxy = wrapper.Proxy;
}
webClient.DownloadDataCompleted += (sender, args) =>
{
if (args.Error != null)
{
ExceptionHandlerOnDownloadData(args.Error, wrapper, runningId, retryCount);
return;
}
//Delete from running
RemoveItemFromRunningCollection(wrapper, runningId);
//Set response
response.Response = args.Result;
//Raise on data arrived event
wrapper.OnDataArrived?.Invoke(response);
//Raise a signal to notify that process was end
CheckIfProcessIsFinished();
};
webClient.DownloadDataAsync(wrapper.Uri);
}
}
private static void OnDataArrived(ScraperDataResponse response)
{
switch (response.ScraperType)
{
case ScraperType.Binary:
var data = (byte[])response.Response;
break;
case ScraperType.String:
var html = (string)response.Response;
break;
default:
throw new Exception("Tipo de scraper inválido");
}
_readCount++;
}
/// <summary>
/// Just get the full requested page in string format
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
public virtual async Task <ScraperDataResponse> GetAsync(ScraperRequest request)
{
var response = new ScraperDataResponse();
try
{
this.httpClient = CreateHttpClient();
var httpRequest = await httpClient.GetAsync(request.Url);
if (!httpRequest.IsSuccessStatusCode)
{
AddNotification("HttpClient", "HTTP response was unsuccessful.");
}
response.Data = await httpRequest.Content.ReadAsStringAsync();
}
catch (Exception ex)
{
AddNotification(ex.GetType().Name, ex.GetMessageConcatenatedWithInner());
}
return(response);
}
private static void OnDequeue(ScraperDataResponse response)
{
}
private static void OnException(ScraperDataResponse response)
{
_errorCount++;
}
