private void ProcessAsBinary(string runningId, ScraperDataWrapper wrapper, ScraperDataResponse response, int retryCount) { //Get HTML from web server using (var webClient = new WebClient()) { //If proxy was configured, it attach to webclient instance if (wrapper.Proxy != null) { webClient.Proxy = wrapper.Proxy; } webClient.DownloadDataCompleted += (sender, args) => { if (args.Error != null) { ExceptionHandlerOnDownloadData(args.Error, wrapper, runningId, retryCount); return; } //Delete from running RemoveItemFromRunningCollection(wrapper, runningId); //Set response response.Response = args.Result; //Raise on data arrived event wrapper.OnDataArrived?.Invoke(response); //Raise a signal to notify that process was end CheckIfProcessIsFinished(); }; webClient.DownloadDataAsync(wrapper.Uri); } }
private static void OnDataArrived(ScraperDataResponse response) { switch (response.ScraperType) { case ScraperType.Binary: var data = (byte[])response.Response; break; case ScraperType.String: var html = (string)response.Response; break; default: throw new Exception("Tipo de scraper inválido"); } _readCount++; }
/// <summary> /// Just get the full requested page in string format /// </summary> /// <param name="request"></param> /// <returns></returns> public virtual async Task <ScraperDataResponse> GetAsync(ScraperRequest request) { var response = new ScraperDataResponse(); try { this.httpClient = CreateHttpClient(); var httpRequest = await httpClient.GetAsync(request.Url); if (!httpRequest.IsSuccessStatusCode) { AddNotification("HttpClient", "HTTP response was unsuccessful."); } response.Data = await httpRequest.Content.ReadAsStringAsync(); } catch (Exception ex) { AddNotification(ex.GetType().Name, ex.GetMessageConcatenatedWithInner()); } return(response); }
private static void OnDequeue(ScraperDataResponse response) { }
private static void OnException(ScraperDataResponse response) { _errorCount++; }