private void ExceptionHandlerOnDownloadData(Exception ex, ScraperDataWrapper item, string runningId, int retryCount) { var response = ScrapperMapper.ToResponse(item); response.Exception = ex; if (ex is WebException) { if (retryCount < MaxRetryCount) { Thread.Sleep(2000); GetDataFromWebServer(runningId, item, retryCount + 1); } else { if (item.OnThrownException != null) { item.OnThrownException?.Invoke(response); RemoveItemFromRunningCollection(item, runningId); } } } else { if (item.OnThrownException != null) { item.OnThrownException?.Invoke(response); RemoveItemFromRunningCollection(item, runningId); } } }
private void GetDataFromWebServer(string runningId, ScraperDataWrapper item, int retryCount = 0) { try { //Create response object var response = ScrapperMapper.ToResponse(item); //Raise on dequeue event item.OnDequeue?.Invoke(response); switch (item.ScraperType) { case ScraperType.String: ProcessAsHtml(runningId, item, response, retryCount); return; case ScraperType.Binary: ProcessAsBinary(runningId, item, response, retryCount); return; default: throw new Exception("ScraperType " + item.ScraperType + " not valid"); } } catch (Exception ex) { ExceptionHandlerOnDownloadData(ex, item, runningId, retryCount); } }
private void RemoveItemFromRunningCollection(ScraperData item, string key, int retryCount = 0) { string dummyValue; var response = ScrapperMapper.ToResponse(item); if (!Running.TryRemove(key, out dummyValue)) { if (retryCount < MaxRetryCount) { RemoveItemFromRunningCollection(item, key, retryCount + 1); } else { response.Exception = new Exception("The scraper data response cannot be deleted from running collection."); item.OnThrownException?.Invoke(response); } } }