public virtual async Task <StoreScrapeResult> ScrapeAsync(IStoreScraper scraper, string appId, bool downloadImages = true) { var sw = new Stopwatch(); sw.Start(); var result = new StoreScrapeResult(); if (scraper == null) { return(result); } result.ScraperType = scraper.GetType(); result.AppId = appId; if (string.IsNullOrEmpty(appId)) { return(result); } try { result.Metadata = await scraper.ScrapeAsync(appId); if (result.Metadata != null) { result.Metadata.ScraperType = result.ScraperType; if (downloadImages) { if (result.Metadata.IconUrl != null && result.Metadata.IconUrl.StartsWith("http", StringComparison.OrdinalIgnoreCase)) { result.Icon = await scraper.DownloadIconAsync(result.Metadata); } else { _logWritter?.Invoke(TraceLevel.Warning, $"{scraper.GetType().Name} scraper did not found valid icon url [{result.Icon}]", null); } } if (string.IsNullOrWhiteSpace(result.Metadata.Name)) { result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed to parse result", scraper, appId, result.Metadata.AppUrl); } } } catch (Exception ex) //should probably only catch WebException { var wex = ex as WebException; var httpResponse = wex?.Response as HttpWebResponse; string url = null; int? statusCode = null; if (httpResponse != null) { statusCode = (int)httpResponse.StatusCode; url = httpResponse.ResponseUri.AbsoluteUri; } _logWritter?.Invoke(TraceLevel.Error, $"{scraper.GetType().Name} scraper threw an exception while scraping; {url}", ex); result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed", scraper, ex, appId, url, statusCode); } result.ParseTime = sw.Elapsed; return(result); }
public ScraperException(string message, IStoreScraper scraper, Exception innerException, string appId = null, string url = null, int?statusCode = null) : base(message, innerException) { ScraperType = scraper.GetType(); AppId = appId; Url = url; StatusCode = statusCode; }
public ScraperException(string message, IStoreScraper scraper, Exception innerException, string appId = null, string url = null, int? statusCode = null) : base(message, innerException) { ScraperType = scraper.GetType(); AppId = appId; Url = url; StatusCode = statusCode; }
public void RegisterScraper(IStoreScraper scraper) { RegisteredScraperInstances[scraper.GetType()] = scraper; }