public ScraperException(string message, IStoreScraper scraper, Exception innerException, string appId = null, string url = null, int?statusCode = null) : base(message, innerException) { ScraperType = scraper.GetType(); AppId = appId; Url = url; StatusCode = statusCode; }
public ScraperException(string message, IStoreScraper scraper, Exception innerException, string appId = null, string url = null, int? statusCode = null) : base(message, innerException) { ScraperType = scraper.GetType(); AppId = appId; Url = url; StatusCode = statusCode; }
public virtual async Task <StoreScrapeResult> ScrapeAsync(IStoreScraper scraper, string appId, bool downloadImages = true) { var sw = new Stopwatch(); sw.Start(); var result = new StoreScrapeResult(); if (scraper == null) { return(result); } result.ScraperType = scraper.GetType(); result.AppId = appId; if (string.IsNullOrEmpty(appId)) { return(result); } try { result.Metadata = await scraper.ScrapeAsync(appId); if (result.Metadata != null) { result.Metadata.ScraperType = result.ScraperType; if (downloadImages) { if (result.Metadata.IconUrl != null && result.Metadata.IconUrl.StartsWith("http", StringComparison.OrdinalIgnoreCase)) { result.Icon = await scraper.DownloadIconAsync(result.Metadata); } else { _logWritter?.Invoke(TraceLevel.Warning, $"{scraper.GetType().Name} scraper did not found valid icon url [{result.Icon}]", null); } } if (string.IsNullOrWhiteSpace(result.Metadata.Name)) { result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed to parse result", scraper, appId, result.Metadata.AppUrl); } } } catch (Exception ex) //should probably only catch WebException { var wex = ex as WebException; var httpResponse = wex?.Response as HttpWebResponse; string url = null; int? statusCode = null; if (httpResponse != null) { statusCode = (int)httpResponse.StatusCode; url = httpResponse.ResponseUri.AbsoluteUri; } _logWritter?.Invoke(TraceLevel.Error, $"{scraper.GetType().Name} scraper threw an exception while scraping; {url}", ex); result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed", scraper, ex, appId, url, statusCode); } result.ParseTime = sw.Elapsed; return(result); }
/// <summary> /// Gets registered scraper instance /// </summary> /// <typeparam name="T">Type of store scraper</typeparam> /// <returns>Scraper instance</returns> public virtual IStoreScraper GetScraper <T>() where T : class, IStoreScraper { IStoreScraper scraper = null; RegisteredScraperInstances.TryGetValue(typeof(T), out scraper); if (scraper == null) { _logWritter?.Invoke(TraceLevel.Warning, $"Scraper {typeof(T).Name} is not registered", null); } return(scraper); }
/// <summary> /// Gets registered scraper instance /// </summary> /// <typeparam name="T">Type of store scraper</typeparam> /// <returns>Scraper instance</returns> public virtual IStoreScraper GetScraper(Type scraperType) { if (scraperType == null) { throw new ArgumentNullException(nameof(scraperType)); } IStoreScraper scraper = null; RegisteredScraperInstances.TryGetValue(scraperType, out scraper); if (scraper == null) { _logWritter?.Invoke(TraceLevel.Warning, $"Scraper {scraperType.Name} is not registered", null); } return(scraper); }
public void RegisterScraper(IStoreScraper scraper) { RegisteredScraperInstances[scraper.GetType()] = scraper; }
public ScraperException(string message, IStoreScraper scraper, string appId = null, string url = null, int?statusCode = null) : this(message, scraper, null, appId, url, statusCode) { }
public ScraperException(string message, IStoreScraper scraper, string appId = null, string url = null, int? statusCode = null) : this (message, scraper, null, appId, url, statusCode) { }