public ScraperException(string message, IStoreScraper scraper, Exception innerException, string appId = null, string url = null, int?statusCode = null) : base(message, innerException)
{
ScraperType = scraper.GetType();
AppId = appId;
Url = url;
StatusCode = statusCode;
}
public ScraperException(string message, IStoreScraper scraper, Exception innerException, string appId = null, string url = null, int? statusCode = null) : base(message, innerException)
{
ScraperType = scraper.GetType();
AppId = appId;
Url = url;
StatusCode = statusCode;
}
public virtual async Task <StoreScrapeResult> ScrapeAsync(IStoreScraper scraper, string appId, bool downloadImages = true)
{
var sw = new Stopwatch();
sw.Start();
var result = new StoreScrapeResult();
if (scraper == null)
{
return(result);
}
result.ScraperType = scraper.GetType();
result.AppId = appId;
if (string.IsNullOrEmpty(appId))
{
return(result);
}
try
{
result.Metadata = await scraper.ScrapeAsync(appId);
if (result.Metadata != null)
{
result.Metadata.ScraperType = result.ScraperType;
if (downloadImages)
{
if (result.Metadata.IconUrl != null && result.Metadata.IconUrl.StartsWith("http", StringComparison.OrdinalIgnoreCase))
{
result.Icon = await scraper.DownloadIconAsync(result.Metadata);
}
else
{
_logWritter?.Invoke(TraceLevel.Warning, $"{scraper.GetType().Name} scraper did not found valid icon url [{result.Icon}]", null);
}
}
if (string.IsNullOrWhiteSpace(result.Metadata.Name))
{
result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed to parse result", scraper, appId, result.Metadata.AppUrl);
}
}
}
catch (Exception ex) //should probably only catch WebException
{
var wex = ex as WebException;
var httpResponse = wex?.Response as HttpWebResponse;
string url = null;
int? statusCode = null;
if (httpResponse != null)
{
statusCode = (int)httpResponse.StatusCode;
url = httpResponse.ResponseUri.AbsoluteUri;
}
_logWritter?.Invoke(TraceLevel.Error, $"{scraper.GetType().Name} scraper threw an exception while scraping; {url}", ex);
result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed", scraper, ex, appId, url, statusCode);
}
result.ParseTime = sw.Elapsed;
return(result);
}
/// <summary>
/// Gets registered scraper instance
/// </summary>
/// <typeparam name="T">Type of store scraper</typeparam>
/// <returns>Scraper instance</returns>
public virtual IStoreScraper GetScraper <T>() where T : class, IStoreScraper
{
IStoreScraper scraper = null;
RegisteredScraperInstances.TryGetValue(typeof(T), out scraper);
if (scraper == null)
{
_logWritter?.Invoke(TraceLevel.Warning, $"Scraper {typeof(T).Name} is not registered", null);
}
return(scraper);
}
/// <summary>
/// Gets registered scraper instance
/// </summary>
/// <typeparam name="T">Type of store scraper</typeparam>
/// <returns>Scraper instance</returns>
public virtual IStoreScraper GetScraper(Type scraperType)
{
if (scraperType == null)
{
throw new ArgumentNullException(nameof(scraperType));
}
IStoreScraper scraper = null;
RegisteredScraperInstances.TryGetValue(scraperType, out scraper);
if (scraper == null)
{
_logWritter?.Invoke(TraceLevel.Warning, $"Scraper {scraperType.Name} is not registered", null);
}
return(scraper);
}
public void RegisterScraper(IStoreScraper scraper)
{
RegisteredScraperInstances[scraper.GetType()] = scraper;
}
public ScraperException(string message, IStoreScraper scraper, string appId = null, string url = null,
int?statusCode = null) : this(message, scraper, null, appId, url, statusCode)
{
}
public ScraperException(string message, IStoreScraper scraper, string appId = null, string url = null, int? statusCode = null) : this (message, scraper, null, appId, url, statusCode)
{
}
