public virtual async Task <StoreScrapeResult> ScrapeAsync(IStoreScraper scraper, string appId, bool downloadImages = true)
{
var sw = new Stopwatch();
sw.Start();
var result = new StoreScrapeResult();
if (scraper == null)
{
return(result);
}
result.ScraperType = scraper.GetType();
result.AppId = appId;
if (string.IsNullOrEmpty(appId))
{
return(result);
}
try
{
result.Metadata = await scraper.ScrapeAsync(appId);
if (result.Metadata != null)
{
result.Metadata.ScraperType = result.ScraperType;
if (downloadImages)
{
if (result.Metadata.IconUrl != null && result.Metadata.IconUrl.StartsWith("http", StringComparison.OrdinalIgnoreCase))
{
result.Icon = await scraper.DownloadIconAsync(result.Metadata);
}
else
{
_logWritter?.Invoke(TraceLevel.Warning, $"{scraper.GetType().Name} scraper did not found valid icon url [{result.Icon}]", null);
}
}
if (string.IsNullOrWhiteSpace(result.Metadata.Name))
{
result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed to parse result", scraper, appId, result.Metadata.AppUrl);
}
}
}
catch (Exception ex) //should probably only catch WebException
{
var wex = ex as WebException;
var httpResponse = wex?.Response as HttpWebResponse;
string url = null;
int? statusCode = null;
if (httpResponse != null)
{
statusCode = (int)httpResponse.StatusCode;
url = httpResponse.ResponseUri.AbsoluteUri;
}
_logWritter?.Invoke(TraceLevel.Error, $"{scraper.GetType().Name} scraper threw an exception while scraping; {url}", ex);
result.Exception = new ScraperException(scraper.GetType().Name + " scraper failed", scraper, ex, appId, url, statusCode);
}
result.ParseTime = sw.Elapsed;
return(result);
}
static void Main(string[] args)
{
var scraperFactory = new StoreScraperFactory();
//Apple store
StoreScrapeResult result = scraperFactory.ScrapeAsync("https://itunes.apple.com/us/app/logic-pro-x/id634148309?mt=12", true).Result;
WriteJson(result);
//Google Play store
result = scraperFactory.ScrapeAsync("https://play.google.com/store/apps/details?id=com.google.android.talk", true).Result;
WriteJson(result);
//Windows store
result = scraperFactory.ScrapeAsync("https://www.microsoft.com/en-us/store/apps/whos-next/9nblggh6d070", true).Result;
WriteJson(result);
//Steam store
result = scraperFactory.ScrapeAsync("http://store.steampowered.com/app/264710/", true).Result;
WriteJson(result);
//Check if Steam store url
if (scraperFactory.GetScraper("http://store.steampowered.com/app/364360") is SteamStoreScraper)
{
//<..>
}
//Get and call parser for specific store
var scraper = scraperFactory.GetScraper <PlayStoreScraper>();
var metadata = scraper.ScrapeAsync("com.android.chrome").Result;
var icon = scraper.DownloadIconAsync(metadata).Result;
ImageToAscii(icon.Content);
var parsed = scraperFactory.ParseUrl("https://play.google.com/store/apps/details?id=com.android.chrome");
WriteJson(parsed);
//Add loggin
scraperFactory = new StoreScraperFactory(logWritter: (level, s, ex) => { Console.WriteLine(level + ": " + s); });
//invalid url
result = scraperFactory.ScrapeAsync("https://itunes.apple.com/us/app/logic-pro-x/id123", true).Result;
if (!result.IsSuccessful)
{
Console.WriteLine("Failed (intended)");
}
Console.ReadKey();
}
