public override async Task <AppMetadata> ScrapeAsync(string appId)
{
var url = string.Format(StoreUrlTemplate, appId);
var msg = new HttpRequestMessage(HttpMethod.Get, url);
msg.Headers.Add("Accept", "text/json");
var response = await _client.SendAsync(msg);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject <dynamic>(content);
if (json?.resultCount == 0)
{
LogWritter?.Invoke(TraceLevel.Warning, $"Apple store (ITunes) url [{url}] returned zero results", null);
return(null);
}
var result = json["results"][0];
var meta = new AppMetadata()
{
Id = result.trackId, ScraperType = this.GetType(), AppUrl = GetUrlFromId(appId)
};
meta.Name = result.trackName;
meta.IconUrl = result.artworkUrl512 ?? result.artworkUrl100 ?? result.artworkUrl60;
meta.Publisher = result.sellerName ?? result.artistName;
meta.Website = result.sellerUrl;
meta.Categories = ArrayToList(result.genres);
meta.Rating = result.averageUserRating;
meta.RatingCount = result.userRatingCount;
meta.Description = result.description;
meta.Version = result.version;
meta.Paid = result.price > 0;
meta.AddValue("Url", result.trackViewUrl.ToString());
meta.AddValue("Price", result.formattedPrice.ToString());
meta.AddValue("artistId", result.artistId);
meta.AddValue("primaryGenreName", result.primaryGenreName);
meta.AddValue("releaseNotes", result.releaseNotes);
meta.AddValue("trackCensoredName", result.trackCensoredName);
meta.AddValue("fileSizeBytes", result.fileSizeBytes);
meta.AddValue("trackId", result.trackId);
meta.AddValue("kind", result.formattedPrice);
meta.AddValue("bundleId", result.bundleId);
DateTime date;
if (DateTime.TryParse((result.currentVersionReleaseDate ?? string.Empty).ToString(), out date))
{
meta.Updated = date;
}
return(meta);
}
public override async Task <AppIcon> DownloadIconAsync(AppMetadata meta)
{
if (string.IsNullOrEmpty(meta.IconUrl))
{
throw new ArgumentException("Metadata has empty icon url", nameof(meta));
}
var result = new AppIcon();
var httpResult = await _client.GetAsync(new Uri(meta.IconUrl));
if (httpResult.Content == null)
{
return(result);
}
result.ContentType = httpResult.Content?.Headers?.ContentType?.MediaType;
if (result.ContentType?.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) == false)
{
//todo: need to find System.Drawing alternative for .Net Core
var imgStream = await httpResult.Content.ReadAsStreamAsync();
var color = GetBgColor(meta);
result.Content = AddBackgroundColor(color, imgStream);
meta.AddValue("BgColorUsed", color);
//result.Content = await httpResult.Content.ReadAsByteArrayAsync();
}
else
{
result.Content = await httpResult.Content.ReadAsByteArrayAsync();
}
return(result);
}
public override async Task<AppMetadata> ScrapeAsync(string appId)
{
var url = string.Format(StoreUrlTemplate, appId);
var msg = new HttpRequestMessage(HttpMethod.Get, url);
msg.Headers.Add("Accept", "text/json");
var response = await _client.SendAsync(msg);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject<dynamic>(content);
if (json?.resultCount == 0)
{
LogWritter?.Invoke(TraceLevel.Warning, $"Apple store (ITunes) url [{url}] returned zero results", null);
return null;
}
var result = json["results"][0];
var meta = new AppMetadata() { Id = result.trackId, ScraperType = this.GetType(), AppUrl = GetUrlFromId(appId) };
meta.Name = result.trackName;
meta.IconUrl = result.artworkUrl512 ?? result.artworkUrl100 ?? result.artworkUrl60;
meta.Publisher = result.sellerName ?? result.artistName;
meta.Website = result.sellerUrl;
meta.Categories = ArrayToList(result.genres);
meta.Rating = result.averageUserRating;
meta.RatingCount = result.userRatingCount;
meta.Description = result.description;
meta.Version = result.version;
meta.Paid = result.price > 0;
meta.AddValue("Url", result.trackViewUrl.ToString());
meta.AddValue("Price", result.formattedPrice.ToString());
meta.AddValue("artistId", result.artistId);
meta.AddValue("primaryGenreName", result.primaryGenreName);
meta.AddValue("releaseNotes", result.releaseNotes);
meta.AddValue("trackCensoredName", result.trackCensoredName);
meta.AddValue("fileSizeBytes", result.fileSizeBytes);
meta.AddValue("trackId", result.trackId);
meta.AddValue("kind", result.formattedPrice);
meta.AddValue("bundleId", result.bundleId);
DateTime date;
if (DateTime.TryParse((result.currentVersionReleaseDate ?? string.Empty).ToString(), out date))
{
meta.Updated = date;
}
return meta;
}
public override async Task<AppMetadata> ScrapeAsync(string appId)
{
var url = GetUrlFromId(appId);
var msg = new HttpRequestMessage(HttpMethod.Get, url);
var response = await _client.SendAsync(msg);
if (response.StatusCode == HttpStatusCode.NotFound)
{
LogWritter?.Invoke(TraceLevel.Warning, $"PlayStore url [{url}] returned 404", null);
return null;
}
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var meta = new AppMetadata() { Id = appId, AppUrl = url, ScraperType = this.GetType() };
meta.Name = AppNameRegex.GetGroup(content);
meta.IconUrl = IconImgRegex.GetGroup(content);
if (meta.IconUrl != null)
meta.IconUrl = Regex.Replace(meta.IconUrl, "(.+?=)w\\d\\d\\d", "$1w900");
if (!string.IsNullOrEmpty(meta.IconUrl) && meta.IconUrl.StartsWith("//"))
meta.IconUrl = "http:" + meta.IconUrl;
DateTime datePublished;
if (DateTime.TryParse(DatePublishedRegex.GetGroup(content), out datePublished))
meta.Updated = datePublished;
meta.Version = VersionRegex.GetGroup(content);
meta.Publisher = Utils.StripHtml(PublisherRegex.GetGroup(content));
meta.PublisherEmail = PublisherEmailRegex.GetGroup(content);
meta.Website = Utils.StripHtml(WebsiteRegex.GetGroup(content));
meta.Description = Utils.StripHtml(DescriptionRegex.GetGroup(content));
decimal rating;
if (decimal.TryParse(RatingValueRegex.GetGroup(content), NumberStyles.Any, CultureInfo.InvariantCulture, out rating))
meta.Rating = rating;
int ratingCount;
if (int.TryParse(RatingCountRegex.GetGroup(content), out ratingCount))
meta.RatingCount = ratingCount;
string price = PriceRegex.GetGroup(content);
if (price != null)
meta.Paid = Regex.IsMatch(price, "[\\d]");
meta.AddValue("Price", price);
meta.Categories = CategoriesRegex.GetGroupMany(content, 1).ToList();
return meta;
}
public override async Task<AppMetadata> ScrapeAsync(string appId)
{
var url = string.Format(StoreUrlTemplate, appId);
var url2 = string.Format(StoreUrlTemplate2, appId);
var msg = new HttpRequestMessage(HttpMethod.Get, url);
msg.Headers.Add("Accept", "text/json");
var response = await _client.SendAsync(msg);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject<dynamic>(content);
if (json == null || json[appId]["success"] != true)
{
LogWritter?.Invoke(TraceLevel.Warning, $"Steam url [{url}] returned unsuccessfull status (app not found)", null);
return null;
}
dynamic result = json[appId].data;
var meta = new AppMetadata() { Id = result.steam_appid, ScraperType = this.GetType(), AppUrl = GetUrlFromId(appId) };
var response2 = await _client.SendAsync(new HttpRequestMessage(HttpMethod.Get, url2));
if (!response2.IsSuccessStatusCode)
{
LogWritter?.Invoke(TraceLevel.Warning, $"Steam url [{url2}] returned status code [{response2.StatusCode}]; This URL is only used for DescriptionShort;", null);
}
else
{
var content2 = await response2.Content.ReadAsStringAsync();
meta.AddValue("DescriptionShort", DescriptionRegex.GetGroup(content2)?.Trim());
}
meta.Name = result.name;
meta.IconUrl = result.header_image;
meta.Publisher = result.publishers[0]?.ToString();
meta.Website = result.website ?? result.support_info?.url;
meta.PublisherEmail = result.support_info.email;
meta.Categories = ArrayToList(result.genres, "description");
meta.Rating = result.metacritic?.score;
meta.Description = Utils.StripHtml(result.about_the_game);
meta.Paid = result.is_free == false;
meta.AddValue("DescriptionLong", Utils.StripHtml(result.detailed_description));
meta.AddValue("DescriptionLongRaw", result.detailed_description?.ToString());
meta.AddValue("Price", result.price_overview?.initial/100m);
meta.AddValue("PriceCurrency", result.price_overview?.currency?.ToString());
meta.AddValue("SupportedLanguages", Utils.StripHtml(result.supported_languages));
meta.AddValue("RequiredAge", result.required_age?.ToString());
meta.AddValue("Type", result.type?.ToString());
meta.AddValue("AboutGameRaw", result.about_the_game?.ToString());
meta.AddValue("LegalNotice", result.legal_notice?.ToString());
meta.AddValue("Developers", ArrayToList(result.developers));
meta.AddValue("Publishers", ArrayToList(result.publishers));
meta.AddValue("Categories", ArrayToList(result.categories, "description"));
meta.AddValue("Screenshots", ArrayToList(result.screenshots, "path_full"));
meta.AddValue("PlatformWindows", result.platforms?.windows);
meta.AddValue("PlatformLinux", result.platforms?.linux);
meta.AddValue("PlatformMac", result.platforms?.mac);
meta.AddValue("ReleaseDate", result.release_date?.date);
meta.AddValue("Background", result.background);
return meta;
}
public override async Task <AppMetadata> ScrapeAsync(string appId)
{
var url = string.Format(StoreUrlTemplate, appId);
var url2 = string.Format(StoreUrlTemplate2, appId);
var msg = new HttpRequestMessage(HttpMethod.Get, url);
msg.Headers.Add("Accept", "text/json");
var response = await _client.SendAsync(msg);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject <dynamic>(content);
if (json == null || json[appId]["success"] != true)
{
LogWritter?.Invoke(TraceLevel.Warning, $"Steam url [{url}] returned unsuccessfull status (app not found)", null);
return(null);
}
dynamic result = json[appId].data;
var meta = new AppMetadata()
{
Id = result.steam_appid, ScraperType = this.GetType(), AppUrl = GetUrlFromId(appId)
};
var response2 = await _client.SendAsync(new HttpRequestMessage(HttpMethod.Get, url2));
if (!response2.IsSuccessStatusCode)
{
LogWritter?.Invoke(TraceLevel.Warning, $"Steam url [{url2}] returned status code [{response2.StatusCode}]; This URL is only used for DescriptionShort;", null);
}
else
{
var content2 = await response2.Content.ReadAsStringAsync();
meta.AddValue("DescriptionShort", DescriptionRegex.GetGroup(content2)?.Trim());
}
meta.Name = result.name;
meta.IconUrl = result.header_image;
meta.Publisher = result.publishers[0]?.ToString();
meta.Website = result.website ?? result.support_info?.url;
meta.PublisherEmail = result.support_info.email;
meta.Categories = ArrayToList(result.genres, "description");
meta.Rating = result.metacritic?.score;
meta.Description = Utils.StripHtml(result.about_the_game);
meta.Paid = result.is_free == false;
meta.AddValue("DescriptionLong", Utils.StripHtml(result.detailed_description));
meta.AddValue("DescriptionLongRaw", result.detailed_description?.ToString());
meta.AddValue("Price", result.price_overview?.initial / 100m);
meta.AddValue("PriceCurrency", result.price_overview?.currency?.ToString());
meta.AddValue("SupportedLanguages", Utils.StripHtml(result.supported_languages));
meta.AddValue("RequiredAge", result.required_age?.ToString());
meta.AddValue("Type", result.type?.ToString());
meta.AddValue("AboutGameRaw", result.about_the_game?.ToString());
meta.AddValue("LegalNotice", result.legal_notice?.ToString());
meta.AddValue("Developers", ArrayToList(result.developers));
meta.AddValue("Publishers", ArrayToList(result.publishers));
meta.AddValue("Categories", ArrayToList(result.categories, "description"));
meta.AddValue("Screenshots", ArrayToList(result.screenshots, "path_full"));
meta.AddValue("PlatformWindows", result.platforms?.windows);
meta.AddValue("PlatformLinux", result.platforms?.linux);
meta.AddValue("PlatformMac", result.platforms?.mac);
meta.AddValue("ReleaseDate", result.release_date?.date);
meta.AddValue("Background", result.background);
return(meta);
}
public override async Task <AppMetadata> ScrapeAsync(string appId)
{
var url = GetUrlFromId(appId);
var msg = new HttpRequestMessage(HttpMethod.Get, url);
var response = await _client.SendAsync(msg);
if (response.StatusCode == HttpStatusCode.NotFound)
{
LogWritter?.Invoke(TraceLevel.Warning, $"PlayStore url [{url}] returned 404", null);
return(null);
}
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var meta = new AppMetadata()
{
Id = appId, AppUrl = url, ScraperType = this.GetType()
};
meta.Name = AppNameRegex.GetGroup(content);
meta.IconUrl = IconImgRegex.GetGroup(content);
if (meta.IconUrl != null)
{
meta.IconUrl = Regex.Replace(meta.IconUrl, "(.+?=)w\\d\\d\\d", "$1w900");
}
if (!string.IsNullOrEmpty(meta.IconUrl) && meta.IconUrl.StartsWith("//"))
{
meta.IconUrl = "http:" + meta.IconUrl;
}
DateTime datePublished;
if (DateTime.TryParse(DatePublishedRegex.GetGroup(content), out datePublished))
{
meta.Updated = datePublished;
}
meta.Version = VersionRegex.GetGroup(content);
meta.Publisher = Utils.StripHtml(PublisherRegex.GetGroup(content));
meta.PublisherEmail = PublisherEmailRegex.GetGroup(content);
meta.Website = Utils.StripHtml(WebsiteRegex.GetGroup(content));
meta.Description = Utils.StripHtml(DescriptionRegex.GetGroup(content));
decimal rating;
if (decimal.TryParse(RatingValueRegex.GetGroup(content), NumberStyles.Any, CultureInfo.InvariantCulture, out rating))
{
meta.Rating = rating;
}
int ratingCount;
if (int.TryParse(RatingCountRegex.GetGroup(content), out ratingCount))
{
meta.RatingCount = ratingCount;
}
string price = PriceRegex.GetGroup(content);
if (price != null)
{
meta.Paid = Regex.IsMatch(price, "[\\d]");
}
meta.AddValue("Price", price);
meta.Categories = CategoriesRegex.GetGroupMany(content, 1).ToList();
return(meta);
}
public override async Task <AppMetadata> ScrapeAsync(string appId)
{
var url = string.Format(StoreUrlTemplate, appId);
var msg = new HttpRequestMessage(HttpMethod.Get, url);
msg.Headers.Add("MS-Contract-Version", "4");
var response = await _client.SendAsync(msg);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
var result = GetAppPayload(content, appId);
if (result == null && content.IndexOf("The thing you're looking for isn't here", StringComparison.OrdinalIgnoreCase) > 0)
{
LogWritter?.Invoke(TraceLevel.Warning, $"Windows store url [{url}] returned not found result", null);
return(null);
}
var meta = new AppMetadata()
{
Id = result.ProductId, ScraperType = this.GetType()
};
meta.AppUrl = string.Format(StoreUrlUserTemplate, meta.Id);
meta.Name = result.Title;
var icon = GetIcon(result.Images);
meta.IconUrl = icon?.Url;
meta.PublisherEmail = GetFirst <string>(result.SupportUris, "Uri", "mailto:", "Uri")?.Replace("mailto:", string.Empty);
meta.Publisher = result.PublisherName;
meta.Website = result.AppWebsiteUrl;
meta.Categories = ArrayToList(result.Categories);
meta.Rating = result.AverageRating;
meta.RatingCount = result.RatingCount;
meta.Description = result.Description;
meta.Version = result.Version;
meta.Paid = result.Price > 0;
meta.AddValue("Price", result.DisplayPrice);
meta.AddValue("IsUniversal", result.IsUniversal);
meta.AddValue("BgColor", icon?.BackgroundColor ?? result.BGColor);
meta.AddValue("HasFreeTrial", result.HasFreeTrial);
meta.AddValue("ProductType", result.ProductType);
meta.AddValue("PackageFamilyName", result.PackageFamilyName);
meta.AddValue("CategoryId", result.CategoryId);
meta.AddValue("ApproximateSizeInBytes", result.ApproximateSizeInBytes);
meta.AddValue("SubcategoryId", result.SubcategoryId);
meta.AddValue("Language", result.Language);
meta.AddValue("ImageType", icon.ImageType);
DateTime date = DateTime.Now;
if (DateTime.TryParse((result.LastUpdateDateUtc ?? string.Empty).ToString(), out date))
{
meta.Updated = date;
}
return(meta);
}