public override async Task <AppMetadata> ScrapeAsync(string appId) { var url = string.Format(StoreUrlTemplate, appId); var msg = new HttpRequestMessage(HttpMethod.Get, url); msg.Headers.Add("MS-Contract-Version", "4"); var response = await _client.SendAsync(msg); response.EnsureSuccessStatusCode(); var content = await response.Content.ReadAsStringAsync(); var result = GetAppPayload(content, appId); if (result == null && content.IndexOf("The thing you're looking for isn't here", StringComparison.OrdinalIgnoreCase) > 0) { LogWritter?.Invoke(TraceLevel.Warning, $"Windows store url [{url}] returned not found result", null); return(null); } var meta = new AppMetadata() { Id = result.ProductId, ScraperType = this.GetType() }; meta.AppUrl = string.Format(StoreUrlUserTemplate, meta.Id); meta.Name = result.Title; var icon = GetIcon(result.Images); meta.IconUrl = icon?.Url; meta.PublisherEmail = GetFirst <string>(result.SupportUris, "Uri", "mailto:", "Uri")?.Replace("mailto:", string.Empty); meta.Publisher = result.PublisherName; meta.Website = result.AppWebsiteUrl; meta.Categories = ArrayToList(result.Categories); meta.Rating = result.AverageRating; meta.RatingCount = result.RatingCount; meta.Description = result.Description; meta.Version = result.Version; meta.Paid = result.Price > 0; meta.AddValue("Price", result.DisplayPrice); meta.AddValue("IsUniversal", result.IsUniversal); meta.AddValue("BgColor", icon?.BackgroundColor ?? result.BGColor); meta.AddValue("HasFreeTrial", result.HasFreeTrial); meta.AddValue("ProductType", result.ProductType); meta.AddValue("PackageFamilyName", result.PackageFamilyName); meta.AddValue("CategoryId", result.CategoryId); meta.AddValue("ApproximateSizeInBytes", result.ApproximateSizeInBytes); meta.AddValue("SubcategoryId", result.SubcategoryId); meta.AddValue("Language", result.Language); meta.AddValue("ImageType", icon.ImageType); DateTime date = DateTime.Now; if (DateTime.TryParse((result.LastUpdateDateUtc ?? string.Empty).ToString(), out date)) { meta.Updated = date; } return(meta); }
public override async Task <AppMetadata> ScrapeAsync(string appId) { var url = string.Format(StoreUrlTemplate, appId); var msg = new HttpRequestMessage(HttpMethod.Get, url); msg.Headers.Add("Accept", "text/json"); var response = await _client.SendAsync(msg); response.EnsureSuccessStatusCode(); var content = await response.Content.ReadAsStringAsync(); var json = JsonConvert.DeserializeObject <dynamic>(content); if (json?.resultCount == 0) { LogWritter?.Invoke(TraceLevel.Warning, $"Apple store (ITunes) url [{url}] returned zero results", null); return(null); } var result = json["results"][0]; var meta = new AppMetadata() { Id = result.trackId, ScraperType = this.GetType(), AppUrl = GetUrlFromId(appId) }; meta.Name = result.trackName; meta.IconUrl = result.artworkUrl512 ?? result.artworkUrl100 ?? result.artworkUrl60; meta.Publisher = result.sellerName ?? result.artistName; meta.Website = result.sellerUrl; meta.Categories = ArrayToList(result.genres); meta.Rating = result.averageUserRating; meta.RatingCount = result.userRatingCount; meta.Description = result.description; meta.Version = result.version; meta.Paid = result.price > 0; meta.AddValue("Url", result.trackViewUrl.ToString()); meta.AddValue("Price", result.formattedPrice.ToString()); meta.AddValue("artistId", result.artistId); meta.AddValue("primaryGenreName", result.primaryGenreName); meta.AddValue("releaseNotes", result.releaseNotes); meta.AddValue("trackCensoredName", result.trackCensoredName); meta.AddValue("fileSizeBytes", result.fileSizeBytes); meta.AddValue("trackId", result.trackId); meta.AddValue("kind", result.formattedPrice); meta.AddValue("bundleId", result.bundleId); DateTime date; if (DateTime.TryParse((result.currentVersionReleaseDate ?? string.Empty).ToString(), out date)) { meta.Updated = date; } return(meta); }
public override async Task <AppMetadata> ScrapeAsync(string appId) { var url = string.Format(StoreUrlTemplate, appId); var url2 = string.Format(StoreUrlTemplate2, appId); var msg = new HttpRequestMessage(HttpMethod.Get, url); msg.Headers.Add("Accept", "text/json"); var response = await Client.SendAsync(msg); response.EnsureSuccessStatusCode(); var content = await response.Content.ReadAsStringAsync(); var json = JsonConvert.DeserializeObject <dynamic>(content); if (json == null || json[appId]["success"] != true) { LogWritter?.Invoke(TraceLevel.Warning, $"Steam url [{url}] returned unsuccessfull status (app not found)", null); return(null); } dynamic result = json[appId].data; var meta = new AppMetadata() { Id = result.steam_appid, ScraperType = this.GetType(), AppUrl = GetUrlFromId(appId) }; var response2 = await Client.SendAsync(new HttpRequestMessage(HttpMethod.Get, url2)); if (!response2.IsSuccessStatusCode) { LogWritter?.Invoke(TraceLevel.Warning, $"Steam url [{url2}] returned status code [{response2.StatusCode}]; This URL is only used for DescriptionShort;", null); } else { var content2 = await response2.Content.ReadAsStringAsync(); meta.AddValue("DescriptionShort", DescriptionRegex.GetGroup(content2)?.Trim()); } meta.Name = result.name; meta.IconUrl = result.header_image; meta.Publisher = result.publishers[0]?.ToString(); meta.Website = result.website ?? result.support_info?.url; meta.PublisherEmail = result.support_info.email; meta.Categories = ArrayToList(result.genres, "description"); meta.Rating = result.metacritic?.score; meta.Description = Utils.StripHtml(result.about_the_game); meta.Paid = result.is_free == false; meta.AddValue("DescriptionLong", Utils.StripHtml(result.detailed_description)); meta.AddValue("DescriptionLongRaw", result.detailed_description?.ToString()); meta.AddValue("Price", result.price_overview?.initial / 100m); meta.AddValue("PriceCurrency", result.price_overview?.currency?.ToString()); meta.AddValue("SupportedLanguages", Utils.StripHtml(result.supported_languages)); meta.AddValue("RequiredAge", result.required_age?.ToString()); meta.AddValue("Type", result.type?.ToString()); meta.AddValue("AboutGameRaw", result.about_the_game?.ToString()); meta.AddValue("LegalNotice", result.legal_notice?.ToString()); meta.AddValue("Developers", ArrayToList(result.developers)); meta.AddValue("Publishers", ArrayToList(result.publishers)); meta.AddValue("Categories", ArrayToList(result.categories, "description")); meta.AddValue("Screenshots", ArrayToList(result.screenshots, "path_full")); meta.AddValue("PlatformWindows", result.platforms?.windows); meta.AddValue("PlatformLinux", result.platforms?.linux); meta.AddValue("PlatformMac", result.platforms?.mac); meta.AddValue("ReleaseDate", result.release_date?.date); meta.AddValue("Background", result.background); return(meta); }
public override async Task <AppMetadata> ScrapeAsync(string appId) { var url = GetUrlFromId(appId); var msg = new HttpRequestMessage(HttpMethod.Get, url); var response = await _client.SendAsync(msg); if (response.StatusCode == HttpStatusCode.NotFound) { LogWritter?.Invoke(TraceLevel.Warning, $"PlayStore url [{url}] returned 404", null); return(null); } response.EnsureSuccessStatusCode(); var content = await response.Content.ReadAsStringAsync(); var meta = new AppMetadata() { Id = appId, AppUrl = url, ScraperType = this.GetType() }; meta.Name = AppNameRegex.GetGroup(content); meta.IconUrl = IconImgRegex.GetGroup(content); if (meta.IconUrl != null) { meta.IconUrl = Regex.Replace(meta.IconUrl, "(.+?=)w\\d\\d\\d", "$1w900"); } if (!string.IsNullOrEmpty(meta.IconUrl) && meta.IconUrl.StartsWith("//")) { meta.IconUrl = "http:" + meta.IconUrl; } DateTime datePublished; if (DateTime.TryParse(DatePublishedRegex.GetGroup(content), out datePublished)) { meta.Updated = datePublished; } meta.Version = VersionRegex.GetGroup(content); meta.Publisher = Utils.StripHtml(PublisherRegex.GetGroup(content)); meta.PublisherEmail = PublisherEmailRegex.GetGroup(content); meta.Website = Utils.StripHtml(WebsiteRegex.GetGroup(content)); meta.Description = Utils.StripHtml(DescriptionRegex.GetGroup(content)); decimal rating; if (decimal.TryParse(RatingValueRegex.GetGroup(content), NumberStyles.Any, CultureInfo.InvariantCulture, out rating)) { meta.Rating = rating; } int ratingCount; if (int.TryParse(RatingCountRegex.GetGroup(content), out ratingCount)) { meta.RatingCount = ratingCount; } string price = PriceRegex.GetGroup(content); if (price != null) { meta.Paid = Regex.IsMatch(price, "[\\d]"); } meta.AddValue("Price", price); meta.Categories = CategoriesRegex.GetGroupMany(content, 1).ToList(); return(meta); }