private static async Task AddOrUpdateThumbnailAsync(string contentId, string name, string url, CancellationToken cancellationToken) { var match = ContentIdMatcher.Match(contentId); if (!match.Success) { return; } var productCode = match.Groups["product_id"].Value; if (!ProductCodeLookup.ProductCode.IsMatch(productCode)) { return; } name = string.IsNullOrEmpty(name) ? null : name; using (var db = new ThumbnailDb()) { var savedItem = db.Thumbnail.FirstOrDefault(t => t.ProductCode == productCode); if (savedItem == null) { var newItem = new Thumbnail { ProductCode = productCode, ContentId = contentId, Name = name, Url = url, Timestamp = DateTime.UtcNow.Ticks, }; db.Thumbnail.Add(newItem); } else if (!string.IsNullOrEmpty(url)) { if (string.IsNullOrEmpty(savedItem.Url)) { savedItem.Url = url; } if (string.IsNullOrEmpty(savedItem.Name) && !string.IsNullOrEmpty(name)) { savedItem.Name = name; } if (!ScrapeStateProvider.IsFresh(savedItem.Timestamp)) { if (savedItem.Url != url) { savedItem.Url = url; savedItem.EmbeddableUrl = null; } if (name != null && savedItem.Name != name) { savedItem.Name = name; } } savedItem.ContentId = contentId; savedItem.Timestamp = DateTime.UtcNow.Ticks; } await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); } }
private static async Task RefreshStoresAsync(CancellationToken cancellationToken) { try { if (ScrapeStateProvider.IsFresh(StoreRefreshTimestamp)) { return; } var result = GetLocalesInPreferredOrder(Client.GetLocales()); await LockObj.WaitAsync(cancellationToken).ConfigureAwait(false); try { if (ScrapeStateProvider.IsFresh(StoreRefreshTimestamp)) { return; } PsnStores = result; StoreRefreshTimestamp = DateTime.UtcNow; } finally { LockObj.Release(); } } catch (Exception e) { PrintError(e); } }
private static bool NeedLookup(string contentId) { using (var db = new ThumbnailDb()) if (db.Thumbnail.FirstOrDefault(t => t.ContentId == contentId) is Thumbnail thumbnail) { if (!string.IsNullOrEmpty(thumbnail.Url)) { if (ScrapeStateProvider.IsFresh(new DateTime(thumbnail.Timestamp, DateTimeKind.Utc))) { return(false); } } } return(true); }
private static async Task DoScrapePassAsync(CancellationToken cancellationToken) { List <string> storesToScrape; await LockObj.WaitAsync(cancellationToken).ConfigureAwait(false); try { storesToScrape = new List <string>(PsnStores); } finally { LockObj.Release(); } var percentPerStore = 1.0 / storesToScrape.Count; for (var storeIdx = 0; storeIdx < storesToScrape.Count; storeIdx++) { var locale = storesToScrape[storeIdx]; if (cancellationToken.IsCancellationRequested) { break; } if (ScrapeStateProvider.IsFresh(locale)) { //Config.Log.Debug($"Cache for {locale} PSN is fresh, skipping"); continue; } Config.Log.Debug($"Scraping {locale} PSN for PS3 games..."); var knownContainers = new HashSet <string>(); // get containers from the left side navigation panel on the main page var containerIds = await Client.GetMainPageNavigationContainerIdsAsync(locale, cancellationToken).ConfigureAwait(false); // get all containers from all the menus var stores = await Client.GetStoresAsync(locale, cancellationToken).ConfigureAwait(false); if (!string.IsNullOrEmpty(stores?.Data.BaseUrl)) { containerIds.Add(Path.GetFileName(stores.Data.BaseUrl)); } foreach (var id in containerIds) { if (cancellationToken.IsCancellationRequested) { return; } await ScrapeContainerIdsAsync(locale, id, knownContainers, cancellationToken).ConfigureAwait(false); } Config.Log.Debug($"\tFound {knownContainers.Count} containers"); // now let's scrape for actual games in every container var defaultFilters = new Dictionary <string, string> { ["platform"] = "ps3", ["game_content_type"] = "games", }; var take = 30; var returned = 0; var containersToScrape = knownContainers.ToList(); //.Where(c => c.Contains("FULL", StringComparison.InvariantCultureIgnoreCase)).ToList(); var percentPerContainer = 1.0 / containersToScrape.Count; for (var containerIdx = 0; containerIdx < containersToScrape.Count; containerIdx++) { var containerId = containersToScrape[containerIdx]; if (cancellationToken.IsCancellationRequested) { return; } if (ScrapeStateProvider.IsFresh(locale, containerId)) { //Config.Log.Debug($"\tCache for {locale} container {containerId} is fresh, skipping"); continue; } var currentPercent = storeIdx * percentPerStore + containerIdx * percentPerStore * percentPerContainer; Config.Log.Debug($"\tScraping {locale} container {containerId} ({currentPercent*100:##0.00}%)..."); var total = -1; var start = 0; do { var tries = 0; Container container = null; bool error = false; do { try { container = await Client.GetGameContainerAsync(locale, containerId, start, take, defaultFilters, cancellationToken).ConfigureAwait(false); } catch (Exception e) { PrintError(e); error = true; } tries++; } while (error && tries < 3 && !cancellationToken.IsCancellationRequested); if (cancellationToken.IsCancellationRequested) { return; } if (container != null) { // this might've changed between the pages for some stupid reason total = container.Data.Attributes.TotalResults; var pages = (int)Math.Ceiling((double)total / take); if (pages > 1) { Config.Log.Debug($"\t\tPage {start / take + 1} of {pages}"); } returned = container.Data?.Relationships?.Children?.Data?.Count(i => i.Type == "game" || i.Type == "legacy-sku") ?? 0; // included contains full data already, so it's wise to get it first await ProcessIncludedGamesAsync(locale, container, cancellationToken).ConfigureAwait(false); // returned items are just ids that need to be resolved if (returned > 0) { foreach (var item in container.Data.Relationships.Children.Data) { if (cancellationToken.IsCancellationRequested) { return; } if (item.Type == "game") { if (!NeedLookup(item.Id)) { continue; } } else if (item.Type != "legacy-sku") { continue; } //need depth=1 in case it's a crossplay title, so ps3 id will be in entitlements instead container = await Client.ResolveContentAsync(locale, item.Id, 1, cancellationToken).ConfigureAwait(false); if (container == null) { PrintError(new InvalidOperationException("No container for " + item.Id)); } else { await ProcessIncludedGamesAsync(locale, container, cancellationToken).ConfigureAwait(false); } } } } start += take; } while ((returned > 0 || (total > -1 && start * take <= total)) && !cancellationToken.IsCancellationRequested); await ScrapeStateProvider.SetLastRunTimestampAsync(locale, containerId).ConfigureAwait(false); Config.Log.Debug($"\tFinished scraping {locale} container {containerId}, processed {start - take + returned} items"); } await ScrapeStateProvider.SetLastRunTimestampAsync(locale).ConfigureAwait(false); } Config.Log.Debug("Finished scraping all the PSN stores"); }
private static async Task UpdateGameTitlesAsync(CancellationToken cancellationToken) { var container = Path.GetFileName(TitleDownloadLink.AbsolutePath); try { if (ScrapeStateProvider.IsFresh(container)) { return; } Config.Log.Debug("Scraping GameTDB for game titles..."); using (var fileStream = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 16384, FileOptions.Asynchronous | FileOptions.RandomAccess | FileOptions.DeleteOnClose)) { using (var downloadStream = await HttpClient.GetStreamAsync(TitleDownloadLink).ConfigureAwait(false)) await downloadStream.CopyToAsync(fileStream, 16384, cancellationToken).ConfigureAwait(false); fileStream.Seek(0, SeekOrigin.Begin); using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read)) { var logEntry = zipArchive.Entries.FirstOrDefault(e => e.Name.EndsWith(".xml", StringComparison.InvariantCultureIgnoreCase)); if (logEntry == null) { throw new InvalidOperationException("No zip entries that match the .xml criteria"); } using (var zipStream = logEntry.Open()) using (var xmlReader = XmlReader.Create(zipStream)) { xmlReader.ReadToFollowing("PS3TDB"); var version = xmlReader.GetAttribute("version"); if (!DateTime.TryParseExact(version, "yyyyMMddHHmmss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var timestamp)) { return; } if (ScrapeStateProvider.IsFresh("PS3TDB", timestamp)) { await ScrapeStateProvider.SetLastRunTimestampAsync("PS3TDB").ConfigureAwait(false); return; } while (!cancellationToken.IsCancellationRequested && xmlReader.ReadToFollowing("game")) { if (xmlReader.ReadToFollowing("id")) { var productId = xmlReader.ReadElementContentAsString().ToUpperInvariant(); if (!ProductCodeLookup.ProductCode.IsMatch(productId)) { continue; } string title = null; if (xmlReader.ReadToFollowing("locale") && xmlReader.ReadToFollowing("title")) { title = xmlReader.ReadElementContentAsString(); } if (!string.IsNullOrEmpty(title)) { using (var db = new ThumbnailDb()) { var item = await db.Thumbnail.FirstOrDefaultAsync(t => t.ProductCode == productId, cancellationToken).ConfigureAwait(false); if (item == null) { await db.Thumbnail.AddAsync(new Thumbnail { ProductCode = productId, Name = title }, cancellationToken).ConfigureAwait(false); await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); } else { if (item.Name != title && item.Timestamp == 0) { item.Name = title; await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); } } } } } } await ScrapeStateProvider.SetLastRunTimestampAsync("PS3TDB").ConfigureAwait(false); } } } await ScrapeStateProvider.SetLastRunTimestampAsync(container).ConfigureAwait(false); } catch (Exception e) { PrintError(e); } finally { Config.Log.Debug("Finished scraping GameTDB for game titles"); } }