private static bool NeedLookup(string contentId)
{
using (var db = new ThumbnailDb())
if (db.Thumbnail.FirstOrDefault(t => t.ContentId == contentId) is Thumbnail thumbnail)
{
if (!string.IsNullOrEmpty(thumbnail.Url))
{
if (ScrapeStateProvider.IsFresh(new DateTime(thumbnail.Timestamp, DateTimeKind.Utc)))
{
return(false);
}
}
}
return(true);
}
public async Task RunAsync(CancellationToken cancellationToken)
{
do
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
await ScrapeStateProvider.CleanAsync(cancellationToken).ConfigureAwait(false);
await RefreshStoresAsync(cancellationToken).ConfigureAwait(false);
try
{
await DoScrapePassAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
PrintError(e);
}
await Task.Delay(TimeSpan.FromHours(1), cancellationToken).ConfigureAwait(false);
} while (!cancellationToken.IsCancellationRequested);
}
private static async Task DoScrapePassAsync(CancellationToken cancellationToken)
{
List <string> storesToScrape;
await LockObj.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
storesToScrape = new List <string>(PsnStores);
}
finally
{
LockObj.Release();
}
var percentPerStore = 1.0 / storesToScrape.Count;
for (var storeIdx = 0; storeIdx < storesToScrape.Count; storeIdx++)
{
var locale = storesToScrape[storeIdx];
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (ScrapeStateProvider.IsFresh(locale))
{
//Config.Log.Debug($"Cache for {locale} PSN is fresh, skipping");
continue;
}
Config.Log.Debug($"Scraping {locale} PSN for PS3 games...");
var knownContainers = new HashSet <string>();
// get containers from the left side navigation panel on the main page
var containerIds = await Client.GetMainPageNavigationContainerIdsAsync(locale, cancellationToken).ConfigureAwait(false);
// get all containers from all the menus
var stores = await Client.GetStoresAsync(locale, cancellationToken).ConfigureAwait(false);
if (!string.IsNullOrEmpty(stores?.Data.BaseUrl))
{
containerIds.Add(Path.GetFileName(stores.Data.BaseUrl));
}
foreach (var id in containerIds)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
await ScrapeContainerIdsAsync(locale, id, knownContainers, cancellationToken).ConfigureAwait(false);
}
Config.Log.Debug($"\tFound {knownContainers.Count} containers");
// now let's scrape for actual games in every container
var defaultFilters = new Dictionary <string, string>
{
["platform"] = "ps3",
["game_content_type"] = "games",
};
var take = 30;
var returned = 0;
var containersToScrape = knownContainers.ToList(); //.Where(c => c.Contains("FULL", StringComparison.InvariantCultureIgnoreCase)).ToList();
var percentPerContainer = 1.0 / containersToScrape.Count;
for (var containerIdx = 0; containerIdx < containersToScrape.Count; containerIdx++)
{
var containerId = containersToScrape[containerIdx];
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (ScrapeStateProvider.IsFresh(locale, containerId))
{
//Config.Log.Debug($"\tCache for {locale} container {containerId} is fresh, skipping");
continue;
}
var currentPercent = storeIdx * percentPerStore + containerIdx * percentPerStore * percentPerContainer;
Config.Log.Debug($"\tScraping {locale} container {containerId} ({currentPercent*100:##0.00}%)...");
var total = -1;
var start = 0;
do
{
var tries = 0;
Container container = null;
bool error = false;
do
{
try
{
container = await Client.GetGameContainerAsync(locale, containerId, start, take, defaultFilters, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
PrintError(e);
error = true;
}
tries++;
} while (error && tries < 3 && !cancellationToken.IsCancellationRequested);
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (container != null)
{
// this might've changed between the pages for some stupid reason
total = container.Data.Attributes.TotalResults;
var pages = (int)Math.Ceiling((double)total / take);
if (pages > 1)
{
Config.Log.Debug($"\t\tPage {start / take + 1} of {pages}");
}
returned = container.Data?.Relationships?.Children?.Data?.Count(i => i.Type == "game" || i.Type == "legacy-sku") ?? 0;
// included contains full data already, so it's wise to get it first
await ProcessIncludedGamesAsync(locale, container, cancellationToken).ConfigureAwait(false);
// returned items are just ids that need to be resolved
if (returned > 0)
{
foreach (var item in container.Data.Relationships.Children.Data)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (item.Type == "game")
{
if (!NeedLookup(item.Id))
{
continue;
}
}
else if (item.Type != "legacy-sku")
{
continue;
}
//need depth=1 in case it's a crossplay title, so ps3 id will be in entitlements instead
container = await Client.ResolveContentAsync(locale, item.Id, 1, cancellationToken).ConfigureAwait(false);
if (container == null)
{
PrintError(new InvalidOperationException("No container for " + item.Id));
}
else
{
await ProcessIncludedGamesAsync(locale, container, cancellationToken).ConfigureAwait(false);
}
}
}
}
start += take;
} while ((returned > 0 || (total > -1 && start * take <= total)) && !cancellationToken.IsCancellationRequested);
await ScrapeStateProvider.SetLastRunTimestampAsync(locale, containerId).ConfigureAwait(false);
Config.Log.Debug($"\tFinished scraping {locale} container {containerId}, processed {start - take + returned} items");
}
await ScrapeStateProvider.SetLastRunTimestampAsync(locale).ConfigureAwait(false);
}
Config.Log.Debug("Finished scraping all the PSN stores");
}
private static async Task UpdateGameTitlesAsync(CancellationToken cancellationToken)
{
var container = Path.GetFileName(TitleDownloadLink.AbsolutePath);
try
{
if (ScrapeStateProvider.IsFresh(container))
{
return;
}
Config.Log.Debug("Scraping GameTDB for game titles...");
using (var fileStream = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 16384, FileOptions.Asynchronous | FileOptions.RandomAccess | FileOptions.DeleteOnClose))
{
using (var downloadStream = await HttpClient.GetStreamAsync(TitleDownloadLink).ConfigureAwait(false))
await downloadStream.CopyToAsync(fileStream, 16384, cancellationToken).ConfigureAwait(false);
fileStream.Seek(0, SeekOrigin.Begin);
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read))
{
var logEntry = zipArchive.Entries.FirstOrDefault(e => e.Name.EndsWith(".xml", StringComparison.InvariantCultureIgnoreCase));
if (logEntry == null)
{
throw new InvalidOperationException("No zip entries that match the .xml criteria");
}
using (var zipStream = logEntry.Open())
using (var xmlReader = XmlReader.Create(zipStream))
{
xmlReader.ReadToFollowing("PS3TDB");
var version = xmlReader.GetAttribute("version");
if (!DateTime.TryParseExact(version, "yyyyMMddHHmmss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var timestamp))
{
return;
}
if (ScrapeStateProvider.IsFresh("PS3TDB", timestamp))
{
await ScrapeStateProvider.SetLastRunTimestampAsync("PS3TDB").ConfigureAwait(false);
return;
}
while (!cancellationToken.IsCancellationRequested && xmlReader.ReadToFollowing("game"))
{
if (xmlReader.ReadToFollowing("id"))
{
var productId = xmlReader.ReadElementContentAsString().ToUpperInvariant();
if (!ProductCodeLookup.ProductCode.IsMatch(productId))
{
continue;
}
string title = null;
if (xmlReader.ReadToFollowing("locale") && xmlReader.ReadToFollowing("title"))
{
title = xmlReader.ReadElementContentAsString();
}
if (!string.IsNullOrEmpty(title))
{
using (var db = new ThumbnailDb())
{
var item = await db.Thumbnail.FirstOrDefaultAsync(t => t.ProductCode == productId, cancellationToken).ConfigureAwait(false);
if (item == null)
{
await db.Thumbnail.AddAsync(new Thumbnail { ProductCode = productId, Name = title }, cancellationToken).ConfigureAwait(false);
await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
}
else
{
if (item.Name != title && item.Timestamp == 0)
{
item.Name = title;
await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
}
}
}
}
}
}
await ScrapeStateProvider.SetLastRunTimestampAsync("PS3TDB").ConfigureAwait(false);
}
}
}
await ScrapeStateProvider.SetLastRunTimestampAsync(container).ConfigureAwait(false);
}
catch (Exception e)
{
PrintError(e);
}
finally
{
Config.Log.Debug("Finished scraping GameTDB for game titles");
}
}