public async Task UpdateNexusCacheAPI()
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting Nexus Update via API");
var api = await _keys.GetClient();
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
var mods = await api.Get <List <NexusUpdateEntry> >(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
return(game, mods);
})
.Select(async rTask =>
{
var(game, mods) = await rTask;
return(mods.Select(mod => new { game = game, mod = mod }));
}).ToList();
_logger.Log(LogLevel.Information, $"Getting update list for {gameTasks.Count} games");
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
_logger.Log(LogLevel.Information, $"Found {purge.Count} updated mods in the last month");
using var queue = new WorkQueue();
var collected = purge.Select(d =>
{
var a = d.mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return(new { Game = d.game.Game, Date = (a > b) ? a : b, ModId = d.mod.ModId });
});
var purged = await collected.PMap(queue, async t =>
{
long purgeCount = 0;
purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return(purgeCount);
});
_logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries");
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
public override async Task <int> Execute()
{
_nexusClient ??= await _nexus.GetClient();
int count = 0;
while (true)
{
var(daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
{
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
}
else
{
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
}
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
{
break;
}
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
ReportStarting(nextDownload.Archive.State.PrimaryKeyString);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State });
await using var tempPath = new TempFile();
if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path))
{
_logger.LogError(
$"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.FileHashAsync();
if (hash == null || (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash))
{
_logger.Log(LogLevel.Warning,
$"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash.Value;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information,
$"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
finally
{
ReportEnding(nextDownload.Archive.State.PrimaryKeyString);
}
count++;
}
if (count > 0)
{
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify <PatchBuilder>();
}
return(count);
}