public override async Task <int> Execute()
{
int downloaded = 0;
var lists = (await ModlistMetadata.LoadFromGithub())
.Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
foreach (var list in lists)
{
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
{
continue;
}
if (!_maintainer.HaveArchive(list.DownloadMetadata !.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await state.Download(new Archive(state) { Name = $"{list.Links.MachineURL}.wabbajack" }, tf.Path);
var hash = await tf.Path.FileHashAsync();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
await using (var fs = await modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
try
{
modlist = entry.FromJson <ModList>();
}
catch (JsonReaderException)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
}
await _sql.IngestModList(list.DownloadMetadata !.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Downloaded {downloaded} new lists" });
}
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Enqueing {fc} files for downloading" });
}
return(downloaded);
}
public override async Task <int> Execute()
{
_nexusClient ??= await NexusApiClient.Get();
int count = 0;
while (true)
{
var(daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
{
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
}
else
{
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
}
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
{
break;
}
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State });
using var tempPath = new TempFile();
await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path);
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
}
count++;
}
return(count);
}
public override async Task <int> Execute()
{
_nexusClient ??= await NexusApiClient.Get();
int count = 0;
while (true)
{
var(daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
{
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
}
else
{
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
}
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
{
break;
}
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State });
await using var tempPath = new TempFile();
if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path))
{
_logger.LogError($"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
count++;
}
if (count > 0)
{
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify <PatchBuilder>();
}
return(count);
}
