public async Task <string[]> GetUsedCDNFiles()
{
var modlists = (await ModlistMetadata.LoadFromGithub())
.Concat((await ModlistMetadata.LoadUnlistedFromGithub()))
.Select(f => f.Links.Download)
.Where(f => f.StartsWith(Consts.WabbajackAuthoredFilesPrefix))
.Select(f => f.Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
var files = (await _sql.ModListArchives())
.Select(a => a.State)
.OfType <WabbajackCDNDownloader.State>()
.Select(s => s.Url.ToString().Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
var names = modlists.Concat(files).Distinct().ToArray();
var namesBoth = names.Concat(names.Select(HttpUtility.UrlDecode))
.Concat(names.Select(HttpUtility.UrlEncode))
.Distinct()
.ToArray();
return(namesBoth);
}
public override async Task <int> Execute()
{
int downloaded = 0;
var lists = (await ModlistMetadata.LoadFromGithub())
.Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
foreach (var list in lists)
{
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
{
continue;
}
if (!_maintainer.HaveArchive(list.DownloadMetadata !.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await state.Download(new Archive(state) { Name = $"{list.Links.MachineURL}.wabbajack" }, tf.Path);
var hash = await tf.Path.FileHashAsync();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
await using (var fs = await modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
try
{
modlist = entry.FromJson <ModList>();
}
catch (JsonReaderException)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
}
await _sql.IngestModList(list.DownloadMetadata !.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Downloaded {downloaded} new lists" });
}
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Enqueing {fc} files for downloading" });
}
return(downloaded);
}