protected override async Task <ExitCode> Run()
{
var state = await DownloadDispatcher.Infer(Url);
if (state == null)
{
return(CLIUtils.Exit($"Could not find download source for URL {Url}", ExitCode.Error));
}
await DownloadDispatcher.PrepareAll(new [] { state });
using var queue = new WorkQueue();
queue.Status
.Where(s => s.ProgressPercent != Percent.Zero)
.Debounce(TimeSpan.FromSeconds(1))
.Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}"));
new[] { state }
.PMap(queue, async s =>
{
await s.Download(new Archive(state: null !)
{
Name = Path.GetFileName(Output)
}, (AbsolutePath)Output);
}).Wait();
public async Task ServerGetsEdgeServerInfo()
{
var service = Fixture.GetService <CDNMirrorList>();
Assert.True(await service.Execute() > 0);
Assert.NotEmpty(service.Mirrors);
Assert.True(DateTime.UtcNow - service.LastUpdate < TimeSpan.FromMinutes(1));
var servers = await ClientAPI.GetCDNMirrorList();
Assert.Equal(service.Mirrors, servers);
var state = new WabbajackCDNDownloader.State(new Uri("https://wabbajack.b-cdn.net/this_file_doesn_t_exist"));
await DownloadDispatcher.PrepareAll(new[] { state });
await using var tmp = new TempFile();
await Assert.ThrowsAsync <HttpException>(async() => await state.Download(new Archive(state)
{
Name = "test"
}, tmp.Path));
var downloader = DownloadDispatcher.GetInstance <WabbajackCDNDownloader>();
Assert.Null(downloader.Mirrors); // Now works through a host remap
}
public static async Task <AbstractDownloadState?> InferDownloadState(Hash hash)
{
if (BuildServerStatus.IsBuildServerDown)
{
return(null);
}
var client = await GetClient();
var results = await client.GetJsonAsync <Archive[]>(
$"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}");
await DownloadDispatcher.PrepareAll(results.Select(r => r.State));
foreach (var result in results)
{
try
{
if (await result.State.Verify(result))
{
return(result.State);
}
}
catch (Exception ex)
{
Utils.Log($"Verification error for failed for inferenced archive {result.State.PrimaryKeyString}");
Utils.Log(ex.ToString());
}
}
return(null);
}
protected override async Task <int> Run()
{
var state = DownloadDispatcher.Infer(Url);
if (state == null)
{
Console.WriteLine($"Could not find download source for URL {Url}");
return(1);
}
DownloadDispatcher.PrepareAll(new [] { state });
using var queue = new WorkQueue();
queue.Status
.Where(s => s.ProgressPercent != Percent.Zero)
.Debounce(TimeSpan.FromSeconds(1))
.Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}"));
new[] { state }
.PMap(queue, async s =>
{
await s.Download(new Archive {
Name = Path.GetFileName(Output)
}, Output);
}).Wait();
File.WriteAllLines(Output + ".meta", state.GetMetaIni());
return(0);
}
public override async Task <int> Execute()
{
var archives = await _sql.GetNonNexusModlistArchives();
_logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives");
using var queue = new WorkQueue();
await DownloadDispatcher.PrepareAll(archives.Select(a => a.State));
var results = await archives.PMap(queue, async archive =>
{
try
{
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
ReportStarting(archive.State.PrimaryKeyString);
bool isValid = false;
switch (archive.State)
{
case WabbajackCDNDownloader.State _:
case GoogleDriveDownloader.State _:
case ManualDownloader.State _:
case ModDBDownloader.State _:
case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):
isValid = true;
break;
default:
isValid = await archive.State.Verify(archive, token.Token);
break;
}
return(Archive : archive, IsValid : isValid);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}");
return(Archive : archive, IsValid : false);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
});
await _sql.UpdateNonNexusModlistArchivesStatus(results);
var failed = results.Count(r => !r.IsValid);
var passed = results.Count() - failed;
foreach (var(archive, _) in results.Where(f => !f.IsValid))
{
_logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}");
}
_logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed");
return(failed);
}
private async Task <(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(string url, string filename, string modName = null)
{
if (modName != null)
{
await utils.AddMod(modName);
}
var src = _downloadFolder.Combine(filename);
if (!src.Exists)
{
var state = DownloadDispatcher.ResolveArchive(url);
await DownloadDispatcher.PrepareAll(new[] { state });
await state.Download(new Archive(state : null !) { Name = "Unknown" }, src);
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlistPath))
{
modlistPath.Delete();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlistPath);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlistPath}");
var installer = AInstaller.LoadFromFile(modlistPath);
Utils.Log($"{installer.Archives.Count} archives to validate");
await DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
var isValid = await IsValid(sql, whitelists, archive);
return(new DetailedStatusItem {
IsFailing = !isValid, Archive = archive
});
}))
.ToList();
var status = new DetailedStatus
{
Name = list.Title,
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
DownloadMetaData = list.DownloadMetadata,
HasFailures = validated.Any(v => v.IsFailing)
};
var dto = new ModListStatus
{
Id = list.Links.MachineURL,
Summary = new ModListSummary
{
Name = status.Name,
MachineURL = list.Links?.MachineURL ?? status.Name,
Checked = status.Checked,
Failed = status.Archives.Count(a => a.IsFailing),
Passed = status.Archives.Count(a => !a.IsFailing),
},
DetailedStatus = status,
Metadata = list
};
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await sql.UpdateModListStatus(dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
}
public override async Task <int> Execute()
{
_nexusClient ??= await NexusApiClient.Get();
int count = 0;
while (true)
{
var(daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
{
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
}
else
{
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
}
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
{
break;
}
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State });
using var tempPath = new TempFile();
await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path);
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
}
count++;
}
return(count);
}
public override async Task <int> Execute()
{
_nexusClient ??= await NexusApiClient.Get();
int count = 0;
while (true)
{
var(daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
{
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
}
else
{
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
}
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
{
break;
}
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State });
await using var tempPath = new TempFile();
if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path))
{
_logger.LogError($"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
count++;
}
if (count > 0)
{
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify <PatchBuilder>();
}
return(count);
}
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlist_path))
{
if (File.Exists(modlist_path))
{
File.Delete(modlist_path);
}
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlist_path}");
var installer = AInstaller.LoadFromFile(modlist_path);
Utils.Log($"{installer.Archives.Count} archives to validate");
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
bool is_failed;
try
{
is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist);
}
catch (Exception)
{
is_failed = false;
}
return(new DetailedStatusItem {
IsFailing = is_failed, Archive = archive
});
}))
.ToList();
var status = new DetailedStatus
{
Name = list.Title,
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
DownloadMetaData = list.DownloadMetadata,
HasFailures = validated.Any(v => v.IsFailing)
};
var dto = new ModListStatus
{
Id = list.Links.MachineURL,
Summary = new ModlistSummary
{
Name = status.Name,
Checked = status.Checked,
Failed = status.Archives.Count(a => a.IsFailing),
Passed = status.Archives.Count(a => !a.IsFailing),
},
DetailedStatus = status,
Metadata = list
};
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await ModListStatus.Update(db, dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
}
protected override async Task <ExitCode> Run()
{
var files = Input.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.ToArray();
Console.WriteLine($"Found {files.Length} files to verify");
using var queue = new WorkQueue();
var states = (await files.PMap(queue, async f =>
{
var ini = f.WithExtension(Consts.MetaFileExtension).LoadIniFile();
var state = (AbstractDownloadState?)await DownloadDispatcher.ResolveArchive(ini, quickMode: true);
if (state == default)
{
Console.WriteLine($"[Skipping] {f.FileName} because no meta could be interpreted");
}
if (!string.IsNullOrWhiteSpace(StateType) && !state !.PrimaryKeyString.StartsWith(StateType + "|"))
{
Console.WriteLine(
$"[Skipping] {f.FileName} because type {state.PrimaryKeyString[0]} does not match filter");
return(f, null);
}
return(f, state);
})).Where(s => s.state != null)
.Select(s => (s.f, s.state !))
.ToArray();
await DownloadDispatcher.PrepareAll(states.Select(s => s.Item2));
Helpers.Init();
Console.WriteLine($"Found {states.Length} states to verify");
int timedOut = 0;
await states.PMap(queue, async p =>
{
try
{
var(f, state) = p;
try
{
var cts = new CancellationTokenSource();
cts.CancelAfter(TimeSpan.FromMinutes(10));
var result =
await state !.Verify(new Archive(state)
{
Name = f.FileName.ToString(), Size = f.Size
},
cts.Token);
Console.WriteLine($"[{(result ? "Failed" : "Passed")}] {f.FileName}");
}
catch (TaskCanceledException)
{
Console.WriteLine($"[Timed Out] {f.FileName} {state!.PrimaryKeyString}");
Interlocked.Increment(ref timedOut);
}
}
catch (Exception ex)
{
Console.WriteLine($"[Exception] {p.f.FileName} {ex.Message}");
}
});
Console.WriteLine($"[Total TimedOut] {timedOut}");
Console.WriteLine("[Done]");
return(ExitCode.Ok);
}