public async Task CanUploadDownloadAndDeleteAuthoredFiles()
{
var cleanup = Fixture.GetService <AuthoredFilesCleanup>();
var sql = Fixture.GetService <SqlService>();
var toDelete = await cleanup.FindFilesToDelete();
await using var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3));
var originalHash = await file.Path.FileHashAsync();
var client = await Client.Create(Fixture.APIKey);
using var queue = new WorkQueue(2);
var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}"));
var data = (await Fixture.GetService <SqlService>().AllAuthoredFiles()).ToArray();
Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName));
var listing = await cleanup.GetCDNMungedNames();
foreach (var d in data)
{
Assert.Contains(d.MungedName, listing);
}
// Just uploaded it, so it shouldn't be marked for deletion
toDelete = await cleanup.FindFilesToDelete();
foreach (var d in data)
{
Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete);
Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete);
}
var result = await _client.GetStringAsync(MakeURL("authored_files"));
Assert.Contains((string)file.Path.FileName, result);
var state = await DownloadDispatcher.Infer(uri);
Assert.IsType <WabbajackCDNDownloader.State>(state);
await state.Download(new Archive(state) { Name = (string)file.Path.FileName }, file.Path);
Assert.Equal(originalHash, await file.Path.FileHashAsync());
// Mark it as old
foreach (var d in data)
{
await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8));
}
// Now it should be marked for deletion
toDelete = await cleanup.FindFilesToDelete();
foreach (var d in data)
{
Assert.Contains(d.MungedName, toDelete.CDNDelete);
Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete);
}
await cleanup.Execute();
toDelete = await cleanup.FindFilesToDelete();
Assert.Empty(toDelete.CDNDelete);
Assert.Empty(toDelete.SQLDelete);
}
public override async Task <int> Execute()
{
int downloaded = 0;
var lists = (await ModlistMetadata.LoadFromGithub())
.Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
foreach (var list in lists)
{
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
{
continue;
}
if (!_maintainer.HaveArchive(list.DownloadMetadata !.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await state.Download(new Archive(state) { Name = $"{list.Links.MachineURL}.wabbajack" }, tf.Path);
var hash = await tf.Path.FileHashAsync();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
await using (var fs = await modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
try
{
modlist = entry.FromJson <ModList>();
}
catch (JsonReaderException)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
}
await _sql.IngestModList(list.DownloadMetadata !.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Downloaded {downloaded} new lists" });
}
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Enqueing {fc} files for downloading" });
}
return(downloaded);
}
public override async Task <int> Execute()
{
_nexusClient ??= await NexusApiClient.Get();
int count = 0;
while (true)
{
var(daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
{
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
}
else
{
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
}
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
{
break;
}
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State });
await using var tempPath = new TempFile();
if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path))
{
_logger.LogError($"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
count++;
}
if (count > 0)
{
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify <PatchBuilder>();
}
return(count);
}
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlist_path))
{
if (File.Exists(modlist_path))
{
File.Delete(modlist_path);
}
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlist_path}");
var installer = AInstaller.LoadFromFile(modlist_path);
Utils.Log($"{installer.Archives.Count} archives to validate");
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
bool is_failed;
try
{
is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist);
}
catch (Exception)
{
is_failed = false;
}
return(new DetailedStatusItem {
IsFailing = is_failed, Archive = archive
});
}))
.ToList();
var status = new DetailedStatus
{
Name = list.Title,
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
DownloadMetaData = list.DownloadMetadata,
HasFailures = validated.Any(v => v.IsFailing)
};
var dto = new ModListStatus
{
Id = list.Links.MachineURL,
Summary = new ModlistSummary
{
Name = status.Name,
Checked = status.Checked,
Failed = status.Archives.Count(a => a.IsFailing),
Passed = status.Archives.Count(a => !a.IsFailing),
},
DetailedStatus = status,
Metadata = list
};
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await ModListStatus.Update(db, dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
}
public async Task TestEndToEndArchiveUpdating()
{
var _sql = Fixture.GetService <SqlService>();
await using var conn = await _sql.Open();
await conn.ExecuteAsync("DELETE FROM dbo.NoPatch");
var settings = Fixture.GetService <AppSettings>();
settings.ValidateModUpgrades = false;
var modLists = await MakeModList("TestEndToEndArchiveUpdating.txt");
Consts.ModlistMetadataURL = modLists.ToString();
var downloader = Fixture.GetService <ArchiveDownloader>();
var archiver = Fixture.GetService <ArchiveMaintainer>();
var patcher = Fixture.GetService <PatchBuilder>();
patcher.NoCleaning = true;
var sql = Fixture.GetService <SqlService>();
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!" + Guid.NewGuid());
var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
var oldDataHash = oldFileData.xxHash();
var newDataHash = newFileData.xxHash();
await "TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(oldFileData);
var oldArchive = new Archive(new HTTPDownloader.State(MakeURL("TestEndToEndArchiveUpdating.txt")))
{
Size = oldFileData.Length,
Hash = oldDataHash
};
await IngestData(archiver, oldFileData);
await sql.EnqueueDownload(oldArchive);
var oldDownload = await sql.GetNextPendingDownload();
await oldDownload.Finish(sql);
// Now update the file
await "TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
await using var tempFile = new TempFile();
var pendingRequest = DownloadDispatcher.DownloadWithPossibleUpgrade(oldArchive, tempFile.Path);
for (var times = 0; await downloader.Execute() == 0 && times < 40; times++)
{
await Task.Delay(TimeSpan.FromMilliseconds(200));
}
for (var times = 0; await patcher.Execute() == 0 && times < 40; times++)
{
await Task.Delay(TimeSpan.FromMilliseconds(200));
}
Assert.Equal(DownloadDispatcher.DownloadResult.Update, await pendingRequest);
Assert.Equal(oldDataHash, await tempFile.Path.FileHashAsync());
}
public override async Task <int> Execute()
{
var archives = await _sql.GetNonNexusModlistArchives();
_logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives");
using var queue = new WorkQueue(10);
await DownloadDispatcher.PrepareAll(archives.Select(a => a.State));
var random = new Random();
var results = await archives.PMap(queue, async archive =>
{
try
{
await Task.Delay(random.Next(1000, 5000));
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
ReportStarting(archive.State.PrimaryKeyString);
bool isValid = false;
switch (archive.State)
{
//case WabbajackCDNDownloader.State _:
//case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
case GameFileSourceDownloader.State _:
isValid = true;
break;
case ManualDownloader.State _:
case ModDBDownloader.State _:
case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):
isValid = true;
break;
default:
isValid = await archive.State.Verify(archive, token.Token);
break;
}
return(Archive : archive, IsValid : isValid);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}");
return(Archive : archive, IsValid : false);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
});
await _sql.UpdateNonNexusModlistArchivesStatus(results);
var failed = results.Count(r => !r.IsValid);
var passed = results.Count() - failed;
foreach (var(archive, _) in results.Where(f => !f.IsValid))
{
_logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}");
}
_logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed");
return(failed);
}
protected override async Task <ExitCode> Run()
{
var files = Input.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.ToArray();
Console.WriteLine($"Found {files.Length} files to verify");
using var queue = new WorkQueue();
var states = (await files.PMap(queue, async f =>
{
var ini = f.WithExtension(Consts.MetaFileExtension).LoadIniFile();
var state = (AbstractDownloadState?)await DownloadDispatcher.ResolveArchive(ini, quickMode: true);
if (state == default)
{
Console.WriteLine($"[Skipping] {f.FileName} because no meta could be interpreted");
}
if (!string.IsNullOrWhiteSpace(StateType) && !state !.PrimaryKeyString.StartsWith(StateType + "|"))
{
Console.WriteLine(
$"[Skipping] {f.FileName} because type {state.PrimaryKeyString[0]} does not match filter");
return(f, null);
}
return(f, state);
})).Where(s => s.state != null)
.Select(s => (s.f, s.state !))
.ToArray();
await DownloadDispatcher.PrepareAll(states.Select(s => s.Item2));
Helpers.Init();
Console.WriteLine($"Found {states.Length} states to verify");
int timedOut = 0;
await states.PMap(queue, async p =>
{
try
{
var(f, state) = p;
try
{
var cts = new CancellationTokenSource();
cts.CancelAfter(TimeSpan.FromMinutes(10));
var result =
await state !.Verify(new Archive(state)
{
Name = f.FileName.ToString(), Size = f.Size
},
cts.Token);
Console.WriteLine($"[{(result ? "Failed" : "Passed")}] {f.FileName}");
}
catch (TaskCanceledException)
{
Console.WriteLine($"[Timed Out] {f.FileName} {state!.PrimaryKeyString}");
Interlocked.Increment(ref timedOut);
}
}
catch (Exception ex)
{
Console.WriteLine($"[Exception] {p.f.FileName} {ex.Message}");
}
});
Console.WriteLine($"[Total TimedOut] {timedOut}");
Console.WriteLine("[Done]");
return(ExitCode.Ok);
}
