public async Task CanUploadDownloadAndDeleteAuthoredFiles() { var cleanup = Fixture.GetService <AuthoredFilesCleanup>(); var sql = Fixture.GetService <SqlService>(); var toDelete = await cleanup.FindFilesToDelete(); await using var file = new TempFile(); await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3)); var originalHash = await file.Path.FileHashAsync(); var client = await Client.Create(Fixture.APIKey); using var queue = new WorkQueue(2); var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}")); var data = (await Fixture.GetService <SqlService>().AllAuthoredFiles()).ToArray(); Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName)); var listing = await cleanup.GetCDNMungedNames(); foreach (var d in data) { Assert.Contains(d.MungedName, listing); } // Just uploaded it, so it shouldn't be marked for deletion toDelete = await cleanup.FindFilesToDelete(); foreach (var d in data) { Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete); Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete); } var result = await _client.GetStringAsync(MakeURL("authored_files")); Assert.Contains((string)file.Path.FileName, result); var state = await DownloadDispatcher.Infer(uri); Assert.IsType <WabbajackCDNDownloader.State>(state); await state.Download(new Archive(state) { Name = (string)file.Path.FileName }, file.Path); Assert.Equal(originalHash, await file.Path.FileHashAsync()); // Mark it as old foreach (var d in data) { await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8)); } // Now it should be marked for deletion toDelete = await cleanup.FindFilesToDelete(); foreach (var d in data) { Assert.Contains(d.MungedName, toDelete.CDNDelete); Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete); } await cleanup.Execute(); toDelete = await cleanup.FindFilesToDelete(); Assert.Empty(toDelete.CDNDelete); Assert.Empty(toDelete.SQLDelete); }
public override async Task <int> Execute() { int downloaded = 0; var lists = (await ModlistMetadata.LoadFromGithub()) .Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList(); foreach (var list in lists) { try { ReportStarting(list.Links.MachineURL); if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash)) { continue; } if (!_maintainer.HaveArchive(list.DownloadMetadata !.Hash)) { _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}"); await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}" }); var tf = new TempFile(); var state = DownloadDispatcher.ResolveArchive(list.Links.Download); if (state == null) { _logger.Log(LogLevel.Error, $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}"); continue; } downloaded += 1; await state.Download(new Archive(state) { Name = $"{list.Links.MachineURL}.wabbajack" }, tf.Path); var hash = await tf.Path.FileHashAsync(); if (hash != list.DownloadMetadata.Hash) { _logger.Log(LogLevel.Error, $"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}"); await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true); continue; } await _maintainer.Ingest(tf.Path); } _maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath); ModList modlist; await using (var fs = await modlistPath.OpenRead()) using (var zip = new ZipArchive(fs, ZipArchiveMode.Read)) await using (var entry = zip.GetEntry("modlist")?.Open()) { if (entry == null) { _logger.LogWarning($"Bad Modlist {list.Links.MachineURL}"); await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}" }); continue; } try { modlist = entry.FromJson <ModList>(); } catch (JsonReaderException) { _logger.LogWarning($"Bad Modlist {list.Links.MachineURL}"); await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}" }); continue; } } await _sql.IngestModList(list.DownloadMetadata !.Hash, list, modlist, false); } catch (Exception ex) { _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}"); await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}" }); } finally { ReportEnding(list.Links.MachineURL); } } _logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists"); if (downloaded > 0) { await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Downloaded {downloaded} new lists" }); } var fc = await _sql.EnqueueModListFilesForIndexing(); _logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading"); if (fc > 0) { await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Enqueing {fc} files for downloading" }); } return(downloaded); }
public override async Task <int> Execute() { _nexusClient ??= await NexusApiClient.Get(); int count = 0; while (true) { var(daily, hourly) = await _nexusClient.GetRemainingApiCalls(); bool ignoreNexus = (daily < 100 && hourly < 10); //var ignoreNexus = true; if (ignoreNexus) { _logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})"); } else { _logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})"); } var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus); if (nextDownload == null) { break; } _logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}"); if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash)) { await nextDownload.Finish(_sql); continue; } if (nextDownload.Archive.State is ManualDownloader.State) { await nextDownload.Finish(_sql); continue; } try { _logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"); if (!(nextDownload.Archive.State is GameFileSourceDownloader.State)) { await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}" }); } await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State }); await using var tempPath = new TempFile(); if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path)) { _logger.LogError($"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Fail(_sql, "Downloader returned false"); continue; } var hash = await tempPath.Path.FileHashAsync(); if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash) { _logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}"); await nextDownload.Fail(_sql, "Invalid Hash"); continue; } if (nextDownload.Archive.Size != default && tempPath.Path.Size != nextDownload.Archive.Size) { await nextDownload.Fail(_sql, "Invalid Size"); continue; } nextDownload.Archive.Hash = hash; nextDownload.Archive.Size = tempPath.Path.Size; _logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}"); await _archiveMaintainer.Ingest(tempPath.Path); _logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Finish(_sql); if (!(nextDownload.Archive.State is GameFileSourceDownloader.State)) { await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}" }); } } catch (Exception ex) { _logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Fail(_sql, ex.ToString()); await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}" }); } count++; } if (count > 0) { // Wake the Patch builder up in case it needs to build a patch now await _quickSync.Notify <PatchBuilder>(); } return(count); }
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) { var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension); if (list.NeedsDownload(modlist_path)) { if (File.Exists(modlist_path)) { File.Delete(modlist_path); } var state = DownloadDispatcher.ResolveArchive(list.Links.Download); Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); await state.Download(modlist_path); } else { Utils.Log($"No changes detected from downloaded modlist"); } Utils.Log($"Loading {modlist_path}"); var installer = AInstaller.LoadFromFile(modlist_path); Utils.Log($"{installer.Archives.Count} archives to validate"); DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); var validated = (await installer.Archives .PMap(queue, async archive => { bool is_failed; try { is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist); } catch (Exception) { is_failed = false; } return(new DetailedStatusItem { IsFailing = is_failed, Archive = archive }); })) .ToList(); var status = new DetailedStatus { Name = list.Title, Archives = validated.OrderBy(v => v.Archive.Name).ToList(), DownloadMetaData = list.DownloadMetadata, HasFailures = validated.Any(v => v.IsFailing) }; var dto = new ModListStatus { Id = list.Links.MachineURL, Summary = new ModlistSummary { Name = status.Name, Checked = status.Checked, Failed = status.Archives.Count(a => a.IsFailing), Passed = status.Archives.Count(a => !a.IsFailing), }, DetailedStatus = status, Metadata = list }; Utils.Log( $"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed"); await ModListStatus.Update(db, dto); Utils.Log( $"Done updating {dto.Summary.Name}"); }
public async Task TestEndToEndArchiveUpdating() { var _sql = Fixture.GetService <SqlService>(); await using var conn = await _sql.Open(); await conn.ExecuteAsync("DELETE FROM dbo.NoPatch"); var settings = Fixture.GetService <AppSettings>(); settings.ValidateModUpgrades = false; var modLists = await MakeModList("TestEndToEndArchiveUpdating.txt"); Consts.ModlistMetadataURL = modLists.ToString(); var downloader = Fixture.GetService <ArchiveDownloader>(); var archiver = Fixture.GetService <ArchiveMaintainer>(); var patcher = Fixture.GetService <PatchBuilder>(); patcher.NoCleaning = true; var sql = Fixture.GetService <SqlService>(); var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!" + Guid.NewGuid()); var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!"); var oldDataHash = oldFileData.xxHash(); var newDataHash = newFileData.xxHash(); await "TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(oldFileData); var oldArchive = new Archive(new HTTPDownloader.State(MakeURL("TestEndToEndArchiveUpdating.txt"))) { Size = oldFileData.Length, Hash = oldDataHash }; await IngestData(archiver, oldFileData); await sql.EnqueueDownload(oldArchive); var oldDownload = await sql.GetNextPendingDownload(); await oldDownload.Finish(sql); // Now update the file await "TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData); await using var tempFile = new TempFile(); var pendingRequest = DownloadDispatcher.DownloadWithPossibleUpgrade(oldArchive, tempFile.Path); for (var times = 0; await downloader.Execute() == 0 && times < 40; times++) { await Task.Delay(TimeSpan.FromMilliseconds(200)); } for (var times = 0; await patcher.Execute() == 0 && times < 40; times++) { await Task.Delay(TimeSpan.FromMilliseconds(200)); } Assert.Equal(DownloadDispatcher.DownloadResult.Update, await pendingRequest); Assert.Equal(oldDataHash, await tempFile.Path.FileHashAsync()); }
public override async Task <int> Execute() { var archives = await _sql.GetNonNexusModlistArchives(); _logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives"); using var queue = new WorkQueue(10); await DownloadDispatcher.PrepareAll(archives.Select(a => a.State)); var random = new Random(); var results = await archives.PMap(queue, async archive => { try { await Task.Delay(random.Next(1000, 5000)); var token = new CancellationTokenSource(); token.CancelAfter(TimeSpan.FromMinutes(10)); ReportStarting(archive.State.PrimaryKeyString); bool isValid = false; switch (archive.State) { //case WabbajackCDNDownloader.State _: //case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021 case GameFileSourceDownloader.State _: isValid = true; break; case ManualDownloader.State _: case ModDBDownloader.State _: case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"): isValid = true; break; default: isValid = await archive.State.Verify(archive, token.Token); break; } return(Archive : archive, IsValid : isValid); } catch (Exception ex) { _logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}"); return(Archive : archive, IsValid : false); } finally { ReportEnding(archive.State.PrimaryKeyString); } }); await _sql.UpdateNonNexusModlistArchivesStatus(results); var failed = results.Count(r => !r.IsValid); var passed = results.Count() - failed; foreach (var(archive, _) in results.Where(f => !f.IsValid)) { _logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}"); } _logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed"); return(failed); }
protected override async Task <ExitCode> Run() { var files = Input.EnumerateFiles() .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists) .ToArray(); Console.WriteLine($"Found {files.Length} files to verify"); using var queue = new WorkQueue(); var states = (await files.PMap(queue, async f => { var ini = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(); var state = (AbstractDownloadState?)await DownloadDispatcher.ResolveArchive(ini, quickMode: true); if (state == default) { Console.WriteLine($"[Skipping] {f.FileName} because no meta could be interpreted"); } if (!string.IsNullOrWhiteSpace(StateType) && !state !.PrimaryKeyString.StartsWith(StateType + "|")) { Console.WriteLine( $"[Skipping] {f.FileName} because type {state.PrimaryKeyString[0]} does not match filter"); return(f, null); } return(f, state); })).Where(s => s.state != null) .Select(s => (s.f, s.state !)) .ToArray(); await DownloadDispatcher.PrepareAll(states.Select(s => s.Item2)); Helpers.Init(); Console.WriteLine($"Found {states.Length} states to verify"); int timedOut = 0; await states.PMap(queue, async p => { try { var(f, state) = p; try { var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromMinutes(10)); var result = await state !.Verify(new Archive(state) { Name = f.FileName.ToString(), Size = f.Size }, cts.Token); Console.WriteLine($"[{(result ? "Failed" : "Passed")}] {f.FileName}"); } catch (TaskCanceledException) { Console.WriteLine($"[Timed Out] {f.FileName} {state!.PrimaryKeyString}"); Interlocked.Increment(ref timedOut); } } catch (Exception ex) { Console.WriteLine($"[Exception] {p.f.FileName} {ex.Message}"); } }); Console.WriteLine($"[Total TimedOut] {timedOut}"); Console.WriteLine("[Done]"); return(ExitCode.Ok); }