protected override async Task <ExitCode> Run() { var state = await DownloadDispatcher.Infer(Url); if (state == null) { return(CLIUtils.Exit($"Could not find download source for URL {Url}", ExitCode.Error)); } await DownloadDispatcher.PrepareAll(new [] { state }); using var queue = new WorkQueue(); queue.Status .Where(s => s.ProgressPercent != Percent.Zero) .Debounce(TimeSpan.FromSeconds(1)) .Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}")); new[] { state } .PMap(queue, async s => { await s.Download(new Archive(state: null !) { Name = Path.GetFileName(Output) }, (AbsolutePath)Output); }).Wait();
public async Task CanUploadDownloadAndDeleteAuthoredFiles() { using var file = new TempFile(); await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3)); var originalHash = await file.Path.FileHashAsync(); var client = await Client.Create(Fixture.APIKey); using var queue = new WorkQueue(2); var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}")); var data = await Fixture.GetService <SqlService>().AllAuthoredFiles(); Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName)); var result = await _client.GetStringAsync(MakeURL("authored_files")); Assert.Contains((string)file.Path.FileName, result); var state = await DownloadDispatcher.Infer(uri); Assert.IsType <WabbajackCDNDownloader.State>(state); await state.Download(new Archive(state) { Name = (string)file.Path.FileName }, file.Path); Assert.Equal(originalHash, await file.Path.FileHashAsync()); }
protected override async Task <int> Run() { var state = DownloadDispatcher.Infer(Url); if (state == null) { Console.WriteLine($"Could not find download source for URL {Url}"); return(1); } DownloadDispatcher.PrepareAll(new [] { state }); using var queue = new WorkQueue(); queue.Status .Where(s => s.ProgressPercent != Percent.Zero) .Debounce(TimeSpan.FromSeconds(1)) .Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}")); new[] { state } .PMap(queue, async s => { await s.Download(new Archive { Name = Path.GetFileName(Output) }, Output); }).Wait(); File.WriteAllLines(Output + ".meta", state.GetMetaIni()); return(0); }
protected override async Task <ExitCode> Run() { using var queue = new WorkQueue(); var state = await DownloadDispatcher.Infer(_Input); if (state == null) { Console.WriteLine("Could not parse URL"); } Console.WriteLine("Performing initial download"); await using var temp = new TempFile(); var archive = new Archive(state !); if (!await state !.Download(archive, temp.Path)) { Console.WriteLine("Failed initial download"); } var hash = await temp.Path.FileHashAsync(); archive.Hash = hash !.Value; archive.Size = temp.Path.Size; Console.WriteLine($"Hash: {archive.Hash} Size: {archive.Size.ToFileSizeString()}"); await Enumerable.Range(0, 100000) .PMap(queue, async idx => { if (!await state.Verify(archive)) { throw new Exception($"{idx} Verification failed"); } Console.WriteLine($"{idx} Verification passed"); }); return(ExitCode.Ok); }
public async Task CanUploadDownloadAndDeleteAuthoredFiles() { var cleanup = Fixture.GetService <AuthoredFilesCleanup>(); var sql = Fixture.GetService <SqlService>(); var toDelete = await cleanup.FindFilesToDelete(); await using var file = new TempFile(); await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3)); var originalHash = await file.Path.FileHashAsync(); var client = await Client.Create(Fixture.APIKey); using var queue = new WorkQueue(2); var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}")); var data = (await Fixture.GetService <SqlService>().AllAuthoredFiles()).ToArray(); Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName)); var listing = await cleanup.GetCDNMungedNames(); foreach (var d in data) { Assert.Contains(d.MungedName, listing); } // Just uploaded it, so it shouldn't be marked for deletion toDelete = await cleanup.FindFilesToDelete(); foreach (var d in data) { Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete); Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete); } var result = await _client.GetStringAsync(MakeURL("authored_files")); Assert.Contains((string)file.Path.FileName, result); var state = await DownloadDispatcher.Infer(uri); Assert.IsType <WabbajackCDNDownloader.State>(state); await state.Download(new Archive(state) { Name = (string)file.Path.FileName }, file.Path); Assert.Equal(originalHash, await file.Path.FileHashAsync()); // Mark it as old foreach (var d in data) { await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8)); } // Now it should be marked for deletion toDelete = await cleanup.FindFilesToDelete(); foreach (var d in data) { Assert.Contains(d.MungedName, toDelete.CDNDelete); Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete); } await cleanup.Execute(); toDelete = await cleanup.FindFilesToDelete(); Assert.Empty(toDelete.CDNDelete); Assert.Empty(toDelete.SQLDelete); }