protected override async Task <ExitCode> Run()
{
var state = await DownloadDispatcher.Infer(Url);
if (state == null)
{
return(CLIUtils.Exit($"Could not find download source for URL {Url}", ExitCode.Error));
}
await DownloadDispatcher.PrepareAll(new [] { state });
using var queue = new WorkQueue();
queue.Status
.Where(s => s.ProgressPercent != Percent.Zero)
.Debounce(TimeSpan.FromSeconds(1))
.Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}"));
new[] { state }
.PMap(queue, async s =>
{
await s.Download(new Archive(state: null !)
{
Name = Path.GetFileName(Output)
}, (AbsolutePath)Output);
}).Wait();
public async Task CanUploadDownloadAndDeleteAuthoredFiles()
{
using var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3));
var originalHash = await file.Path.FileHashAsync();
var client = await Client.Create(Fixture.APIKey);
using var queue = new WorkQueue(2);
var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}"));
var data = await Fixture.GetService <SqlService>().AllAuthoredFiles();
Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName));
var result = await _client.GetStringAsync(MakeURL("authored_files"));
Assert.Contains((string)file.Path.FileName, result);
var state = await DownloadDispatcher.Infer(uri);
Assert.IsType <WabbajackCDNDownloader.State>(state);
await state.Download(new Archive(state) { Name = (string)file.Path.FileName }, file.Path);
Assert.Equal(originalHash, await file.Path.FileHashAsync());
}
protected override async Task <int> Run()
{
var state = DownloadDispatcher.Infer(Url);
if (state == null)
{
Console.WriteLine($"Could not find download source for URL {Url}");
return(1);
}
DownloadDispatcher.PrepareAll(new [] { state });
using var queue = new WorkQueue();
queue.Status
.Where(s => s.ProgressPercent != Percent.Zero)
.Debounce(TimeSpan.FromSeconds(1))
.Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}"));
new[] { state }
.PMap(queue, async s =>
{
await s.Download(new Archive {
Name = Path.GetFileName(Output)
}, Output);
}).Wait();
File.WriteAllLines(Output + ".meta", state.GetMetaIni());
return(0);
}
protected override async Task <ExitCode> Run()
{
using var queue = new WorkQueue();
var state = await DownloadDispatcher.Infer(_Input);
if (state == null)
{
Console.WriteLine("Could not parse URL");
}
Console.WriteLine("Performing initial download");
await using var temp = new TempFile();
var archive = new Archive(state !);
if (!await state !.Download(archive, temp.Path))
{
Console.WriteLine("Failed initial download");
}
var hash = await temp.Path.FileHashAsync();
archive.Hash = hash !.Value;
archive.Size = temp.Path.Size;
Console.WriteLine($"Hash: {archive.Hash} Size: {archive.Size.ToFileSizeString()}");
await Enumerable.Range(0, 100000)
.PMap(queue, async idx =>
{
if (!await state.Verify(archive))
{
throw new Exception($"{idx} Verification failed");
}
Console.WriteLine($"{idx} Verification passed");
});
return(ExitCode.Ok);
}
public async Task CanUploadDownloadAndDeleteAuthoredFiles()
{
var cleanup = Fixture.GetService <AuthoredFilesCleanup>();
var sql = Fixture.GetService <SqlService>();
var toDelete = await cleanup.FindFilesToDelete();
await using var file = new TempFile();
await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3));
var originalHash = await file.Path.FileHashAsync();
var client = await Client.Create(Fixture.APIKey);
using var queue = new WorkQueue(2);
var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}"));
var data = (await Fixture.GetService <SqlService>().AllAuthoredFiles()).ToArray();
Assert.Contains((string)file.Path.FileName, data.Select(f => f.OriginalFileName));
var listing = await cleanup.GetCDNMungedNames();
foreach (var d in data)
{
Assert.Contains(d.MungedName, listing);
}
// Just uploaded it, so it shouldn't be marked for deletion
toDelete = await cleanup.FindFilesToDelete();
foreach (var d in data)
{
Assert.DoesNotContain(d.MungedName, toDelete.CDNDelete);
Assert.DoesNotContain(d.ServerAssignedUniqueId, toDelete.SQLDelete);
}
var result = await _client.GetStringAsync(MakeURL("authored_files"));
Assert.Contains((string)file.Path.FileName, result);
var state = await DownloadDispatcher.Infer(uri);
Assert.IsType <WabbajackCDNDownloader.State>(state);
await state.Download(new Archive(state) { Name = (string)file.Path.FileName }, file.Path);
Assert.Equal(originalHash, await file.Path.FileHashAsync());
// Mark it as old
foreach (var d in data)
{
await sql.TouchAuthoredFile(await sql.GetCDNFileDefinition(d.ServerAssignedUniqueId), DateTime.Now - TimeSpan.FromDays(8));
}
// Now it should be marked for deletion
toDelete = await cleanup.FindFilesToDelete();
foreach (var d in data)
{
Assert.Contains(d.MungedName, toDelete.CDNDelete);
Assert.Contains(d.ServerAssignedUniqueId, toDelete.SQLDelete);
}
await cleanup.Execute();
toDelete = await cleanup.FindFilesToDelete();
Assert.Empty(toDelete.CDNDelete);
Assert.Empty(toDelete.SQLDelete);
}
