public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
Utils.Log("Starting Modlist Validation");
var modlists = await ModlistMetadata.LoadFromGithub();
using (var queue = new WorkQueue())
{
var whitelists = new ValidateModlist();
await whitelists.LoadListsFromGithub();
foreach (var list in modlists)
{
try
{
await ValidateList(sql, list, queue, whitelists);
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
}
}
return(JobResult.Success());
}
public async Task CanValidateModLists()
{
var sql = Fixture.GetService <SqlService>();
await using var conn = await sql.Open();
await conn.ExecuteAsync("DELETE from Patches");
var modlists = await MakeModList("CanValidateModlistsFile.txt");
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists(true);
Utils.Log("Checking validated results");
var data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
await CheckListFeeds(0, 1);
Utils.Log("Break List");
var archive = "CanValidateModlistsFile.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
// Run the non-nexus validator
await RevalidateLists(true);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(1, data.ValidationSummary.Failed);
Assert.Equal(0, data.ValidationSummary.Passed);
await CheckListFeeds(1, 0);
Utils.Log("Fix List");
await archive.WithExtension(new Extension(".moved")).MoveToAsync(archive, false);
await RevalidateLists(true);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
await CheckListFeeds(0, 1);
}
public async Task CanLoadMetadataFromTestServer()
{
var modlist = await MakeModList();
Consts.ModlistMetadataURL = modlist.ToString();
var data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal("test_list", data.First().Links.MachineURL);
}
public async Task CanLoadMetadataFromTestServer()
{
var modlist = await MakeModList("CanLoadMetadataFromTestServer.txt");
Consts.ModlistMetadataURL = modlist.ToString();
var data = await ModlistMetadata.LoadFromGithub();
Assert.Equal(3, data.Count);
Assert.Equal("test_list", data.OrderByDescending(x => x.Links.MachineURL).First().Links.MachineURL);
}
public async Task VerifyLogoURLs()
{
var modlists = await ModlistMetadata.LoadFromGithub();
foreach (var modlist in modlists.Select(m => m.Links))
{
var logo_state = DownloadDispatcher.ResolveArchive(modlist.ImageUri);
Assert.IsNotNull(logo_state);
Assert.IsTrue(await logo_state.Verify(), $"{modlist.ImageUri} is not valid");
}
}
public async Task CanHealLists()
{
var modlists = await MakeModList("CanHealLists.txt");
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists(true);
Utils.Log("Checking validated results");
var data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
await CheckListFeeds(0, 1);
Utils.Log("Break List by changing the file");
var archive = "CanHealLists.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.WriteAllTextAsync("broken");
// We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
// Run the non-nexus validator
await RevalidateLists(true);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(0, data.ValidationSummary.Passed);
Assert.Equal(1, data.ValidationSummary.Updating);
var patcher = Fixture.GetService <PatchBuilder>();
Assert.True(await patcher.Execute() > 1);
await RevalidateLists(false);
data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.NotNull(data);
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
Assert.Equal(0, data.ValidationSummary.Updating);
}
public async Task CanValidateModLists()
{
await ClearJobQueue();
var modlists = await MakeModList();
Consts.ModlistMetadataURL = modlists.ToString();
Utils.Log("Updating modlists");
await RevalidateLists();
Utils.Log("Checking validated results");
var data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(0, data.First().ValidationSummary.Failed);
Assert.Equal(1, data.First().ValidationSummary.Passed);
await CheckListFeeds(0, 1);
Utils.Log("Break List");
var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
await RevalidateLists();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(1, data.First().ValidationSummary.Failed);
Assert.Equal(0, data.First().ValidationSummary.Passed);
await CheckListFeeds(1, 0);
Utils.Log("Fix List");
await archive.WithExtension(new Extension(".moved")).MoveToAsync(archive, false);
await RevalidateLists();
data = await ModlistMetadata.LoadFromGithub();
Assert.Single(data);
Assert.Equal(0, data.First().ValidationSummary.Failed);
Assert.Equal(1, data.First().ValidationSummary.Passed);
await CheckListFeeds(0, 1);
}
public ModListGalleryVM(MainWindowVM mainWindowVM)
: base(mainWindowVM)
{
MWVM = mainWindowVM;
Observable.Return(Unit.Default)
.ObserveOn(RxApp.TaskpoolScheduler)
.SelectTask(async _ =>
{
try
{
Error = null;
var list = await ModlistMetadata.LoadFromGithub();
return(list.AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? $"Fallback{missingHashFallbackCounter++}"));
}
catch (Exception ex)
{
Utils.Error(ex);
Error = ErrorResponse.Fail(ex);
return(Observable.Empty <IChangeSet <ModlistMetadata, string> >());
}
})
// Unsubscribe and release when not active
.FlowSwitch(
this.WhenAny(x => x.IsActive),
valueWhenOff: Observable.Return(ChangeSet <ModlistMetadata, string> .Empty))
// Convert to VM and bind to resulting list
.Switch()
.ObserveOnGuiThread()
.Transform(m => new ModListMetadataVM(this, m))
.DisposeMany()
.Bind(ModLists)
.Subscribe()
.DisposeWith(CompositeDisposable);
// Extra GC when navigating away, just to immediately clean up modlist metadata
this.WhenAny(x => x.IsActive)
.Where(x => !x)
.Skip(1)
.Delay(TimeSpan.FromMilliseconds(50), RxApp.MainThreadScheduler)
.Subscribe(_ =>
{
GC.Collect();
})
.DisposeWith(CompositeDisposable);
}
public override async Task <JobResult> Execute(DBContext db, AppSettings settings)
{
Utils.Log("Starting ModList indexing");
var modlists = await ModlistMetadata.LoadFromGithub();
using (var queue = new WorkQueue())
{
foreach (var list in modlists)
{
try
{
await EnqueueFromList(db, list, queue);
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
}
}
return(JobResult.Success());
}
public ModListGalleryVM(MainWindowVM mainWindowVM)
{
MWVM = mainWindowVM;
BackCommand = ReactiveCommand.Create(
execute: () => mainWindowVM.ActivePane = mainWindowVM.ModeSelectionVM);
RefreshCommand = ReactiveCommand.Create(() => { });
RefreshCommand.StartingExecution()
.StartWith(Unit.Default)
.ObserveOn(RxApp.TaskpoolScheduler)
.SelectTask(async _ =>
{
return((await ModlistMetadata.LoadFromGithub())
.AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? $"Fallback{missingHashFallbackCounter++}"));
})
.Switch()
.ObserveOnGuiThread()
.Transform(m => new ModListMetadataVM(this, m))
.Bind(ModLists)
.Subscribe()
.DisposeWith(CompositeDisposable);
}
public async Task <string[]> GetUsedCDNFiles()
{
var modlists = (await ModlistMetadata.LoadFromGithub())
.Concat((await ModlistMetadata.LoadUnlistedFromGithub()))
.Select(f => f.Links.Download)
.Where(f => f.StartsWith(Consts.WabbajackAuthoredFilesPrefix))
.Select(f => f.Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
var files = (await _sql.ModListArchives())
.Select(a => a.State)
.OfType <WabbajackCDNDownloader.State>()
.Select(s => s.Url.ToString().Substring(Consts.WabbajackAuthoredFilesPrefix.Length));
var names = modlists.Concat(files).Distinct().ToArray();
var namesBoth = names.Concat(names.Select(HttpUtility.UrlDecode))
.Concat(names.Select(HttpUtility.UrlEncode))
.Distinct()
.ToArray();
return(namesBoth);
}
public void TestLoadingModlists()
{
var modlists = ModlistMetadata.LoadFromGithub();
Assert.IsTrue(modlists.Count > 0);
}
public async Task TestLoadingModlists()
{
var modlists = await ModlistMetadata.LoadFromGithub();
Assert.IsTrue(modlists.Count > 0);
}
public override async Task <int> Execute()
{
int downloaded = 0;
var lists = (await ModlistMetadata.LoadFromGithub())
.Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
foreach (var list in lists)
{
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
{
continue;
}
if (!_maintainer.HaveArchive(list.DownloadMetadata !.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await state.Download(new Archive(state) { Name = $"{list.Links.MachineURL}.wabbajack" }, tf.Path);
var hash = await tf.Path.FileHashAsync();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
await using (var fs = await modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
try
{
modlist = entry.FromJson <ModList>();
}
catch (JsonReaderException)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
}
await _sql.IngestModList(list.DownloadMetadata !.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Downloaded {downloaded} new lists" });
}
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Enqueing {fc} files for downloading" });
}
return(downloaded);
}
public ModListGalleryVM(MainWindowVM mainWindowVM)
: base(mainWindowVM)
{
MWVM = mainWindowVM;
ClearFiltersCommand = ReactiveCommand.Create(
() =>
{
OnlyInstalled = false;
Search = string.Empty;
});
var random = new Random();
var sourceList = Observable.Return(Unit.Default)
.ObserveOn(RxApp.TaskpoolScheduler)
.SelectTask(async _ =>
{
try
{
Error = null;
var list = await ModlistMetadata.LoadFromGithub();
Error = ErrorResponse.Success;
return(list
// Sort randomly initially, just to give each list a fair shake
.Shuffle(random)
.AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? $"Fallback{missingHashFallbackCounter++}"));
}
catch (Exception ex)
{
Utils.Error(ex);
Error = ErrorResponse.Fail(ex);
return(Observable.Empty <IChangeSet <ModlistMetadata, string> >());
}
})
// Unsubscribe and release when not active
.FlowSwitch(
this.WhenAny(x => x.IsActive),
valueWhenOff: Observable.Return(ChangeSet <ModlistMetadata, string> .Empty))
.Switch()
.RefCount();
_Loaded = sourceList.CollectionCount()
.Select(c => c > 0)
.ToProperty(this, nameof(Loaded));
// Convert to VM and bind to resulting list
sourceList
.ObserveOnGuiThread()
.Transform(m => new ModListMetadataVM(this, m))
.DisposeMany()
// Filter only installed
.Filter(predicateChanged: this.WhenAny(x => x.OnlyInstalled)
.Select <bool, Func <ModListMetadataVM, bool> >(onlyInstalled => (vm) =>
{
if (!onlyInstalled)
{
return(true);
}
if (!GameRegistry.Games.TryGetValue(vm.Metadata.Game, out var gameMeta))
{
return(false);
}
return(gameMeta.IsInstalled);
}))
// Filter on search box
.Filter(predicateChanged: this.WhenAny(x => x.Search)
.Debounce(TimeSpan.FromMilliseconds(150), RxApp.MainThreadScheduler)
.Select <string, Func <ModListMetadataVM, bool> >(search => (vm) =>
{
if (string.IsNullOrWhiteSpace(search))
{
return(true);
}
return(vm.Metadata.Title.ContainsCaseInsensitive(search));
}))
// Put broken lists at bottom
.Sort(Comparer <ModListMetadataVM> .Create((a, b) => a.IsBroken.CompareTo(b.IsBroken)))
.Bind(ModLists)
.Subscribe()
.DisposeWith(CompositeDisposable);
// Extra GC when navigating away, just to immediately clean up modlist metadata
this.WhenAny(x => x.IsActive)
.Where(x => !x)
.Skip(1)
.Delay(TimeSpan.FromMilliseconds(50), RxApp.MainThreadScheduler)
.Subscribe(_ =>
{
GC.Collect();
})
.DisposeWith(CompositeDisposable);
}
public ModListGalleryVM(MainWindowVM mainWindowVM)
: base(mainWindowVM)
{
MWVM = mainWindowVM;
// load persistent filter settings
if (settings.IsPersistent)
{
GameType = !string.IsNullOrEmpty(settings.Game) ? settings.Game : ALL_GAME_TYPE;
ShowNSFW = settings.ShowNSFW;
ShowUtilityLists = settings.ShowUtilityLists;
OnlyInstalled = settings.OnlyInstalled;
Search = settings.Search;
}
else
{
GameType = ALL_GAME_TYPE;
}
// subscribe to save signal
MWVM.Settings.SaveSignal
.Subscribe(_ => UpdateFiltersSettings())
.DisposeWith(this.CompositeDisposable);
ClearFiltersCommand = ReactiveCommand.Create(
() =>
{
OnlyInstalled = false;
ShowNSFW = false;
ShowUtilityLists = false;
Search = string.Empty;
GameType = ALL_GAME_TYPE;
});
this.WhenAny(x => x.OnlyInstalled)
.Subscribe(val =>
{
if (val)
{
GameType = ALL_GAME_TYPE;
}
})
.DisposeWith(CompositeDisposable);
var sourceList = Observable.Return(Unit.Default)
.ObserveOn(RxApp.TaskpoolScheduler)
.SelectTask(async _ =>
{
try
{
Error = null;
var list = await ModlistMetadata.LoadFromGithub();
Error = ErrorResponse.Success;
return(list
.AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? Hash.Empty));
}
catch (Exception ex)
{
Utils.Error(ex);
Error = ErrorResponse.Fail(ex);
return(Observable.Empty <IChangeSet <ModlistMetadata, Hash> >());
}
})
// Unsubscribe and release when not active
.FlowSwitch(
this.WhenAny(x => x.IsActive),
valueWhenOff: Observable.Return(ChangeSet <ModlistMetadata, Hash> .Empty))
.Switch()
.RefCount();
_Loaded = sourceList.CollectionCount()
.Select(c => c > 0)
.ToProperty(this, nameof(Loaded));
// Convert to VM and bind to resulting list
sourceList
.ObserveOnGuiThread()
.Transform(m => new ModListMetadataVM(this, m))
.DisposeMany()
// Filter only installed
.Filter(this.WhenAny(x => x.OnlyInstalled)
.Select <bool, Func <ModListMetadataVM, bool> >(onlyInstalled => (vm) =>
{
if (!onlyInstalled)
{
return(true);
}
if (!GameRegistry.Games.TryGetValue(vm.Metadata.Game, out var gameMeta))
{
return(false);
}
return(gameMeta.IsInstalled);
}))
// Filter on search box
.Filter(this.WhenAny(x => x.Search)
.Debounce(TimeSpan.FromMilliseconds(150), RxApp.MainThreadScheduler)
.Select <string, Func <ModListMetadataVM, bool> >(search => (vm) =>
{
if (string.IsNullOrWhiteSpace(search))
{
return(true);
}
return(vm.Metadata.Title.ContainsCaseInsensitive(search) || vm.Metadata.tags.Any(t => t.ContainsCaseInsensitive(search)));
}))
.Filter(this.WhenAny(x => x.ShowNSFW)
.Select <bool, Func <ModListMetadataVM, bool> >(showNSFW => vm =>
{
if (!vm.Metadata.NSFW)
{
return(true);
}
return(vm.Metadata.NSFW && showNSFW);
}))
.Filter(this.WhenAny(x => x.ShowUtilityLists)
.Select <bool, Func <ModListMetadataVM, bool> >(showUtilityLists => vm => showUtilityLists ? vm.Metadata.UtilityList : !vm.Metadata.UtilityList))
// Filter by Game
.Filter(this.WhenAny(x => x.GameType)
.Debounce(TimeSpan.FromMilliseconds(150), RxApp.MainThreadScheduler)
.Select <string, Func <ModListMetadataVM, bool> >(GameType => (vm) =>
{
if (GameType == ALL_GAME_TYPE)
{
return(true);
}
if (string.IsNullOrEmpty(GameType))
{
return(false);
}
return(GameType == vm.Metadata.Game.GetDescription <Game>().ToString());
}))
.Bind(ModLists)
.Subscribe()
.DisposeWith(CompositeDisposable);
// Extra GC when navigating away, just to immediately clean up modlist metadata
this.WhenAny(x => x.IsActive)
.Where(x => !x)
.Skip(1)
.Delay(TimeSpan.FromMilliseconds(50), RxApp.MainThreadScheduler)
.Subscribe(_ =>
{
GC.Collect();
})
.DisposeWith(CompositeDisposable);
}
