private static async Task EnqueueFromList(DBContext db, ModlistMetadata list, WorkQueue queue) { var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension); if (list.NeedsDownload(modlist_path)) { if (File.Exists(modlist_path)) { File.Delete(modlist_path); } var state = DownloadDispatcher.ResolveArchive(list.Links.Download); Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); await state.Download(modlist_path); } else { Utils.Log($"No changes detected from downloaded ModList"); } Utils.Log($"Loading {modlist_path}"); var installer = AInstaller.LoadFromFile(modlist_path); var archives = installer.Archives; Utils.Log($"Found {archives.Count} archives in {installer.Name} to index"); var searching = archives.Select(a => a.Hash).Distinct().ToArray(); Utils.Log($"Looking for missing archives"); var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash)) .Select(d => d.Hash).ToListAsync()).ToDictionary(a => a); Utils.Log($"Found {knownArchives.Count} pre-existing archives"); var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList(); Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); var jobs = missing.Select(a => new Job { Payload = new IndexJob { Archive = a }, Priority = Job.JobPriority.Low }); Utils.Log($"Writing jobs to the database"); await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions { IsOrdered = false }); Utils.Log($"Done adding archives for {installer.Name}"); }
private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue) { var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); if (list.NeedsDownload(modlistPath)) { modlistPath.Delete(); var state = DownloadDispatcher.ResolveArchive(list.Links.Download); Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); await state.Download(modlistPath); } else { Utils.Log($"No changes detected from downloaded ModList"); } Utils.Log($"Loading {modlistPath}"); var installer = AInstaller.LoadFromFile(modlistPath); var archives = installer.Archives; Utils.Log($"Found {archives.Count} archives in {installer.Name} to index"); var searching = archives.Select(a => a.Hash).ToHashSet(); Utils.Log($"Looking for missing archives"); var knownArchives = await sql.FilterByExistingIndexedArchives(searching); Utils.Log($"Found {knownArchives.Count} pre-existing archives"); var missing = archives.Where(a => !knownArchives.Contains(a.Hash)).ToList(); Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); var jobs = missing.Select(a => new Job { Payload = new IndexJob { Archive = a }, Priority = Job.JobPriority.Low }); Utils.Log($"Writing jobs to the database"); foreach (var job in jobs) { await sql.EnqueueJob(job); } Utils.Log($"Done adding archives for {installer.Name}"); }
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) { var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); if (list.NeedsDownload(modlistPath)) { modlistPath.Delete(); var state = DownloadDispatcher.ResolveArchive(list.Links.Download); Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); await state.Download(modlistPath); } else { Utils.Log($"No changes detected from downloaded modlist"); } Utils.Log($"Loading {modlistPath}"); var installer = AInstaller.LoadFromFile(modlistPath); Utils.Log($"{installer.Archives.Count} archives to validate"); await DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); var validated = (await installer.Archives .PMap(queue, async archive => { var isValid = await IsValid(sql, whitelists, archive); return(new DetailedStatusItem { IsFailing = !isValid, Archive = archive }); })) .ToList(); var status = new DetailedStatus { Name = list.Title, Archives = validated.OrderBy(v => v.Archive.Name).ToList(), DownloadMetaData = list.DownloadMetadata, HasFailures = validated.Any(v => v.IsFailing) }; var dto = new ModListStatus { Id = list.Links.MachineURL, Summary = new ModListSummary { Name = status.Name, MachineURL = list.Links?.MachineURL ?? status.Name, Checked = status.Checked, Failed = status.Archives.Count(a => a.IsFailing), Passed = status.Archives.Count(a => !a.IsFailing), }, DetailedStatus = status, Metadata = list }; Utils.Log( $"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed"); await sql.UpdateModListStatus(dto); Utils.Log( $"Done updating {dto.Summary.Name}"); }
public ModListMetadataVM(ModListGalleryVM parent, ModlistMetadata metadata) { _parent = parent; Metadata = metadata; Location = Consts.ModListDownloadFolder.Combine(Metadata.Links.MachineURL + (string)Consts.ModListExtension); ModListTagList = new List <ModListTag>(); Metadata.tags.ForEach(tag => { ModListTagList.Add(new ModListTag(tag)); }); DownloadSizeText = "Download size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfArchives); InstallSizeText = "Installation size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfInstalledFiles); IsBroken = metadata.ValidationSummary.HasFailures; //https://www.wabbajack.org/#/modlists/info?machineURL=eldersouls OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.Links.MachineURL}"))); ExecuteCommand = ReactiveCommand.CreateFromObservable <Unit, Unit>( canExecute: this.WhenAny(x => x.IsBroken).Select(x => !x), execute: (unit) => Observable.Return(unit) .WithLatestFrom( this.WhenAny(x => x.Exists), (_, e) => e) // Do any download work on background thread .ObserveOn(RxApp.TaskpoolScheduler) .SelectTask(async(exists) => { if (!exists) { try { var success = await Download(); if (!success) { Error = ErrorResponse.Fail("Download was marked unsuccessful"); return(false); } } catch (Exception ex) { Error = ErrorResponse.Fail(ex); return(false); } // Return an updated check on exists return(Location.Exists); } return(exists); }) .Where(exists => exists) // Do any install page swap over on GUI thread .ObserveOnGuiThread() .Select(_ => { _parent.MWVM.OpenInstaller(Location); // Wait for modlist member to be filled, then open its readme return(_parent.MWVM.Installer.Value.WhenAny(x => x.ModList) .NotNull() .Take(1) .Do(modList => { try { modList.OpenReadme(); } catch (Exception ex) { Utils.Error(ex); } })); }) .Switch() .Unit()); _Exists = Observable.Interval(TimeSpan.FromSeconds(0.5)) .Unit() .StartWith(Unit.Default) .FlowSwitch(_parent.WhenAny(x => x.IsActive)) .SelectAsync(async _ => { try { return(!IsDownloading && !(await metadata.NeedsDownload(Location))); } catch (Exception) { return(true); } }) .ToGuiProperty(this, nameof(Exists)); var imageObs = Observable.Return(Metadata.Links.ImageUri) .DownloadBitmapImage((ex) => Utils.Log($"Error downloading modlist image {Metadata.Title}")); _Image = imageObs .ToGuiProperty(this, nameof(Image)); _LoadingImage = imageObs .Select(x => false) .StartWith(true) .ToGuiProperty(this, nameof(LoadingImage)); }
public ModListMetadataVM(ModListGalleryVM parent, ModlistMetadata metadata) { _parent = parent; Metadata = metadata; Location = Path.Combine(Consts.ModListDownloadFolder, Metadata.Links.MachineURL + ExtensionManager.Extension); IsBroken = metadata.ValidationSummary.HasFailures; OpenWebsiteCommand = ReactiveCommand.Create(() => Process.Start($"https://www.wabbajack.org/modlist/{Metadata.Links.MachineURL}")); ExecuteCommand = ReactiveCommand.CreateFromObservable <Unit, Unit>( canExecute: this.WhenAny(x => x.IsBroken).Select(x => !x), execute: (unit) => Observable.Return(unit) .WithLatestFrom( this.WhenAny(x => x.Exists), (_, e) => e) // Do any download work on background thread .ObserveOn(RxApp.TaskpoolScheduler) .SelectTask(async(exists) => { if (!exists) { try { var success = await Download(); if (!success) { Error = ErrorResponse.Fail("Download was marked unsuccessful"); return(false); } } catch (Exception ex) { Error = ErrorResponse.Fail(ex); return(false); } // Return an updated check on exists return(File.Exists(Location)); } return(exists); }) .Where(exists => exists) // Do any install page swap over on GUI thread .ObserveOnGuiThread() .Select(_ => { _parent.MWVM.OpenInstaller(Path.GetFullPath(Location)); // Wait for modlist member to be filled, then open its readme return(_parent.MWVM.Installer.Value.WhenAny(x => x.ModList) .NotNull() .Take(1) .Do(modList => { try { modList.OpenReadmeWindow(); } catch (Exception ex) { Utils.Error(ex); } })); }) .Switch() .Unit()); _Exists = Observable.Interval(TimeSpan.FromSeconds(0.5)) .Unit() .StartWith(Unit.Default) .FlowSwitch(_parent.WhenAny(x => x.IsActive)) .Select(_ => { try { return(!metadata.NeedsDownload(Location)); } catch (Exception) { return(true); } }) .ToGuiProperty(this, nameof(Exists)); var imageObs = Observable.Return(Metadata.Links.ImageUri) .DownloadBitmapImage((ex) => Utils.Log($"Error downloading modlist image {Metadata.Title}")); _Image = imageObs .ToGuiProperty(this, nameof(Image)); _LoadingImage = imageObs .Select(x => false) .StartWith(true) .ToGuiProperty(this, nameof(LoadingImage)); }
public ModListMetadataVM(ModListGalleryVM parent, ModlistMetadata metadata) { _parent = parent; Metadata = metadata; IsBroken = metadata.ValidationSummary.HasFailures; OpenWebsiteCommand = ReactiveCommand.Create(() => Process.Start($"https://www.wabbajack.org/modlist/{Metadata.Links.MachineURL}")); ExecuteCommand = ReactiveCommand.CreateFromObservable <Unit, Unit>( canExecute: this.WhenAny(x => x.IsBroken).Select(x => !x), execute: (unit) => Observable.Return(unit) .WithLatestFrom( this.WhenAny(x => x.Exists), (_, e) => e) // Do any download work on background thread .ObserveOn(RxApp.TaskpoolScheduler) .SelectTask(async(exists) => { if (!exists) { await Download(); // Return an updated check on exists return(File.Exists(Location)); } return(exists); }) .Where(exists => exists) // Do any install page swap over on GUI thread .ObserveOnGuiThread() .Select(_ => { _parent.MWVM.OpenInstaller(Path.GetFullPath(Location)); // Wait for modlist member to be filled, then open its readme return(_parent.MWVM.Installer.Value.WhenAny(x => x.ModList) .NotNull() .Take(1) .Do(modList => { try { modList.OpenReadmeWindow(); } catch (Exception ex) { Utils.Error(ex); } })); }) .Switch() .Unit()); _Exists = Observable.Interval(TimeSpan.FromSeconds(0.5)) .Unit() .StartWith(Unit.Default) .Select(_ => { try { return(!metadata.NeedsDownload(Location)); } catch (Exception) { return(true); } }) .ToProperty(this, nameof(Exists)); }
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) { var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension); if (list.NeedsDownload(modlist_path)) { if (File.Exists(modlist_path)) { File.Delete(modlist_path); } var state = DownloadDispatcher.ResolveArchive(list.Links.Download); Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); await state.Download(modlist_path); } else { Utils.Log($"No changes detected from downloaded modlist"); } Utils.Log($"Loading {modlist_path}"); var installer = AInstaller.LoadFromFile(modlist_path); Utils.Log($"{installer.Archives.Count} archives to validate"); DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); var validated = (await installer.Archives .PMap(queue, async archive => { bool is_failed; try { is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist); } catch (Exception) { is_failed = false; } return(new DetailedStatusItem { IsFailing = is_failed, Archive = archive }); })) .ToList(); var status = new DetailedStatus { Name = list.Title, Archives = validated.OrderBy(v => v.Archive.Name).ToList(), DownloadMetaData = list.DownloadMetadata, HasFailures = validated.Any(v => v.IsFailing) }; var dto = new ModListStatus { Id = list.Links.MachineURL, Summary = new ModlistSummary { Name = status.Name, Checked = status.Checked, Failed = status.Archives.Count(a => a.IsFailing), Passed = status.Archives.Count(a => !a.IsFailing), }, DetailedStatus = status, Metadata = list }; Utils.Log( $"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed"); await ModListStatus.Update(db, dto); Utils.Log( $"Done updating {dto.Summary.Name}"); }