private static async Task EnqueueFromList(DBContext db, ModlistMetadata list, WorkQueue queue)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlist_path = Path.Combine(Consts.ModListDownloadFolder,
list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlist_path))
{
if (File.Exists(modlist_path))
{
File.Delete(modlist_path);
}
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
}
else
{
Utils.Log($"No changes detected from downloaded ModList");
}
Utils.Log($"Loading {modlist_path}");
var installer = AInstaller.LoadFromFile(modlist_path);
var archives = installer.Archives;
Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
var searching = archives.Select(a => a.Hash).Distinct().ToArray();
Utils.Log($"Looking for missing archives");
var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash))
.Select(d => d.Hash).ToListAsync()).ToDictionary(a => a);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {
Payload = new IndexJob {
Archive = a
}, Priority = Job.JobPriority.Low
});
Utils.Log($"Writing jobs to the database");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions { IsOrdered = false });
Utils.Log($"Done adding archives for {installer.Name}");
}
private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlistPath))
{
modlistPath.Delete();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlistPath);
}
else
{
Utils.Log($"No changes detected from downloaded ModList");
}
Utils.Log($"Loading {modlistPath}");
var installer = AInstaller.LoadFromFile(modlistPath);
var archives = installer.Archives;
Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
var searching = archives.Select(a => a.Hash).ToHashSet();
Utils.Log($"Looking for missing archives");
var knownArchives = await sql.FilterByExistingIndexedArchives(searching);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.Contains(a.Hash)).ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {
Payload = new IndexJob {
Archive = a
}, Priority = Job.JobPriority.Low
});
Utils.Log($"Writing jobs to the database");
foreach (var job in jobs)
{
await sql.EnqueueJob(job);
}
Utils.Log($"Done adding archives for {installer.Name}");
}
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlistPath))
{
modlistPath.Delete();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlistPath);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlistPath}");
var installer = AInstaller.LoadFromFile(modlistPath);
Utils.Log($"{installer.Archives.Count} archives to validate");
await DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
var isValid = await IsValid(sql, whitelists, archive);
return(new DetailedStatusItem {
IsFailing = !isValid, Archive = archive
});
}))
.ToList();
var status = new DetailedStatus
{
Name = list.Title,
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
DownloadMetaData = list.DownloadMetadata,
HasFailures = validated.Any(v => v.IsFailing)
};
var dto = new ModListStatus
{
Id = list.Links.MachineURL,
Summary = new ModListSummary
{
Name = status.Name,
MachineURL = list.Links?.MachineURL ?? status.Name,
Checked = status.Checked,
Failed = status.Archives.Count(a => a.IsFailing),
Passed = status.Archives.Count(a => !a.IsFailing),
},
DetailedStatus = status,
Metadata = list
};
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await sql.UpdateModListStatus(dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
}
public ModListMetadataVM(ModListGalleryVM parent, ModlistMetadata metadata)
{
_parent = parent;
Metadata = metadata;
Location = Consts.ModListDownloadFolder.Combine(Metadata.Links.MachineURL + (string)Consts.ModListExtension);
ModListTagList = new List <ModListTag>();
Metadata.tags.ForEach(tag =>
{
ModListTagList.Add(new ModListTag(tag));
});
DownloadSizeText = "Download size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfArchives);
InstallSizeText = "Installation size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfInstalledFiles);
IsBroken = metadata.ValidationSummary.HasFailures;
//https://www.wabbajack.org/#/modlists/info?machineURL=eldersouls
OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.Links.MachineURL}")));
ExecuteCommand = ReactiveCommand.CreateFromObservable <Unit, Unit>(
canExecute: this.WhenAny(x => x.IsBroken).Select(x => !x),
execute: (unit) =>
Observable.Return(unit)
.WithLatestFrom(
this.WhenAny(x => x.Exists),
(_, e) => e)
// Do any download work on background thread
.ObserveOn(RxApp.TaskpoolScheduler)
.SelectTask(async(exists) =>
{
if (!exists)
{
try
{
var success = await Download();
if (!success)
{
Error = ErrorResponse.Fail("Download was marked unsuccessful");
return(false);
}
}
catch (Exception ex)
{
Error = ErrorResponse.Fail(ex);
return(false);
}
// Return an updated check on exists
return(Location.Exists);
}
return(exists);
})
.Where(exists => exists)
// Do any install page swap over on GUI thread
.ObserveOnGuiThread()
.Select(_ =>
{
_parent.MWVM.OpenInstaller(Location);
// Wait for modlist member to be filled, then open its readme
return(_parent.MWVM.Installer.Value.WhenAny(x => x.ModList)
.NotNull()
.Take(1)
.Do(modList =>
{
try
{
modList.OpenReadme();
}
catch (Exception ex)
{
Utils.Error(ex);
}
}));
})
.Switch()
.Unit());
_Exists = Observable.Interval(TimeSpan.FromSeconds(0.5))
.Unit()
.StartWith(Unit.Default)
.FlowSwitch(_parent.WhenAny(x => x.IsActive))
.SelectAsync(async _ =>
{
try
{
return(!IsDownloading && !(await metadata.NeedsDownload(Location)));
}
catch (Exception)
{
return(true);
}
})
.ToGuiProperty(this, nameof(Exists));
var imageObs = Observable.Return(Metadata.Links.ImageUri)
.DownloadBitmapImage((ex) => Utils.Log($"Error downloading modlist image {Metadata.Title}"));
_Image = imageObs
.ToGuiProperty(this, nameof(Image));
_LoadingImage = imageObs
.Select(x => false)
.StartWith(true)
.ToGuiProperty(this, nameof(LoadingImage));
}
public ModListMetadataVM(ModListGalleryVM parent, ModlistMetadata metadata)
{
_parent = parent;
Metadata = metadata;
Location = Path.Combine(Consts.ModListDownloadFolder, Metadata.Links.MachineURL + ExtensionManager.Extension);
IsBroken = metadata.ValidationSummary.HasFailures;
OpenWebsiteCommand = ReactiveCommand.Create(() => Process.Start($"https://www.wabbajack.org/modlist/{Metadata.Links.MachineURL}"));
ExecuteCommand = ReactiveCommand.CreateFromObservable <Unit, Unit>(
canExecute: this.WhenAny(x => x.IsBroken).Select(x => !x),
execute: (unit) =>
Observable.Return(unit)
.WithLatestFrom(
this.WhenAny(x => x.Exists),
(_, e) => e)
// Do any download work on background thread
.ObserveOn(RxApp.TaskpoolScheduler)
.SelectTask(async(exists) =>
{
if (!exists)
{
try
{
var success = await Download();
if (!success)
{
Error = ErrorResponse.Fail("Download was marked unsuccessful");
return(false);
}
}
catch (Exception ex)
{
Error = ErrorResponse.Fail(ex);
return(false);
}
// Return an updated check on exists
return(File.Exists(Location));
}
return(exists);
})
.Where(exists => exists)
// Do any install page swap over on GUI thread
.ObserveOnGuiThread()
.Select(_ =>
{
_parent.MWVM.OpenInstaller(Path.GetFullPath(Location));
// Wait for modlist member to be filled, then open its readme
return(_parent.MWVM.Installer.Value.WhenAny(x => x.ModList)
.NotNull()
.Take(1)
.Do(modList =>
{
try
{
modList.OpenReadmeWindow();
}
catch (Exception ex)
{
Utils.Error(ex);
}
}));
})
.Switch()
.Unit());
_Exists = Observable.Interval(TimeSpan.FromSeconds(0.5))
.Unit()
.StartWith(Unit.Default)
.FlowSwitch(_parent.WhenAny(x => x.IsActive))
.Select(_ =>
{
try
{
return(!metadata.NeedsDownload(Location));
}
catch (Exception)
{
return(true);
}
})
.ToGuiProperty(this, nameof(Exists));
var imageObs = Observable.Return(Metadata.Links.ImageUri)
.DownloadBitmapImage((ex) => Utils.Log($"Error downloading modlist image {Metadata.Title}"));
_Image = imageObs
.ToGuiProperty(this, nameof(Image));
_LoadingImage = imageObs
.Select(x => false)
.StartWith(true)
.ToGuiProperty(this, nameof(LoadingImage));
}
public ModListMetadataVM(ModListGalleryVM parent, ModlistMetadata metadata)
{
_parent = parent;
Metadata = metadata;
IsBroken = metadata.ValidationSummary.HasFailures;
OpenWebsiteCommand = ReactiveCommand.Create(() => Process.Start($"https://www.wabbajack.org/modlist/{Metadata.Links.MachineURL}"));
ExecuteCommand = ReactiveCommand.CreateFromObservable <Unit, Unit>(
canExecute: this.WhenAny(x => x.IsBroken).Select(x => !x),
execute: (unit) =>
Observable.Return(unit)
.WithLatestFrom(
this.WhenAny(x => x.Exists),
(_, e) => e)
// Do any download work on background thread
.ObserveOn(RxApp.TaskpoolScheduler)
.SelectTask(async(exists) =>
{
if (!exists)
{
await Download();
// Return an updated check on exists
return(File.Exists(Location));
}
return(exists);
})
.Where(exists => exists)
// Do any install page swap over on GUI thread
.ObserveOnGuiThread()
.Select(_ =>
{
_parent.MWVM.OpenInstaller(Path.GetFullPath(Location));
// Wait for modlist member to be filled, then open its readme
return(_parent.MWVM.Installer.Value.WhenAny(x => x.ModList)
.NotNull()
.Take(1)
.Do(modList =>
{
try
{
modList.OpenReadmeWindow();
}
catch (Exception ex)
{
Utils.Error(ex);
}
}));
})
.Switch()
.Unit());
_Exists = Observable.Interval(TimeSpan.FromSeconds(0.5))
.Unit()
.StartWith(Unit.Default)
.Select(_ =>
{
try
{
return(!metadata.NeedsDownload(Location));
}
catch (Exception)
{
return(true);
}
})
.ToProperty(this, nameof(Exists));
}
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlist_path))
{
if (File.Exists(modlist_path))
{
File.Delete(modlist_path);
}
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlist_path}");
var installer = AInstaller.LoadFromFile(modlist_path);
Utils.Log($"{installer.Archives.Count} archives to validate");
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
bool is_failed;
try
{
is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist);
}
catch (Exception)
{
is_failed = false;
}
return(new DetailedStatusItem {
IsFailing = is_failed, Archive = archive
});
}))
.ToList();
var status = new DetailedStatus
{
Name = list.Title,
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
DownloadMetaData = list.DownloadMetadata,
HasFailures = validated.Any(v => v.IsFailing)
};
var dto = new ModListStatus
{
Id = list.Links.MachineURL,
Summary = new ModlistSummary
{
Name = status.Name,
Checked = status.Checked,
Failed = status.Archives.Count(a => a.IsFailing),
Passed = status.Archives.Count(a => !a.IsFailing),
},
DetailedStatus = status,
Metadata = list
};
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await ModListStatus.Update(db, dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
}
