public override async Task <bool> Verify(Archive a)
{
try
{
var client = await NexusApiClient.Get();
var modInfo = await client.GetModInfo(Game, ModID);
if (!modInfo.available)
{
return(false);
}
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return(found != null);
}
catch (Exception ex)
{
Utils.Log($"{Name} - {Game} - {ModID} - {FileID} - Error getting Nexus download URL - {ex}");
return(false);
}
}
public async Task <AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
{
var general = archiveINI?.General;
if (general.modID != null && general.fileID != null && general.gameName != null)
{
var name = (string)general.gameName;
var gameMeta = GameRegistry.GetByMO2ArchiveName(name);
var game = gameMeta != null?GameRegistry.GetByMO2ArchiveName(name).Game : GameRegistry.GetByNexusName(name).Game;
var client = await NexusApiClient.Get();
var info = await client.GetModInfo(game, general.modID);
return(new State
{
GameName = general.gameName,
FileID = general.fileID,
ModID = general.modID,
Version = general.version ?? "0.0.0.0",
Author = info.author,
UploadedBy = info.uploaded_by,
UploaderProfile = info.uploaded_users_profile_url,
ModName = info.name,
SlideShowPic = info.picture_url,
NexusURL = NexusApiUtils.GetModURL(game, info.mod_id),
Summary = info.summary,
Adult = info.contains_adult_content
});
}
return(null);
}
public override async Task <bool> Verify(Archive a)
{
try
{
var gameMeta = GameRegistry.GetByMO2ArchiveName(GameName) ?? GameRegistry.GetByNexusName(GameName);
if (gameMeta == null)
{
return(false);
}
var game = gameMeta.Game;
if (!int.TryParse(ModID, out var modID))
{
return(false);
}
var client = await NexusApiClient.Get();
var modFiles = await client.GetModFiles(game, modID);
if (!ulong.TryParse(FileID, out var fileID))
{
return(false);
}
var found = modFiles.files
.FirstOrDefault(file => file.file_id == fileID && file.category_name != null);
return(found != null);
}
catch (Exception ex)
{
Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error getting Nexus download URL - {ex}");
return(false);
}
}
public async Task <NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
{
_logger.Log(LogLevel.Information, $"{GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetModFiles(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
result = await api.GetModFiles(game, ModId, false);
var date = result.files.Select(f => f.uploaded_time).OrderByDescending(o => o).FirstOrDefault();
date = date == default ? DateTime.UtcNow : date;
await _sql.AddNexusModFiles(game, ModId, date, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return(result);
}
private static async Task <AbsolutePath> DownloadMod(Game game, int mod)
{
using var client = await NexusApiClient.Get();
var results = await client.GetModFiles(game, mod);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = _stagingFolder.Combine(file.file_name);
if (src.Exists)
{
return(src);
}
var state = new NexusDownloader.State
{
ModID = mod,
Game = game,
FileID = file.file_id
};
await state.Download(src);
return(src);
}
public async Task <NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
{
Utils.Log($"Nexus Mod Files {GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await SQL.GetModFiles(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
result = await api.GetModFiles(game, ModId, false);
await SQL.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return(result);
}
public async Task <NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, string ModId)
{
var result = await Db.NexusModFiles.FindOneAsync(info => info.Game == GameName && info.ModId == ModId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/games/{GameName}/mods/{ModId}/files.json";
var body = await api.Get <NexusApiClient.GetModFilesResponse>(path);
result = new NexusCacheData <NexusApiClient.GetModFilesResponse>
{
Data = body, Path = path, Game = GameName, ModId = ModId
};
try
{
await Db.NexusModFiles.InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response.Headers.Add("x-cache-result", method);
return(result.Data);
}
public async Task Prepare()
{
if (!_prepared)
{
using var _ = await _lock.WaitAsync();
// Could have become prepared while we waited for the lock
if (!_prepared)
{
_client = await NexusApiClient.Get();
_status = await _client.GetUserStatus();
if (!_client.IsAuthenticated)
{
Utils.ErrorThrow(new UnconvertedError(
$"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
return;
}
if (!await _client.IsPremium())
{
var result = await Utils.Log(new YesNoIntervention(
"Wabbajack can operate without a premium account, but downloads will be slower and the install process will require more user interactions (you will have to start each download by hand). Are you sure you wish to continue?",
"Continue without Premium?")).Task;
if (result == ConfirmationIntervention.Choice.Abort)
{
Utils.ErrorThrow(new UnconvertedError($"Aborting at the request of the user"));
}
}
_prepared = true;
}
}
}
private async Task <object> HandleGetFiles(dynamic arg)
{
Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}");
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
return(api.GetModFiles(GameRegistry.GetByNexusName((string)arg.GameName).Game, (int)arg.ModID).ToJSON());
}
public async Task <NexusApiClient> GetClient()
{
var keys = await _sql.GetNexusApiKeysWithCounts(1500);
foreach (var key in keys.Where(k => k.Key != _selfKey))
{
try
{
var client = new TrackingClient(_sql, key);
if (await client.IsPremium())
{
return(client);
}
_logger.LogWarning($"Purging non premium key");
await _sql.DeleteNexusAPIKey(key.Key);
continue;
}
catch (Exception ex)
{
Utils.Log($"Error getting tracking client: {ex}");
}
}
var bclient = await NexusApiClient.Get();
await bclient.GetUserStatus();
return(bclient);
}
public override async Task <bool> Verify(Archive a)
{
try
{
var client = await NexusApiClient.Get();
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
if (found != null)
{
return(true);
}
Utils.Log($"Could not validate {URL} with cache, validating manually");
modFiles = await client.GetModFiles(Game, ModID, false);
found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return(found != null);
}
catch (Exception ex)
{
Utils.Log($"{Name} - {Game} - {ModID} - {FileID} - Error getting Nexus download URL - {ex}");
return(false);
}
}
public async Task <object> UpdateCache(object arg)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
await api.ClearUpdatedModsInCache();
return("Done");
}
public async Task UpdateNexusCacheRSS()
{
using var _ = _logger.BeginScope("Nexus Update via RSS");
_logger.Log(LogLevel.Information, "Starting");
var results = await NexusUpdatesFeeds.GetUpdates();
NexusApiClient client = null;
long updated = 0;
foreach (var result in results)
{
try
{
var purgedMods =
await _sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var purgedFiles =
await _sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var totalPurged = purgedFiles + purgedMods;
if (totalPurged > 0)
{
_logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items {result.Game} {result.ModId} {result.TimeStamp}");
}
if (await _sql.GetNexusModInfoString(result.Game, result.ModId) != null)
{
continue;
}
// Lazily create the client
client ??= await NexusApiClient.Get();
// Cache the info
var files = await client.GetModFiles(result.Game, result.ModId, false);
await _sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
var modInfo = await client.GetModInfo(result.Game, result.ModId, useCache : false);
await _sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
updated++;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed Nexus update for {result.Game} - {result.ModId} - {result.TimeStamp}");
}
}
if (updated > 0)
{
_logger.Log(LogLevel.Information, $"Primed {updated} nexus cache entries");
}
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
public override async Task <(Archive?Archive, TempFile NewFile)> FindUpgrade(Archive a, Func <Archive, Task <AbsolutePath> > downloadResolver)
{
var client = await NexusApiClient.Get();
var mod = await client.GetModInfo(Game, ModID);
if (!mod.available)
{
return(default);
public async Task <AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var general = archiveINI.General;
if (general.modID != null && general.fileID != null && general.gameName != null)
{
var game = GameRegistry.GetByFuzzyName((string)general.gameName).Game;
if (quickMode)
{
return(new State
{
Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game,
ModID = long.Parse(general.modID),
FileID = long.Parse(general.fileID),
});
}
var client = await NexusApiClient.Get();
ModInfo info;
try
{
info = await client.GetModInfo(game, long.Parse((string)general.modID));
}
catch (Exception)
{
Utils.Error($"Error getting mod info for Nexus mod with {general.modID}");
throw;
}
try
{
return(new State
{
Name = NexusApiUtils.FixupSummary(info.name),
Author = NexusApiUtils.FixupSummary(info.author),
Version = general.version ?? "0.0.0.0",
ImageURL = info.picture_url,
IsNSFW = info.contains_adult_content,
Description = NexusApiUtils.FixupSummary(info.summary),
Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game,
ModID = long.Parse(general.modID),
FileID = long.Parse(general.fileID)
});
}
catch (FormatException)
{
Utils.Log(
$"Cannot parse ModID/FileID from {(string)general.gameName} {(string)general.modID} {(string)general.fileID}");
throw;
}
}
return(null);
}
private async Task <AbsolutePath> DownloadMod(Game game, int mod, int fileId)
{
using var client = await NexusApiClient.Get();
var results = await client.GetModFiles(game, mod);
var file = results.files.FirstOrDefault(f => f.file_id == fileId);
return(await DownloadNexusFile(game, mod, file));
}
private static async Task <AbsolutePath> DownloadMod(Game game, int mod)
{
using var client = await NexusApiClient.Get();
var results = await client.GetModFiles(game, mod);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
return(await DownloadNexusFile(game, mod, file));
}
public async Task <NexusApiClient> GetClient()
{
var keys = await _sql.GetNexusApiKeysWithCounts(1500);
foreach (var key in keys)
{
return(new TrackingClient(_sql, key));
}
return(await NexusApiClient.Get());
}
public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
var api = await NexusApiClient.Get();
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
var mods = await api.Get <List <NexusUpdateEntry> >(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
var entry = new NexusCacheData <List <NexusUpdateEntry> >();
entry.Game = game.NexusName;
entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m";
entry.Data = mods;
return(game, mods);
})
.Select(async rTask =>
{
var(game, mods) = await rTask;
return(mods.Select(mod => new { game = game, mod = mod }));
}).ToList();
Utils.Log($"Getting update list for {gameTasks.Count} games");
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
Utils.Log($"Found {purge.Count} updated mods in the last month");
using (var queue = new WorkQueue())
{
var collected = purge.Select(d =>
{
var a = d.mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return(new { Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId });
});
var purged = await collected.PMap(queue, async t =>
{
var resultA = await sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
var resultB = await sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return(resultA + resultB);
});
Utils.Log($"Purged {purged.Sum()} cache entries");
}
return(JobResult.Success());
}
private async Task <object> HandleFileID(dynamic arg)
{
Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}");
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
return(api.GetFileInfo(new NexusDownloader.State
{
GameName = arg.GameName,
ModID = arg.ModID,
FileID = arg.FileID
}).ToJSON());
}
public async Task TestCanGetModInfo()
{
var sqlService = Fixture.GetService <SqlService>();
var modId = long.MaxValue >> 1;
await sqlService.AddNexusModInfo(Game.SkyrimSpecialEdition, modId, DateTime.Now,
new ModInfo { author = "Buzz", uploaded_by = "bille" });
var api = await NexusApiClient.Get();
var modInfoResponse = await api.GetModInfo(Game.SkyrimSpecialEdition, modId);
Assert.Equal("Buzz", modInfoResponse.author);
Assert.Equal("bille", modInfoResponse.uploaded_by);
}
public async Task UpdateNexusCacheAPI()
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting");
var api = await NexusApiClient.Get();
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
var mods = await api.Get <List <NexusUpdateEntry> >(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
return(game, mods);
})
.Select(async rTask =>
{
var(game, mods) = await rTask;
return(mods.Select(mod => new { game = game, mod = mod }));
}).ToList();
_logger.Log(LogLevel.Information, $"Getting update list for {gameTasks.Count} games");
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
_logger.Log(LogLevel.Information, $"Found {purge.Count} updated mods in the last month");
using var queue = new WorkQueue();
var collected = purge.Select(d =>
{
var a = d.mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return(new { Game = d.game.Game, Date = (a > b) ? a : b, ModId = d.mod.ModId });
});
var purged = await collected.PMap(queue, async t =>
{
long purgeCount = 0;
purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return(purgeCount);
});
_logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries");
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
public async Task <(Archive?Archive, TempFile NewFile)> FindUpgrade(Archive a)
{
var client = await NexusApiClient.Get();
var mod = await client.GetModInfo(Game, ModID);
var files = await client.GetModFiles(Game, ModID);
var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID);
var nl = new Levenshtein();
var newFile = files.files.Where(f => f.category_name != null)
.OrderBy(f => nl.Distance(oldFile.name.ToLowerInvariant(), f.name.ToLowerInvariant())).FirstOrDefault();
if (!mod.available || oldFile == default || newFile == default)
{
return(default);
public static async Task <long> UpdateNexusCacheFast(SqlService sql)
{
var results = await NexusUpdatesFeeds.GetUpdates();
NexusApiClient client = null;
long updated = 0;
foreach (var result in results)
{
var purgedMods = await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var purgedFiles = await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var totalPurged = purgedFiles + purgedMods;
if (totalPurged > 0)
{
Utils.Log($"Purged {totalPurged} cache items");
}
if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null)
{
continue;
}
// Lazily create the client
client ??= await NexusApiClient.Get();
// Cache the info
var files = await client.GetModFiles(result.Game, result.ModId, false);
await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
var modInfo = await client.GetModInfo(result.Game, result.ModId);
await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
updated++;
}
if (updated > 0)
{
Utils.Log($"Primed {updated} nexus cache entries");
}
LastNexusSync = DateTime.Now;
return(updated);
}
private async Task <Archive> FindAlternatives(NexusDownloader.State state, Hash srcHash)
{
var origSize = _settings.PathForArchive(srcHash).Size;
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var allMods = await api.GetModFiles(state.Game, state.ModID);
var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
.OrderBy(s => Math.Abs((long)s.size - origSize))
.Select(s =>
new Archive(
new NexusDownloader.State
{
Game = state.Game,
ModID = state.ModID,
FileID = s.file_id
})
{
Name = s.file_name,
Size = (long)s.size,
})
.FirstOrDefault();
if (archive == null)
{
Utils.Log($"No alternative for {srcHash}");
return(null);
}
Utils.Log($"Found alternative for {srcHash}");
var indexed = await SQL.DownloadStateByPrimaryKey(archive.State.PrimaryKeyString);
if (indexed == null)
{
return(archive);
}
Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}");
archive.Hash = indexed.Hash;
return(archive);
}
public override async Task <bool> Download(Archive a, AbsolutePath destination)
{
string url;
try
{
var client = await NexusApiClient.Get();
url = await client.GetNexusDownloadLink(this);
}
catch (Exception ex)
{
Utils.Log($"{a.Name} - Error getting Nexus download URL - {ex.Message}");
return(false);
}
Utils.Log($"Downloading Nexus Archive - {a.Name} - {Game} - {ModID} - {FileID}");
return(await new HTTPDownloader.State(url).Download(a, destination));
}
public async Task <NexusApiClient> GetClient()
{
var keys = await _sql.GetNexusApiKeysWithCounts(1500);
foreach (var key in keys.Where(k => k.Key != _selfKey))
{
var client = new TrackingClient(_sql, key);
if (!await client.IsPremium())
{
_logger.LogWarning($"Purging non premium key");
await _sql.DeleteNexusAPIKey(key.Key);
continue;
}
return(client);
}
return(await NexusApiClient.Get());
}
private async Task <(string Download, string ModFolder)> DownloadAndInstall(Game game, int modid, string mod_name)
{
utils.AddMod(mod_name);
var client = await NexusApiClient.Get();
var resp = await client.GetModFiles(game, modid);
var file = resp.files.FirstOrDefault(f => f.is_primary) ?? resp.files.FirstOrDefault(f => !string.IsNullOrEmpty(f.category_name));
var src = Path.Combine(DOWNLOAD_FOLDER, file.file_name);
var ini = string.Join("\n",
new List <string>
{
"[General]",
$"gameName={game.MetaData().MO2ArchiveName}",
$"modID={modid}",
$"fileID={file.file_id}"
});
if (!File.Exists(src))
{
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
await state.Download(src);
}
if (!Directory.Exists(utils.DownloadsFolder))
{
Directory.CreateDirectory(utils.DownloadsFolder);
}
var dest = Path.Combine(utils.DownloadsFolder, file.file_name);
await Utils.CopyFileAsync(src, dest);
var modFolder = Path.Combine(utils.ModsFolder, mod_name);
await FileExtractor.ExtractAll(Queue, src, modFolder);
File.WriteAllText(dest + Consts.MetaFileExtension, ini);
return(dest, modFolder);
}
private async Task AskToEndorse()
{
var mods = ModList.Archives
.Select(m => m.State)
.OfType <NexusDownloader.State>()
.GroupBy(f => (f.GameName, f.ModID))
.Select(mod => mod.First())
.ToArray();
var result = MessageBox.Show(
$"Installation has completed, but you have installed {mods.Length} from the Nexus, would you like to" +
" endorse these mods to show support to the authors? It will only take a few moments.", "Endorse Mods?",
MessageBoxButton.YesNo, MessageBoxImage.Question);
if (result != MessageBoxResult.Yes)
{
return;
}
// Shuffle mods so that if we hit a API limit we don't always miss the same mods
var r = new Random();
for (var i = 0; i < mods.Length; i++)
{
var a = r.Next(mods.Length);
var b = r.Next(mods.Length);
var tmp = mods[a];
mods[a] = mods[b];
mods[b] = tmp;
}
await mods.PMap(Queue, async mod =>
{
var client = await NexusApiClient.Get();
var er = await client.EndorseMod(mod);
Utils.Log($"Endorsed {mod.GameName} - {mod.ModID} - Result: {er.message}");
});
Info("Done! You may now exit the application!");
}
public async Task TestCanGetModFiles()
{
var sqlService = Fixture.GetService <SqlService>();
var modId = long.MaxValue >> 1;
await sqlService.AddNexusModFiles(Game.SkyrimSpecialEdition, modId, DateTime.Now,
new NexusApiClient.GetModFilesResponse {
files = new List <NexusFileInfo>
{
new NexusFileInfo
{
file_name = "blerg"
}
}
});
var api = await NexusApiClient.Get();
var modInfoResponse = await api.GetModFiles(Game.SkyrimSpecialEdition, modId);
Assert.Single(modInfoResponse.files);
Assert.Equal("blerg", modInfoResponse.files.First().file_name);
}
