public override async Task <int> Execute()
{
var report = await _quickSync.Report();
foreach (var service in report)
{
if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 4)
{
await _discord.Send(Channel.Spam,
new DiscordMessage { Content = $"Service {service.Key.Name} has missed it's scheduled execution window. \n Current work: \n {string.Join("\n", service.Value.ActiveWork)}" });
}
}
return(report.Count);
}
public override async Task <int> Execute()
{
var toDelete = await FindFilesToDelete();
var log = new[] { $"CDNDelete ({toDelete.CDNDelete.Length}):\n\n" }
.Concat(toDelete.CDNDelete)
.Concat(new[] { $"SQLDelete ({toDelete.SQLDelete.Length}" })
.Concat(toDelete.SQLDelete)
.Concat(new[] { $"CDNRemain ({toDelete.CDNNotDeleted.Length}" })
.Concat(toDelete.CDNNotDeleted)
.Concat(new[] { $"SQLRemain ({toDelete.SQLNotDeleted.Length}" })
.Concat(toDelete.SQLNotDeleted)
.ToArray();
//await AbsolutePath.EntryPoint.Combine("cdn_delete_log.txt").WriteAllLinesAsync(log);
foreach (var sqlFile in toDelete.SQLDelete)
{
Utils.Log($"Deleting {sqlFile} from SQL");
await _sql.DeleteFileDefinition(await _sql.GetCDNFileDefinition(sqlFile));
}
using var queue = new WorkQueue(6);
await toDelete.CDNDelete.Select((d, idx) => (d, idx)).PMap(queue, async cdnFile =>
{
using var conn = await(await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles)).GetClient();
Utils.Log($"Deleting {cdnFile} from CDN");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"({cdnFile.idx}/{toDelete.CDNDelete.Length}) {cdnFile.d} is no longer referenced by any modlist and will be removed from the CDN"
});
if (await conn.DirectoryExistsAsync(cdnFile.d))
{
await conn.DeleteDirectoryAsync(cdnFile.d);
}
if (await conn.FileExistsAsync(cdnFile.d))
{
await conn.DeleteFileAsync(cdnFile.d);
}
});
return(toDelete.CDNDelete.Length + toDelete.SQLDelete.Length);
}
public override async Task <int> Execute()
{
await _sql.UpdateGameMetadata();
var data = await _sql.ModListArchives();
var nexusArchives = data.Select(a => a.State).OfType <NexusDownloader.State>().Select(d => (d.Game, d.ModID))
.Where(g => g.Game.MetaData().NexusGameId != 0)
.Distinct()
.ToList();
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue();
var prev = await _sql.GetHiddenNexusMods();
_logger.LogInformation($"Found {prev.Count} hidden nexus mods to check");
await prev.PMap(queue, async archive =>
{
var(game, modID) = archive.Key;
_logger.LogInformation($"Checking permissions for {game} {modID}");
var result = await HTMLInterface.GetUploadPermissions(game, modID);
await _sql.SetNexusPermission(game, modID, result);
if (archive.Value != result)
{
await _discord.Send(Channel.Ham,
new DiscordMessage {
Content = $"Permissions status of {game} {modID} was {archive.Value} is now {result}"
});
await _sql.PurgeNexusCache(modID);
await _quickSync.Notify <ListValidator>();
}
});
return(1);
}
public override async Task <int> Execute()
{
int downloaded = 0;
var lists = (await ModlistMetadata.LoadFromGithub())
.Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
foreach (var list in lists)
{
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
{
continue;
}
if (!_maintainer.HaveArchive(list.DownloadMetadata !.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await state.Download(new Archive(state) { Name = $"{list.Links.MachineURL}.wabbajack" }, tf.Path);
var hash = await tf.Path.FileHashAsync();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
await using (var fs = await modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open())
{
if (entry == null)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
try
{
modlist = entry.FromJson <ModList>();
}
catch (JsonReaderException)
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
}
await _sql.IngestModList(list.DownloadMetadata !.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Downloaded {downloaded} new lists" });
}
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
{
await _discord.Send(Channel.Ham,
new DiscordMessage { Content = $"Enqueing {fc} files for downloading" });
}
return(downloaded);
}
public override async Task <int> Execute()
{
int uploaded = 0;
TOP:
var toUpload = await _sql.GetNextMirroredFile();
if (toUpload == default)
{
return(uploaded);
}
uploaded += 1;
try
{
var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
using var queue = new WorkQueue();
if (_archives.TryGetPath(toUpload.Hash, out var path))
{
_logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}");
bool exists = false;
using (var client = await GetClient(creds))
{
exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz");
}
if (exists)
{
_logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server");
await toUpload.Finish(_sql);
goto TOP;
}
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}"
});
var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
using (var client = await GetClient(creds))
{
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
}
string MakePath(long idx)
{
return($"{definition.Hash.ToHex()}/parts/{idx}");
}
await definition.Parts.PMap(queue, async part =>
{
_logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})");
var buffer = new byte[part.Size];
await using (var fs = await path.OpenShared())
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);
}
using var client = await GetClient(creds);
var name = MakePath(part.Index);
await client.UploadAsync(new MemoryStream(buffer), name);
});
using (var client = await GetClient(creds))
{
_logger.LogInformation($"Finishing mirror upload");
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
definition.ToJson(gz);
}
ms.Position = 0;
var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz";
await client.UploadAsync(ms, remoteName);
}
await toUpload.Finish(_sql);
}
else
{
await toUpload.Fail(_sql, "Archive not found");
}
}
catch (Exception ex)
{
_logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
_logger.LogError(ex, "Error uploading");
await toUpload.Fail(_sql, ex.ToString());
}
goto TOP;
}
public override async Task <int> Execute()
{
_nexusClient ??= await NexusApiClient.Get();
int count = 0;
while (true)
{
var(daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = (daily < 100 && hourly < 10);
//var ignoreNexus = true;
if (ignoreNexus)
{
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
}
else
{
_logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
}
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == null)
{
break;
}
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is ManualDownloader.State)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
await DownloadDispatcher.PrepareAll(new[] { nextDownload.Archive.State });
await using var tempPath = new TempFile();
if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path))
{
_logger.LogError($"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
{
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}" });
}
count++;
}
if (count > 0)
{
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify <PatchBuilder>();
}
return(count);
}
public override async Task <int> Execute()
{
int count = 0;
while (true)
{
count++;
var patch = await _sql.GetPendingPatch();
if (patch == default)
{
break;
}
try
{
_logger.LogInformation(
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString == patch.Dest.Archive.State.PrimaryKeyString)
{
await patch.Fail(_sql, "Hashes match");
continue;
}
if (patch.Src.Archive.Size > 2_500_000_000 || patch.Dest.Archive.Size > 2_500_000_000)
{
await patch.Fail(_sql, "Too large to patch");
continue;
}
_maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
_maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
await using var sigFile = new TempFile();
await using var patchFile = new TempFile();
await using var srcStream = await srcPath.OpenShared();
await using var destStream = await destPath.OpenShared();
await using var sigStream = await sigFile.Path.Create();
await using var patchOutput = await patchFile.Path.Create();
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput, new OctoDiff.ProgressReporter(TimeSpan.FromSeconds(1), (s, p) => _logger.LogInformation($"Patch Builder: {p} {s}")));
await patchOutput.DisposeAsync();
var size = patchFile.Path.Size;
await UploadToCDN(patchFile.Path, PatchName(patch));
await patch.Finish(_sql, size);
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while building patch");
await patch.Fail(_sql, ex.ToString());
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
}
if (count > 0)
{
// Notify the List Validator that we may have more patches
await _quickSync.Notify <ListValidator>();
}
if (!NoCleaning)
{
await CleanupOldPatches();
}
return(count);
}
