public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
using (var queue = new WorkQueue(4))
{
Utils.Log($"Indexing game files");
var states = GameRegistry.Games.Values
.Where(game => game.TryGetGameLocation() != null && game.MainExecutable != null)
.SelectMany(game => game.GameLocation().EnumerateFiles()
.Select(file => new GameFileSourceDownloader.State(game.InstalledVersion)
{
Game = game.Game,
GameFile = file.RelativeTo(game.GameLocation()),
}))
.ToList();
var pks = states.Select(s => s.PrimaryKeyString).ToHashSet();
Utils.Log($"Found {pks.Count} archives to cross-reference with the database");
var found = await sql.FilterByExistingPrimaryKeys(pks);
states = states.Where(s => !found.Contains(s.PrimaryKeyString)).ToList();
Utils.Log($"Found {states.Count} archives to index");
await states.PMap(queue, async state =>
{
var path = state.Game.MetaData().GameLocation().Combine(state.GameFile);
Utils.Log($"Hashing Game file {path}");
try
{
state.Hash = await path.FileHashAsync();
}
catch (IOException)
{
Utils.Log($"Unable to hash {path}");
}
});
var with_hash = states.Where(state => state.Hash != default).ToList();
Utils.Log($"Inserting {with_hash.Count} jobs.");
var jobs = states.Select(state => new IndexJob {
Archive = new Archive(state)
{
Name = state.GameFile.FileName.ToString()
}
})
.Select(j => new Job {
Payload = j, RequiresNexus = j.UsesNexus
})
.ToList();
foreach (var job in jobs)
{
await sql.EnqueueJob(job);
}
return(JobResult.Success());
}
}
public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
int retries = 0;
TOP:
var file = await sql.UploadedFileById(FileId);
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
{
return(JobResult.Success());
}
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
await client.ConnectAsync();
using (var stream = File.OpenRead(Path.Combine("public", "files", file.MungedName)))
{
try
{
await client.UploadAsync(stream, file.MungedName, progress : new Progress((RelativePath)file.MungedName));
}
catch (Exception ex)
{
if (retries > 10)
{
throw;
}
Utils.Log(ex.ToString());
Utils.Log("Retrying FTP Upload");
retries++;
goto TOP;
}
}
await sql.EnqueueJob(new Job
{
Priority = Job.JobPriority.High,
Payload = new IndexJob
{
Archive = new Archive(new HTTPDownloader.State(file.Uri))
{
Name = file.MungedName,
Size = file.Size,
Hash = file.Hash,
}
}
});
}
return(JobResult.Success());
}
private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlistPath))
{
modlistPath.Delete();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlistPath);
}
else
{
Utils.Log($"No changes detected from downloaded ModList");
}
Utils.Log($"Loading {modlistPath}");
var installer = AInstaller.LoadFromFile(modlistPath);
var archives = installer.Archives;
Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
var searching = archives.Select(a => a.Hash).ToHashSet();
Utils.Log($"Looking for missing archives");
var knownArchives = await sql.FilterByExistingIndexedArchives(searching);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.Contains(a.Hash)).ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {
Payload = new IndexJob {
Archive = a
}, Priority = Job.JobPriority.Low
});
Utils.Log($"Writing jobs to the database");
foreach (var job in jobs)
{
await sql.EnqueueJob(job);
}
Utils.Log($"Done adding archives for {installer.Name}");
}
public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
var doc = new HtmlDocument();
var body = await new HttpClient().GetStringAsync(new Uri(
"https://forum.step-project.com/topic/13894-dyndolod-beta-for-skyrim-special-edition-and-skyrim-vr-279/"));
doc.LoadHtml(body);
var matches =
doc.DocumentNode
.Descendants()
.Where(d => d.NodeType == HtmlNodeType.Element && d.Attributes.Contains("href"))
.Select(d => d.Attributes["href"].Value)
.Select(m => Uri.TryCreate(m.ToString(), UriKind.Absolute, out var result) ? result : null)
.Where(uri => uri != null && uri.Host == "mega.nz")
.Select(url => new Job()
{
Payload = new IndexJob
{
Archive = new Archive(new MegaDownloader.State(url.ToString()))
{
Name = Guid.NewGuid() + ".7z",
}
}
})
.ToList();
foreach (var job in matches)
{
var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString;
var found = await sql.DownloadStateByPrimaryKey(key);
if (found != null)
{
continue;
}
Utils.Log($"Queuing {key} for indexing");
await sql.EnqueueJob(job);
}
return(JobResult.Success());
}
private async Task ScheduledJob <T>(TimeSpan span, Job.JobPriority priority) where T : AJobPayload, new()
{
if (!Settings.RunBackEndJobs && typeof(T).ImplementsInterface(typeof(IBackEndJob)))
{
return;
}
if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob)))
{
return;
}
try
{
var jobs = (await Sql.GetUnfinishedJobs())
.Where(j => j.Payload is T)
.OrderByDescending(j => j.Created)
.Take(10);
foreach (var job in jobs)
{
if (job.Started == null || job.Ended == null)
{
return;
}
if (DateTime.Now - job.Ended < span)
{
return;
}
}
await Sql.EnqueueJob(new Job
{
Priority = priority,
Payload = new T()
});
}
catch (Exception ex)
{
Logger.Log(LogLevel.Error, ex, $"Error in JobScheduler when scheduling {typeof(T).Name}");
}
}