public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
var doc = new HtmlDocument();
var body = await new HttpClient().GetStringAsync(new Uri(
"https://forum.step-project.com/topic/13894-dyndolod-beta-for-skyrim-special-edition-and-skyrim-vr-279/"));
doc.LoadHtml(body);
var matches =
doc.DocumentNode
.Descendants()
.Where(d => d.NodeType == HtmlNodeType.Element && d.Attributes.Contains("href"))
.Select(d => d.Attributes["href"].Value)
.Select(m => Uri.TryCreate(m.ToString(), UriKind.Absolute, out var result) ? result : null)
.Where(uri => uri != null && uri.Host == "mega.nz")
.Select(url => new Job()
{
Payload = new IndexJob
{
Archive = new Archive(new MegaDownloader.State(url.ToString()))
{
Name = Guid.NewGuid() + ".7z",
}
}
})
.ToList();
foreach (var job in matches)
{
var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString;
var found = await sql.DownloadStateByPrimaryKey(key);
if (found != null)
{
continue;
}
Utils.Log($"Queuing {key} for indexing");
await sql.EnqueueJob(job);
}
return(JobResult.Success());
}
public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
var srcPath = settings.PathForArchive(Src);
var destHash = (await sql.DownloadStateByPrimaryKey(DestPK)).Hash;
var destPath = settings.PathForArchive(destHash);
if (Src == destHash)
{
return(JobResult.Success());
}
Utils.Log($"Creating Patch ({Src} -> {DestPK})");
var cdnPath = CdnPath(Src, destHash);
cdnPath.Parent.CreateDirectory();
if (cdnPath.Exists)
{
return(JobResult.Success());
}
Utils.Log($"Calculating Patch ({Src} -> {DestPK})");
await using var fs = cdnPath.Create();
await using (var srcStream = srcPath.OpenRead())
await using (var destStream = destPath.OpenRead())
await using (var sigStream = cdnPath.WithExtension(Consts.OctoSig).Create())
{
OctoDiff.Create(destStream, srcStream, sigStream, fs);
}
fs.Position = 0;
Utils.Log($"Uploading Patch ({Src} -> {DestPK})");
int retries = 0;
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
{
return(JobResult.Success());
}
TOP:
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
await client.ConnectAsync();
try
{
await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress : new UploadToCDN.Progress(cdnPath.FileName));
}
catch (Exception ex)
{
if (retries > 10)
{
throw;
}
Utils.Log(ex.ToString());
Utils.Log("Retrying FTP Upload");
retries++;
goto TOP;
}
}
return(JobResult.Success());
}