public async Task DiffCreateAndApply(byte[] src, byte[] dest, DiffMethod method)
{
await using var ms = new MemoryStream();
switch (method)
{
case DiffMethod.Default:
await Utils.CreatePatch(src, dest, ms);
break;
case DiffMethod.BSDiff:
BSDiff.Create(src, dest, ms);
break;
case DiffMethod.OctoDiff:
OctoDiff.Create(src, dest, ms);
break;
default:
throw new ArgumentOutOfRangeException(nameof(method), method, null);
}
ms.Position = 0;
var patch = ms.ToArray();
await using var resultStream = new MemoryStream();
Utils.ApplyPatch(new MemoryStream(src), () => new MemoryStream(patch), resultStream);
Assert.Equal(dest, resultStream.ToArray());
}
public static async Task <bool> DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination)
{
var success = await Download(archive, destination);
if (success)
{
await destination.FileHashCachedAsync();
return(true);
}
Utils.Log($"Download failed, looking for upgrade");
var upgrade = await ClientAPI.GetModUpgrade(archive.Hash);
if (upgrade == null)
{
Utils.Log($"No upgrade found for {archive.Hash}");
return(false);
}
Utils.Log($"Upgrading via {upgrade.State.PrimaryKeyString}");
Utils.Log($"Upgrading {archive.Hash}");
var upgradePath = destination.Parent.Combine("_Upgrade_" + archive.Name);
var upgradeResult = await Download(upgrade, upgradePath);
if (!upgradeResult)
{
return(false);
}
var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}";
var patchPath = destination.Parent.Combine("_Patch_" + patchName);
var patchState = new Archive(new HTTPDownloader.State($"https://wabbajackcdn.b-cdn.net/updates/{patchName}"))
{
Name = patchName,
};
var patchResult = await Download(patchState, patchPath);
if (!patchResult)
{
return(false);
}
Utils.Status($"Applying Upgrade to {archive.Hash}");
await using (var patchStream = patchPath.OpenRead())
await using (var srcStream = upgradePath.OpenRead())
await using (var destStream = destination.Create())
{
OctoDiff.Apply(srcStream, patchStream, destStream);
}
await destination.FileHashCachedAsync();
return(true);
}
public override async Task <int> Execute()
{
int count = 0;
while (true)
{
count++;
var patch = await _sql.GetPendingPatch();
if (patch == default)
{
break;
}
try
{
_logger.LogInformation(
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString == patch.Dest.Archive.State.PrimaryKeyString)
{
await patch.Fail(_sql, "Hashes match");
continue;
}
if (patch.Src.Archive.Size > 2_500_000_000 || patch.Dest.Archive.Size > 2_500_000_000)
{
await patch.Fail(_sql, "Too large to patch");
continue;
}
_maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
_maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
await using var sigFile = new TempFile();
await using var patchFile = new TempFile();
await using var srcStream = await srcPath.OpenShared();
await using var destStream = await destPath.OpenShared();
await using var sigStream = await sigFile.Path.Create();
await using var patchOutput = await patchFile.Path.Create();
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput, new OctoDiff.ProgressReporter(TimeSpan.FromSeconds(1), (s, p) => _logger.LogInformation($"Patch Builder: {p} {s}")));
await patchOutput.DisposeAsync();
var size = patchFile.Path.Size;
await UploadToCDN(patchFile.Path, PatchName(patch));
await patch.Finish(_sql, size);
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while building patch");
await patch.Fail(_sql, ex.ToString());
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
}
if (count > 0)
{
// Notify the List Validator that we may have more patches
await _quickSync.Notify <ListValidator>();
}
if (!NoCleaning)
{
await CleanupOldPatches();
}
return(count);
}
public override async Task <JobResult> Execute(SqlService sql, AppSettings settings)
{
var srcPath = settings.PathForArchive(Src);
var destHash = (await sql.DownloadStateByPrimaryKey(DestPK)).Hash;
var destPath = settings.PathForArchive(destHash);
if (Src == destHash)
{
return(JobResult.Success());
}
Utils.Log($"Creating Patch ({Src} -> {DestPK})");
var cdnPath = CdnPath(Src, destHash);
cdnPath.Parent.CreateDirectory();
if (cdnPath.Exists)
{
return(JobResult.Success());
}
Utils.Log($"Calculating Patch ({Src} -> {DestPK})");
await using var fs = cdnPath.Create();
await using (var srcStream = srcPath.OpenRead())
await using (var destStream = destPath.OpenRead())
await using (var sigStream = cdnPath.WithExtension(Consts.OctoSig).Create())
{
OctoDiff.Create(destStream, srcStream, sigStream, fs);
}
fs.Position = 0;
Utils.Log($"Uploading Patch ({Src} -> {DestPK})");
int retries = 0;
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
{
return(JobResult.Success());
}
TOP:
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
await client.ConnectAsync();
try
{
await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress : new UploadToCDN.Progress(cdnPath.FileName));
}
catch (Exception ex)
{
if (retries > 10)
{
throw;
}
Utils.Log(ex.ToString());
Utils.Log("Retrying FTP Upload");
retries++;
goto TOP;
}
}
return(JobResult.Success());
}
