public static string PerformCompress(string oldfile, string newfile, string destfile, bool IgnoreSizeLimit)
{
try
{
if (IgnoreSizeLimit)
{
var dest = destfile.Replace("[]", Path.GetFileNameWithoutExtension(newfile));
var compression = new MsDeltaCompression();
compression.CreateDelta(oldfile, newfile, dest);
return(dest);
}
else
{
var dest = destfile.Replace("[]", Path.GetFileNameWithoutExtension(newfile));
var compression = new PatchApiCompression();
compression.CreateDelta(oldfile, newfile, dest);
return(dest);
}
}
catch (Win32Exception ex)
{
Tools.TrueOrDie(false, "Error when creating delta. Native error code " + ex.NativeErrorCode);
return("");
}
}
public void CreateAndApplyDelta()
{
var compression = new MsDeltaCompression();
compression.CreateDelta(_privateNotepadPath, _privateCalcPath, _privateDeltaPath);
compression.ApplyDelta(_privateDeltaPath, _privateNotepadPath, _privateFinalPath);
var finalHash = HashFile(_privateFinalPath);
CollectionAssert.AreEqual(_calcHash, finalHash);
}
private void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, Dictionary <string, string> baseFileListing)
{
string key = targetFile.FullName.Replace(workingDirectory.FullName, "");
if (!baseFileListing.ContainsKey(key))
{
this.Log <DeltaPackageBuilder>().Info <string>("{0} not found in base package, marking as new", key);
}
else
{
byte[] oldData = File.ReadAllBytes(baseFileListing[key]);
byte[] newData = File.ReadAllBytes(targetFile.FullName);
if (this.bytesAreIdentical(oldData, newData))
{
this.Log <DeltaPackageBuilder>().Info <string>("{0} hasn't changed, writing dummy file", key);
File.Create(targetFile.FullName + ".diff").Dispose();
File.Create(targetFile.FullName + ".shasum").Dispose();
targetFile.Delete();
}
else
{
this.Log <DeltaPackageBuilder>().Info <string, string>("Delta patching {0} => {1}", baseFileListing[key], targetFile.FullName);
MsDeltaCompression compression = new MsDeltaCompression();
try
{
compression.CreateDelta(baseFileListing[key], targetFile.FullName, targetFile.FullName + ".diff");
}
catch (Win32Exception)
{
this.Log <DeltaPackageBuilder>().Warn <string>("We couldn't create a delta for {0}, attempting to create bsdiff", targetFile.Name);
FileStream stream = null;
try
{
BinaryPatchUtility.Create(oldData, newData, File.Create(targetFile.FullName + ".bsdiff"));
File.WriteAllText(targetFile.FullName + ".diff", "1");
}
catch (Exception exception)
{
this.Log <DeltaPackageBuilder>().WarnException($"We really couldn't create a delta for {targetFile.Name}", exception);
return;
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
ReleaseEntry entry = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum", null);
File.WriteAllText(targetFile.FullName + ".shasum", entry.EntryAsString, Encoding.UTF8);
targetFile.Delete();
}
}
}
public void CreateAndApplyDelta()
{
var compression = new MsDeltaCompression();
compression.CreateDelta(_privateNotepadPath, _privateCalcPath, _privateDeltaPath);
compression.ApplyDelta(_privateDeltaPath, _privateNotepadPath, _privateFinalPath);
var finalHash = HashFile(_privateFinalPath);
CollectionAssert.AreEqual(_calcHash, finalHash);
}
void applyDiffToFile(string deltaPath, string relativeFilePath, string workingDirectory)
{
var inputFile = Path.Combine(deltaPath, relativeFilePath);
var finalTarget = Path.Combine(workingDirectory, Regex.Replace(relativeFilePath, @".diff$", ""));
var tempTargetFile = default(string);
Utility.WithTempFile(out tempTargetFile, localAppDirectory);
try {
// NB: Zero-length diffs indicate the file hasn't actually changed
if (new FileInfo(inputFile).Length == 0)
{
this.Log().Info("{0} exists unchanged, skipping", relativeFilePath);
return;
}
if (relativeFilePath.EndsWith(".diff", StringComparison.InvariantCultureIgnoreCase))
{
this.Log().Info("Applying Diff to {0}", relativeFilePath);
var msDelta = new MsDeltaCompression();
msDelta.ApplyDelta(inputFile, finalTarget, tempTargetFile);
verifyPatchedFile(relativeFilePath, inputFile, tempTargetFile);
}
else
{
using (var of = File.OpenWrite(tempTargetFile))
using (var inf = File.OpenRead(inputFile)) {
this.Log().Info("Adding new file: {0}", relativeFilePath);
inf.CopyTo(of);
}
}
if (File.Exists(finalTarget))
{
File.Delete(finalTarget);
}
var targetPath = Directory.GetParent(finalTarget);
if (!targetPath.Exists)
{
targetPath.Create();
}
File.Move(tempTargetFile, finalTarget);
} finally {
if (File.Exists(tempTargetFile))
{
Utility.DeleteFileHarder(tempTargetFile, true);
}
}
}
private void ModifyFiles(string diffPath, string targetPath, IEnumerable <string> modifiedFilePaths)
{
var delta = new MsDeltaCompression();
foreach (var partialFilePath in modifiedFilePaths)
{
var oldPath = targetPath + partialFilePath;
var deltaPath = diffPath + partialFilePath;
var newPath = oldPath + "1";
delta.ApplyDelta(deltaPath, oldPath, newPath);
File.Delete(oldPath);
File.Move(newPath, oldPath);
}
}
void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, Dictionary <string, string> baseFileListing)
{
// NB: There are three cases here that we'll handle:
//
// 1. Exists only in new => leave it alone, we'll use it directly.
// 2. Exists in both old and new => write a dummy file so we know
// to keep it.
// 3. Exists in old but changed in new => create a delta file
//
// The fourth case of "Exists only in old => delete it in new"
// is handled when we apply the delta package
var relativePath = targetFile.FullName.Replace(workingDirectory.FullName, "");
if (!baseFileListing.ContainsKey(relativePath))
{
this.Log().Info("{0} not found in base package, marking as new", relativePath);
return;
}
var oldData = File.ReadAllBytes(baseFileListing[relativePath]);
var newData = File.ReadAllBytes(targetFile.FullName);
if (bytesAreIdentical(oldData, newData))
{
this.Log().Info("{0} hasn't changed, writing dummy file", relativePath);
File.Create(targetFile.FullName + ".diff").Dispose();
File.Create(targetFile.FullName + ".shasum").Dispose();
targetFile.Delete();
return;
}
this.Log().Info("Delta patching {0} => {1}", baseFileListing[relativePath], targetFile.FullName);
var msDelta = new MsDeltaCompression();
try {
msDelta.CreateDelta(baseFileListing[relativePath], targetFile.FullName, targetFile.FullName + ".diff");
} catch (Win32Exception ex) {
this.Log().Warn("We couldn't create a delta for {0}, writing full file", targetFile.Name);
return;
}
var rl = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum");
File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8);
targetFile.Delete();
}
public Task <Unit> Handle(Request request, CancellationToken cancellationToken)
{
var baseDirectory = Directory.GetCurrentDirectory();
var delta = new MsDeltaCompression();
foreach (var relativePath in request.ModifiedFiles)
{
var sourcePath = $"{request.SourcePath}{relativePath}";
var targetPath = $"{request.TargetPath}{relativePath}";
var deltaFilePath = $@"{baseDirectory}\output{relativePath}";
Directory.CreateDirectory(Path.GetDirectoryName(deltaFilePath));
delta.CreateDelta(sourcePath, targetPath, deltaFilePath);
}
return(Unit.Task);
}
void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, Dictionary <string, string> baseFileListing)
{
// NB: There are three cases here that we'll handle:
//
// 1. Exists only in new => leave it alone, we'll use it directly.
// 2. Exists in both old and new => write a dummy file so we know
// to keep it.
// 3. Exists in old but changed in new => create a delta file
//
// The fourth case of "Exists only in old => delete it in new"
// is handled when we apply the delta package
var relativePath = targetFile.FullName.Replace(workingDirectory.FullName, "");
if (!baseFileListing.ContainsKey(relativePath))
{
this.Log().Info("{0} not found in base package, marking as new", relativePath);
return;
}
var oldData = File.ReadAllBytes(baseFileListing[relativePath]);
var newData = File.ReadAllBytes(targetFile.FullName);
if (bytesAreIdentical(oldData, newData))
{
this.Log().Info("{0} hasn't changed, writing dummy file", relativePath);
File.Create(targetFile.FullName + ".diff").Dispose();
File.Create(targetFile.FullName + ".shasum").Dispose();
targetFile.Delete();
return;
}
this.Log().Info("Delta patching {0} => {1}", baseFileListing[relativePath], targetFile.FullName);
var msDelta = new MsDeltaCompression();
if (targetFile.Extension.Equals(".exe", StringComparison.OrdinalIgnoreCase) ||
targetFile.Extension.Equals(".dll", StringComparison.OrdinalIgnoreCase) ||
targetFile.Extension.Equals(".node", StringComparison.OrdinalIgnoreCase))
{
try {
msDelta.CreateDelta(baseFileListing[relativePath], targetFile.FullName, targetFile.FullName + ".diff");
goto exit;
} catch (Exception) {
this.Log().Warn("We couldn't create a delta for {0}, attempting to create bsdiff", targetFile.Name);
}
}
var of = default(FileStream);
try {
of = File.Create(targetFile.FullName + ".bsdiff");
BinaryPatchUtility.Create(oldData, newData, of);
// NB: Create a dummy corrupt .diff file so that older
// versions which don't understand bsdiff will fail out
// until they get upgraded, instead of seeing the missing
// file and just removing it.
File.WriteAllText(targetFile.FullName + ".diff", "1");
} catch (Exception ex) {
this.Log().WarnException(String.Format("We really couldn't create a delta for {0}", targetFile.Name), ex);
return;
} finally {
if (of != null)
{
of.Dispose();
}
}
exit:
var rl = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum");
File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8);
targetFile.Delete();
}
public ReleasePackage ApplyDeltaPackage(ReleasePackage basePackage, ReleasePackage deltaPackage, string outputFile, Action <int> progress = null)
{
Contract.Requires(deltaPackage != null);
Contract.Requires(!String.IsNullOrEmpty(outputFile) && !File.Exists(outputFile));
if (progress == null)
{
progress = i => { }
}
;
string workingPath;
string deltaPath;
using (Utility.WithTempDirectory(out deltaPath, localAppDirectory))
using (Utility.WithTempDirectory(out workingPath, localAppDirectory)) {
var fz = new FastZip();
fz.ExtractZip(deltaPackage.InputPackageFile, deltaPath, null);
// The percentages here are somewhat arbitrary and reflect typical Bloom file sizes.
// Probably not appropriate to push upstream.
// If you are thinking about improving this using FastZip's Progress event, check with JohnT.
// I experimented with it and found that the PercentCompleted which it reports is useless,
// toggling between 0 and 100% repeatedly...possibly it applies to a particular file?
// I also found that it is VERY expensive to turn on progress reporting while unzipping;
// seems to slow the process down to the point of making it take two to three times as long.
// This may of course improve in a later version of the library.
progress(10);
fz.ExtractZip(basePackage.InputPackageFile, workingPath, null);
progress(35);
var pathsVisited = new List <string>();
var deltaPathRelativePaths = new DirectoryInfo(deltaPath).GetAllFilesRecursively()
.Select(x => x.FullName.Replace(deltaPath + Path.DirectorySeparatorChar, ""))
.ToArray();
// Apply all of the .diff files
deltaPathRelativePaths
.Where(x => x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".shasum", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".diff", StringComparison.InvariantCultureIgnoreCase) ||
!deltaPathRelativePaths.Contains(x.Replace(".diff", ".bsdiff")))
.ForEach(file => {
pathsVisited.Add(Regex.Replace(file, @"\.(bs)?diff$", "").ToLowerInvariant());
applyDiffToFile(deltaPath, file, workingPath);
});
// Delete all of the files that were in the old package but
// not in the new one.
new DirectoryInfo(workingPath).GetAllFilesRecursively()
.Select(x => x.FullName.Replace(workingPath + Path.DirectorySeparatorChar, "").ToLowerInvariant())
.Where(x => x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase) && !pathsVisited.Contains(x))
.ForEach(x => {
this.Log().Info("{0} was in old package but not in new one, deleting", x);
File.Delete(Path.Combine(workingPath, x));
});
// Update all the files that aren't in 'lib' with the delta
// package's versions (i.e. the nuspec file, etc etc).
deltaPathRelativePaths
.Where(x => !x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase))
.ForEach(x => {
this.Log().Info("Updating metadata file: {0}", x);
File.Copy(Path.Combine(deltaPath, x), Path.Combine(workingPath, x), true);
});
progress(50);
this.Log().Info("Repacking into full package: {0}", outputFile);
fz.CreateZip(outputFile, workingPath, true, null);
progress(100);
}
return(new ReleasePackage(outputFile));
}
void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, Dictionary <string, string> baseFileListing)
{
// NB: There are three cases here that we'll handle:
//
// 1. Exists only in new => leave it alone, we'll use it directly.
// 2. Exists in both old and new => write a dummy file so we know
// to keep it.
// 3. Exists in old but changed in new => create a delta file
//
// The fourth case of "Exists only in old => delete it in new"
// is handled when we apply the delta package
var relativePath = targetFile.FullName.Replace(workingDirectory.FullName, "");
if (!baseFileListing.ContainsKey(relativePath))
{
this.Log().Info("{0} not found in base package, marking as new", relativePath);
return;
}
var oldData = File.ReadAllBytes(baseFileListing[relativePath]);
var newData = File.ReadAllBytes(targetFile.FullName);
if (bytesAreIdentical(oldData, newData))
{
this.Log().Info("{0} hasn't changed, writing dummy file", relativePath);
File.Create(targetFile.FullName + ".diff").Dispose();
File.Create(targetFile.FullName + ".shasum").Dispose();
targetFile.Delete();
return;
}
this.Log().Info("Delta patching {0} => {1}", baseFileListing[relativePath], targetFile.FullName);
var msDelta = new MsDeltaCompression();
try {
msDelta.CreateDelta(baseFileListing[relativePath], targetFile.FullName, targetFile.FullName + ".diff");
} catch (Win32Exception) {
this.Log().Warn("We couldn't create a delta for {0}, attempting to create bsdiff", targetFile.Name);
var of = default(FileStream);
try {
of = File.Create(targetFile.FullName + ".bsdiff");
BinaryPatchUtility.Create(oldData, newData, of);
// NB: Create a dummy corrupt .diff file so that older
// versions which don't understand bsdiff will fail out
// until they get upgraded, instead of seeing the missing
// file and just removing it.
File.WriteAllText(targetFile.FullName + ".diff", "1");
} catch (Exception ex) {
this.Log().WarnException(String.Format("We really couldn't create a delta for {0}", targetFile.Name), ex);
return;
} finally {
if (of != null)
{
of.Dispose();
}
}
}
var rl = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum");
File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8);
targetFile.Delete();
}
void applyDiffToFile(string deltaPath, string relativeFilePath, string workingDirectory)
{
var inputFile = Path.Combine(deltaPath, relativeFilePath);
var finalTarget = Path.Combine(workingDirectory, Regex.Replace(relativeFilePath, @"\.(bs)?diff$", ""));
var tempTargetFile = default(string);
Utility.WithTempFile(out tempTargetFile, localAppDirectory);
try {
// NB: Zero-length diffs indicate the file hasn't actually changed
if (new FileInfo(inputFile).Length == 0)
{
this.Log().Info("{0} exists unchanged, skipping", relativeFilePath);
return;
}
if (relativeFilePath.EndsWith(".bsdiff", StringComparison.InvariantCultureIgnoreCase))
{
using (var of = File.OpenWrite(tempTargetFile))
using (var inf = File.OpenRead(finalTarget)) {
this.Log().Info("Applying BSDiff to {0}", relativeFilePath);
BinaryPatchUtility.Apply(inf, () => File.OpenRead(inputFile), of);
}
verifyPatchedFile(relativeFilePath, inputFile, tempTargetFile);
}
else if (relativeFilePath.EndsWith(".diff", StringComparison.InvariantCultureIgnoreCase))
{
this.Log().Info("Applying MSDiff to {0}", relativeFilePath);
var msDelta = new MsDeltaCompression();
msDelta.ApplyDelta(inputFile, finalTarget, tempTargetFile);
verifyPatchedFile(relativeFilePath, inputFile, tempTargetFile);
}
else
{
using (var of = File.OpenWrite(tempTargetFile))
using (var inf = File.OpenRead(inputFile)) {
this.Log().Info("Adding new file: {0}", relativeFilePath);
inf.CopyTo(of);
}
}
if (File.Exists(finalTarget))
{
File.Delete(finalTarget);
}
var targetPath = Directory.GetParent(finalTarget);
if (!targetPath.Exists)
{
targetPath.Create();
}
File.Move(tempTargetFile, finalTarget);
} finally {
if (File.Exists(tempTargetFile))
{
Utility.DeleteFileHarder(tempTargetFile, true);
}
}
}
void verifyPatchedFile(string relativeFilePath, string inputFile, string tempTargetFile)
{
var shaFile = Regex.Replace(inputFile, @"\.(bs)?diff$", ".shasum");
var expectedReleaseEntry = ReleaseEntry.ParseReleaseEntry(File.ReadAllText(shaFile, Encoding.UTF8));
var actualReleaseEntry = ReleaseEntry.GenerateFromFile(tempTargetFile);
if (expectedReleaseEntry.Filesize != actualReleaseEntry.Filesize)
{
this.Log().Warn("Patched file {0} has incorrect size, expected {1}, got {2}", relativeFilePath,
expectedReleaseEntry.Filesize, actualReleaseEntry.Filesize);
throw new ChecksumFailedException()
{
Filename = relativeFilePath
};
}
if (expectedReleaseEntry.SHA1 != actualReleaseEntry.SHA1)
{
this.Log().Warn("Patched file {0} has incorrect SHA1, expected {1}, got {2}", relativeFilePath,
expectedReleaseEntry.SHA1, actualReleaseEntry.SHA1);
throw new ChecksumFailedException()
{
Filename = relativeFilePath
};
}
}
bool bytesAreIdentical(byte[] oldData, byte[] newData)
{
if (oldData == null || newData == null)
{
return(oldData == newData);
}
if (oldData.LongLength != newData.LongLength)
{
return(false);
}
for (long i = 0; i < newData.LongLength; i++)
{
if (oldData[i] != newData[i])
{
return(false);
}
}
return(true);
}
}
/// <summary>
/// Applies a delta byte array to a file
/// </summary>
/// <param name="sourceFile">The path where the source file is located</param>
/// <param name="deltaFile">The path where the delta file is located</param>
/// <param name="finalFile">The path where the file with delta applied is going to be saved</param>
/// <param name="deltaCompressionType">The delta compression used to compare</param>
public static bool Apply(string sourceFile, string deltaFile, string finalFile, DeltaCompressionType deltaCompressionType = DeltaCompressionType.Core)
{
// File checks
if (!File.Exists(sourceFile))
{
throw new FileNotFoundException("Source file not found.", sourceFile);
}
if (!File.Exists(deltaFile))
{
throw new FileNotFoundException("Delta file not found.", deltaFile);
}
Directory.CreateDirectory(Path.GetDirectoryName(finalFile)); // Create directory if not found
// Use native methods
if (deltaCompressionType == DeltaCompressionType.Native)
{
// Create final file if not found
if (!File.Exists(finalFile))
{
using (var fs = File.Create(finalFile)) { }
}
// Create ms delta instance
var msDelta = new MsDeltaCompression();
try
{
// Apply delta
msDelta.ApplyDelta(deltaFile, sourceFile, finalFile);
return(true);
}
catch (Win32Exception)
{
return(false);
}
}
// Use core methods
else
{
// Read files
byte[] sourceFileBytes = File.ReadAllBytes(sourceFile);
byte[] deltaBytes = File.ReadAllBytes(deltaFile);
byte[] finalBytesBuffer = default; // Create buffer to store the final bytes
// Apply delta
finalBytesBuffer = CoreDeltaCompression.Apply(sourceFileBytes, deltaBytes);
try
{
// Create stream to write to the file
using (var fs = new FileStream(finalFile, FileMode.OpenOrCreate, FileAccess.Write))
using (var binaryWriter = new BinaryWriter(fs))
binaryWriter.Write(finalBytesBuffer); // Write the buffer
}
catch (Exception)
{
return(false);
}
return(true);
}
}
/// <summary>
/// Creates a delta file from a source file. Returns true if the delta buffer could be created.
/// </summary>
/// <param name="sourceFile">The path where the source file is located</param>
/// <param name="newVersionFile">The path where the target file is located</param>
/// <param name="outputDeltaFile">The path where the delta file is going to be saved</param>
/// <param name="deltaCompressionType">The delta compression used to compare</param>
public static bool Create(string sourceFile, string newVersionFile, string outputDeltaFile, DeltaCompressionType deltaCompressionType = DeltaCompressionType.Core)
{
// File checks
if (!File.Exists(sourceFile))
{
throw new FileNotFoundException("File not found.", sourceFile);
}
if (!File.Exists(newVersionFile))
{
throw new FileNotFoundException("File not found.", newVersionFile);
}
Directory.CreateDirectory(Path.GetDirectoryName(outputDeltaFile)); // Create output directory if no exists
// Use native methods
if (deltaCompressionType == DeltaCompressionType.Native)
{
// Create output delta file if not found
if (!File.Exists(outputDeltaFile))
{
using (var fs = File.Create(outputDeltaFile)) { }
}
// Create msDelta
var msDelta = new MsDeltaCompression();
try
{
// Create delta
msDelta.CreateDelta(sourceFile, newVersionFile, outputDeltaFile);
return(true);
}
catch (Win32Exception)
{
return(false);
}
}
// Use core methods
else
{
// Get bytes
byte[] sourceFileBytes = File.ReadAllBytes(sourceFile);
byte[] newVersionFileBytes = File.ReadAllBytes(sourceFile);
byte[] deltaBuffer = default; // Create array to store the delta
// Create delta
deltaBuffer = CoreDeltaCompression.Create(sourceFileBytes, newVersionFileBytes);
// If the delta buffer is 0, return false
if (deltaBuffer.Length <= 0)
{
return(false);
}
// Create stream to write
using (var fs = new FileStream(outputDeltaFile, FileMode.OpenOrCreate, FileAccess.Write))
using (var binaryWriter = new BinaryWriter(fs))
binaryWriter.Write(deltaBuffer); // Write the buffer
// Return success
return(true);
}
}
