public void ChecksumHashLarge_block_Test()
{
using var srcStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}a.test");
using var targetStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}b.test");
using var md5 = MD5.Create();
var originalHash = md5.ComputeHash(targetStream);
targetStream.Position = 0;
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
using VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream, blockSize: 32);
VCDiffResult result = coder.Encode(checksumFormat: ChecksumFormat.SDCH); //encodes with no checksum and not interleaved
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
using VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream);
Assert.Equal(VCDiffResult.SUCCESS, decoder.Decode(out long bytesWritten));
outputStream.Position = 0;
var outputHash = md5.ComputeHash(outputStream);
Assert.Equal(originalHash, outputHash);
}
public void MaxFileSize_Test()
{
using var srcStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}a.test");
using var targetStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}b.test");
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
using VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream);
VCDiffResult result = coder.Encode();
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
long bytesWritten = 0;
using VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream, -1);
ArgumentException ex = Assert.Throws <ArgumentException>(() => decoder.Decode(out bytesWritten));
Assert.Matches(@"maxWindowSize must be a positive value", ex.Message);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
using VcDecoder decoder1 = new VcDecoder(srcStream, deltaStream, outputStream, 2);
InvalidOperationException ex1 = Assert.Throws <InvalidOperationException>(() => decoder1.Decode(out bytesWritten));
Assert.Matches(@"Length of target window \(\d*\) exceeds limit of 2 bytes", ex1.Message);
}
public void Interleaved_Test()
{
using var srcStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}a.test");
using var targetStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}b.test");
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
using var md5 = MD5.Create();
var originalHash = md5.ComputeHash(targetStream);
targetStream.Position = 0;
using VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream);
VCDiffResult result = coder.Encode(interleaved: true); //encodes with no checksum and not interleaved
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
using VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream);
long bytesWritten = 0;
while (bytesWritten < targetStream.Length)
{
Assert.Equal(VCDiffResult.SUCCESS, decoder.Decode(out long chunk));
bytesWritten += chunk;
}
outputStream.Position = 0;
var outputHash = md5.ComputeHash(outputStream);
Assert.Equal(originalHash, outputHash);
}
public void Interleaved_Test()
{
using var srcStream = new MemoryStream(ADiffData.ToArray());
using var targetStream = new MemoryStream(BDiffData.ToArray());
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream);
VCDiffResult result = coder.Encode(interleaved: true); //encodes with no checksum and not interleaved
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream);
long bytesWritten = 0;
while (bytesWritten < BDiffData.Length)
{
Assert.Equal(VCDiffResult.SUCCESS, decoder.Decode(out long chunk));
bytesWritten += chunk;
}
Assert.Equal("Goodbye World", Encoding.UTF8.GetString(outputStream.ToArray()));
}
public void MaxFileSize_Test()
{
using var srcStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}a.test");
using var targetStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}b.test");
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
using VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream);
VCDiffResult result = coder.Encode();
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
long bytesWritten = 0;
using VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream, -1);
Assert.Throws <ArgumentException>(() => decoder.Decode(out bytesWritten));
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
using VcDecoder decoder1 = new VcDecoder(srcStream, deltaStream, outputStream, 2);
Assert.Throws <InvalidOperationException>(() => decoder1.Decode(out bytesWritten));
}
public void NoChecksumGoogleTo_Test()
{
using var srcStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}size-overflow-64");
using var targetStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}size-overflow-32");
using var md5 = MD5.Create();
var originalHash = md5.ComputeHash(targetStream);
targetStream.Position = 0;
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
using VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream);
VCDiffResult result = coder.Encode();
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
using VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream);
Assert.Equal(VCDiffResult.SUCCESS, decoder.Decode(out long bytesWritten));
outputStream.Position = 0;
var outputHash = md5.ComputeHash(outputStream);
Assert.Equal(originalHash, outputHash);
}
/// <summary>
/// Function to assemble a file from delta and original and output it to the "output" file
/// </summary>
/// <param name="delta">Path to delta file</param>
/// <param name="original">Path to original file</param>
/// <param name="output">Path to output file</param>
private static void DoDecode(string delta, string original, string output)
{
using (FileStream outputS = new FileStream(output, FileMode.Create, FileAccess.Write))
using (FileStream dictS = new FileStream(original, FileMode.Open, FileAccess.Read))
using (FileStream targetS = new FileStream(delta, FileMode.Open, FileAccess.Read))
{
VCDecoder decoder = new VCDecoder(dictS, targetS, outputS);
//You must call decoder.Start() first. The header of the delta file must be available before calling decoder.Start()
VCDiffResult result = decoder.Start();
if (result != VCDiffResult.SUCCESS)
{
//error abort
}
result = decoder.Decode(out long bytesWritten);
if (result != VCDiffResult.SUCCESS)
{
//error decoding
}
//if success bytesWritten will contain the number of bytes that were decoded
}
}
/// <summary>
/// Writes the patched file into the output stream asynchronously.
/// This method is only asynchronous for the final step of writing the patched data into the output stream.
/// For large outputs, this may be beneficial.
/// </summary>
/// <returns></returns>
public async Task <(VCDiffResult result, long bytesWritten)> DecodeAsync()
{
if (!Decode_Init(out var bytesWritten, out var result, out var decodeAsync))
{
return(decodeAsync);
}
while (delta.CanRead)
{
//delta is streamed in order aka not random access
using var w = new WindowDecoder <TDeltaBuffer>(source.Length, delta, maxTargetFileSize);
if (w.Decode(this.IsSDCHFormat))
{
using var body = new BodyDecoder <TDeltaBuffer, TSourceBuffer, TDeltaBuffer>(w, source, delta, outputStream);
if (this.IsSDCHFormat && w.AddRunLength == 0 && w.AddressesForCopyLength == 0 && w.InstructionAndSizesLength > 0)
{
//interleaved
//decodedinterleave actually has an internal loop for waiting and streaming the incoming rest of the interleaved window
result = await body.DecodeInterleaveAsync();
if (result != VCDiffResult.SUCCESS && result != VCDiffResult.EOD)
{
return(result, bytesWritten);
}
bytesWritten += body.TotalBytesDecoded;
}
//technically add could be 0 if it is all copy instructions
//so do an or check on those two
else if (!this.IsSDCHFormat || (this.IsSDCHFormat && (w.AddRunLength > 0 || w.AddressesForCopyLength > 0) &&
w.InstructionAndSizesLength > 0))
{
//not interleaved
//expects the full window to be available
//in the stream
result = await body.DecodeAsync();
if (result != VCDiffResult.SUCCESS)
{
return(result, bytesWritten);
}
bytesWritten += body.TotalBytesDecoded;
}
else
{
//invalid file
return(VCDiffResult.ERROR, bytesWritten);
}
}
else
{
return((VCDiffResult)w.Result, bytesWritten);
}
}
return(result, bytesWritten);
}
internal static void SaveDiffToFile(Stream origStream, Stream modifiedStream, Stream diffStream)
{
VCCoder coder = new VCCoder(origStream, modifiedStream, diffStream);
VCDiffResult result = coder.Encode(); //encodes with no checksum and not interleaved
if (result != VCDiffResult.SUCCESS)
{
//error was not able to encode properly
}
}
void DoDecode(string outputFile, string oldFile, string patchFile)
{
using (FileStream target = new FileStream(patchFile, FileMode.Open, FileAccess.Read))
{
byte[] oldHash = new byte[20];
byte[] newHash = new byte[20];
target.Read(oldHash, 0, oldHash.Length);
target.Read(newHash, 0, newHash.Length);
byte[] realHash = GetSha1FromFile(oldFile);
bool oldHashMatches = CompareHashes(oldHash, realHash);
if (!oldHashMatches)
{
if (CompareHashes(realHash, newHash))
{
File.Copy(oldFile, outputFile);
return;
}
else
{
throw new Exception("file hash mismatch");
}
}
using (FileStream dict = new FileStream(oldFile, FileMode.Open, FileAccess.Read))
using (FileStream output = new FileStream(outputFile, FileMode.Create, FileAccess.Write))
{
VCDecoder decoder = new VCDecoder(dict, target, output);
//You must call decoder.Start() first. The header of the delta file must be available before calling decoder.Start()
VCDiffResult result = decoder.Start();
if (result != VCDiffResult.SUCCESS)
{
//error abort
throw new Exception("abort while decoding");
}
long bytesWritten = 0;
result = decoder.Decode(out bytesWritten);
if (result != VCDiffResult.SUCCESS)
{
//error decoding
throw new Exception("Error decoding");
}
//if success bytesWritten will contain the number of bytes that were decoded
}
}
}
/// <summary>
/// Encodes the file
/// </summary>
/// <param name="interleaved">Set this to true to enable SDHC interleaved vcdiff google format</param>
/// <param name="checksum">Set this to true to add checksum for encoded data windows</param>
/// <returns></returns>
public VCDiffResult Encode(bool interleaved = false, bool checksum = false)
{
if (newData.Length == 0 || oldData.Length == 0)
{
return(VCDiffResult.Error);
}
VCDiffResult result = VCDiffResult.Succes;
oldData.Position = 0;
newData.Position = 0;
//file header
//write magic bytes
if (!interleaved && !checksum)
{
sout.writeBytes(MagicBytes);
}
else
{
sout.writeBytes(MagicBytesExtended);
}
//buffer the whole olddata (dictionary)
//otherwise it will be a slow process
//even Google's version reads in the entire dictionary file to memory
//it is just faster that way because of having to move the memory pointer around
//to find all the hash comparisons and stuff.
//It is much slower trying to random access read from file with FileStream class
//however the newData is read in chunks and processed for memory efficiency and speed
oldData.BufferAll();
//read in all the dictionary it is the only thing that needs to be
BlockHash dictionary = new BlockHash(oldData, 0, hasher);
dictionary.AddAllBlocks();
oldData.Position = 0;
ChunkEncoder chunker = new ChunkEncoder(dictionary, oldData, hasher, interleaved, checksum);
while (newData.CanRead)
{
using (ByteBuffer ntarget = new ByteBuffer(newData.ReadBytes(bufferSize)))
{
chunker.EncodeChunk(ntarget, sout);
}
//just in case
// System.GC.Collect();
}
return(result);
}
/// <summary>
/// Do not run on main thread
/// </summary>
/// <param name="id"></param>
/// <param name="region"></param>
public static bool Merge(int id, string region)
{
if (!HasLocalData(id, region) || !HasLocalDataDiff(id, region))
{
return(false);
}
byte[] dictData = GetLocalData(id, region);
byte[] deltaData = GetLocalDataDiff(id, region);
if (dictData == null || deltaData == null)
{
return(false);
}
try
{
using (ByteBuffer dictBuffer = new ByteBuffer(dictData))
using (ByteBuffer deltaBuffer = new ByteBuffer(deltaData))
using (MemoryStream ms = new MemoryStream())
{
VCDecoder decoder = new VCDecoder(dictBuffer, deltaBuffer, ms);
VCDiffResult result = decoder.Start();
if (result != VCDiffResult.SUCCESS)
{
return(false);
}
long bytesWritten = 0;
result = decoder.Decode(out bytesWritten);
if (result != VCDiffResult.SUCCESS)
{
return(false);
}
string fname = string.Format(id >= 0 ? DB_FILE : DB_REGION_FILE, region, id);
string fpath = Path.Combine(DirectoryPath, fname);
File.WriteAllBytes(fpath, ms.ToArray());
return(true);
}
}
catch (Exception e)
{
Debug.WriteLine(e.ToString());
}
return(false);
}
/// <summary>
/// Calculate and write a diff for the file.
/// </summary>
/// <param name="interleaved">Whether to output in SDCH interleaved diff format.</param>
/// <param name="checksumFormat">
/// Whether to include Adler32 checksums for encoded data windows. If interleaved is true, <see cref="ChecksumFormat.Xdelta3"/>
/// is not supported.
/// </param>
/// <returns>
/// <see cref="VCDiffResult.SUCCESS"/> if successful, <see cref="VCDiffResult.ERROR"/> if the sourceStream or target are zero-length.</returns>
/// <exception cref="ArgumentException">If interleaved is true, and <see cref="ChecksumFormat.Xdelta3"/> is chosen.</exception>
public VCDiffResult Encode(bool interleaved = false,
ChecksumFormat checksumFormat = ChecksumFormat.None)
{
if (oldData == null)
{
this.oldData = new ByteBuffer(sourceStream);
}
if (interleaved && checksumFormat == ChecksumFormat.Xdelta3)
{
throw new ArgumentException("Interleaved diffs can not have an xdelta3 checksum!");
}
if (targetData.Length == 0 || oldData.Length == 0)
{
return(VCDiffResult.ERROR);
}
VCDiffResult result = VCDiffResult.SUCCESS;
oldData.Position = 0;
targetData.Position = 0;
// file header
// write magic bytes
if (!interleaved && checksumFormat != ChecksumFormat.SDCH)
{
outputStream.Write(MagicBytes);
}
else
{
outputStream.Write(MagicBytesExtended);
}
//read in all the dictionary it is the only thing that needs to be
BlockHash dictionary = new BlockHash(oldData, 0, hasher, blockSize);
dictionary.AddAllBlocks();
oldData.Position = 0;
ChunkEncoder chunker = new ChunkEncoder(dictionary, oldData, hasher, checksumFormat, interleaved, chunkSize);
while (targetData.CanRead)
{
using ByteBuffer ntarget = new ByteBuffer(targetData.ReadBytesAsBuf(bufferSize));
chunker.EncodeChunk(ntarget, outputStream);
}
return(result);
}
public void NoChecksumNoInterleaved_Test()
{
using var srcStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}a.test");
using var targetStream = File.OpenRead($"patches{Path.DirectorySeparatorChar}b.test");
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
using VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream);
VCDiffResult result = coder.Encode(); //encodes with no checksum and not interleaved
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
using VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream);
Assert.Equal(VCDiffResult.SUCCESS, decoder.Decode(out long bytesWritten));
Assert.NotEqual(0, bytesWritten);
}
private void DoEncode(string outPatchFile, string oldFile, string newFile)
{
byte[] hash = GetSha1FromFile(oldFile);
byte[] newHash = GetSha1FromFile(newFile);
using (FileStream output = new FileStream(outPatchFile, FileMode.Create, FileAccess.Write))
using (FileStream dict = new FileStream(oldFile, FileMode.Open, FileAccess.Read))
using (FileStream target = new FileStream(newFile, FileMode.Open, FileAccess.Read))
{
output.Write(hash, 0, 20);
output.Write(newHash, 0, 20);
VCCoder coder = new VCCoder(dict, target, output);
VCDiffResult result = coder.Encode(); //encodes with no checksum and not interleaved
if (result != VCDiffResult.SUCCESS)
{
throw new Exception("DoEncode was not able to encode properly file: " + Path.GetFileName(oldFile));
}
}
}
internal static void MergeDiffToDat(Stream origStream, Stream modifiedStream, Stream mergedStream)
{
VCDecoder decoder = new VCDecoder(origStream, modifiedStream, mergedStream);
VCDiffResult result = decoder.Start(); //encodes with no checksum and not interleaved
if (result != VCDiffResult.SUCCESS)
{
//error was not able to encode properly
}
else
{
long bytesWritten = 0;
result = decoder.Decode(out bytesWritten);
if (result != VCDiffResult.SUCCESS)
{
}
}
}
/// <summary>
/// Create delta file from 2 files
/// </summary>
/// <param name="modified">File that the delta will update to</param>
/// <param name="original">File that the delta file will use as the base</param>
/// <param name="output">Output path for the delta file</param>
private static void DoEncode(string modified, string original, string output)
{
using (FileStream outputS = new FileStream(output, FileMode.CreateNew, FileAccess.Write))
using (FileStream dictS = new FileStream(original, FileMode.Open, FileAccess.Read))
using (FileStream targetS = new FileStream(modified, FileMode.Open, FileAccess.Read))
{
VCCoder coder = new VCCoder(dictS, targetS, outputS);
VCDiffResult result = coder.Encode(true, true); //encodes with no checksum and not interleaved
if (result != VCDiffResult.SUCCESS)
{
Debug.WriteLine($"Something got f****d up, how to check please help");
//error was not able to encode properly
}
if (outputS.Length <= 5)
{
outputS.Close();
File.Delete(output);
}
}
}
public void Checksum_Test()
{
using var srcStream = new MemoryStream(ADiffData.ToArray());
using var targetStream = new MemoryStream(BDiffData.ToArray());
using var deltaStream = new MemoryStream();
using var outputStream = new MemoryStream();
VcEncoder coder = new VcEncoder(srcStream, targetStream, deltaStream);
VCDiffResult result = coder.Encode(checksumFormat: ChecksumFormat.SDCH); //encodes with no checksum and not interleaved
Assert.Equal(VCDiffResult.SUCCESS, result);
srcStream.Position = 0;
targetStream.Position = 0;
deltaStream.Position = 0;
VcDecoder decoder = new VcDecoder(srcStream, deltaStream, outputStream);
Assert.Equal(VCDiffResult.SUCCESS, decoder.Decode(out long bytesWritten));
Assert.Equal("Goodbye World", Encoding.UTF8.GetString(outputStream.ToArray()));
}
/// <summary>
/// Use this after calling Start
/// Each time the decode is called it is expected
/// that at least 1 Window header is available in the stream
/// </summary>
/// <param name="bytesWritten">bytes decoded for all available windows</param>
/// <returns></returns>
public VCDiffResult Decode(out long bytesWritten)
{
if (!isStarted)
{
bytesWritten = 0;
return(VCDiffResult.ERRROR);
}
VCDiffResult result = VCDiffResult.SUCCESS;
bytesWritten = 0;
if (!delta.CanRead)
{
return(VCDiffResult.EOD);
}
while (delta.CanRead)
{
//delta is streamed in order aka not random access
WindowDecoder w = new WindowDecoder(dict.Length, delta);
if (w.Decode(googleVersion))
{
using (BodyDecoder body = new BodyDecoder(w, dict, delta, sout))
{
if (googleVersion && w.AddRunLength == 0 && w.AddressesForCopyLength == 0 && w.InstructionAndSizesLength > 0)
{
//interleaved
//decodedinterleave actually has an internal loop for waiting and streaming the incoming rest of the interleaved window
result = body.DecodeInterleave();
if (result != VCDiffResult.SUCCESS && result != VCDiffResult.EOD)
{
return(result);
}
bytesWritten += body.Decoded;
}
//technically add could be 0 if it is all copy instructions
//so do an or check on those two
else if (googleVersion && (w.AddRunLength > 0 || w.AddressesForCopyLength > 0) && w.InstructionAndSizesLength > 0)
{
//not interleaved
//expects the full window to be available
//in the stream
result = body.Decode();
if (result != VCDiffResult.SUCCESS)
{
return(result);
}
bytesWritten += body.Decoded;
}
else if (!googleVersion)
{
//not interleaved
//expects the full window to be available
//in the stream
result = body.Decode();
if (result != VCDiffResult.SUCCESS)
{
return(result);
}
bytesWritten += body.Decoded;
}
else
{
//invalid file
return(VCDiffResult.ERRROR);
}
}
}
else
{
return((VCDiffResult)w.Result);
}
}
return(result);
}
private VCDiffResult DecodeInterleaveCore()
{
VCDiffResult result = VCDiffResult.SUCCESS;
//since interleave expected then the last point that was most likely decoded was the lengths section
//so following is all data for the add run copy etc
long interleaveLength = window.InstructionAndSizesLength;
using var previous = new MemoryStream();
int lastDecodedSize = 0;
VCDiffInstructionType lastDecodedInstruction = VCDiffInstructionType.NOOP;
while (interleaveLength > 0)
{
if (!delta.CanRead)
{
continue;
}
//read in
var didBreakBeforeComplete = false;
//try to read in all interleaved bytes
//if not then it will buffer for next time
previous.Write(delta.ReadBytes((int)interleaveLength).Span);
using ByteBuffer incoming = new ByteBuffer(previous.ToArray());
previous.SetLength(0);
long initialLength = incoming.Length;
InstructionDecoder instrDecoder = new InstructionDecoder(incoming, customTable);
while (incoming.CanRead && TotalBytesDecoded < window.TargetWindowLength)
{
int decodedSize = 0;
byte mode = 0;
VCDiffInstructionType instruction = VCDiffInstructionType.NOOP;
if (lastDecodedSize > 0 && lastDecodedInstruction != VCDiffInstructionType.NOOP)
{
decodedSize = lastDecodedSize;
instruction = lastDecodedInstruction;
}
else
{
instruction = instrDecoder.Next(out decodedSize, out mode);
switch (instruction)
{
case VCDiffInstructionType.EOD:
didBreakBeforeComplete = true;
break;
case VCDiffInstructionType.ERROR:
targetData.SetLength(0);
return(VCDiffResult.ERROR);
}
}
//if instruction is EOD then decodedSize will be 0 as well
//the last part of the buffer containing the instruction will be
//buffered for the next loop
lastDecodedInstruction = instruction;
lastDecodedSize = decodedSize;
if (didBreakBeforeComplete)
{
//we don't have all the data so store this pointer into a temporary list to resolve next loop
didBreakBeforeComplete = true;
interleaveLength -= incoming.Position;
if (initialLength - incoming.Position > 0)
{
previous.Write(incoming.ReadBytes((int)(initialLength - incoming.Position)).Span);
}
break;
}
switch (instruction)
{
case VCDiffInstructionType.ADD:
result = DecodeAdd(decodedSize, incoming);
break;
case VCDiffInstructionType.RUN:
result = DecodeRun(decodedSize, incoming);
break;
case VCDiffInstructionType.COPY:
result = DecodeCopy(decodedSize, mode, incoming);
break;
default:
targetData.SetLength(0);
return(VCDiffResult.ERROR);
}
if (result == VCDiffResult.EOD)
{
//we don't have all the data so store this pointer into a temporary list to resolve next loop
didBreakBeforeComplete = true;
interleaveLength -= incoming.Position;
if (initialLength - incoming.Position > 0)
{
previous.Write(incoming.ReadBytes((int)(initialLength - incoming.Position)).Span);
}
break;
}
//reset these as we have successfully used them
lastDecodedInstruction = VCDiffInstructionType.NOOP;
lastDecodedSize = 0;
}
if (!didBreakBeforeComplete)
{
interleaveLength -= initialLength;
}
}
if (window.ChecksumFormat == ChecksumFormat.SDCH)
{
uint adler = Checksum.ComputeGoogleAdler32(targetData.GetBuffer().AsMemory(0, (int)targetData.Length));
if (adler != window.Checksum)
{
result = VCDiffResult.ERROR;
}
}
return(result);
}
private VCDiffResult DecodeCore()
{
using ByteBuffer instructionBuffer = new ByteBuffer(window.InstructionsAndSizesData);
using ByteBuffer addressBuffer = new ByteBuffer(window.AddressesForCopyData);
using ByteBuffer addRunBuffer = new ByteBuffer(window.AddRunData);
InstructionDecoder instrDecoder = new InstructionDecoder(instructionBuffer, customTable);
VCDiffResult result = VCDiffResult.SUCCESS;
while (this.TotalBytesDecoded < window.TargetWindowLength)
{
VCDiffInstructionType instruction = instrDecoder.Next(out int decodedSize, out byte mode);
switch (instruction)
{
case VCDiffInstructionType.EOD:
targetData.SetLength(0);
return(VCDiffResult.EOD);
case VCDiffInstructionType.ERROR:
targetData.SetLength(0);
return(VCDiffResult.ERROR);
}
switch (instruction)
{
case VCDiffInstructionType.ADD:
result = DecodeAdd(decodedSize, addRunBuffer);
break;
case VCDiffInstructionType.RUN:
result = DecodeRun(decodedSize, addRunBuffer);
break;
case VCDiffInstructionType.COPY:
result = DecodeCopy(decodedSize, mode, addressBuffer);
break;
default:
targetData.SetLength(0);
return(VCDiffResult.ERROR);
}
}
if (window.ChecksumFormat == ChecksumFormat.SDCH)
{
uint adler = Checksum.ComputeGoogleAdler32(targetData.GetBuffer().AsMemory(0, (int)targetData.Length));
if (adler != window.Checksum)
{
result = VCDiffResult.ERROR;
}
}
else if (window.ChecksumFormat == ChecksumFormat.Xdelta3)
{
uint adler = Checksum.ComputeXdelta3Adler32(targetData.GetBuffer().AsMemory(0, (int)targetData.Length));
if (adler != window.Checksum)
{
result = VCDiffResult.ERROR;
}
}
return(result);
}
public async override Task <ISaveGame> CreateSave(IIndelibleDirectory saveContents)
{
if (!this.ProfileRoot.ContainsDirectory("base"))
{
await this.CreateBaseSave(saveContents);
}
using var rollingHash = new RollingHash(32);
// setup
var newGuid = Guid.NewGuid();
var saveName = $"{DateTimeOffset.UtcNow.ToString(DateFormat)}-{newGuid}";
var saveDirectory = this.ProfileRoot.OpenDirectory(saveName);
var contentDirectory = saveDirectory.OpenDirectory("content");
// diff is for anything that exists in the base directory
var diffDir = contentDirectory.OpenDirectory("diff");
// copy is for anything that does not and can not be diffed.
var copyDir = contentDirectory.OpenDirectory("copy");
// Traverse base directory in tandem with saveContents
var baseDir = this.ProfileRoot.OpenDirectory("base/content").AsReadOnly();
foreach (var f in saveContents.EnumerateFiles())
{
if (!baseDir.ContainsFile(f.Name))
{
await copyDir.CopyFromAsync(f);
continue;
}
using var targetStream = f.OpenReadStream();
using var baseStream = baseDir.OpenFile(f.Name).OpenReadStream();
using var outStream = diffDir.OpenFile(f.Name).OpenStream();
using var decoder = new VcEncoder(baseStream, targetStream, outStream,
rollingHash: rollingHash, blockSize: 32);
VCDiffResult result = await decoder.EncodeAsync();
if (result != VCDiffResult.SUCCESS)
{
throw new IOException($"Failed to encode delta for {f.Name}");
}
}
foreach (var d in saveContents.EnumerateDirectories().Where(d => !baseDir.ContainsDirectory(d.Name)))
{
// Copy all directories not in the base.
await foreach (var _ in copyDir.OpenDirectory(d.Name).CopyFromDirectory(d))
{
}
;
}
var queuedDirs = (from targetDir in saveContents.EnumerateDirectories()
where baseDir.ContainsDirectory(targetDir.Name)
select(diffDir, baseDir.OpenDirectory(targetDir.Name), targetDir)).ToList();
Queue <(IDirectory parentDir, IReadOnlyDirectory baseDir, IDirectory targetDir)> dirsToProcess =
new Queue <(IDirectory, IReadOnlyDirectory, IDirectory)>(queuedDirs);
while (dirsToProcess.Count > 0)
{
var(parent, src, diff) = dirsToProcess.Dequeue();
var dst = parent.OpenDirectory(src.Name);
foreach (var f in src.EnumerateFiles())
{
if (!diff.ContainsFile(f.Name))
{
continue;
}
using var baseStream = f.OpenReadStream();
using var targetStream = diff.OpenFile(f.Name).OpenReadStream();
using var outStream = dst.OpenFile(f.Name).OpenStream();
using var decoder = new VcEncoder(baseStream, targetStream, outStream,
rollingHash: rollingHash, blockSize: 32);
VCDiffResult result = await decoder.EncodeAsync();
if (result != VCDiffResult.SUCCESS)
{
throw new IOException($"Failed to decode delta for {f.Name}");
}
}
var children = from targetDir in diff.EnumerateDirectories()
where src.ContainsDirectory(targetDir.Name)
select(dst, src.OpenDirectory(targetDir.Name), targetDir);
foreach (var childDirectory in children)
{
dirsToProcess.Enqueue(childDirectory);
}
}
this.ProfileRoot.OpenFile("latest").WriteAllText(saveName, Encoding.UTF8);
return(this.GetSave(saveDirectory) !);
}
/// <summary>
/// Decode if as expecting interleave
/// </summary>
/// <returns></returns>
public VCDiffResult DecodeInterleave()
{
VCDiffResult result = VCDiffResult.SUCCESS;
//since interleave expected then the last point that was most likely decoded was the lengths section
//so following is all data for the add run copy etc
long interleaveLength = window.InstructionAndSizesLength;
List <byte> previous = new List <byte>();
bool didBreakBeforeComplete = false;
int lastDecodedSize = 0;
VCDiffInstructionType lastDecodedInstruction = VCDiffInstructionType.NOOP;
while (interleaveLength > 0)
{
if (target.CanRead)
{
//read in
didBreakBeforeComplete = false;
//try to read in all interleaved bytes
//if not then it will buffer for next time
previous.AddRange(target.ReadBytes((int)interleaveLength));
ByteBuffer incoming = new ByteBuffer(previous.ToArray());
previous.Clear();
long initialLength = incoming.Length;
InstructionDecoder instrDecoder = new InstructionDecoder(incoming, this.customTable);
while (incoming.CanRead && decodedOnly < window.DecodedDeltaLength)
{
int decodedSize = 0;
byte mode = 0;
VCDiffInstructionType instruction = VCDiffInstructionType.NOOP;
if (lastDecodedSize > 0 && lastDecodedInstruction != VCDiffInstructionType.NOOP)
{
decodedSize = lastDecodedSize;
instruction = lastDecodedInstruction;
}
else
{
instruction = instrDecoder.Next(out decodedSize, out mode);
switch (instruction)
{
case VCDiffInstructionType.EOD:
didBreakBeforeComplete = true;
break;
case VCDiffInstructionType.ERROR:
targetData.Clear();
return(VCDiffResult.ERRROR);
default:
break;
}
}
//if instruction is EOD then decodedSize will be 0 as well
//the last part of the buffer containing the instruction will be
//buffered for the next loop
lastDecodedInstruction = instruction;
lastDecodedSize = decodedSize;
if (didBreakBeforeComplete)
{
//we don't have all the data so store this pointer into a temporary list to resolve next loop
didBreakBeforeComplete = true;
interleaveLength -= incoming.Position;
if (initialLength - incoming.Position > 0)
{
previous.AddRange(incoming.ReadBytes((int)(initialLength - incoming.Position)));
}
break;
}
switch (instruction)
{
case VCDiffInstructionType.ADD:
result = DecodeAdd(decodedSize, incoming);
break;
case VCDiffInstructionType.RUN:
result = DecodeRun(decodedSize, incoming);
break;
case VCDiffInstructionType.COPY:
result = DecodeCopy(decodedSize, mode, incoming);
break;
default:
targetData.Clear();
return(VCDiffResult.ERRROR);
}
if (result == VCDiffResult.EOD)
{
//we don't have all the data so store this pointer into a temporary list to resolve next loop
didBreakBeforeComplete = true;
interleaveLength -= incoming.Position;
if (initialLength - incoming.Position > 0)
{
previous.AddRange(incoming.ReadBytes((int)(initialLength - incoming.Position)));
}
break;
}
//reset these as we have successfully used them
lastDecodedInstruction = VCDiffInstructionType.NOOP;
lastDecodedSize = 0;
}
if (!didBreakBeforeComplete)
{
interleaveLength -= initialLength;
}
}
}
if (window.HasChecksum)
{
uint adler = Checksum.ComputeAdler32(targetData.ToArray());
if (adler != window.Checksum)
{
result = VCDiffResult.ERRROR;
}
}
targetData.Clear();
return(result);
}
/// <summary>
/// Decode normally
/// </summary>
/// <returns></returns>
public VCDiffResult Decode()
{
ByteBuffer instructionBuffer = new ByteBuffer(window.InstructionsAndSizesData);
ByteBuffer addressBuffer = new ByteBuffer(window.AddressesForCopyData);
ByteBuffer addRunBuffer = new ByteBuffer(window.AddRunData);
InstructionDecoder instrDecoder = new InstructionDecoder(instructionBuffer, this.customTable);
VCDiffResult result = VCDiffResult.SUCCESS;
while (decodedOnly < window.DecodedDeltaLength && instructionBuffer.CanRead)
{
int decodedSize = 0;
byte mode = 0;
VCDiffInstructionType instruction = instrDecoder.Next(out decodedSize, out mode);
switch (instruction)
{
case VCDiffInstructionType.EOD:
targetData.Clear();
return(VCDiffResult.EOD);
case VCDiffInstructionType.ERROR:
targetData.Clear();
return(VCDiffResult.ERRROR);
default:
break;
}
switch (instruction)
{
case VCDiffInstructionType.ADD:
result = DecodeAdd(decodedSize, addRunBuffer);
break;
case VCDiffInstructionType.RUN:
result = DecodeRun(decodedSize, addRunBuffer);
break;
case VCDiffInstructionType.COPY:
result = DecodeCopy(decodedSize, mode, addressBuffer);
break;
default:
targetData.Clear();
return(VCDiffResult.ERRROR);
}
}
if (window.HasChecksum)
{
uint adler = Checksum.ComputeAdler32(targetData.ToArray());
if (adler != window.Checksum)
{
result = VCDiffResult.ERRROR;
}
}
targetData.Clear();
return(result);
}
private bool Decode_Init(out long bytesWritten, out VCDiffResult result, out (VCDiffResult result, long bytesWritten) decodeAsync)
/// <summary>
/// Writes the patched file into the output stream.
/// </summary>
/// <param name="bytesWritten">Number of bytes written into the output stream.</param>
/// <returns></returns>
public VCDiffResult Decode(out long bytesWritten)
{
if (!this.IsInitialized)
{
var initializeResult = this.Initialize();
if (initializeResult != VCDiffResult.SUCCESS || !this.IsInitialized)
{
bytesWritten = 0;
return initializeResult;
}
}
VCDiffResult result = VCDiffResult.SUCCESS;
bytesWritten = 0;
if (!delta.CanRead) return VCDiffResult.EOD;
while (delta.CanRead)
{
//delta is streamed in order aka not random access
WindowDecoder w = new WindowDecoder(source.Length, delta);
if (w.Decode(this.IsSDCHFormat))
{
using BodyDecoder body = new BodyDecoder(w, source, delta, outputStream);
if (this.IsSDCHFormat && w.AddRunLength == 0 && w.AddressesForCopyLength == 0 && w.InstructionAndSizesLength > 0)
{
//interleaved
//decodedinterleave actually has an internal loop for waiting and streaming the incoming rest of the interleaved window
result = body.DecodeInterleave();
if (result != VCDiffResult.SUCCESS && result != VCDiffResult.EOD)
{
return result;
}
bytesWritten += body.TotalBytesDecoded;
}
//technically add could be 0 if it is all copy instructions
//so do an or check on those two
else if (!this.IsSDCHFormat || (this.IsSDCHFormat && (w.AddRunLength > 0 || w.AddressesForCopyLength > 0) && w.InstructionAndSizesLength > 0))
{
//not interleaved
//expects the full window to be available
//in the stream
result = body.Decode();
if (result != VCDiffResult.SUCCESS)
{
return result;
}
bytesWritten += body.TotalBytesDecoded;
}
else
{
//invalid file
return VCDiffResult.ERROR;
}
}
else
{
return (VCDiffResult)w.Result;
}
}
return result;
}
/// <summary>
/// Call this before calling decode
/// This expects at least the header part of the delta file
/// is available in the stream
/// </summary>
/// <returns></returns>
private VCDiffResult Initialize()
{
if (!delta.CanRead) return VCDiffResult.EOD;
byte V = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte C = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte D = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte version = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte hdr = delta.ReadByte();
if (V != MagicBytes[0])
{
return VCDiffResult.ERROR;
}
if (C != MagicBytes[1])
{
return VCDiffResult.ERROR;
}
if (D != MagicBytes[2])
{
return VCDiffResult.ERROR;
}
if (version != 0x00 && version != 'S')
{
return VCDiffResult.ERROR;
}
//compression not supported
if ((hdr & (int)VCDiffCodeFlags.VCDDECOMPRESS) != 0)
{
return VCDiffResult.ERROR;
}
//custom code table!
if ((hdr & (int)VCDiffCodeFlags.VCDCODETABLE) != 0)
{
if (!delta.CanRead) return VCDiffResult.EOD;
//try decoding the custom code table
//since we don't support the compress the next line should be the length of the code table
customTable = new CustomCodeTableDecoder();
VCDiffResult result = customTable.Decode(delta);
if (result != VCDiffResult.SUCCESS)
{
return result;
}
}
if ((hdr & (int)VCDiffCodeFlags.VCDAPPHEADER) != 0)
{
if (!delta.CanRead) return VCDiffResult.EOD;
int headerLength = VarIntBE.ParseInt32(delta);
// skip the app header
delta.ReadBytes(headerLength);
}
this.IsSDCHFormat = version == 'S';
this.IsInitialized = true;
return VCDiffResult.SUCCESS;
}
/// <summary>
/// Call this before calling decode
/// This expects at least the header part of the delta file
/// is available in the stream
/// </summary>
/// <returns></returns>
public VCDiffResult Start()
{
if (!delta.CanRead)
{
return(VCDiffResult.EOD);
}
byte V = delta.ReadByte();
if (!delta.CanRead)
{
return(VCDiffResult.EOD);
}
byte C = delta.ReadByte();
if (!delta.CanRead)
{
return(VCDiffResult.EOD);
}
byte D = delta.ReadByte();
if (!delta.CanRead)
{
return(VCDiffResult.EOD);
}
byte version = delta.ReadByte();
if (!delta.CanRead)
{
return(VCDiffResult.EOD);
}
byte hdr = delta.ReadByte();
if (V != MagicBytes[0])
{
return(VCDiffResult.ERRROR);
}
if (C != MagicBytes[1])
{
return(VCDiffResult.ERRROR);
}
if (D != MagicBytes[2])
{
return(VCDiffResult.ERRROR);
}
if (version != 0x00 && version != 'S')
{
return(VCDiffResult.ERRROR);
}
//compression not supported
if ((hdr & (int)VCDiffCodeFlags.VCDDECOMPRESS) != 0)
{
return(VCDiffResult.ERRROR);
}
//custom code table!
if ((hdr & (int)VCDiffCodeFlags.VCDCODETABLE) != 0)
{
if (!delta.CanRead)
{
return(VCDiffResult.EOD);
}
//try decoding the custom code table
//since we don't support the compress the next line should be the length of the code table
customTable = new CustomCodeTableDecoder();
VCDiffResult result = customTable.Decode(delta);
if (result != VCDiffResult.SUCCESS)
{
return(result);
}
}
googleVersion = version == 'S';
isStarted = true;
//buffer all the dictionary up front
dict.BufferAll();
return(VCDiffResult.SUCCESS);
}
public VCDiffResult Decode(IByteBuffer source)
{
VCDiffResult result = VCDiffResult.Succes;
//the custom codetable itself is a VCDiff file but it is required to be encoded with the standard table
//the length should be the first thing after the hdr_indicator if not supporting compression
//at least according to the RFC specs.
int lengthOfCodeTable = VarIntBE.ParseInt32(source);
if (lengthOfCodeTable == 0)
{
return(VCDiffResult.Error);
}
ByteBuffer codeTable = new ByteBuffer(source.ReadBytes(lengthOfCodeTable));
//according to the RFC specifications the next two items will be the size of near and size of same
//they are bytes in the RFC spec, but for some reason Google uses the varint to read which does
//the same thing if it is a single byte
//but I am going to just read in bytes because it is the RFC standard
NearSize = codeTable.ReadByte();
SameSize = codeTable.ReadByte();
if (NearSize == 0 || SameSize == 0 || NearSize > byte.MaxValue || SameSize > byte.MaxValue)
{
return(VCDiffResult.Error);
}
CustomTable = new CodeTable();
//get the original bytes of the default codetable to use as a dictionary
IByteBuffer dictionary = CustomTable.GetBytes();
//Decode the code table VCDiff file itself
//stream the decoded output into a memory stream
using (MemoryStream sout = new MemoryStream())
{
VCDecoder decoder = new VCDecoder(dictionary, codeTable, sout);
result = decoder.Start();
if (result != VCDiffResult.Succes)
{
return(result);
}
long bytesWritten = 0;
result = decoder.Decode(out bytesWritten);
if (result != VCDiffResult.Succes || bytesWritten == 0)
{
return(VCDiffResult.Error);
}
//set the new table data that was decoded
if (!CustomTable.SetBytes(sout.ToArray()))
{
result = VCDiffResult.Error;
}
}
return(result);
}
