private bool ParseInt32(out int value)
{
if ((int)VCDiffResult.Succes != Result)
{
value = 0;
return(false);
}
if (chunk.IsEmpty)
{
value = 0;
Result = (int)VCDiffResult.EOD;
return(false);
}
int parsed = VarIntBE.ParseInt32(buffer);
switch (parsed)
{
case (int)VCDiffResult.Error:
value = 0;
return(false);
case (int)VCDiffResult.EOD:
value = 0;
return(false);
default:
break;
}
chunk.Position = buffer.Position;
value = parsed;
return(true);
}
private bool ParseInt32(out int value)
{
if ((int)VCDiffResult.SUCCESS != returnCode)
{
value = 0;
return(false);
}
if (chunk.IsEmpty)
{
value = 0;
returnCode = (int)VCDiffResult.EOD;
return(false);
}
int parsed = VarIntBE.ParseInt32(buffer);
switch (parsed)
{
case (int)VCDiffResult.ERROR:
value = 0;
return(false);
case (int)VCDiffResult.EOD:
value = 0;
return(false);
}
chunk.Position = buffer.Position;
value = parsed;
return(true);
}
private int CalculateLengthOfTheDeltaEncoding()
{
if (IsInterleaved)
{
return(VarIntBE.CalcInt32Length((int)targetLength) +
1 +
VarIntBE.CalcInt32Length(0) +
VarIntBE.CalcInt32Length((int)instructionAndSizes.Length) +
VarIntBE.CalcInt32Length(0) +
0 +
(int)instructionAndSizes.Length
// interleaved implies SDCH checksum if any.
+ (this.ChecksumFormat == ChecksumFormat.SDCH ? VarIntBE.CalcInt64Length(Checksum) : 0));
}
int lengthOfDelta = VarIntBE.CalcInt32Length((int)targetLength) +
1 +
VarIntBE.CalcInt32Length((int)dataForAddAndRun.Length) +
VarIntBE.CalcInt32Length((int)instructionAndSizes.Length) +
VarIntBE.CalcInt32Length((int)addressForCopy.Length) +
(int)dataForAddAndRun.Length +
(int)instructionAndSizes.Length +
(int)addressForCopy.Length;
if (this.ChecksumFormat == ChecksumFormat.SDCH)
{
lengthOfDelta += VarIntBE.CalcInt64Length(Checksum);
}
else if (this.ChecksumFormat == ChecksumFormat.Xdelta3)
{
lengthOfDelta += 4;
}
return(lengthOfDelta);
}
void EncodeInstruction(VCDiffInstructionType inst, int size, byte mode = 0)
{
if (lastOpcodeIndex >= 0)
{
int lastOp = instructionAndSizes[lastOpcodeIndex];
if (inst == VCDiffInstructionType.ADD && (table.inst1[lastOp] == CodeTable.A))
{
//warning adding two in a row
Console.WriteLine("Warning: performing two ADD instructions in a row.");
}
int compoundOp = CodeTable.kNoOpcode;
if (size <= byte.MaxValue)
{
compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, (byte)size, mode);
if (compoundOp != CodeTable.kNoOpcode)
{
instructionAndSizes[lastOpcodeIndex] = (byte)compoundOp;
lastOpcodeIndex = -1;
return;
}
}
compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, (byte)0, mode);
if (compoundOp != CodeTable.kNoOpcode)
{
instructionAndSizes[lastOpcodeIndex] = (byte)compoundOp;
//append size to instructionAndSizes
VarIntBE.AppendInt32(size, instructionAndSizes);
lastOpcodeIndex = -1;
}
}
int opcode = CodeTable.kNoOpcode;
if (size <= byte.MaxValue)
{
opcode = instrMap.LookFirstOpcode((byte)inst, (byte)size, mode);
if (opcode != CodeTable.kNoOpcode)
{
instructionAndSizes.Add((byte)opcode);
lastOpcodeIndex = instructionAndSizes.Count - 1;
return;
}
}
opcode = instrMap.LookFirstOpcode((byte)inst, 0, mode);
if (opcode == CodeTable.kNoOpcode)
{
return;
}
instructionAndSizes.Add((byte)opcode);
lastOpcodeIndex = instructionAndSizes.Count - 1;
VarIntBE.AppendInt32(size, instructionAndSizes);
}
private void EncodeInstruction(VCDiffInstructionType inst, int size, byte mode = 0)
{
if (lastOpcodeIndex >= 0)
{
int lastOp = instructionAndSizes.GetBuffer()[lastOpcodeIndex];
int compoundOp;
if (size <= byte.MaxValue)
{
compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, (byte)size, mode);
if (compoundOp != CodeTable.kNoOpcode)
{
instructionAndSizes.GetBuffer()[lastOpcodeIndex] = (byte)compoundOp;
lastOpcodeIndex = -1;
return;
}
}
compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, 0, mode);
if (compoundOp != CodeTable.kNoOpcode)
{
instructionAndSizes.GetBuffer()[lastOpcodeIndex] = (byte)compoundOp;
//append size to instructionAndSizes
VarIntBE.AppendInt32(size, instructionAndSizes);
lastOpcodeIndex = -1;
}
}
int opcode;
if (size <= byte.MaxValue)
{
opcode = instrMap.LookFirstOpcode((byte)inst, (byte)size, mode);
if (opcode != CodeTable.kNoOpcode)
{
instructionAndSizes.WriteByte((byte)opcode);
lastOpcodeIndex = (int)instructionAndSizes.Length - 1;
return;
}
}
opcode = instrMap.LookFirstOpcode((byte)inst, 0, mode);
if (opcode == CodeTable.kNoOpcode)
{
return;
}
instructionAndSizes.WriteByte((byte)opcode);
lastOpcodeIndex = (int)instructionAndSizes.Length - 1;
VarIntBE.AppendInt32(size, instructionAndSizes);
}
internal VCDiffResult Decode(IByteBuffer source)
{
//the custom codetable itself is a VCDiff file but it is required to be encoded with the standard table
//the length should be the first thing after the hdr_indicator if not supporting compression
//at least according to the RFC specs.
int lengthOfCodeTable = VarIntBE.ParseInt32(source);
if (lengthOfCodeTable == 0)
{
return(VCDiffResult.ERROR);
}
using ByteBuffer codeTable = new ByteBuffer(source.ReadBytes(lengthOfCodeTable));
//according to the RFC specifications the next two items will be the size of near and size of same
//they are bytes in the RFC spec, but for some reason Google uses the varint to read which does
//the same thing if it is a single byte
//but I am going to just read in bytes because it is the RFC standard
NearSize = codeTable.ReadByte();
SameSize = codeTable.ReadByte();
if (NearSize == 0 || SameSize == 0 || NearSize > byte.MaxValue || SameSize > byte.MaxValue)
{
return(VCDiffResult.ERROR);
}
CustomTable = new CodeTable();
//get the original bytes of the default codetable to use as a dictionary
using ByteBuffer dictionary = CustomTable.GetBytes();
//Decode the code table VCDiff file itself
//stream the decoded output into a memory stream
using MemoryStream sout = new MemoryStream();
VcDecoder decoder = new VcDecoder(dictionary, codeTable, sout);
var result = decoder.Decode(out long bytesWritten);
if (result != VCDiffResult.SUCCESS || bytesWritten == 0)
{
return(VCDiffResult.ERROR);
}
//set the new table data that was decoded
if (!CustomTable.SetBytes(sout.ToArray()))
{
result = VCDiffResult.ERROR;
}
return(result);
}
public void Copy(int offset, int length)
{
byte mode = addrCache.EncodeAddress(offset, dictionarySize + targetLength, out long encodedAddr);
EncodeInstruction(VCDiffInstructionType.COPY, length, mode);
if (addrCache.WriteAddressAsVarint(mode))
{
VarIntBE.AppendInt64(encodedAddr, addressForCopy);
}
else
{
addressForCopy.WriteByte((byte)encodedAddr);
}
targetLength += length;
}
int CalculateLengthOfTheDeltaEncoding()
{
int extraLength = 0;
if (hasChecksum)
{
extraLength += VarIntBE.CalcInt64Length(checksum);
}
if (!interleaved)
{
int lengthOfDelta = VarIntBE.CalcInt32Length((int)targetLength) +
1 +
VarIntBE.CalcInt32Length(dataForAddAndRun.Count) +
VarIntBE.CalcInt32Length(instructionAndSizes.Count) +
VarIntBE.CalcInt32Length(addressForCopy.Count) +
dataForAddAndRun.Count +
instructionAndSizes.Count +
addressForCopy.Count;
lengthOfDelta += extraLength;
return(lengthOfDelta);
}
else
{
int lengthOfDelta = VarIntBE.CalcInt32Length((int)targetLength) +
1 +
VarIntBE.CalcInt32Length(0) +
VarIntBE.CalcInt32Length(instructionAndSizes.Count) +
VarIntBE.CalcInt32Length(0) +
0 +
instructionAndSizes.Count;
lengthOfDelta += extraLength;
return(lengthOfDelta);
}
}
bool ParseUInt32(out uint value)
{
if ((int)VCDiffResult.SUCCESS != returnCode)
{
value = 0;
return(false);
}
if (chunk.IsEmpty)
{
value = 0;
returnCode = (int)VCDiffResult.EOD;
return(false);
}
long parsed = VarIntBE.ParseInt64(buffer);
switch (parsed)
{
case (int)VCDiffResult.ERRROR:
value = 0;
return(false);
case (int)VCDiffResult.EOD:
value = 0;
return(false);
default:
break;
}
if (parsed > 0xFFFFFFFF)
{
returnCode = (int)VCDiffResult.ERRROR;
value = 0;
return(false);
}
chunk.Position = buffer.Position;
value = (uint)parsed;
return(true);
}
/// <summary>
/// Gets the next instruction from the buffer
/// </summary>
/// <param name="size">the size</param>
/// <param name="mode">the mode</param>
/// <returns></returns>
public unsafe VCDiffInstructionType Next(out int size, out byte mode)
{
byte opcode = 0;
byte instructionType = CodeTable.N;
int instructionSize = 0;
byte instructionMode = 0;
int start = (int)source.Position;
do
{
if (pendingSecond != CodeTable.kNoOpcode)
{
opcode = (byte)pendingSecond;
pendingSecond = CodeTable.kNoOpcode;
instructionType = table.inst2.Pointer[opcode];
instructionSize = table.size2.Pointer[opcode];
instructionMode = table.mode2.Pointer[opcode];
break;
}
if (!source.CanRead)
{
size = 0;
mode = 0;
return(VCDiffInstructionType.EOD);
}
opcode = source.PeekByte();
if (table.inst2.Pointer[opcode] != CodeTable.N)
{
pendingSecond = source.PeekByte();
}
source.Next();
instructionType = table.inst1.Pointer[opcode];
instructionSize = table.size1.Pointer[opcode];
instructionMode = table.mode1.Pointer[opcode];
} while (instructionType == CodeTable.N);
if (instructionSize == 0)
{
switch (size = VarIntBE.ParseInt32(source))
{
case (int)VCDiffResult.ERROR:
mode = 0;
size = 0;
return(VCDiffInstructionType.ERROR);
case (int)VCDiffResult.EOD:
mode = 0;
size = 0;
//reset it back before we read the instruction
//otherwise when parsing interleave we will miss data
source.Position = start;
return(VCDiffInstructionType.EOD);
}
}
else
{
size = instructionSize;
}
mode = instructionMode;
return((VCDiffInstructionType)instructionType);
}
public void Output(Stream outputStream)
{
int lengthOfDelta = CalculateLengthOfTheDeltaEncoding();
//Google's Checksum Implementation Support
if (this.ChecksumFormat != ChecksumFormat.None)
{
outputStream.WriteByte((byte)VCDiffWindowFlags.VCDSOURCE | (byte)VCDiffWindowFlags.VCDCHECKSUM); //win indicator
}
else
{
outputStream.WriteByte((byte)VCDiffWindowFlags.VCDSOURCE); //win indicator
}
VarIntBE.AppendInt32((int)dictionarySize, outputStream); //dictionary size
VarIntBE.AppendInt32(0, outputStream); //dictionary start position 0 is default aka encompass the whole dictionary
VarIntBE.AppendInt32(lengthOfDelta, outputStream); //length of delta
//begin of delta encoding
long sizeBeforeDelta = outputStream.Position;
VarIntBE.AppendInt32((int)targetLength, outputStream); //final target length after decoding
outputStream.WriteByte(0x00); // uncompressed
// [Here is where a secondary compressor would be used
// if the encoder and decoder supported that feature.]
//non interleaved then it is separeat areas for each type
if (!IsInterleaved)
{
VarIntBE.AppendInt32((int)dataForAddAndRun.Length, outputStream); //length of add/run
VarIntBE.AppendInt32((int)instructionAndSizes.Length, outputStream); //length of instructions and sizes
VarIntBE.AppendInt32((int)addressForCopy.Length, outputStream); //length of addresses for copys
switch (this.ChecksumFormat)
{
//Google Checksum Support
case ChecksumFormat.SDCH:
VarIntBE.AppendInt64(this.Checksum, outputStream);
break;
// Xdelta checksum support.
case ChecksumFormat.Xdelta3:
{
Span <byte> checksumBytes = stackalloc [] {
(byte)(this.Checksum >> 24), (byte)(this.Checksum >> 16), (byte)(this.Checksum >> 8), (byte)(this.Checksum & 0x000000FF)
};
outputStream.Write(checksumBytes);
break;
}
}
outputStream.Write(dataForAddAndRun.GetBuffer().AsSpanFast((int)dataForAddAndRun.Length)); //data section for adds and runs
outputStream.Write(instructionAndSizes.GetBuffer().AsSpanFast((int)instructionAndSizes.Length)); //data for instructions and sizes
outputStream.Write(addressForCopy.GetBuffer().AsSpanFast((int)addressForCopy.Length)); //data for addresses section copys
}
else
{
//interleaved everything is woven in and out in one block
VarIntBE.AppendInt32(0, outputStream); //length of add/run
VarIntBE.AppendInt32((int)instructionAndSizes.Length, outputStream); //length of instructions and sizes + other data for interleaved
VarIntBE.AppendInt32(0, outputStream); //length of addresses for copys
//Google Checksum Support
if (this.ChecksumFormat == ChecksumFormat.SDCH)
{
VarIntBE.AppendInt64(Checksum, outputStream);
}
outputStream.Write(instructionAndSizes.GetBuffer().AsSpan(0, (int)instructionAndSizes.Length)); //data for instructions and sizes, in interleaved it is everything
}
//end of delta encoding
long sizeAfterDelta = outputStream.Position;
if (lengthOfDelta != sizeAfterDelta - sizeBeforeDelta)
{
throw new IOException("Delta output length does not match");
}
dataForAddAndRun.SetLength(0);
instructionAndSizes.SetLength(0);
addressForCopy.SetLength(0);
if (targetLength == 0)
{
throw new IOException("Empty target window");
}
}
/// <summary>
/// Call this before calling decode
/// This expects at least the header part of the delta file
/// is available in the stream
/// </summary>
/// <returns></returns>
private VCDiffResult Initialize()
{
if (!delta.CanRead) return VCDiffResult.EOD;
byte V = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte C = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte D = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte version = delta.ReadByte();
if (!delta.CanRead) return VCDiffResult.EOD;
byte hdr = delta.ReadByte();
if (V != MagicBytes[0])
{
return VCDiffResult.ERROR;
}
if (C != MagicBytes[1])
{
return VCDiffResult.ERROR;
}
if (D != MagicBytes[2])
{
return VCDiffResult.ERROR;
}
if (version != 0x00 && version != 'S')
{
return VCDiffResult.ERROR;
}
//compression not supported
if ((hdr & (int)VCDiffCodeFlags.VCDDECOMPRESS) != 0)
{
return VCDiffResult.ERROR;
}
//custom code table!
if ((hdr & (int)VCDiffCodeFlags.VCDCODETABLE) != 0)
{
if (!delta.CanRead) return VCDiffResult.EOD;
//try decoding the custom code table
//since we don't support the compress the next line should be the length of the code table
customTable = new CustomCodeTableDecoder();
VCDiffResult result = customTable.Decode(delta);
if (result != VCDiffResult.SUCCESS)
{
return result;
}
}
if ((hdr & (int)VCDiffCodeFlags.VCDAPPHEADER) != 0)
{
if (!delta.CanRead) return VCDiffResult.EOD;
int headerLength = VarIntBE.ParseInt32(delta);
// skip the app header
delta.ReadBytes(headerLength);
}
this.IsSDCHFormat = version == 'S';
this.IsInitialized = true;
return VCDiffResult.SUCCESS;
}
public void Output(ByteStreamWriter sout)
{
int lengthOfDelta = CalculateLengthOfTheDeltaEncoding();
int windowSize = lengthOfDelta +
1 +
VarIntBE.CalcInt32Length((int)dictionarySize) +
VarIntBE.CalcInt32Length(0);
VarIntBE.CalcInt32Length(lengthOfDelta);
//Google's Checksum Implementation Support
if (hasChecksum)
{
sout.writeByte((byte)VCDiffWindowFlags.VCDSOURCE | (byte)VCDiffWindowFlags.VCDCHECKSUM); //win indicator
}
else
{
sout.writeByte((byte)VCDiffWindowFlags.VCDSOURCE); //win indicator
}
VarIntBE.AppendInt32((int)dictionarySize, sout); //dictionary size
VarIntBE.AppendInt32(0, sout); //dictionary start position 0 is default aka encompass the whole dictionary
VarIntBE.AppendInt32(lengthOfDelta, sout); //length of delta
//begin of delta encoding
Int64 sizeBeforeDelta = sout.Position;
VarIntBE.AppendInt32((int)targetLength, sout); //final target length after decoding
sout.writeByte(0x00); //uncompressed
// [Here is where a secondary compressor would be used
// if the encoder and decoder supported that feature.]
//non interleaved then it is separata areas for each type
if (!interleaved)
{
VarIntBE.AppendInt32(dataForAddAndRun.Count, sout); //length of add/run
VarIntBE.AppendInt32(instructionAndSizes.Count, sout); //length of instructions and sizes
VarIntBE.AppendInt32(addressForCopy.Count, sout); //length of addresses for copys
//Google Checksum Support
if (hasChecksum)
{
VarIntBE.AppendInt64(checksum, sout);
}
sout.writeBytes(dataForAddAndRun.ToArray()); //data section for adds and runs
sout.writeBytes(instructionAndSizes.ToArray()); //data for instructions and sizes
sout.writeBytes(addressForCopy.ToArray()); //data for addresses section copys
}
else
{
//interleaved everything is woven in and out in one block
VarIntBE.AppendInt32(0, sout); //length of add/run
VarIntBE.AppendInt32(instructionAndSizes.Count, sout); //length of instructions and sizes + other data for interleaved
VarIntBE.AppendInt32(0, sout); //length of addresses for copys
//Google Checksum Support
if (hasChecksum)
{
VarIntBE.AppendInt64(checksum, sout);
}
sout.writeBytes(instructionAndSizes.ToArray()); //data for instructions and sizes, in interleaved it is everything
}
//end of delta encoding
Int64 sizeAfterDelta = sout.Position;
if (lengthOfDelta != sizeAfterDelta - sizeBeforeDelta)
{
Console.WriteLine("Delta output length does not match");
}
dataForAddAndRun.Clear();
instructionAndSizes.Clear();
addressForCopy.Clear();
if (targetLength == 0)
{
Console.WriteLine("Empty target window");
}
addrCache = new AddressCache();
}
