private bool ParseInt32(out int value) { if ((int)VCDiffResult.Succes != Result) { value = 0; return(false); } if (chunk.IsEmpty) { value = 0; Result = (int)VCDiffResult.EOD; return(false); } int parsed = VarIntBE.ParseInt32(buffer); switch (parsed) { case (int)VCDiffResult.Error: value = 0; return(false); case (int)VCDiffResult.EOD: value = 0; return(false); default: break; } chunk.Position = buffer.Position; value = parsed; return(true); }
private bool ParseInt32(out int value) { if ((int)VCDiffResult.SUCCESS != returnCode) { value = 0; return(false); } if (chunk.IsEmpty) { value = 0; returnCode = (int)VCDiffResult.EOD; return(false); } int parsed = VarIntBE.ParseInt32(buffer); switch (parsed) { case (int)VCDiffResult.ERROR: value = 0; return(false); case (int)VCDiffResult.EOD: value = 0; return(false); } chunk.Position = buffer.Position; value = parsed; return(true); }
private int CalculateLengthOfTheDeltaEncoding() { if (IsInterleaved) { return(VarIntBE.CalcInt32Length((int)targetLength) + 1 + VarIntBE.CalcInt32Length(0) + VarIntBE.CalcInt32Length((int)instructionAndSizes.Length) + VarIntBE.CalcInt32Length(0) + 0 + (int)instructionAndSizes.Length // interleaved implies SDCH checksum if any. + (this.ChecksumFormat == ChecksumFormat.SDCH ? VarIntBE.CalcInt64Length(Checksum) : 0)); } int lengthOfDelta = VarIntBE.CalcInt32Length((int)targetLength) + 1 + VarIntBE.CalcInt32Length((int)dataForAddAndRun.Length) + VarIntBE.CalcInt32Length((int)instructionAndSizes.Length) + VarIntBE.CalcInt32Length((int)addressForCopy.Length) + (int)dataForAddAndRun.Length + (int)instructionAndSizes.Length + (int)addressForCopy.Length; if (this.ChecksumFormat == ChecksumFormat.SDCH) { lengthOfDelta += VarIntBE.CalcInt64Length(Checksum); } else if (this.ChecksumFormat == ChecksumFormat.Xdelta3) { lengthOfDelta += 4; } return(lengthOfDelta); }
void EncodeInstruction(VCDiffInstructionType inst, int size, byte mode = 0) { if (lastOpcodeIndex >= 0) { int lastOp = instructionAndSizes[lastOpcodeIndex]; if (inst == VCDiffInstructionType.ADD && (table.inst1[lastOp] == CodeTable.A)) { //warning adding two in a row Console.WriteLine("Warning: performing two ADD instructions in a row."); } int compoundOp = CodeTable.kNoOpcode; if (size <= byte.MaxValue) { compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, (byte)size, mode); if (compoundOp != CodeTable.kNoOpcode) { instructionAndSizes[lastOpcodeIndex] = (byte)compoundOp; lastOpcodeIndex = -1; return; } } compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, (byte)0, mode); if (compoundOp != CodeTable.kNoOpcode) { instructionAndSizes[lastOpcodeIndex] = (byte)compoundOp; //append size to instructionAndSizes VarIntBE.AppendInt32(size, instructionAndSizes); lastOpcodeIndex = -1; } } int opcode = CodeTable.kNoOpcode; if (size <= byte.MaxValue) { opcode = instrMap.LookFirstOpcode((byte)inst, (byte)size, mode); if (opcode != CodeTable.kNoOpcode) { instructionAndSizes.Add((byte)opcode); lastOpcodeIndex = instructionAndSizes.Count - 1; return; } } opcode = instrMap.LookFirstOpcode((byte)inst, 0, mode); if (opcode == CodeTable.kNoOpcode) { return; } instructionAndSizes.Add((byte)opcode); lastOpcodeIndex = instructionAndSizes.Count - 1; VarIntBE.AppendInt32(size, instructionAndSizes); }
private void EncodeInstruction(VCDiffInstructionType inst, int size, byte mode = 0) { if (lastOpcodeIndex >= 0) { int lastOp = instructionAndSizes.GetBuffer()[lastOpcodeIndex]; int compoundOp; if (size <= byte.MaxValue) { compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, (byte)size, mode); if (compoundOp != CodeTable.kNoOpcode) { instructionAndSizes.GetBuffer()[lastOpcodeIndex] = (byte)compoundOp; lastOpcodeIndex = -1; return; } } compoundOp = instrMap.LookSecondOpcode((byte)lastOp, (byte)inst, 0, mode); if (compoundOp != CodeTable.kNoOpcode) { instructionAndSizes.GetBuffer()[lastOpcodeIndex] = (byte)compoundOp; //append size to instructionAndSizes VarIntBE.AppendInt32(size, instructionAndSizes); lastOpcodeIndex = -1; } } int opcode; if (size <= byte.MaxValue) { opcode = instrMap.LookFirstOpcode((byte)inst, (byte)size, mode); if (opcode != CodeTable.kNoOpcode) { instructionAndSizes.WriteByte((byte)opcode); lastOpcodeIndex = (int)instructionAndSizes.Length - 1; return; } } opcode = instrMap.LookFirstOpcode((byte)inst, 0, mode); if (opcode == CodeTable.kNoOpcode) { return; } instructionAndSizes.WriteByte((byte)opcode); lastOpcodeIndex = (int)instructionAndSizes.Length - 1; VarIntBE.AppendInt32(size, instructionAndSizes); }
internal VCDiffResult Decode(IByteBuffer source) { //the custom codetable itself is a VCDiff file but it is required to be encoded with the standard table //the length should be the first thing after the hdr_indicator if not supporting compression //at least according to the RFC specs. int lengthOfCodeTable = VarIntBE.ParseInt32(source); if (lengthOfCodeTable == 0) { return(VCDiffResult.ERROR); } using ByteBuffer codeTable = new ByteBuffer(source.ReadBytes(lengthOfCodeTable)); //according to the RFC specifications the next two items will be the size of near and size of same //they are bytes in the RFC spec, but for some reason Google uses the varint to read which does //the same thing if it is a single byte //but I am going to just read in bytes because it is the RFC standard NearSize = codeTable.ReadByte(); SameSize = codeTable.ReadByte(); if (NearSize == 0 || SameSize == 0 || NearSize > byte.MaxValue || SameSize > byte.MaxValue) { return(VCDiffResult.ERROR); } CustomTable = new CodeTable(); //get the original bytes of the default codetable to use as a dictionary using ByteBuffer dictionary = CustomTable.GetBytes(); //Decode the code table VCDiff file itself //stream the decoded output into a memory stream using MemoryStream sout = new MemoryStream(); VcDecoder decoder = new VcDecoder(dictionary, codeTable, sout); var result = decoder.Decode(out long bytesWritten); if (result != VCDiffResult.SUCCESS || bytesWritten == 0) { return(VCDiffResult.ERROR); } //set the new table data that was decoded if (!CustomTable.SetBytes(sout.ToArray())) { result = VCDiffResult.ERROR; } return(result); }
public void Copy(int offset, int length) { byte mode = addrCache.EncodeAddress(offset, dictionarySize + targetLength, out long encodedAddr); EncodeInstruction(VCDiffInstructionType.COPY, length, mode); if (addrCache.WriteAddressAsVarint(mode)) { VarIntBE.AppendInt64(encodedAddr, addressForCopy); } else { addressForCopy.WriteByte((byte)encodedAddr); } targetLength += length; }
int CalculateLengthOfTheDeltaEncoding() { int extraLength = 0; if (hasChecksum) { extraLength += VarIntBE.CalcInt64Length(checksum); } if (!interleaved) { int lengthOfDelta = VarIntBE.CalcInt32Length((int)targetLength) + 1 + VarIntBE.CalcInt32Length(dataForAddAndRun.Count) + VarIntBE.CalcInt32Length(instructionAndSizes.Count) + VarIntBE.CalcInt32Length(addressForCopy.Count) + dataForAddAndRun.Count + instructionAndSizes.Count + addressForCopy.Count; lengthOfDelta += extraLength; return(lengthOfDelta); } else { int lengthOfDelta = VarIntBE.CalcInt32Length((int)targetLength) + 1 + VarIntBE.CalcInt32Length(0) + VarIntBE.CalcInt32Length(instructionAndSizes.Count) + VarIntBE.CalcInt32Length(0) + 0 + instructionAndSizes.Count; lengthOfDelta += extraLength; return(lengthOfDelta); } }
bool ParseUInt32(out uint value) { if ((int)VCDiffResult.SUCCESS != returnCode) { value = 0; return(false); } if (chunk.IsEmpty) { value = 0; returnCode = (int)VCDiffResult.EOD; return(false); } long parsed = VarIntBE.ParseInt64(buffer); switch (parsed) { case (int)VCDiffResult.ERRROR: value = 0; return(false); case (int)VCDiffResult.EOD: value = 0; return(false); default: break; } if (parsed > 0xFFFFFFFF) { returnCode = (int)VCDiffResult.ERRROR; value = 0; return(false); } chunk.Position = buffer.Position; value = (uint)parsed; return(true); }
/// <summary> /// Gets the next instruction from the buffer /// </summary> /// <param name="size">the size</param> /// <param name="mode">the mode</param> /// <returns></returns> public unsafe VCDiffInstructionType Next(out int size, out byte mode) { byte opcode = 0; byte instructionType = CodeTable.N; int instructionSize = 0; byte instructionMode = 0; int start = (int)source.Position; do { if (pendingSecond != CodeTable.kNoOpcode) { opcode = (byte)pendingSecond; pendingSecond = CodeTable.kNoOpcode; instructionType = table.inst2.Pointer[opcode]; instructionSize = table.size2.Pointer[opcode]; instructionMode = table.mode2.Pointer[opcode]; break; } if (!source.CanRead) { size = 0; mode = 0; return(VCDiffInstructionType.EOD); } opcode = source.PeekByte(); if (table.inst2.Pointer[opcode] != CodeTable.N) { pendingSecond = source.PeekByte(); } source.Next(); instructionType = table.inst1.Pointer[opcode]; instructionSize = table.size1.Pointer[opcode]; instructionMode = table.mode1.Pointer[opcode]; } while (instructionType == CodeTable.N); if (instructionSize == 0) { switch (size = VarIntBE.ParseInt32(source)) { case (int)VCDiffResult.ERROR: mode = 0; size = 0; return(VCDiffInstructionType.ERROR); case (int)VCDiffResult.EOD: mode = 0; size = 0; //reset it back before we read the instruction //otherwise when parsing interleave we will miss data source.Position = start; return(VCDiffInstructionType.EOD); } } else { size = instructionSize; } mode = instructionMode; return((VCDiffInstructionType)instructionType); }
public void Output(Stream outputStream) { int lengthOfDelta = CalculateLengthOfTheDeltaEncoding(); //Google's Checksum Implementation Support if (this.ChecksumFormat != ChecksumFormat.None) { outputStream.WriteByte((byte)VCDiffWindowFlags.VCDSOURCE | (byte)VCDiffWindowFlags.VCDCHECKSUM); //win indicator } else { outputStream.WriteByte((byte)VCDiffWindowFlags.VCDSOURCE); //win indicator } VarIntBE.AppendInt32((int)dictionarySize, outputStream); //dictionary size VarIntBE.AppendInt32(0, outputStream); //dictionary start position 0 is default aka encompass the whole dictionary VarIntBE.AppendInt32(lengthOfDelta, outputStream); //length of delta //begin of delta encoding long sizeBeforeDelta = outputStream.Position; VarIntBE.AppendInt32((int)targetLength, outputStream); //final target length after decoding outputStream.WriteByte(0x00); // uncompressed // [Here is where a secondary compressor would be used // if the encoder and decoder supported that feature.] //non interleaved then it is separeat areas for each type if (!IsInterleaved) { VarIntBE.AppendInt32((int)dataForAddAndRun.Length, outputStream); //length of add/run VarIntBE.AppendInt32((int)instructionAndSizes.Length, outputStream); //length of instructions and sizes VarIntBE.AppendInt32((int)addressForCopy.Length, outputStream); //length of addresses for copys switch (this.ChecksumFormat) { //Google Checksum Support case ChecksumFormat.SDCH: VarIntBE.AppendInt64(this.Checksum, outputStream); break; // Xdelta checksum support. case ChecksumFormat.Xdelta3: { Span <byte> checksumBytes = stackalloc [] { (byte)(this.Checksum >> 24), (byte)(this.Checksum >> 16), (byte)(this.Checksum >> 8), (byte)(this.Checksum & 0x000000FF) }; outputStream.Write(checksumBytes); break; } } outputStream.Write(dataForAddAndRun.GetBuffer().AsSpanFast((int)dataForAddAndRun.Length)); //data section for adds and runs outputStream.Write(instructionAndSizes.GetBuffer().AsSpanFast((int)instructionAndSizes.Length)); //data for instructions and sizes outputStream.Write(addressForCopy.GetBuffer().AsSpanFast((int)addressForCopy.Length)); //data for addresses section copys } else { //interleaved everything is woven in and out in one block VarIntBE.AppendInt32(0, outputStream); //length of add/run VarIntBE.AppendInt32((int)instructionAndSizes.Length, outputStream); //length of instructions and sizes + other data for interleaved VarIntBE.AppendInt32(0, outputStream); //length of addresses for copys //Google Checksum Support if (this.ChecksumFormat == ChecksumFormat.SDCH) { VarIntBE.AppendInt64(Checksum, outputStream); } outputStream.Write(instructionAndSizes.GetBuffer().AsSpan(0, (int)instructionAndSizes.Length)); //data for instructions and sizes, in interleaved it is everything } //end of delta encoding long sizeAfterDelta = outputStream.Position; if (lengthOfDelta != sizeAfterDelta - sizeBeforeDelta) { throw new IOException("Delta output length does not match"); } dataForAddAndRun.SetLength(0); instructionAndSizes.SetLength(0); addressForCopy.SetLength(0); if (targetLength == 0) { throw new IOException("Empty target window"); } }
/// <summary> /// Call this before calling decode /// This expects at least the header part of the delta file /// is available in the stream /// </summary> /// <returns></returns> private VCDiffResult Initialize() { if (!delta.CanRead) return VCDiffResult.EOD; byte V = delta.ReadByte(); if (!delta.CanRead) return VCDiffResult.EOD; byte C = delta.ReadByte(); if (!delta.CanRead) return VCDiffResult.EOD; byte D = delta.ReadByte(); if (!delta.CanRead) return VCDiffResult.EOD; byte version = delta.ReadByte(); if (!delta.CanRead) return VCDiffResult.EOD; byte hdr = delta.ReadByte(); if (V != MagicBytes[0]) { return VCDiffResult.ERROR; } if (C != MagicBytes[1]) { return VCDiffResult.ERROR; } if (D != MagicBytes[2]) { return VCDiffResult.ERROR; } if (version != 0x00 && version != 'S') { return VCDiffResult.ERROR; } //compression not supported if ((hdr & (int)VCDiffCodeFlags.VCDDECOMPRESS) != 0) { return VCDiffResult.ERROR; } //custom code table! if ((hdr & (int)VCDiffCodeFlags.VCDCODETABLE) != 0) { if (!delta.CanRead) return VCDiffResult.EOD; //try decoding the custom code table //since we don't support the compress the next line should be the length of the code table customTable = new CustomCodeTableDecoder(); VCDiffResult result = customTable.Decode(delta); if (result != VCDiffResult.SUCCESS) { return result; } } if ((hdr & (int)VCDiffCodeFlags.VCDAPPHEADER) != 0) { if (!delta.CanRead) return VCDiffResult.EOD; int headerLength = VarIntBE.ParseInt32(delta); // skip the app header delta.ReadBytes(headerLength); } this.IsSDCHFormat = version == 'S'; this.IsInitialized = true; return VCDiffResult.SUCCESS; }
public void Output(ByteStreamWriter sout) { int lengthOfDelta = CalculateLengthOfTheDeltaEncoding(); int windowSize = lengthOfDelta + 1 + VarIntBE.CalcInt32Length((int)dictionarySize) + VarIntBE.CalcInt32Length(0); VarIntBE.CalcInt32Length(lengthOfDelta); //Google's Checksum Implementation Support if (hasChecksum) { sout.writeByte((byte)VCDiffWindowFlags.VCDSOURCE | (byte)VCDiffWindowFlags.VCDCHECKSUM); //win indicator } else { sout.writeByte((byte)VCDiffWindowFlags.VCDSOURCE); //win indicator } VarIntBE.AppendInt32((int)dictionarySize, sout); //dictionary size VarIntBE.AppendInt32(0, sout); //dictionary start position 0 is default aka encompass the whole dictionary VarIntBE.AppendInt32(lengthOfDelta, sout); //length of delta //begin of delta encoding Int64 sizeBeforeDelta = sout.Position; VarIntBE.AppendInt32((int)targetLength, sout); //final target length after decoding sout.writeByte(0x00); //uncompressed // [Here is where a secondary compressor would be used // if the encoder and decoder supported that feature.] //non interleaved then it is separata areas for each type if (!interleaved) { VarIntBE.AppendInt32(dataForAddAndRun.Count, sout); //length of add/run VarIntBE.AppendInt32(instructionAndSizes.Count, sout); //length of instructions and sizes VarIntBE.AppendInt32(addressForCopy.Count, sout); //length of addresses for copys //Google Checksum Support if (hasChecksum) { VarIntBE.AppendInt64(checksum, sout); } sout.writeBytes(dataForAddAndRun.ToArray()); //data section for adds and runs sout.writeBytes(instructionAndSizes.ToArray()); //data for instructions and sizes sout.writeBytes(addressForCopy.ToArray()); //data for addresses section copys } else { //interleaved everything is woven in and out in one block VarIntBE.AppendInt32(0, sout); //length of add/run VarIntBE.AppendInt32(instructionAndSizes.Count, sout); //length of instructions and sizes + other data for interleaved VarIntBE.AppendInt32(0, sout); //length of addresses for copys //Google Checksum Support if (hasChecksum) { VarIntBE.AppendInt64(checksum, sout); } sout.writeBytes(instructionAndSizes.ToArray()); //data for instructions and sizes, in interleaved it is everything } //end of delta encoding Int64 sizeAfterDelta = sout.Position; if (lengthOfDelta != sizeAfterDelta - sizeBeforeDelta) { Console.WriteLine("Delta output length does not match"); } dataForAddAndRun.Clear(); instructionAndSizes.Clear(); addressForCopy.Clear(); if (targetLength == 0) { Console.WriteLine("Empty target window"); } addrCache = new AddressCache(); }