//This is a window encoder for the VCDIFF format //if you are not including a checksum simply pass 0 to checksum //it will be ignored public WindowEncoder(long dictionarySize, uint checksum, ChecksumFormat checksumFormat, bool interleaved) { this.Checksum = checksum; this.ChecksumFormat = checksumFormat; this.IsInterleaved = interleaved; this.dictionarySize = dictionarySize; // The encoder currently doesn't support encoding with a custom table // will be added in later since it will be easy as decoding is already implemented maxMode = AddressCache.DefaultLast; table = CodeTable.DefaultTable; addrCache = new AddressCache(); targetLength = 0; lastOpcodeIndex = -1; instrMap = InstructionMap.Instance; //Separate buffers for each type if not interleaved if (!interleaved) { instructionAndSizes = new MemoryStream(); dataForAddAndRun = new MemoryStream(); addressForCopy = new MemoryStream(); } else { instructionAndSizes = dataForAddAndRun = addressForCopy = new MemoryStream(); } }
/// <summary> /// Decodes the window header. /// </summary> /// <param name="isSdch">If the delta uses SDCH extensions.</param> /// <returns></returns> public bool Decode(bool isSdch) { if (!ParseWindowIndicatorAndSegment(dictionarySize, 0, false, out winIndicator, out sourceSegmentLength, out sourceSegmentOffset)) { return(false); } if (!ParseWindowLengths(out _targetLength)) { return(false); } if (!ParseDeltaIndicator()) { return(false); } this.ChecksumFormat = ChecksumFormat.None; if ((winIndicator & (int)VCDiffWindowFlags.VCDCHECKSUM) != 0) { this.ChecksumFormat = isSdch ? ChecksumFormat.SDCH : ChecksumFormat.Xdelta3; } if (!ParseSectionLengths(this.ChecksumFormat, out addRunLength, out instructionAndSizesLength, out addressForCopyLength, out checksum)) { return(false); } if (isSdch && addRunLength == 0 && addressForCopyLength == 0 && instructionAndSizesLength > 0) { //interleave format return(true); } // Note: Copied required here due to caching behaviour. if (buffer.CanRead) { AddRunData = new PinnedArrayRental((int)addRunLength); Debug.Assert(addRunLength <= int.MaxValue); buffer.ReadBytesToSpan(AddRunData.AsSpan()); } if (buffer.CanRead) { InstructionsAndSizesData = new PinnedArrayRental((int)instructionAndSizesLength); Debug.Assert(instructionAndSizesLength <= int.MaxValue); buffer.ReadBytesToSpan(InstructionsAndSizesData.AsSpan()); } if (buffer.CanRead) { AddressesForCopyData = new PinnedArrayRental((int)addressForCopyLength); Debug.Assert(addressForCopyLength <= int.MaxValue); buffer.ReadBytesToSpan(AddressesForCopyData.AsSpan()); } return(true); }
/// <summary> /// Performs the actual encoding of a chunk of data into the VCDiff format /// </summary> /// <param name="dictionary">The dictionary hash table</param> /// <param name="oldData">The data for the dictionary hash table</param> /// <param name="hash">The rolling hash object</param> /// <param name="interleaved">Whether to interleave the data or not</param> /// <param name="checksumFormat">The format of the checksums for each window.</param> /// <param name="minBlockSize">The minimum block size to use. Defaults to 32, and must be a power of 2. /// This value must also be smaller than the block size of the dictionary.</param> public ChunkEncoder(BlockHash dictionary, ByteBuffer oldData, RollingHash hash, ChecksumFormat checksumFormat, bool interleaved = false, int minBlockSize = 32) { this.checksumFormat = checksumFormat; this.hasher = hash; this.oldData = oldData; this.dictionary = dictionary; this.minBlockSize = minBlockSize; this.interleaved = interleaved; }
/// <summary> /// Calculate and write a diff for the file. /// </summary> /// <param name="interleaved">Whether to output in SDCH interleaved diff format.</param> /// <param name="checksumFormat"> /// Whether to include Adler32 checksums for encoded data windows. If interleaved is true, <see cref="ChecksumFormat.Xdelta3"/> /// is not supported. /// </param> /// <returns> /// <see cref="VCDiffResult.SUCCESS"/> if successful, <see cref="VCDiffResult.ERROR"/> if the sourceStream or target are zero-length.</returns> /// <exception cref="ArgumentException">If interleaved is true, and <see cref="ChecksumFormat.Xdelta3"/> is chosen.</exception> public VCDiffResult Encode(bool interleaved = false, ChecksumFormat checksumFormat = ChecksumFormat.None) { if (oldData == null) { this.oldData = new ByteBuffer(sourceStream); } if (interleaved && checksumFormat == ChecksumFormat.Xdelta3) { throw new ArgumentException("Interleaved diffs can not have an xdelta3 checksum!"); } if (targetData.Length == 0 || oldData.Length == 0) { return(VCDiffResult.ERROR); } VCDiffResult result = VCDiffResult.SUCCESS; oldData.Position = 0; targetData.Position = 0; // file header // write magic bytes if (!interleaved && checksumFormat != ChecksumFormat.SDCH) { outputStream.Write(MagicBytes); } else { outputStream.Write(MagicBytesExtended); } //read in all the dictionary it is the only thing that needs to be BlockHash dictionary = new BlockHash(oldData, 0, hasher, blockSize); dictionary.AddAllBlocks(); oldData.Position = 0; ChunkEncoder chunker = new ChunkEncoder(dictionary, oldData, hasher, checksumFormat, interleaved, chunkSize); while (targetData.CanRead) { using ByteBuffer ntarget = new ByteBuffer(targetData.ReadBytesAsBuf(bufferSize)); chunker.EncodeChunk(ntarget, outputStream); } return(result); }
/// <summary> /// Decodes the window header. /// </summary> /// <param name="isSdch">If the delta uses SDCH extensions.</param> /// <returns></returns> public bool Decode(bool isSdch) { if (!ParseWindowIndicatorAndSegment(dictionarySize, 0, false, out winIndicator, out sourceSegmentLength, out sourceSegmentOffset)) { return(false); } if (!ParseWindowLengths(out targetLength)) { return(false); } if (!ParseDeltaIndicator()) { return(false); } this.ChecksumFormat = ChecksumFormat.None; if ((winIndicator & (int)VCDiffWindowFlags.VCDCHECKSUM) != 0) { this.ChecksumFormat = isSdch ? ChecksumFormat.SDCH : ChecksumFormat.Xdelta3; } if (!ParseSectionLengths(this.ChecksumFormat, out addRunLength, out instructionAndSizesLength, out addressForCopyLength, out checksum)) { return(false); } if (isSdch && addRunLength == 0 && addressForCopyLength == 0 && instructionAndSizesLength > 0) { //interleave format return(true); } if (buffer.CanRead) { AddRunData = buffer.ReadBytes((int)addRunLength); } if (buffer.CanRead) { InstructionsAndSizesData = buffer.ReadBytes((int)instructionAndSizesLength); } if (buffer.CanRead) { AddressesForCopyData = buffer.ReadBytes((int)addressForCopyLength); } return(true); }
public bool ParseSectionLengths(ChecksumFormat checksumFormat, out long addRunLength, out long instructionsLength, out long addressLength, out uint checksum) { ParseInt32(out int outAdd); ParseInt32(out int outInstruct); ParseInt32(out int outAddress); checksum = 0; if (checksumFormat == ChecksumFormat.SDCH) { ParseUInt32(out checksum); } else if (checksumFormat == ChecksumFormat.Xdelta3) { // xdelta checksum is stored as a 4-part byte array ParseByte(out byte chk0); ParseByte(out byte chk1); ParseByte(out byte chk2); ParseByte(out byte chk3); checksum = (uint)(chk0 << 24 | chk1 << 16 | chk2 << 8 | chk3); } addRunLength = outAdd; addressLength = outAddress; instructionsLength = outInstruct; if (returnCode != (int)VCDiffResult.SUCCESS) { return(false); } long deltaHeaderLength = chunk.ParsedSize - deltaEncodingStart; long totalLen = deltaHeaderLength + addRunLength + instructionsLength + addressLength; if (deltaEncodingLength == totalLen) { return(true); } returnCode = (int)VCDiffResult.ERROR; return(false); }
/// <summary> /// Calculate and write a diff for the file. /// </summary> /// <param name="interleaved">Whether to output in SDCH interleaved diff format.</param> /// <param name="checksumFormat"> /// Whether to include Adler32 checksums for encoded data windows. If interleaved is true, <see cref="ChecksumFormat.Xdelta3"/> /// is not supported. /// </param> /// <param name="progress">Reports an estimate of the encoding progress. Value if 0 to 1.</param> /// <returns> /// <see cref="VCDiffResult.SUCCESS"/> if successful, <see cref="VCDiffResult.ERROR"/> if the sourceStream or target are zero-length.</returns> /// <exception cref="ArgumentException">If interleaved is true, and <see cref="ChecksumFormat.Xdelta3"/> is chosen.</exception> public VCDiffResult Encode(bool interleaved = false, ChecksumFormat checksumFormat = ChecksumFormat.None, IProgress <float>?progress = null) { Task WriteBytes(byte[] bytes) { outputStream.Write(bytes); return(Task.CompletedTask); } ValidateParameters(interleaved, checksumFormat); if (!Encode_Init(interleaved, checksumFormat, WriteBytes).Result) { return(VCDiffResult.ERROR); } // Read in all the dictionary it is the only thing that needs to be Encode_Setup(interleaved, checksumFormat, out var chunker, out var buf); try { var bufSpan = buf.Span; while (targetData.CanRead) { int bytesRead = targetData.ReadBytesIntoBuf(bufSpan); using ByteBuffer ntarget = new ByteBuffer(buf[..bytesRead]);