void FlushCurrentCopyCommand()
        {
            if (bufferedCopy.Length <= 0)
            {
                return;
            }

            decorated.WriteCopyCommand(bufferedCopy);
            bufferedCopy = new DataRange();
        }
Beispiel #2
0
        public void BuildDelta(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter)
        {
            var signature = signatureReader.ReadSignature();
            var chunks    = signature.Chunks;

            var newFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5();
            var newFileHash = newFileVerificationHashAlgorithm.ComputeHash(newFileStream);

            newFileStream.Seek(0, SeekOrigin.Begin);

            deltaWriter.WriteMetadata(new DeltaMetadata
            {
                HashAlgorithm             = signature.HashAlgorithm.Name,
                ExpectedFileHashAlgorithm = newFileVerificationHashAlgorithm.Name,
                ExpectedFileHash          = Convert.ToBase64String(newFileHash)
            });

            chunks = OrderChunksByChecksum(chunks);

            var chunkMap = CreateChunkMap(chunks, out int maxChunkSize, out int minChunkSize);

            var  buffer            = new byte[readBufferSize];
            long lastMatchPosition = 0;

            var fileSize = newFileStream.Length;

            ProgressReport?.Report(new ProgressReport
            {
                Operation       = ProgressOperationType.BuildingDelta,
                CurrentPosition = 0,
                Total           = fileSize
            });

            while (true)
            {
                var startPosition = newFileStream.Position;
                var read          = newFileStream.Read(buffer, 0, buffer.Length);
                if (read < 0)
                {
                    break;
                }

                var  checksumAlgorithm = signature.RollingChecksumAlgorithm;
                uint checksum          = 0;

                var remainingPossibleChunkSize = maxChunkSize;

                for (var i = 0; i < read - minChunkSize + 1; i++)
                {
                    var readSoFar = startPosition + i;

                    var remainingBytes = read - i;
                    if (remainingBytes < maxChunkSize)
                    {
                        remainingPossibleChunkSize = minChunkSize;
                    }

                    if (i == 0 || remainingBytes < maxChunkSize)
                    {
                        checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize);
                    }
                    else
                    {
                        var remove = buffer[i - 1];
                        var add    = buffer[i + remainingPossibleChunkSize - 1];
                        checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize);
                    }

                    ProgressReport?.Report(new ProgressReport
                    {
                        Operation       = ProgressOperationType.BuildingDelta,
                        CurrentPosition = readSoFar,
                        Total           = fileSize
                    });

                    if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize)
                    {
                        continue;
                    }

                    if (!chunkMap.ContainsKey(checksum))
                    {
                        continue;
                    }

                    var startIndex = chunkMap[checksum];

                    for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++)
                    {
                        var chunk = chunks[j];
                        var hash  = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize);

                        if (StructuralComparisons.StructuralEqualityComparer.Equals(hash, chunks[j].Hash))
                        {
                            readSoFar = readSoFar + remainingPossibleChunkSize;

                            var missing = readSoFar - lastMatchPosition;
                            if (missing > remainingPossibleChunkSize)
                            {
                                deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize);
                            }

                            deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length));
                            lastMatchPosition = readSoFar;
                            break;
                        }
                    }
                }

                if (read < buffer.Length)
                {
                    break;
                }

                newFileStream.Position = newFileStream.Position - maxChunkSize + 1;
            }

            if (newFileStream.Length != lastMatchPosition)
            {
                deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition);
            }

            deltaWriter.Finish();
        }
Beispiel #3
0
        public void BuildDelta(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter)
        {
            var signature = signatureReader.ReadSignature();
            var chunks = signature.Chunks;

            var hash = signature.HashAlgorithm.ComputeHash(newFileStream);
            newFileStream.Seek(0, SeekOrigin.Begin);

            deltaWriter.WriteMetadata(signature.HashAlgorithm, hash);

            chunks = OrderChunksByChecksum(chunks);

            int minChunkSize;
            int maxChunkSize;
            var chunkMap = CreateChunkMap(chunks, out maxChunkSize, out minChunkSize);

            var buffer = new byte[ReadBufferSize];
            long lastMatchPosition = 0;

            var fileSize = newFileStream.Length;
            ProgressReporter.ReportProgress("Building delta", 0, fileSize);

            while (true)
            {
                var startPosition = newFileStream.Position;
                var read = newFileStream.Read(buffer, 0, buffer.Length);
                if (read < 0)
                    break;

                var checksumAlgorithm = signature.RollingChecksumAlgorithm;
                uint checksum = 0;

                var remainingPossibleChunkSize = maxChunkSize;

                for (var i = 0; i < read - minChunkSize + 1; i++)
                {
                    var readSoFar = startPosition + i;

                    var remainingBytes = read - i;
                    if (remainingBytes < maxChunkSize)
                    {
                        remainingPossibleChunkSize = minChunkSize;
                    }

                    if (i == 0 || remainingBytes < maxChunkSize)
                    {
                        checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize);
                    }
                    else
                    {
                        var remove = buffer[i- 1];
                        var add = buffer[i + remainingPossibleChunkSize - 1];
                        checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize);
                    }

                    ProgressReporter.ReportProgress("Building delta", readSoFar, fileSize);

                    if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize)
                        continue;

                    if (!chunkMap.ContainsKey(checksum)) 
                        continue;

                    var startIndex = chunkMap[checksum];

                    for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++)
                    {
                        var chunk = chunks[j];

                        var sha = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize);

                        if (StructuralComparisons.StructuralEqualityComparer.Equals(sha, chunks[j].Hash))
                        {
                            readSoFar = readSoFar + remainingPossibleChunkSize;

                            var missing = readSoFar - lastMatchPosition;
                            if (missing > remainingPossibleChunkSize)
                            {
                                deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize);
                            }

                            deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length));
                            lastMatchPosition = readSoFar;
                            break;
                        }
                    }
                }

                if (read < buffer.Length)
                {
                    break;
                }

                newFileStream.Position = newFileStream.Position - maxChunkSize + 1;
            }

            if (newFileStream.Length != lastMatchPosition)
            {
                deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition);
            }

            deltaWriter.Finish();
        }
Beispiel #4
0
        public void BuildDelta(IHashAlgorithm hashAlgorithm, Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter)
        {
            var signature = signatureReader.ReadSignature();

            var hash = hashAlgorithm.ComputeHash(newFileStream);

            newFileStream.Seek(0, SeekOrigin.Begin);

            deltaWriter.WriteMetadata(hashAlgorithm, hash, newFileStream.Length, signature.ChunkSize);

            var fileSize = newFileStream.Length;

            ProgressReporter.ReportProgress("Building delta", 0, fileSize);

            var chunkSize = signature.ChunkSize;

            var        missingPosition   = newFileStream.Position;
            RingBuffer buffer            = new RingBuffer(newFileStream, chunkSize);
            var        checksumAlgorithm = signature.RollingChecksumAlgorithm;
            uint       checksum          = checksumAlgorithm.Calculate(buffer.GetBuffer(), 0, chunkSize);

            while (buffer.IsEnd == false)
            {
                ProgressReporter.ReportProgress("Building delta", newFileStream.Position, fileSize);

                var list = signature.FindChecksum(checksum);

                if (list != null)
                {
                    var sha       = signature.HashAlgorithm.ComputeHash(buffer.GetBuffer(), 0, chunkSize);
                    var chunkHash = list.Find(sha);
                    if (chunkHash != null)
                    {
                        long offset = chunkHash.FindBestOffset(deltaWriter.LastOffset);

                        if (missingPosition < newFileStream.Position - chunkSize)
                        {
                            deltaWriter.WriteDataCommand(newFileStream, missingPosition, newFileStream.Position - chunkSize - missingPosition);
                        }

                        deltaWriter.WriteCopyCommand(new DataRange(offset, chunkSize));
                        missingPosition = newFileStream.Position;

                        buffer.ReadNew();
                        checksum = checksumAlgorithm.Calculate(buffer.GetBuffer(), 0, chunkSize);

                        continue;
                    }
                }

                var remove = buffer.Peek();
                var add    = buffer.ReadContinue();
                checksum = checksumAlgorithm.Rotate(checksum, remove, add, chunkSize);
            }

            if (newFileStream.Length != missingPosition)
            {
                deltaWriter.WriteDataCommand(newFileStream, missingPosition, newFileStream.Length - missingPosition);
            }
            ProgressReporter.ReportProgress("Building delta", newFileStream.Position, fileSize);

            deltaWriter.Finish();
        }
        public void BuildDelta(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter)
        {
            var signature = signatureReader.ReadSignature();
            var chunks    = signature.Chunks;

            var hash = signature.HashAlgorithm.ComputeHash(newFileStream);

            newFileStream.Seek(0, SeekOrigin.Begin);

            deltaWriter.WriteMetadata(signature.HashAlgorithm, hash);

            chunks = OrderChunksByChecksum(chunks);

            int minChunkSize;
            int maxChunkSize;
            var chunkMap = CreateChunkMap(chunks, out maxChunkSize, out minChunkSize);

            var  buffer            = new byte[ReadBufferSize];
            long lastMatchPosition = 0;

            var fileSize = newFileStream.Length;

            ProgressReporter.ReportProgress("Building delta", 0, fileSize);

            while (true)
            {
                var startPosition = newFileStream.Position;
                var read          = newFileStream.Read(buffer, 0, buffer.Length);
                if (read < 0)
                {
                    break;
                }

                var  checksumAlgorithm = signature.RollingChecksumAlgorithm;
                uint checksum          = 0;

                var remainingPossibleChunkSize = maxChunkSize;

                for (var i = 0; i < read - minChunkSize + 1; i++)
                {
                    var readSoFar = startPosition + i;

                    var remainingBytes = read - i;
                    if (remainingBytes < maxChunkSize)
                    {
                        remainingPossibleChunkSize = minChunkSize;
                    }

                    if (i == 0 || remainingBytes < maxChunkSize)
                    {
                        checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize);
                    }
                    else
                    {
                        var remove = buffer[i - 1];
                        var add    = buffer[i + remainingPossibleChunkSize - 1];
                        checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize);
                    }

                    ProgressReporter.ReportProgress("Building delta", readSoFar, fileSize);

                    if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize)
                    {
                        continue;
                    }

                    if (!chunkMap.ContainsKey(checksum))
                    {
                        continue;
                    }

                    var startIndex = chunkMap[checksum];

                    for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++)
                    {
                        var chunk = chunks[j];

                        var sha = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize);

                        if (BinaryComparer.CompareArray(sha, chunks[j].Hash))
                        {
                            readSoFar = readSoFar + remainingPossibleChunkSize;

                            var missing = readSoFar - lastMatchPosition;
                            if (missing > remainingPossibleChunkSize)
                            {
                                deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize);
                            }

                            deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length));
                            lastMatchPosition = readSoFar;
                            break;
                        }
                    }
                }

                if (read < buffer.Length)
                {
                    break;
                }

                newFileStream.Position = newFileStream.Position - maxChunkSize + 1;
            }

            if (newFileStream.Length != lastMatchPosition)
            {
                deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition);
            }

            deltaWriter.Finish();
        }