public void MoveForward_shifts_index_for_byte_retrieval() { int window = 100; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock) .Returns((byte[] s, int o, int l) => { int actualLength = MathEx.Bounded(0, Math.Min(l, buffer.Length), buffer.Length); Array.Copy(buffer, 0, s, o, actualLength); return(actualLength); }); var buf = new SlidingStreamBuffer(streamMock.Object, window); buf.Warmup(); Assert.AreEqual(buffer[5], buf[5]); buf.MoveForward(1); buf.MoveForward(3); Assert.AreEqual(buffer[9], buf[5]); }
public void MoveForward_pulls_at_most_windowSize_plus_extrabuffer_minus_totalValidBytes() { int window = 100; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock) .Returns((byte[] s, int o, int l) => { if (l > buffer.Length - o) { Assert.Fail("Too much data being read"); } return(l); }); var buf = new SlidingStreamBuffer(streamMock.Object, window, window); buf.Warmup(); buf.MoveForward(window + 1); buf.MoveForward(1); }
public void MoveForward_pulls_data_to_the_correct_offset_when_less_than_windowSize_bytes_remain() { int window = 100; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } bool warmupFillComplete = false; var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock) .Returns((byte[] s, int o, int l) => { if (warmupFillComplete && o != window - 1) { Assert.Fail("Wrong offset being read to"); } warmupFillComplete = true; return(l); }); var buf = new SlidingStreamBuffer(streamMock.Object, window, window); buf.Warmup(); buf.MoveForward(window + 1); }
public void MoveForward_throws_for_zero_size() { int window = 100; var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => l); var buf = new SlidingStreamBuffer(streamMock.Object, window, 0); buf.Warmup(); buf.MoveForward(0); }
public void GetBuffer_returns_a_buffer_corresponding_to_GetNumBytesAvailable() { int window = 100; var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => 50); var buf = new SlidingStreamBuffer(streamMock.Object, window); buf.Warmup(); streamMock.VerifyAll(); Assert.AreEqual(buf.GetNumBytesAvailable(), buf.GetBuffer().Length); }
public void GetNumBytesAvailable_returns_windowSize_for_full_read() { int window = 100; var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock) .Returns((byte[] s, int o, int l) => l); var buf = new SlidingStreamBuffer(streamMock.Object, window); buf.Warmup(); Assert.AreEqual(window, buf.GetNumBytesAvailable()); }
public void MoveForward_stops_at_end_of_stream() { int window = 10; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window); buf.Warmup(); buf.MoveForward(window + 1); }
public void GetBuffer_when_moved_beyond_end_of_stream_yields_empty_buffer() { int window = 10; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window, window); buf.Warmup(); buf.MoveForward(buffer.Length + 1); Assert.IsTrue(buf.GetBuffer().Length == 0); }
public void MoveForward_beyond_windowSize_with_no_more_data_in_stream_yields_NumAvailable_lower_than_windowSize() { int window = 10; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window, window); buf.Warmup(); buf.MoveForward(window + 1); Assert.AreEqual(window - 1, buf.GetNumBytesAvailable()); }
public void GetNumBytesAvailable_when_moved_to_end_of_stream_returns_zero() { int window = 10; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window, window); buf.Warmup(); buf.MoveForward(buffer.Length); Assert.AreEqual(0, buf.GetNumBytesAvailable()); }
public void Warmup_pulls_atleast_windowSize_data_from_stream() { int window = 100; var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); streamMock.Setup(x => x.Read( It.IsAny <byte[]>(), It.IsAny <int>(), It.Is <int>(i => i >= window))) .Returns((byte[] s, int o, int l) => l); var buf = new SlidingStreamBuffer(streamMock.Object, window); buf.Warmup(); streamMock.VerifyAll(); }
public void MoveForward_causes_more_data_to_be_read_when_no_padding_buffer_is_supplied() { int window = 100; var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => l); var buf = new SlidingStreamBuffer(streamMock.Object, window, 0); buf.Warmup(); buf.MoveForward(1); buf.GetByteAt(0); streamMock.Verify(x => x.Read( It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()), Times.Exactly(2)); }
public void GetBuffer_returns_a_buffer_with_valid_data() { int window = 100; var buffer = new byte[window * 2]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock) .Returns((byte[] s, int o, int l) => { int actualLength = MathEx.Bounded(0, Math.Min(l, buffer.Length), buffer.Length); Array.Copy(buffer, 0, s, o, actualLength); return(actualLength); }); var buf = new SlidingStreamBuffer(streamMock.Object, window); buf.Warmup(); VerifyBufferPartialEquality(buffer, buf.GetBuffer()); }
public void GetByteAt_retrieves_correct_byte() { int window = 100; var buffer = new byte[window]; for (int i = 0; i < buffer.Length; ++i) { buffer[i] = (byte)(200 - i); } var streamMock = new Mock <MemoryStream>(MockBehavior.Strict); SetupMockStreamRead(streamMock) .Returns((byte[] s, int o, int l) => { int actualLength = MathEx.Bounded(0, Math.Min(l, buffer.Length), buffer.Length); Array.Copy(buffer, 0, s, o, actualLength); return(actualLength); }); var buf = new SlidingStreamBuffer(streamMock.Object, window); buf.Warmup(); Assert.AreEqual(buffer[5], buf.GetByteAt(5)); }
public IEnumerable<IDelta> GetDeltas(Stream inputStream) { if (inputStream == null) throw new ArgumentNullException("inputStream"); if (_initialized == false) throw new InvalidOperationException("Initialize must be called"); ChecksumProvider.Reset(); var slidingBuffer = new SlidingStreamBuffer(inputStream, _blockSize); slidingBuffer.Warmup(); bool startingNewBlock = true; long offset = 0; var deltas = new List<IDelta>(); var currentByteDelta = new ByteDelta(); #if DEBUG Statistics.Matching = 0; Statistics.PossibleMatches = 0; Statistics.NonMatching = 0; #endif int currentBlockSize; while ((currentBlockSize = (int) slidingBuffer.GetNumBytesAvailable()) > 0) { // Deal with signed integer limits if (IsSignedIntLength(offset - currentByteDelta.Offset)) { currentByteDelta.Length = (int) (offset - currentByteDelta.Offset); deltas.Add(currentByteDelta); startingNewBlock = true; } if (startingNewBlock) { currentByteDelta = new ByteDelta {Offset = offset}; ChecksumProvider.ProcessBlock(slidingBuffer.GetBuffer(), 0, currentBlockSize); } else if (currentBlockSize < _blockSize) ChecksumProvider.TrimFront(); // remaining bytes < block_size, so read nothing new - just trim else ChecksumProvider.RollByte(slidingBuffer[(int) (currentBlockSize - 1)]); // at this point, sigGen needs the last byte of the current block uint currentBlockChecksum = ChecksumProvider.Value; ushort currentBlockChecksumHash = RollingChecksum.HashChecksum(currentBlockChecksum); if (_remoteBlocksIndexTable.ContainsKey(currentBlockChecksumHash)) { List<HashBlock> possibleRemoteBlockMatches = _remoteBlocksIndexTable[currentBlockChecksumHash]; if (possibleRemoteBlockMatches.Any(entry => entry.Checksum == currentBlockChecksum)) { #if DEBUG ++Statistics.PossibleMatches; #endif byte[] currentBlockHash = HashProvider.ComputeHash(slidingBuffer.GetBuffer(), 0, (int) currentBlockSize); HashBlock matchingTargetBlock; if ((matchingTargetBlock = possibleRemoteBlockMatches.FirstOrDefault( entry => entry.Hash.SequenceEqual(currentBlockHash))) != null) { #if DEBUG Statistics.Matching += 1; #endif if ((currentByteDelta.Length = (int) (offset - currentByteDelta.Offset)) > 0) { deltas.Add(currentByteDelta); } deltas.Add(new CopyDelta { Offset = matchingTargetBlock.Offset, Length = matchingTargetBlock.Length }); slidingBuffer.MoveForward((int) currentBlockSize); offset += currentBlockSize; startingNewBlock = true; continue; } } } #if DEBUG ++Statistics.NonMatching; #endif slidingBuffer.MoveForward(1); ++offset; startingNewBlock = false; } Statistics.FileLength = offset; if (!startingNewBlock && (currentByteDelta.Length = (int) (offset - currentByteDelta.Offset)) > 0) { deltas.Add(currentByteDelta); } #if DEBUG Statistics.FileLength = offset; #endif return deltas; }
public IEnumerable <IDelta> GetDeltas(Stream inputStream) { if (inputStream == null) { throw new ArgumentNullException("inputStream"); } if (_initialized == false) { throw new InvalidOperationException("Initialize must be called"); } ChecksumProvider.Reset(); var slidingBuffer = new SlidingStreamBuffer(inputStream, _blockSize); slidingBuffer.Warmup(); bool startingNewBlock = true; long offset = 0; var deltas = new List <IDelta>(); var currentByteDelta = new ByteDelta(); #if DEBUG Statistics.Matching = 0; Statistics.PossibleMatches = 0; Statistics.NonMatching = 0; #endif int currentBlockSize; while ((currentBlockSize = (int)slidingBuffer.GetNumBytesAvailable()) > 0) { // Deal with signed integer limits if (IsSignedIntLength(offset - currentByteDelta.Offset)) { currentByteDelta.Length = (int)(offset - currentByteDelta.Offset); deltas.Add(currentByteDelta); startingNewBlock = true; } if (startingNewBlock) { currentByteDelta = new ByteDelta { Offset = offset }; ChecksumProvider.ProcessBlock(slidingBuffer.GetBuffer(), 0, currentBlockSize); } else if (currentBlockSize < _blockSize) { ChecksumProvider.TrimFront(); // remaining bytes < block_size, so read nothing new - just trim } else { ChecksumProvider.RollByte(slidingBuffer[(int)(currentBlockSize - 1)]); } // at this point, sigGen needs the last byte of the current block uint currentBlockChecksum = ChecksumProvider.Value; ushort currentBlockChecksumHash = RollingChecksum.HashChecksum(currentBlockChecksum); if (_remoteBlocksIndexTable.ContainsKey(currentBlockChecksumHash)) { List <HashBlock> possibleRemoteBlockMatches = _remoteBlocksIndexTable[currentBlockChecksumHash]; if (possibleRemoteBlockMatches.Any(entry => entry.Checksum == currentBlockChecksum)) { #if DEBUG ++Statistics.PossibleMatches; #endif byte[] currentBlockHash = HashProvider.ComputeHash(slidingBuffer.GetBuffer(), 0, (int)currentBlockSize); HashBlock matchingTargetBlock; if ((matchingTargetBlock = possibleRemoteBlockMatches.FirstOrDefault( entry => entry.Hash.SequenceEqual(currentBlockHash))) != null) { #if DEBUG Statistics.Matching += 1; #endif if ((currentByteDelta.Length = (int)(offset - currentByteDelta.Offset)) > 0) { deltas.Add(currentByteDelta); } deltas.Add(new CopyDelta { Offset = matchingTargetBlock.Offset, Length = matchingTargetBlock.Length }); slidingBuffer.MoveForward((int)currentBlockSize); offset += currentBlockSize; startingNewBlock = true; continue; } } } #if DEBUG ++Statistics.NonMatching; #endif slidingBuffer.MoveForward(1); ++offset; startingNewBlock = false; } Statistics.FileLength = offset; if (!startingNewBlock && (currentByteDelta.Length = (int)(offset - currentByteDelta.Offset)) > 0) { deltas.Add(currentByteDelta); } #if DEBUG Statistics.FileLength = offset; #endif return(deltas); }