public void MoveForward_pulls_at_most_windowSize_plus_extrabuffer_minus_totalValidBytes()
{
int window = 100;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock)
.Returns((byte[] s, int o, int l) =>
{
if (l > buffer.Length - o)
{
Assert.Fail("Too much data being read");
}
return(l);
});
var buf = new SlidingStreamBuffer(streamMock.Object, window, window);
buf.Warmup();
buf.MoveForward(window + 1);
buf.MoveForward(1);
}
public void MoveForward_pulls_data_to_the_correct_offset_when_less_than_windowSize_bytes_remain()
{
int window = 100;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
bool warmupFillComplete = false;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock)
.Returns((byte[] s, int o, int l) =>
{
if (warmupFillComplete && o != window - 1)
{
Assert.Fail("Wrong offset being read to");
}
warmupFillComplete = true;
return(l);
});
var buf = new SlidingStreamBuffer(streamMock.Object, window, window);
buf.Warmup();
buf.MoveForward(window + 1);
}
public void MoveForward_shifts_index_for_byte_retrieval()
{
int window = 100;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock)
.Returns((byte[] s, int o, int l) =>
{
int actualLength = MathEx.Bounded(0, Math.Min(l, buffer.Length), buffer.Length);
Array.Copy(buffer, 0, s, o, actualLength);
return(actualLength);
});
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.Warmup();
Assert.AreEqual(buffer[5], buf[5]);
buf.MoveForward(1);
buf.MoveForward(3);
Assert.AreEqual(buffer[9], buf[5]);
}
public void Indexer_throws_exception_for_index_over_windowSize()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => window);
var buf = new SlidingStreamBuffer(streamMock.Object, window);
int i = buf[window];
}
public void GetByteAt_throws_exception_for_negative_index()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => 50);
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.GetByteAt(-1);
}
public void MoveForward_throws_for_zero_size()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => l);
var buf = new SlidingStreamBuffer(streamMock.Object, window, 0);
buf.Warmup();
buf.MoveForward(0);
}
public void GetBuffer_calls_warmup_if_necessary()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => 50);
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.GetBuffer();
streamMock.VerifyAll();
}
public void GetBuffer_returns_a_buffer_corresponding_to_GetNumBytesAvailable()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => 50);
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.Warmup();
streamMock.VerifyAll();
Assert.AreEqual(buf.GetNumBytesAvailable(), buf.GetBuffer().Length);
}
public void GetNumBytesAvailable_returns_windowSize_for_full_read()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock)
.Returns((byte[] s, int o, int l) => l);
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.Warmup();
Assert.AreEqual(window, buf.GetNumBytesAvailable());
}
public void MoveForward_stops_at_end_of_stream()
{
int window = 10;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window);
buf.Warmup();
buf.MoveForward(window + 1);
}
public void MoveForward_beyond_windowSize_with_no_more_data_in_stream_yields_NumAvailable_lower_than_windowSize()
{
int window = 10;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window, window);
buf.Warmup();
buf.MoveForward(window + 1);
Assert.AreEqual(window - 1, buf.GetNumBytesAvailable());
}
public void GetNumBytesAvailable_when_moved_to_end_of_stream_returns_zero()
{
int window = 10;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window, window);
buf.Warmup();
buf.MoveForward(buffer.Length);
Assert.AreEqual(0, buf.GetNumBytesAvailable());
}
public void GetBuffer_when_moved_beyond_end_of_stream_yields_empty_buffer()
{
int window = 10;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var buf = new SlidingStreamBuffer(new MemoryStream(buffer), window, window);
buf.Warmup();
buf.MoveForward(buffer.Length + 1);
Assert.IsTrue(buf.GetBuffer().Length == 0);
}
public void Warmup_pulls_atleast_windowSize_data_from_stream()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
streamMock.Setup(x =>
x.Read(
It.IsAny <byte[]>(),
It.IsAny <int>(),
It.Is <int>(i => i >= window)))
.Returns((byte[] s, int o, int l) => l);
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.Warmup();
streamMock.VerifyAll();
}
public void MoveForward_causes_more_data_to_be_read_when_no_padding_buffer_is_supplied()
{
int window = 100;
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock).Returns((byte[] s, int o, int l) => l);
var buf = new SlidingStreamBuffer(streamMock.Object, window, 0);
buf.Warmup();
buf.MoveForward(1);
buf.GetByteAt(0);
streamMock.Verify(x =>
x.Read(
It.IsAny <byte[]>(),
It.IsAny <int>(),
It.IsAny <int>()),
Times.Exactly(2));
}
public void GetBuffer_returns_a_buffer_with_valid_data()
{
int window = 100;
var buffer = new byte[window * 2];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock)
.Returns((byte[] s, int o, int l) =>
{
int actualLength = MathEx.Bounded(0, Math.Min(l, buffer.Length), buffer.Length);
Array.Copy(buffer, 0, s, o, actualLength);
return(actualLength);
});
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.Warmup();
VerifyBufferPartialEquality(buffer, buf.GetBuffer());
}
public void GetByteAt_retrieves_correct_byte()
{
int window = 100;
var buffer = new byte[window];
for (int i = 0; i < buffer.Length; ++i)
{
buffer[i] = (byte)(200 - i);
}
var streamMock = new Mock <MemoryStream>(MockBehavior.Strict);
SetupMockStreamRead(streamMock)
.Returns((byte[] s, int o, int l) =>
{
int actualLength = MathEx.Bounded(0, Math.Min(l, buffer.Length), buffer.Length);
Array.Copy(buffer, 0, s, o, actualLength);
return(actualLength);
});
var buf = new SlidingStreamBuffer(streamMock.Object, window);
buf.Warmup();
Assert.AreEqual(buffer[5], buf.GetByteAt(5));
}
public IEnumerable<IDelta> GetDeltas(Stream inputStream)
{
if (inputStream == null) throw new ArgumentNullException("inputStream");
if (_initialized == false) throw new InvalidOperationException("Initialize must be called");
ChecksumProvider.Reset();
var slidingBuffer = new SlidingStreamBuffer(inputStream, _blockSize);
slidingBuffer.Warmup();
bool startingNewBlock = true;
long offset = 0;
var deltas = new List<IDelta>();
var currentByteDelta = new ByteDelta();
#if DEBUG
Statistics.Matching = 0;
Statistics.PossibleMatches = 0;
Statistics.NonMatching = 0;
#endif
int currentBlockSize;
while ((currentBlockSize = (int) slidingBuffer.GetNumBytesAvailable()) > 0)
{
// Deal with signed integer limits
if (IsSignedIntLength(offset - currentByteDelta.Offset))
{
currentByteDelta.Length = (int) (offset - currentByteDelta.Offset);
deltas.Add(currentByteDelta);
startingNewBlock = true;
}
if (startingNewBlock)
{
currentByteDelta = new ByteDelta {Offset = offset};
ChecksumProvider.ProcessBlock(slidingBuffer.GetBuffer(), 0, currentBlockSize);
}
else if (currentBlockSize < _blockSize)
ChecksumProvider.TrimFront(); // remaining bytes < block_size, so read nothing new - just trim
else
ChecksumProvider.RollByte(slidingBuffer[(int) (currentBlockSize - 1)]);
// at this point, sigGen needs the last byte of the current block
uint currentBlockChecksum = ChecksumProvider.Value;
ushort currentBlockChecksumHash = RollingChecksum.HashChecksum(currentBlockChecksum);
if (_remoteBlocksIndexTable.ContainsKey(currentBlockChecksumHash))
{
List<HashBlock> possibleRemoteBlockMatches = _remoteBlocksIndexTable[currentBlockChecksumHash];
if (possibleRemoteBlockMatches.Any(entry => entry.Checksum == currentBlockChecksum))
{
#if DEBUG
++Statistics.PossibleMatches;
#endif
byte[] currentBlockHash = HashProvider.ComputeHash(slidingBuffer.GetBuffer(), 0,
(int) currentBlockSize);
HashBlock matchingTargetBlock;
if ((matchingTargetBlock =
possibleRemoteBlockMatches.FirstOrDefault(
entry => entry.Hash.SequenceEqual(currentBlockHash))) != null)
{
#if DEBUG
Statistics.Matching += 1;
#endif
if ((currentByteDelta.Length = (int) (offset - currentByteDelta.Offset)) > 0)
{
deltas.Add(currentByteDelta);
}
deltas.Add(new CopyDelta
{
Offset = matchingTargetBlock.Offset,
Length = matchingTargetBlock.Length
});
slidingBuffer.MoveForward((int) currentBlockSize);
offset += currentBlockSize;
startingNewBlock = true;
continue;
}
}
}
#if DEBUG
++Statistics.NonMatching;
#endif
slidingBuffer.MoveForward(1);
++offset;
startingNewBlock = false;
}
Statistics.FileLength = offset;
if (!startingNewBlock && (currentByteDelta.Length = (int) (offset - currentByteDelta.Offset)) > 0)
{
deltas.Add(currentByteDelta);
}
#if DEBUG
Statistics.FileLength = offset;
#endif
return deltas;
}
public IEnumerable <IDelta> GetDeltas(Stream inputStream)
{
if (inputStream == null)
{
throw new ArgumentNullException("inputStream");
}
if (_initialized == false)
{
throw new InvalidOperationException("Initialize must be called");
}
ChecksumProvider.Reset();
var slidingBuffer = new SlidingStreamBuffer(inputStream, _blockSize);
slidingBuffer.Warmup();
bool startingNewBlock = true;
long offset = 0;
var deltas = new List <IDelta>();
var currentByteDelta = new ByteDelta();
#if DEBUG
Statistics.Matching = 0;
Statistics.PossibleMatches = 0;
Statistics.NonMatching = 0;
#endif
int currentBlockSize;
while ((currentBlockSize = (int)slidingBuffer.GetNumBytesAvailable()) > 0)
{
// Deal with signed integer limits
if (IsSignedIntLength(offset - currentByteDelta.Offset))
{
currentByteDelta.Length = (int)(offset - currentByteDelta.Offset);
deltas.Add(currentByteDelta);
startingNewBlock = true;
}
if (startingNewBlock)
{
currentByteDelta = new ByteDelta {
Offset = offset
};
ChecksumProvider.ProcessBlock(slidingBuffer.GetBuffer(), 0, currentBlockSize);
}
else if (currentBlockSize < _blockSize)
{
ChecksumProvider.TrimFront(); // remaining bytes < block_size, so read nothing new - just trim
}
else
{
ChecksumProvider.RollByte(slidingBuffer[(int)(currentBlockSize - 1)]);
}
// at this point, sigGen needs the last byte of the current block
uint currentBlockChecksum = ChecksumProvider.Value;
ushort currentBlockChecksumHash = RollingChecksum.HashChecksum(currentBlockChecksum);
if (_remoteBlocksIndexTable.ContainsKey(currentBlockChecksumHash))
{
List <HashBlock> possibleRemoteBlockMatches = _remoteBlocksIndexTable[currentBlockChecksumHash];
if (possibleRemoteBlockMatches.Any(entry => entry.Checksum == currentBlockChecksum))
{
#if DEBUG
++Statistics.PossibleMatches;
#endif
byte[] currentBlockHash = HashProvider.ComputeHash(slidingBuffer.GetBuffer(), 0,
(int)currentBlockSize);
HashBlock matchingTargetBlock;
if ((matchingTargetBlock =
possibleRemoteBlockMatches.FirstOrDefault(
entry => entry.Hash.SequenceEqual(currentBlockHash))) != null)
{
#if DEBUG
Statistics.Matching += 1;
#endif
if ((currentByteDelta.Length = (int)(offset - currentByteDelta.Offset)) > 0)
{
deltas.Add(currentByteDelta);
}
deltas.Add(new CopyDelta
{
Offset = matchingTargetBlock.Offset,
Length = matchingTargetBlock.Length
});
slidingBuffer.MoveForward((int)currentBlockSize);
offset += currentBlockSize;
startingNewBlock = true;
continue;
}
}
}
#if DEBUG
++Statistics.NonMatching;
#endif
slidingBuffer.MoveForward(1);
++offset;
startingNewBlock = false;
}
Statistics.FileLength = offset;
if (!startingNewBlock && (currentByteDelta.Length = (int)(offset - currentByteDelta.Offset)) > 0)
{
deltas.Add(currentByteDelta);
}
#if DEBUG
Statistics.FileLength = offset;
#endif
return(deltas);
}
