/// <summary>
/// Writes a raw BSON array.
/// </summary>
/// <param name="slice">The byte buffer containing the raw BSON array.</param>
public virtual void WriteRawBsonArray(IByteBuffer slice)
{
// overridden in BsonBinaryWriter to write the raw bytes to the stream
// for all other streams, deserialize the raw bytes and serialize the resulting array instead
using (var chunkSource = new InputBufferChunkSource(BsonChunkPool.Default))
using (var buffer = new MultiChunkBuffer(chunkSource))
using (var stream = new ByteBufferStream(buffer))
{
// wrap the array in a fake document so we can deserialize it
var documentLength = slice.Length + 8;
buffer.EnsureCapacity(documentLength);
stream.WriteInt32(documentLength);
stream.WriteBsonType(BsonType.Array);
stream.WriteByte((byte)'x');
stream.WriteByte(0);
stream.WriteSlice(slice);
stream.WriteByte(0);
buffer.MakeReadOnly();
stream.Position = 0;
using (var reader = new BsonBinaryReader(stream, BsonBinaryReaderSettings.Defaults))
{
var deserializationContext = BsonDeserializationContext.CreateRoot(reader);
reader.ReadStartDocument();
reader.ReadName("x");
var array = BsonArraySerializer.Instance.Deserialize(deserializationContext);
reader.ReadEndDocument();
var serializationContext = BsonSerializationContext.CreateRoot(this);
BsonArraySerializer.Instance.Serialize(serializationContext, array);
}
}
}
public void AccessBackingBytes_should_return_expected_result_when_there_are_zero_chunks()
{
var mockChunkSource = new Mock<IBsonChunkSource>();
var subject = new MultiChunkBuffer(mockChunkSource.Object);
var result = subject.AccessBackingBytes(0);
result.Array.Should().HaveCount(0);
result.Offset.Should().Be(0);
result.Count.Should().Be(0);
}
public void AccessBackingBytes_should_adjust_count_when_multiple_chunks_are_present()
{
var arrays = new[] { new byte[] { 1, 2 }, new byte[] { 3, 4 } };
var chunks = arrays.Select(a => new ByteArrayChunk(a));
var buffer = new MultiChunkBuffer(chunks, isReadOnly: true);
var subject = new ByteBufferSlice(buffer, 1, 2);
var result = subject.AccessBackingBytes(0);
result.Array.Should().BeSameAs(arrays[0]);
result.Offset.Should().Be(1);
result.Count.Should().Be(1); // not 2 or 3
}
/// <inheritdoc/>
public IByteBuffer GetSlice(int position, int length)
{
ThrowIfDisposed();
if (position < 0 || position > _length)
{
throw new ArgumentOutOfRangeException("position");
}
if (length < 0 || position + length > _length)
{
throw new ArgumentOutOfRangeException("length");
}
EnsureIsReadOnly();
if (length == 0)
{
return(new ByteArrayBuffer(new byte[0]));
}
var firstChunkIndex = GetChunkIndex(position);
var lastChunkIndex = GetChunkIndex(position + length - 1);
IByteBuffer forkedBuffer;
if (firstChunkIndex == lastChunkIndex)
{
var forkedChunk = _chunks[firstChunkIndex].Fork();
forkedBuffer = new SingleChunkBuffer(forkedChunk, forkedChunk.Bytes.Count, isReadOnly: true);
}
else
{
var forkedChunks = _chunks.Skip(firstChunkIndex).Take(lastChunkIndex - firstChunkIndex + 1).Select(c => c.Fork());
var forkedBufferLength = _positions[lastChunkIndex + 1] - _positions[firstChunkIndex];
forkedBuffer = new MultiChunkBuffer(forkedChunks, forkedBufferLength, isReadOnly: true);
}
var offset = position - _positions[firstChunkIndex];
return(new ByteBufferSlice(forkedBuffer, offset, length));
}
/// <summary>
/// Writes a raw BSON array.
/// </summary>
/// <param name="slice">The byte buffer containing the raw BSON array.</param>
public virtual void WriteRawBsonArray(IByteBuffer slice)
{
// overridden in BsonBinaryWriter to write the raw bytes to the stream
// for all other streams, deserialize the raw bytes and serialize the resulting array instead
using (var chunkSource = new InputBufferChunkSource(BsonChunkPool.Default))
using (var buffer = new MultiChunkBuffer(chunkSource))
using (var stream = new ByteBufferStream(buffer))
{
// wrap the array in a fake document so we can deserialize it
var documentLength = slice.Length + 8;
buffer.EnsureCapacity(documentLength);
stream.WriteInt32(documentLength);
stream.WriteBsonType(BsonType.Array);
stream.WriteByte((byte)'x');
stream.WriteByte(0);
stream.WriteSlice(slice);
stream.WriteByte(0);
buffer.MakeReadOnly();
stream.Position = 0;
using (var reader = new BsonBinaryReader(stream, BsonBinaryReaderSettings.Defaults))
{
var deserializationContext = BsonDeserializationContext.CreateRoot(reader);
reader.ReadStartDocument();
reader.ReadName("x");
var array = BsonArraySerializer.Instance.Deserialize(deserializationContext);
reader.ReadEndDocument();
var serializationContext = BsonSerializationContext.CreateRoot(this);
BsonArraySerializer.Instance.Serialize(serializationContext, array);
}
}
}
public async Task SendMessagesAsync(IEnumerable<RequestMessage> messages, MessageEncoderSettings messageEncoderSettings, CancellationToken cancellationToken)
{
Ensure.IsNotNull(messages, nameof(messages));
ThrowIfDisposedOrNotOpen();
var messagesToSend = messages.ToList();
var requestIds = messagesToSend.Select(x => x.RequestId).ToList();
try
{
if (_sendingMessagesEventHandler != null)
{
_sendingMessagesEventHandler(new ConnectionSendingMessagesEvent(_connectionId, requestIds, EventContext.OperationId));
}
cancellationToken.ThrowIfCancellationRequested();
var stopwatch = Stopwatch.StartNew();
var outputBufferChunkSource = new OutputBufferChunkSource(BsonChunkPool.Default);
using (var buffer = new MultiChunkBuffer(outputBufferChunkSource))
{
using (var stream = new ByteBufferStream(buffer, ownsBuffer: false))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings);
foreach (var message in messagesToSend)
{
if (message.ShouldBeSent == null || message.ShouldBeSent())
{
var encoder = message.GetEncoder(encoderFactory);
encoder.WriteMessage(message);
message.WasSent = true;
}
// Encoding messages includes serializing the
// documents, so encoding message could be expensive
// and worthy of us honoring cancellation here.
cancellationToken.ThrowIfCancellationRequested();
}
buffer.Length = (int)stream.Length;
buffer.MakeReadOnly();
}
var serializationDuration = stopwatch.Elapsed;
if (_commandEventHelper.ShouldCallBeforeSending)
{
_commandEventHelper.BeforeSending(
messages,
_connectionId,
buffer,
messageEncoderSettings,
stopwatch);
}
var stopwatchCheckPoint = stopwatch.Elapsed;
await SendBufferAsync(buffer, cancellationToken).ConfigureAwait(false);
var networkDuration = stopwatch.Elapsed - stopwatchCheckPoint;
if (_commandEventHelper.ShouldCallAfterSending)
{
_commandEventHelper.AfterSending(messages, _connectionId);
}
if (_sentMessagesEventHandler != null)
{
_sentMessagesEventHandler(new ConnectionSentMessagesEvent(_connectionId, requestIds, buffer.Length, networkDuration, serializationDuration, EventContext.OperationId));
}
}
}
catch (Exception ex)
{
if (_commandEventHelper.ShouldCallErrorSending)
{
_commandEventHelper.ErrorSending(messages, _connectionId, ex);
}
if (_failedSendingMessagesEvent != null)
{
_failedSendingMessagesEvent(new ConnectionSendingMessagesFailedEvent(_connectionId, requestIds, ex, EventContext.OperationId));
}
throw;
}
}
private ByteBufferStream CreateSubject(int length, IEnumerable<IBsonChunk> chunks)
{
IByteBuffer buffer;
if (chunks.Count() == 1)
{
var chunk = chunks.First();
buffer = new SingleChunkBuffer(chunk, length);
}
else
{
buffer = new MultiChunkBuffer(chunks, length);
}
return new ByteBufferStream(buffer, ownsBuffer: true);
}
public void ExpandCapacity_should_throw_when_expanded_capacity_exceeds_2GB()
{
RequireProcess.Is64Bit();
using (var subject = new MultiChunkBuffer(BsonChunkPool.Default))
{
subject.EnsureCapacity(int.MaxValue - 128 * 1024 * 1024);
Action action = () => subject.EnsureCapacity(int.MaxValue); // indirectly calls private ExpandCapacity method
action.ShouldThrow<InvalidOperationException>();
}
}
public void EnsureCapacity_should_have_expected_effect(int minimumCapacity, int[] expectedChunkSizes)
{
var chunkSource = Substitute.For<IBsonChunkSource>();
var subject = new MultiChunkBuffer(chunkSource);
var chunkSize = 1;
chunkSource.GetChunk(Arg.Any<int>()).Returns(x => new ByteArrayChunk(chunkSize++));
subject.EnsureCapacity(minimumCapacity);
var reflector = new Reflector(subject);
subject.Capacity.Should().BeGreaterOrEqualTo(minimumCapacity);
reflector._chunks.Select(c => c.Bytes.Count).Should().Equal(expectedChunkSizes);
}
public Reflector(MultiChunkBuffer instance)
{
_instance = instance;
}
public void EnsureCapacity_should_have_expected_effect(int minimumCapacity, int[] expectedChunkSizes)
{
var mockChunkSource = new Mock<IBsonChunkSource>();
var subject = new MultiChunkBuffer(mockChunkSource.Object);
var chunkSize = 1;
mockChunkSource.Setup(s => s.GetChunk(It.IsAny<int>())).Returns(() => new ByteArrayChunk(chunkSize++));
subject.EnsureCapacity(minimumCapacity);
var reflector = new Reflector(subject);
subject.Capacity.Should().BeGreaterOrEqualTo(minimumCapacity);
reflector._chunks.Select(c => c.Bytes.Count).Should().Equal(expectedChunkSizes);
}
public void WriteSlice_should_have_expected_effect(
[Values(0, 1, 2, 16)]
int length,
[Values(1, 2, 3)]
int numberOfChunks)
{
numberOfChunks = length == 0 ? 1 : length < numberOfChunks ? length : numberOfChunks;
using (var memoryStream = new MemoryStream())
using (var stream = new BsonStreamAdapter(memoryStream))
{
IByteBuffer slice;
var bytes = Enumerable.Range(0, length).Select(n => (byte)n).ToArray();
if (numberOfChunks == 1)
{
slice = new ByteArrayBuffer(bytes, isReadOnly: true);
}
else
{
var chunkSize = length / numberOfChunks;
var chunks = Enumerable.Range(0, numberOfChunks)
.Select(i => bytes.Skip(i * chunkSize).Take(i < numberOfChunks - 1 ? chunkSize : int.MaxValue).ToArray())
.Select(b => new ByteArrayChunk(b));
slice = new MultiChunkBuffer(chunks);
}
stream.WriteSlice(slice);
memoryStream.ToArray().Should().Equal(bytes);
}
}
/// <inheritdoc/>
public IByteBuffer GetSlice(int position, int length)
{
ThrowIfDisposed();
if (position < 0 || position > _length)
{
throw new ArgumentOutOfRangeException("position");
}
if (length < 0 || position + length > _length)
{
throw new ArgumentOutOfRangeException("length");
}
EnsureIsReadOnly();
if (length == 0)
{
return new ByteArrayBuffer(new byte[0]);
}
var firstChunkIndex = GetChunkIndex(position);
var lastChunkIndex = GetChunkIndex(position + length - 1);
IByteBuffer forkedBuffer;
if (firstChunkIndex == lastChunkIndex)
{
var forkedChunk = _chunks[firstChunkIndex].Fork();
forkedBuffer = new SingleChunkBuffer(forkedChunk, forkedChunk.Bytes.Count, isReadOnly: true);
}
else
{
var forkedChunks = _chunks.Skip(firstChunkIndex).Take(lastChunkIndex - firstChunkIndex + 1).Select(c => c.Fork());
var forkedBufferLength = _positions[lastChunkIndex + 1] - _positions[firstChunkIndex];
forkedBuffer = new MultiChunkBuffer(forkedChunks, forkedBufferLength, isReadOnly: true);
}
var offset = position - _positions[firstChunkIndex];
return new ByteBufferSlice(forkedBuffer, offset, length);
}
public void constructor_with_chunks_should_default_isReadOnly_to_false()
{
var chunks = Enumerable.Empty<IBsonChunk>();
var subject = new MultiChunkBuffer(chunks);
subject.IsReadOnly.Should().BeFalse();
}
public void constructor_with_chunks_should_default_length_to_capacity(int numberOfChunks)
{
var chunkSizes = Enumerable.Range(1, numberOfChunks);
var chunks = CreateChunks(chunkSizes);
var subject = new MultiChunkBuffer(chunks);
subject.Length.Should().Be(subject.Capacity);
}
public void constructor_with_chunks_should_compute_positions(int numberOfChunks, int[] expectedPositions)
{
var chunkSizes = Enumerable.Range(1, numberOfChunks);
var chunks = CreateChunks(chunkSizes);
var subject = new MultiChunkBuffer(chunks);
var reflector = new Reflector(subject);
reflector._positions.Should().Equal(expectedPositions);
}
public void constructor_with_chunks_should_compute_capacity(int numberOfChunks, int expectedCapacity)
{
var chunkSizes = Enumerable.Range(1, numberOfChunks);
var chunks = CreateChunks(chunkSizes);
var subject = new MultiChunkBuffer(chunks);
subject.Capacity.Should().Be(expectedCapacity);
}
public void ChunkSource_get_should_return_expected_result(
[Values(false, true)]
bool disposed)
{
var mockChunkSource = new Mock<IBsonChunkSource>();
var subject = new MultiChunkBuffer(mockChunkSource.Object);
if (disposed)
{
subject.Dispose();
}
var result = subject.ChunkSource;
result.Should().BeSameAs(mockChunkSource.Object);
}
public void Dispose_should_dispose_chunks(
[Values(0, 1, 2, 3)]
int numberOfChunks)
{
var chunks = Enumerable.Range(1, numberOfChunks).Select(_ => Substitute.For<IBsonChunk>()).ToList();
var subject = new MultiChunkBuffer(chunks);
subject.Dispose();
foreach (var chunk in chunks)
{
chunk.Received(1).Dispose();
}
}
public void ChunkSource_get_should_return_expected_result(
[Values(false, true)]
bool disposed)
{
var chunkSource = Substitute.For<IBsonChunkSource>();
var subject = new MultiChunkBuffer(chunkSource);
if (disposed)
{
subject.Dispose();
}
var result = subject.ChunkSource;
result.Should().BeSameAs(chunkSource);
}
public void constructor_with_chunks_should_initialize_subject()
{
var chunks = Enumerable.Empty<IBsonChunk>();
var subject = new MultiChunkBuffer(chunks, 0, false);
var reflector = new Reflector(subject);
subject.Capacity.Should().Be(0);
subject.ChunkSource.Should().BeNull();
subject.IsReadOnly.Should().BeFalse();
subject.Length.Should().Be(0);
reflector._chunks.Should().HaveCount(0);
reflector._disposed.Should().BeFalse();
reflector._positions.Should().Equal(new[] { 0 });
}
public void PrepareToWrite_should_throw_when_stream_would_exceed_2GB()
{
using (var buffer = new MultiChunkBuffer(BsonChunkPool.Default))
using (var subject = new ByteBufferStream(buffer))
{
var bytes = new byte[int.MaxValue / 2 + 1024];
subject.Write(bytes, 0, bytes.Length);
Action action = () => subject.Write(bytes, 0, bytes.Length); // indirectly calls private PrepareToWrite method
action.ShouldThrow<IOException>();
}
}
public IByteBuffer EncodeMessages(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var serializationStopwatch = Stopwatch.StartNew();
var outputBufferChunkSource = new OutputBufferChunkSource(BsonChunkPool.Default);
var buffer = new MultiChunkBuffer(outputBufferChunkSource);
using (var stream = new ByteBufferStream(buffer, ownsBuffer: false))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, _messageEncoderSettings);
foreach (var message in _messages)
{
if (message.ShouldBeSent == null || message.ShouldBeSent())
{
var encoder = message.GetEncoder(encoderFactory);
encoder.WriteMessage(message);
message.WasSent = true;
}
// Encoding messages includes serializing the
// documents, so encoding message could be expensive
// and worthy of us honoring cancellation here.
cancellationToken.ThrowIfCancellationRequested();
}
buffer.Length = (int)stream.Length;
buffer.MakeReadOnly();
}
serializationStopwatch.Stop();
_serializationDuration = serializationStopwatch.Elapsed;
return buffer;
}
public void constructor_with_chunkSource_should_initialize_subject()
{
var mockChunkSource = new Mock<IBsonChunkSource>();
var subject = new MultiChunkBuffer(mockChunkSource.Object);
var reflector = new Reflector(subject);
subject.Capacity.Should().Be(0);
subject.ChunkSource.Should().BeSameAs(mockChunkSource.Object);
subject.IsReadOnly.Should().BeFalse();
subject.Length.Should().Be(0);
reflector._chunks.Should().HaveCount(0);
reflector._disposed.Should().BeFalse();
reflector._positions.Should().Equal(new[] { 0 });
}
public void Dispose_should_dispose_chunks(
[Values(0, 1, 2, 3)]
int numberOfChunks)
{
var chunks = Enumerable.Range(1, numberOfChunks).Select(_ => new Mock<IBsonChunk>().Object).ToList();
var subject = new MultiChunkBuffer(chunks);
subject.Dispose();
foreach (var chunk in chunks)
{
var mockChunk = Mock.Get(chunk);
mockChunk.Verify(c => c.Dispose(), Times.Once);
}
}
public void Test20KDocument()
{
// manufacture an approximately 20K document using 200 strings each 100 characters long
// it's enough to cause the document to straddle a chunk boundary
var document = new BsonDocument();
var value = new string('x', 100);
for (int i = 0; i < 200; i++)
{
var name = i.ToString();
document.Add(name, value);
}
// round trip tests
var bson = document.ToBson();
var rehydrated = BsonSerializer.Deserialize<BsonDocument>(bson);
Assert.IsTrue(bson.SequenceEqual(rehydrated.ToBson()));
// test failure mode when 20 bytes are truncated from the buffer
using (var byteBuffer = new MultiChunkBuffer(BsonChunkPool.Default))
using (var byteBufferStream = new ByteBufferStream(byteBuffer, ownsBuffer: true))
{
using (var memoryStream = new MemoryStream(bson))
{
memoryStream.CopyTo(byteBufferStream);
}
byteBufferStream.SetLength(byteBufferStream.Length - 20);
byteBufferStream.Position = 0;
using (var bsonReader = new BsonBinaryReader(byteBufferStream))
{
Assert.Throws<EndOfStreamException>(() => BsonSerializer.Deserialize<BsonDocument>(bsonReader));
}
}
}