public void BeforeSending(IEnumerable <RequestMessage> messages, ConnectionId connectionId, IByteBuffer buffer, MessageEncoderSettings encoderSettings, Stopwatch stopwatch)
{
using (var stream = new ByteBufferStream(buffer, ownsBuffer: false))
{
var messageQueue = new Queue <RequestMessage>(messages);
while (messageQueue.Count > 0)
{
ProcessRequestMessages(messageQueue, connectionId, stream, encoderSettings, stopwatch);
}
}
}
public void ReaderThreadMethod(object o)
{
ByteBufferStream p = (ByteBufferStream)o;
byte[] readBlock = new byte[8 * 1024];
int readSize;
while ((readSize = p.Read(readBlock, 0, readBlock.Length)) != 0)
{
Interlocked.Add(ref dataRead, readSize);
}
}
private void CompressMessage(
RequestMessage message,
ByteBufferStream uncompressedMessageStream,
ByteBufferStream compressedStream,
MessageEncoderSettings messageEncoderSettings)
{
var compressedMessage = new CompressedMessage(message, uncompressedMessageStream, _sendCompressorType.Value);
var compressedMessageEncoderFactory = new BinaryMessageEncoderFactory(compressedStream, messageEncoderSettings, _compressorSource);
var compressedMessageEncoder = compressedMessageEncoderFactory.GetCompressedMessageEncoder(null);
compressedMessageEncoder.WriteMessage(compressedMessage);
}
public async Task SendMessagesAsync(IEnumerable <RequestMessage> messages, TimeSpan timeout, CancellationToken cancellationToken)
{
Ensure.IsNotNull(messages, "messages");
Ensure.IsInfiniteOrGreaterThanOrEqualToZero(timeout, "timeout");
ThrowIfDisposedOrNotOpen();
var messagesToSend = messages.ToList();
try
{
if (_listener != null)
{
_listener.ConnectionBeforeSendingMessages(_connectionId, messagesToSend);
}
using (var buffer = new MultiChunkBuffer(BsonChunkPool.Default))
{
using (var stream = new ByteBufferStream(buffer, ownsByteBuffer: false))
{
var writerSettings = BsonBinaryWriterSettings.Defaults; // TODO: where are writer settings supposed to come from?
var binaryWriter = new BsonBinaryWriter(stream, writerSettings);
var encoderFactory = new BinaryMessageEncoderFactory(binaryWriter);
foreach (var message in messagesToSend)
{
var encoder = message.GetEncoder(encoderFactory);
encoder.WriteMessage(message);
}
buffer.Length = (int)stream.Length;
}
var stopwatch = Stopwatch.StartNew();
var entry = new OutboundQueueEntry(buffer, cancellationToken);
_outboundQueue.Enqueue(entry);
await entry.Task;
stopwatch.Stop();
if (_listener != null)
{
_listener.ConnectionAfterSendingMessages(_connectionId, messagesToSend, buffer.Length, stopwatch.Elapsed);
}
}
}
catch (Exception ex)
{
if (_listener != null)
{
_listener.ConnectionErrorSendingMessages(_connectionId, messagesToSend, ex);
}
throw;
}
}
public async Task <ResponseMessage> ReceiveMessageAsync(
int responseTo,
IMessageEncoderSelector encoderSelector,
MessageEncoderSettings messageEncoderSettings,
CancellationToken cancellationToken)
{
Ensure.IsNotNull(encoderSelector, nameof(encoderSelector));
ThrowIfDisposedOrNotOpen();
try
{
if (_receivingMessageEventHandler != null)
{
_receivingMessageEventHandler(new ConnectionReceivingMessageEvent(_connectionId, responseTo));
}
ResponseMessage reply;
var stopwatch = Stopwatch.StartNew();
using (var buffer = await ReceiveBufferAsync(responseTo, cancellationToken).ConfigureAwait(false))
{
stopwatch.Stop();
var networkDuration = stopwatch.Elapsed;
cancellationToken.ThrowIfCancellationRequested();
stopwatch.Restart();
using (var stream = new ByteBufferStream(buffer))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings);
var encoder = encoderSelector.GetEncoder(encoderFactory);
reply = (ResponseMessage)encoder.ReadMessage();
}
stopwatch.Stop();
if (_receivedMessageEventHandler != null)
{
_receivedMessageEventHandler(new ConnectionReceivedMessageEvent(_connectionId, responseTo, buffer.Length, networkDuration, stopwatch.Elapsed));
}
}
return(reply);
}
catch (Exception ex)
{
if (_failedReceivingMessageEventHandler != null)
{
_failedReceivingMessageEventHandler(new ConnectionReceivingMessageFailedEvent(_connectionId, responseTo, ex));
}
throw;
}
}
public async Task <ReplyMessage <TDocument> > ReceiveMessageAsync <TDocument>(
int responseTo,
IBsonSerializer <TDocument> serializer,
MessageEncoderSettings messageEncoderSettings,
TimeSpan timeout,
CancellationToken cancellationToken)
{
Ensure.IsNotNull(serializer, "serializer");
Ensure.IsInfiniteOrGreaterThanOrEqualToZero(timeout, "timeout");
ThrowIfDisposedOrNotOpen();
var slidingTimeout = new SlidingTimeout(timeout);
try
{
if (_listener != null)
{
_listener.ConnectionBeforeReceivingMessage(_connectionId, responseTo);
}
var stopwatch = Stopwatch.StartNew();
var buffer = await _inboundDropbox.ReceiveAsync(responseTo, slidingTimeout, cancellationToken).ConfigureAwait(false);
int length = buffer.Length;
ReplyMessage <TDocument> reply;
using (var stream = new ByteBufferStream(buffer, ownsByteBuffer: true))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings);
var encoder = encoderFactory.GetReplyMessageEncoder <TDocument>(serializer);
reply = encoder.ReadMessage();
}
stopwatch.Stop();
if (_listener != null)
{
_listener.ConnectionAfterReceivingMessage <TDocument>(_connectionId, reply, length, stopwatch.Elapsed);
}
return(reply);
}
catch (Exception ex)
{
if (_listener != null)
{
_listener.ConnectionErrorReceivingMessage(_connectionId, responseTo, ex);
}
throw;
}
}
private void Assert(byte[] bytes, Action <ByteBufferStream, MemoryStream> test, Action <ByteBufferStream, MemoryStream> assertResult = null)
{
using (var buffer = new ByteArrayBuffer(bytes))
{
var memoryStream = new MemoryStream();
var byteBufferStream = new ByteBufferStream(buffer);
using (new NonDisposingStream(memoryStream))
using (new NonDisposingStream(byteBufferStream))
{
test(byteBufferStream, memoryStream);
assertResult?.Invoke(byteBufferStream, memoryStream);
}
}
}
public void SetContentFromStream()
{
// definedSize=0
TestHttpData test = new TestHttpData("test", Encoding.UTF8, 0);
string contentStr = "foo_test";
var buf = Unpooled.WrappedBuffer(Encoding.UTF8.GetBytes(contentStr));
buf.MarkReaderIndex();
var bs = new ByteBufferStream(buf);
try
{
test.SetContent(bs);
Assert.False(buf.IsReadable());
Assert.Equal(test.GetString(Encoding.UTF8), contentStr);
buf.ResetReaderIndex();
Assert.True(ByteBufferUtil.Equals(buf, test.GetByteBuffer()));
}
finally
{
bs.Close();
}
var random = new Random();
for (int i = 0; i < 20; i++)
{
// Generate input data bytes.
int size = random.Next(short.MaxValue);
var bytes = new byte[size];
random.NextBytes(bytes);
// Generate parsed HTTP data block.
var httpData = new TestHttpData("name", Encoding.UTF8, 0);
httpData.SetContent(new MemoryStream(bytes));
// Validate stored data.
IByteBuffer buffer = httpData.GetByteBuffer();
Assert.Equal(0, buffer.ReaderIndex);
Assert.Equal(bytes.Length, buffer.WriterIndex);
var data = new byte[bytes.Length];
buffer.GetBytes(buffer.ReaderIndex, data);
Assert.True(data.AsSpan().SequenceEqual(bytes));
}
}
public async Task <ReplyMessage <TDocument> > ReceiveMessageAsync <TDocument>(
int responseTo,
IBsonSerializer <TDocument> serializer,
MessageEncoderSettings messageEncoderSettings,
CancellationToken cancellationToken)
{
Ensure.IsNotNull(serializer, "serializer");
ThrowIfDisposedOrNotOpen();
try
{
if (_listener != null)
{
_listener.ConnectionBeforeReceivingMessage(new ConnectionBeforeReceivingMessageEvent(_connectionId, responseTo));
}
var stopwatch = Stopwatch.StartNew();
ReplyMessage <TDocument> reply;
int length;
using (var buffer = await ReceiveBufferAsync(responseTo, cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
length = buffer.Length;
using (var stream = new ByteBufferStream(buffer))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings);
var encoder = encoderFactory.GetReplyMessageEncoder <TDocument>(serializer);
reply = (ReplyMessage <TDocument>)encoder.ReadMessage();
}
}
stopwatch.Stop();
if (_listener != null)
{
_listener.ConnectionAfterReceivingMessage <TDocument>(new ConnectionAfterReceivingMessageEvent <TDocument>(_connectionId, reply, length, stopwatch.Elapsed));
}
return(reply);
}
catch (Exception ex)
{
if (_listener != null)
{
_listener.ConnectionErrorReceivingMessage(new ConnectionErrorReceivingMessageEvent(_connectionId, responseTo, ex));
}
throw;
}
}
private TResult ProcessCommandResult(ConnectionId connectionId, RawBsonDocument rawBsonDocument)
{
var binaryReaderSettings = new BsonBinaryReaderSettings
{
Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict),
GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy)
};
using (var stream = new ByteBufferStream(rawBsonDocument.Slice, ownsBuffer: false))
using (var reader = new BsonBinaryReader(stream, binaryReaderSettings))
{
var context = BsonDeserializationContext.CreateRoot(reader);
return(_resultSerializer.Deserialize(context));
}
}
public async Task CopyToAsync()
{
var byteBuffer = Unpooled.Buffer(4096);
var bufferStream = new ByteBufferStream(byteBuffer, true);
var text = "庄生晓梦迷蝴蝶，望帝春心托杜鹃。";
var bytes = Encoding.UTF8.GetBytes(text);
await bufferStream.WriteAsync(bytes, 0, bytes.Length);
Assert.Equal(bytes.Length, bufferStream.Length);
var ms = new MemoryStream();
await bufferStream.CopyToAsync(ms);
ms.Position = 0;
Assert.Equal(text, Encoding.UTF8.GetString(ms.ToArray()));
bufferStream.Close();
}
public void CopyTo()
{
var byteBuffer = Unpooled.Buffer(4096);
var bufferStream = new ByteBufferStream(byteBuffer, true);
var text = "Hello World";
var bytes = Encoding.UTF8.GetBytes(text);
bufferStream.Write(bytes, 0, bytes.Length);
Assert.Equal(bytes.Length, bufferStream.Length);
var ms = new MemoryStream();
bufferStream.CopyTo(ms);
ms.Position = 0;
Assert.Equal(text, Encoding.UTF8.GetString(ms.ToArray()));
bufferStream.Close();
}
public override void CopyTo(object[] array, int arrayIndex)
{
ThrowIfDisposed();
using (var stream = new ByteBufferStream(_slice, ownsBuffer: false))
using (var bsonReader = new BsonBinaryReader(stream, _readerSettings))
{
var context = BsonDeserializationContext.CreateRoot(bsonReader);
bsonReader.ReadStartDocument();
while (bsonReader.ReadBsonType() != BsonType.EndOfDocument)
{
bsonReader.SkipName();
array[arrayIndex++] = DeserializeBsonValue(context).RawValue;
}
bsonReader.ReadEndDocument();
}
}
/// <summary>
/// Gets an enumerator that can enumerate the elements of the array.
/// </summary>
/// <returns>An enumerator.</returns>
public override IEnumerator <BsonValue> GetEnumerator()
{
ThrowIfDisposed();
using (var stream = new ByteBufferStream(_slice, ownsBuffer: false))
using (var bsonReader = new BsonBinaryReader(stream, _readerSettings))
{
var context = BsonDeserializationContext.CreateRoot(bsonReader);
bsonReader.ReadStartDocument();
while (bsonReader.ReadBsonType() != BsonType.EndOfDocument)
{
bsonReader.SkipName();
yield return(DeserializeBsonValue(context));
}
bsonReader.ReadEndDocument();
}
}
private CompressedMessage GetCompressedMessage(MongoDBMessage message, CompressorType?compressorType = null)
{
if (!compressorType.HasValue)
{
compressorType = CompressorType.Zlib;
}
using (var memoryStream = new MemoryStream())
{
var encoderFactory = new BinaryMessageEncoderFactory(memoryStream, _messageEncoderSettings);
message.GetEncoder(encoderFactory).WriteMessage(message);
var byteBuffer = new ByteArrayBuffer(memoryStream.ToArray());
var stream = new ByteBufferStream(byteBuffer);
return(new CompressedMessage(
message,
stream,
compressorType.Value));
}
}
public ResponseMessage DecodeMessage(IByteBuffer buffer, IMessageEncoderSelector encoderSelector, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
_stopwatch.Stop();
_networkDuration = _stopwatch.Elapsed;
ResponseMessage message;
_stopwatch.Restart();
using (var stream = new ByteBufferStream(buffer, ownsBuffer: false))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, _messageEncoderSettings);
var encoder = encoderSelector.GetEncoder(encoderFactory);
message = (ResponseMessage)encoder.ReadMessage();
}
_stopwatch.Stop();
_deserializationDuration = _stopwatch.Elapsed;
return(message);
}
/// <summary>
/// Gets the index of a value in the array.
/// </summary>
/// <param name="value">The value to search for.</param>
/// <param name="index">The zero based index at which to start the search.</param>
/// <param name="count">The number of elements to search.</param>
/// <returns>The zero based index of the value (or -1 if not found).</returns>
public override int IndexOf(BsonValue value, int index, int count)
{
ThrowIfDisposed();
using (var stream = new ByteBufferStream(_slice, ownsBuffer: false))
using (var bsonReader = new BsonBinaryReader(stream, _readerSettings))
{
var context = BsonDeserializationContext.CreateRoot(bsonReader);
bsonReader.ReadStartDocument();
var i = 0;
while (bsonReader.ReadBsonType() != BsonType.EndOfDocument)
{
bsonReader.SkipName();
if (i >= index)
{
if (count == 0)
{
return(-1);
}
if (DeserializeBsonValue(context).Equals(value))
{
return(i);
}
count--;
}
else
{
bsonReader.SkipValue();
}
i++;
}
bsonReader.ReadEndDocument();
return(-1);
}
}
/// <summary>
/// Tests whether the document contains an element with the specified value.
/// </summary>
/// <param name="value">The value of the element to look for.</param>
/// <returns>
/// True if the document contains an element with the specified value.
/// </returns>
public override bool ContainsValue(BsonValue value)
{
ThrowIfDisposed();
using (var stream = new ByteBufferStream(_slice, ownsByteBuffer: false))
using (var bsonReader = new BsonBinaryReader(stream, _readerSettings))
{
var context = BsonDeserializationContext.CreateRoot <RawBsonDocument>(bsonReader);
bsonReader.ReadStartDocument();
while (bsonReader.ReadBsonType() != BsonType.EndOfDocument)
{
bsonReader.SkipName();
if (DeserializeBsonValue(context).Equals(value))
{
return(true);
}
}
bsonReader.ReadEndDocument();
return(false);
}
}
public void TestSerialWrites()
{
var str = new ByteBufferStream(16 * 1024);
byte[] writeBlock = new byte[8 * 1024];
byte[] readBlock = new byte[7 * 1024];
int detritusSize = 0;
for (int i = 0; i < 73; i++)
{
str.Write(writeBlock, 0, writeBlock.Length);
detritusSize += writeBlock.Length;
Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream);
detritusSize -= str.Read(readBlock, 0, readBlock.Length);
Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream);
while (detritusSize >= readBlock.Length)
{
detritusSize -= str.Read(readBlock, 0, readBlock.Length);
Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream);
}
}
Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream);
}
/// <summary>
/// Reads the message.
/// </summary>
/// <returns>A message.</returns>
public CompressedMessage ReadMessage()
{
var reader = CreateBinaryReader();
var stream = reader.BsonStream;
var messageStartPosition = stream.Position;
var messageLength = stream.ReadInt32();
EnsureMessageLengthIsValid(messageLength);
var requestId = stream.ReadInt32();
var responseTo = stream.ReadInt32();
var opcode = (Opcode)stream.ReadInt32();
EnsureOpcodeIsValid(opcode);
var originalOpcode = (Opcode)stream.ReadInt32();
var uncompressedSize = stream.ReadInt32();
var compressorType = (CompressorType)stream.ReadByte();
var compressor = _compressorSource.Get(compressorType);
using (var uncompressedBuffer = new MultiChunkBuffer(new OutputBufferChunkSource(BsonChunkPool.Default)))
using (var uncompressedStream = new ByteBufferStream(uncompressedBuffer, ownsBuffer: false))
{
uncompressedStream.WriteInt32(uncompressedSize + MessageHeaderLength);
uncompressedStream.WriteInt32(requestId);
uncompressedStream.WriteInt32(responseTo);
uncompressedStream.WriteInt32((int)originalOpcode);
compressor.Decompress(stream, uncompressedStream);
uncompressedStream.Position = 0;
uncompressedBuffer.MakeReadOnly();
var originalMessageEncoderFactory = new BinaryMessageEncoderFactory(uncompressedStream, _encoderSettings, _compressorSource);
var originalMessageEncoder = _originalEncoderSelector.GetEncoder(originalMessageEncoderFactory);
var originalMessage = originalMessageEncoder.ReadMessage();
return(new CompressedMessage(originalMessage, null, compressorType));
}
}
// public indexers
/// <summary>
/// Gets or sets a value by position.
/// </summary>
/// <param name="index">The position.</param>
/// <returns>The value.</returns>
public override BsonValue this[int index]
{
get
{
if (index < 0)
{
throw new ArgumentOutOfRangeException("index");
}
ThrowIfDisposed();
using (var stream = new ByteBufferStream(_slice, ownsBuffer: false))
using (var bsonReader = new BsonBinaryReader(stream, _readerSettings))
{
bsonReader.ReadStartDocument();
var i = 0;
while (bsonReader.ReadBsonType() != BsonType.EndOfDocument)
{
bsonReader.SkipName();
if (i == index)
{
var context = BsonDeserializationContext.CreateRoot(bsonReader);
return(DeserializeBsonValue(context));
}
bsonReader.SkipValue();
i++;
}
bsonReader.ReadEndDocument();
throw new ArgumentOutOfRangeException("index");
}
}
set
{
throw new NotSupportedException("RawBsonArray instances are immutable.");
}
}
public void TwoReaderTwoWriterTest()
{
var str = new ByteBufferStream(1 * 1024 * 1024);
Thread writerThread = new Thread(WriterThreadMethod)
{
Name = "WriterThread2"
};
Thread writerThread2 = new Thread(WriterThreadMethod)
{
Name = "WriterThread2"
};
Thread clientThread = new Thread(ReaderThreadMethod)
{
Name = "ReaderThread1"
};
Thread clientThread2 = new Thread(ReaderThreadMethod)
{
Name = "ReaderThread2"
};
clientThread.Start(str);
clientThread2.Start(str);
writerThread.Start(str);
writerThread2.Start(str);
writerThread.Join();
writerThread2.Join();
str.Dispose();
clientThread.Join();
clientThread2.Join();
Interlocked.Read(ref dataWritten);
Interlocked.Read(ref dataRead);
Assert.IsTrue(dataWritten == dataRead, "size comparison: data written ({0}) != data read ({1})", dataWritten, dataRead);
}
public void Test20KDocument()
{
// manufacture an approximately 20K document using 200 strings each 100 characters long
// it's enough to cause the document to straddle a chunk boundary
var document = new BsonDocument();
var value = new string('x', 100);
for (int i = 0; i < 200; i++)
{
var name = i.ToString();
document.Add(name, value);
}
// round trip tests
var bson = document.ToBson();
var rehydrated = BsonSerializer.Deserialize <BsonDocument>(bson);
Assert.IsTrue(bson.SequenceEqual(rehydrated.ToBson()));
// test failure mode when 20 bytes are truncated from the buffer
using (var byteBuffer = new MultiChunkBuffer(BsonChunkPool.Default))
using (var byteBufferStream = new ByteBufferStream(byteBuffer, ownsBuffer: true))
{
using (var memoryStream = new MemoryStream(bson))
{
memoryStream.CopyTo(byteBufferStream);
}
byteBufferStream.SetLength(byteBufferStream.Length - 20);
byteBufferStream.Position = 0;
using (var bsonReader = new BsonBinaryReader(byteBufferStream))
{
Assert.Throws <EndOfStreamException>(() => BsonSerializer.Deserialize <BsonDocument>(bsonReader));
}
}
}
private TCommandResult ProcessReply(ConnectionId connectionId, ReplyMessage <RawBsonDocument> reply)
{
if (reply.NumberReturned == 0)
{
throw new MongoCommandException(connectionId, "Command returned no documents.", _command);
}
if (reply.NumberReturned > 1)
{
throw new MongoCommandException(connectionId, "Command returned multiple documents.", _command);
}
if (reply.QueryFailure)
{
var failureDocument = reply.QueryFailureDocument;
throw ExceptionMapper.Map(connectionId, failureDocument) ?? new MongoCommandException(connectionId, "Command failed.", _command, failureDocument);
}
using (var rawDocument = reply.Documents[0])
{
if (!rawDocument.GetValue("ok", false).ToBoolean())
{
var binaryReaderSettings = new BsonBinaryReaderSettings();
if (_messageEncoderSettings != null)
{
binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict);
binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy);
}
;
var materializedDocument = rawDocument.Materialize(binaryReaderSettings);
var commandName = _command.GetElement(0).Name;
if (commandName == "$query")
{
commandName = _command["$query"].AsBsonDocument.GetElement(0).Name;
}
var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, materializedDocument, "errmsg");
if (notPrimaryOrNodeIsRecoveringException != null)
{
throw notPrimaryOrNodeIsRecoveringException;
}
string message;
BsonValue errmsgBsonValue;
if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString)
{
var errmsg = errmsgBsonValue.ToString();
message = string.Format("Command {0} failed: {1}.", commandName, errmsg);
}
else
{
message = string.Format("Command {0} failed.", commandName);
}
var mappedException = ExceptionMapper.Map(connectionId, materializedDocument);
if (mappedException != null)
{
throw mappedException;
}
throw new MongoCommandException(connectionId, message, _command, materializedDocument);
}
using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, _messageEncoderSettings);
var encoder = (ReplyMessageBinaryEncoder <TCommandResult>)encoderFactory.GetReplyMessageEncoder <TCommandResult>(_resultSerializer);
using (var reader = encoder.CreateBinaryReader())
{
var context = BsonDeserializationContext.CreateRoot(reader);
return(_resultSerializer.Deserialize(context));
}
}
}
}
private void ProcessReplyMessage(CommandState state, ResponseMessage message, IByteBuffer buffer, ConnectionId connectionId, MessageEncoderSettings encoderSettings)
{
state.Stopwatch.Stop();
bool disposeOfDocuments = false;
var replyMessage = message as ReplyMessage <RawBsonDocument>;
if (replyMessage == null)
{
// ReplyMessage is generic, which means that we can't use it here, so, we need to use a different one...
using (var stream = new ByteBufferStream(buffer, ownsBuffer: false))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, encoderSettings);
replyMessage = (ReplyMessage <RawBsonDocument>)encoderFactory
.GetReplyMessageEncoder(RawBsonDocumentSerializer.Instance)
.ReadMessage();
disposeOfDocuments = true;
}
}
try
{
if (replyMessage.CursorNotFound ||
replyMessage.QueryFailure ||
(state.ExpectedResponseType != ExpectedResponseType.Query && replyMessage.Documents.Count == 0))
{
var queryFailureDocument = replyMessage.QueryFailureDocument;
if (__securitySensitiveCommands.Contains(state.CommandName))
{
queryFailureDocument = new BsonDocument();
}
if (_failedEvent != null)
{
_failedEvent(new CommandFailedEvent(
state.CommandName,
new MongoCommandException(
connectionId,
string.Format("{0} command failed", state.CommandName),
null,
queryFailureDocument),
state.OperationId,
replyMessage.ResponseTo,
connectionId,
state.Stopwatch.Elapsed));
}
}
else
{
switch (state.ExpectedResponseType)
{
case ExpectedResponseType.Command:
ProcessCommandReplyMessage(state, replyMessage, connectionId);
break;
case ExpectedResponseType.GLE:
ProcessGLEReplyMessage(state, replyMessage, connectionId);
break;
case ExpectedResponseType.Query:
ProcessQueryReplyMessage(state, replyMessage, connectionId);
break;
}
}
}
finally
{
if (disposeOfDocuments && replyMessage.Documents != null)
{
replyMessage.Documents.ForEach(d => d.Dispose());
}
}
}
public async Task SendMessagesAsync(IEnumerable <RequestMessage> messages, MessageEncoderSettings messageEncoderSettings, CancellationToken cancellationToken)
{
Ensure.IsNotNull(messages, nameof(messages));
ThrowIfDisposedOrNotOpen();
var messagesToSend = messages.ToList();
var requestIds = messagesToSend.Select(x => x.RequestId).ToList();
try
{
if (_sendingMessagesEventHandler != null)
{
_sendingMessagesEventHandler(new ConnectionSendingMessagesEvent(_connectionId, requestIds));
}
cancellationToken.ThrowIfCancellationRequested();
var stopwatch = Stopwatch.StartNew();
var outputBufferChunkSource = new OutputBufferChunkSource(BsonChunkPool.Default);
using (var buffer = new MultiChunkBuffer(outputBufferChunkSource))
{
using (var stream = new ByteBufferStream(buffer, ownsBuffer: false))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings);
foreach (var message in messagesToSend)
{
if (message.ShouldBeSent == null || message.ShouldBeSent())
{
var encoder = message.GetEncoder(encoderFactory);
encoder.WriteMessage(message);
message.WasSent = true;
}
// Encoding messages includes serializing the
// documents, so encoding message could be expensive
// and worthy of us honoring cancellation here.
cancellationToken.ThrowIfCancellationRequested();
}
buffer.Length = (int)stream.Length;
}
stopwatch.Stop();
var serializationDuration = stopwatch.Elapsed;
stopwatch.Restart();
await SendBufferAsync(buffer, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
if (_sentMessagesEventHandler != null)
{
_sentMessagesEventHandler(new ConnectionSentMessagesEvent(_connectionId, requestIds, buffer.Length, stopwatch.Elapsed, serializationDuration));
}
}
}
catch (Exception ex)
{
if (_failedSendingMessagesEvent != null)
{
_failedSendingMessagesEvent(new ConnectionSendingMessagesFailedEvent(_connectionId, requestIds, ex));
}
throw;
}
}
private TCommandResult ProcessResponse(ConnectionId connectionId, CommandMessage responseMessage)
{
using (new CommandMessageDisposer(responseMessage))
{
var rawDocument = responseMessage.Sections.OfType <Type0CommandMessageSection <RawBsonDocument> >().Single().Document;
var binaryReaderSettings = new BsonBinaryReaderSettings();
if (_messageEncoderSettings != null)
{
binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict);
binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy);
}
;
BsonValue clusterTime;
if (rawDocument.TryGetValue("$clusterTime", out clusterTime))
{
// note: we are assuming that _session is an instance of ClusterClockAdvancingClusterTime
// and that calling _session.AdvanceClusterTime will have the side effect of advancing the cluster's ClusterTime also
var materializedClusterTime = ((RawBsonDocument)clusterTime).Materialize(binaryReaderSettings);
_session.AdvanceClusterTime(materializedClusterTime);
}
BsonValue operationTime;
if (rawDocument.TryGetValue("operationTime", out operationTime))
{
_session.AdvanceOperationTime(operationTime.AsBsonTimestamp);
}
if (!rawDocument.GetValue("ok", false).ToBoolean())
{
var materializedDocument = rawDocument.Materialize(binaryReaderSettings);
var commandName = _command.GetElement(0).Name;
if (commandName == "$query")
{
commandName = _command["$query"].AsBsonDocument.GetElement(0).Name;
}
var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, materializedDocument, "errmsg");
if (notPrimaryOrNodeIsRecoveringException != null)
{
throw notPrimaryOrNodeIsRecoveringException;
}
var mappedException = ExceptionMapper.Map(connectionId, materializedDocument);
if (mappedException != null)
{
throw mappedException;
}
string message;
BsonValue errmsgBsonValue;
if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString)
{
var errmsg = errmsgBsonValue.ToString();
message = string.Format("Command {0} failed: {1}.", commandName, errmsg);
}
else
{
message = string.Format("Command {0} failed.", commandName);
}
throw new MongoCommandException(connectionId, message, _command, materializedDocument);
}
using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false))
{
using (var reader = new BsonBinaryReader(stream, binaryReaderSettings))
{
var context = BsonDeserializationContext.CreateRoot(reader);
return(_resultSerializer.Deserialize(context));
}
}
}
}
private TCommandResult ProcessResponse(ConnectionId connectionId, CommandMessage responseMessage)
{
using (new CommandMessageDisposer(responseMessage))
{
var rawDocument = responseMessage.Sections.OfType <Type0CommandMessageSection <RawBsonDocument> >().Single().Document;
var binaryReaderSettings = new BsonBinaryReaderSettings();
if (_messageEncoderSettings != null)
{
binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict);
#pragma warning disable 618
if (BsonDefaults.GuidRepresentationMode == GuidRepresentationMode.V2)
{
binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy);
}
#pragma warning restore 618
}
;
BsonValue clusterTime;
if (rawDocument.TryGetValue("$clusterTime", out clusterTime))
{
// note: we are assuming that _session is an instance of ClusterClockAdvancingClusterTime
// and that calling _session.AdvanceClusterTime will have the side effect of advancing the cluster's ClusterTime also
var materializedClusterTime = ((RawBsonDocument)clusterTime).Materialize(binaryReaderSettings);
_session.AdvanceClusterTime(materializedClusterTime);
}
BsonValue operationTime;
if (rawDocument.TryGetValue("operationTime", out operationTime))
{
_session.AdvanceOperationTime(operationTime.AsBsonTimestamp);
}
if (rawDocument.GetValue("ok", false).ToBoolean())
{
if (rawDocument.TryGetValue("recoveryToken", out var rawRecoveryToken))
{
var recoveryToken = ((RawBsonDocument)rawRecoveryToken).Materialize(binaryReaderSettings);
_session.CurrentTransaction.RecoveryToken = recoveryToken;
}
}
else
{
var materializedDocument = rawDocument.Materialize(binaryReaderSettings);
var commandName = _command.GetElement(0).Name;
if (commandName == "$query")
{
commandName = _command["$query"].AsBsonDocument.GetElement(0).Name;
}
var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, _command, materializedDocument, "errmsg");
if (notPrimaryOrNodeIsRecoveringException != null)
{
throw notPrimaryOrNodeIsRecoveringException;
}
var mappedException = ExceptionMapper.Map(connectionId, materializedDocument);
if (mappedException != null)
{
throw mappedException;
}
string message;
BsonValue errmsgBsonValue;
if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString)
{
var errmsg = errmsgBsonValue.ToString();
message = string.Format("Command {0} failed: {1}.", commandName, errmsg);
}
else
{
message = string.Format("Command {0} failed.", commandName);
}
var exception = new MongoCommandException(connectionId, message, _command, materializedDocument);
// https://jira.mongodb.org/browse/CSHARP-2678
if (IsRetryableWriteExceptionAndDeploymentDoesNotSupportRetryableWrites(exception))
{
throw WrapNotSupportedRetryableWriteException(exception);
}
else
{
throw exception;
}
}
if (rawDocument.Contains("writeConcernError"))
{
var materializedDocument = rawDocument.Materialize(binaryReaderSettings);
var writeConcernError = materializedDocument["writeConcernError"].AsBsonDocument;
var message = writeConcernError.AsBsonDocument.GetValue("errmsg", null)?.AsString;
var writeConcernResult = new WriteConcernResult(materializedDocument);
throw new MongoWriteConcernException(connectionId, message, writeConcernResult);
}
using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false))
{
using (var reader = new BsonBinaryReader(stream, binaryReaderSettings))
{
var context = BsonDeserializationContext.CreateRoot(reader);
return(_resultSerializer.Deserialize(context));
}
}
}
}
public async Task Execute(IJobExecutionContext context)
{
int jobId = int.Parse(context.JobDetail.Key.Name);
int historyId = 0;
IDisposableList disposableList = new IDisposableList();
try
{
// load the job from the database and create a history point for this scheduled execution
BackupJob job = await backupJobRepository.Get(jobId);
DateTime?lastRun = await backupJobRepository.GetLastRun(jobId);
historyId = await backupJobRepository.AddHistory(job.ID);
// sort the providers so they are executed in the correct order
var providers = job.Providers.OrderBy(p => p.Order);
// load and create the backup and storage providers
IBackupProvider backupProvider = await providerMappingService.CreateProvider <IBackupProvider>(providers.FirstOrDefault());
IStorageProvider storageProvider = await providerMappingService.CreateProvider <IStorageProvider>(providers.LastOrDefault());
disposableList.AddRange(new IDisposable[] { backupProvider, storageProvider });
// load all the transform providers
List <ITransformProvider> transformProviders = new List <ITransformProvider>();
foreach (var tp in providers.Where(p => p.Provider.Type == ProviderType.Transform))
{
transformProviders.Add(await providerMappingService.CreateProvider <ITransformProvider>(tp));
}
disposableList.AddRange(transformProviders);
// fetch all items from the backup providers
var items = await backupProvider.GetItems(lastRun);
List <List <TransformBackupItem> > transformExecuteList = new List <List <TransformBackupItem> >(transformProviders.Count());
Dictionary <ITransformProvider, IEnumerable <MappedBackupItem> > transformers = new Dictionary <ITransformProvider, IEnumerable <MappedBackupItem> >(transformProviders.Count());
for (int i = 0; i < transformProviders.Count(); i++)
{
if (i > 0)
{
var mappedItems = await transformProviders[i].MapInput(transformers.Last().Value.Select(x => x.Output));
transformers.Add(transformProviders[i], mappedItems);
List <TransformBackupItem> subTransformExecuteList = new List <TransformBackupItem>();
foreach (var mappedItem in mappedItems)
{
int currentMappedIndex = i;
Func <Stream, Task> action = async(stream) =>
{
//Dictionary<BackupItem, Stream> dictionary = new Dictionary<BackupItem, Stream>();
//foreach (var backupItem in mappedItem.Input)
//{
// PassThroughStream backupItemStream = disposableList.CreateAndAdd<PassThroughStream>();
// dictionary.Add(backupItem, backupItemStream);
//}
//Task transformTask = transformProviders[currentMappedIndex].TransformItem(mappedItem.Output, stream, dictionary);
//foreach (var kvp in dictionary)
//{
// PassThroughStream passThroughStream = kvp.Value as PassThroughStream;
// var transformBackupItem = transformExecuteList[currentMappedIndex - 1].FirstOrDefault(x => x.MappedBackupItem.Output == kvp.Key);
// await transformBackupItem.Execute(passThroughStream);
// passThroughStream.SetComplete();
//}
//transformTask.Wait();
Dictionary <BackupItem, Stream> dictionary = new Dictionary <BackupItem, Stream>();
foreach (var backupItem in mappedItem.Input)
{
ByteBufferStream ms = disposableList.CreateAndAdd <ByteBufferStream>();
var transformBackupItem = transformExecuteList[currentMappedIndex - 1].FirstOrDefault(x => x.MappedBackupItem.Output == backupItem);
await transformBackupItem.Execute(ms);
dictionary.Add(backupItem, ms);
}
await transformProviders[currentMappedIndex].TransformItem(mappedItem.Output, stream, dictionary);
};
subTransformExecuteList.Add(new TransformBackupItem {
MappedBackupItem = mappedItem, Execute = action
});
}
transformExecuteList.Add(subTransformExecuteList);
}
else
{
var mappedItems = await transformProviders[i].MapInput(items);
transformers.Add(transformProviders[i], mappedItems);
List <TransformBackupItem> subTransformExecuteList = new List <TransformBackupItem>();
foreach (var mappedItem in mappedItems)
{
int currentMappedIndex = i;
Func <Stream, Task> action = async(stream) =>
{
Dictionary <BackupItem, Stream> dictionary = new Dictionary <BackupItem, Stream>();
foreach (var backupItem in mappedItem.Input)
{
Stream itemStream = await disposableList.CreateAndAdd(async() => await backupProvider.OpenRead(backupItem));
dictionary.Add(backupItem, itemStream);
}
await transformProviders[currentMappedIndex].TransformItem(mappedItem.Output, stream, dictionary);
};
subTransformExecuteList.Add(new TransformBackupItem {
MappedBackupItem = mappedItem, Execute = action
});
}
transformExecuteList.Add(subTransformExecuteList);
}
}
if (transformProviders.Count() > 0)
{
foreach (var mappedItem in transformExecuteList.Last())
{
using (WaitableByteBufferStream outputStream = new WaitableByteBufferStream())
{
Task <bool> storeItem = storageProvider.StoreItem(mappedItem.MappedBackupItem.Output, outputStream);
await mappedItem.Execute(outputStream);
outputStream.SetComplete();
storeItem.Wait();
}
}
}
else
{
foreach (var item in items)
{
using (Stream itemStream = await backupProvider.OpenRead(item))
{
await storageProvider.StoreItem(item, itemStream);
}
}
}
await backupJobRepository.UpdateHistory(historyId, ExitCode.Success, "Backup completed successfully.");
}
catch (Exception ex)
{
logger.LogError(ex, $"Backup failed with ID {jobId} and History ID {historyId}.");
if (historyId > 0)
{
await backupJobRepository.UpdateHistory(historyId, ExitCode.Failed, $"Backup failed with message: {ex.Message} ({ex.GetType()})");
}
}
finally
{
foreach (IDisposable item in disposableList)
{
try
{
item?.Dispose();
}
catch (NotImplementedException)
{
// ignore this exception
}
catch (Exception ex)
{
// log every other error
logger.LogError(ex, $"Failed to dispose item.");
}
}
disposableList.Clear();
disposableList = null;
}
}
private TCommandResult ProcessReply(ConnectionId connectionId, ReplyMessage <RawBsonDocument> reply)
{
if (reply.NumberReturned == 0)
{
throw new MongoCommandException(connectionId, "Command returned no documents.", _command);
}
if (reply.NumberReturned > 1)
{
throw new MongoCommandException(connectionId, "Command returned multiple documents.", _command);
}
if (reply.QueryFailure)
{
var failureDocument = reply.QueryFailureDocument;
throw ExceptionMapper.Map(connectionId, failureDocument) ?? new MongoCommandException(connectionId, "Command failed.", _command, failureDocument);
}
using (var rawDocument = reply.Documents[0])
{
var binaryReaderSettings = new BsonBinaryReaderSettings();
if (_messageEncoderSettings != null)
{
binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict);
#pragma warning disable 618
if (BsonDefaults.GuidRepresentationMode == GuidRepresentationMode.V2)
{
binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy);
}
#pragma warning restore 618
}
;
BsonValue clusterTime;
if (rawDocument.TryGetValue("$clusterTime", out clusterTime))
{
// note: we are assuming that _session is an instance of ClusterClockAdvancingClusterTime
// and that calling _session.AdvanceClusterTime will have the side effect of advancing the cluster's ClusterTime also
var materializedClusterTime = ((RawBsonDocument)clusterTime).Materialize(binaryReaderSettings);
_session.AdvanceClusterTime(materializedClusterTime);
}
BsonValue operationTime;
if (rawDocument.TryGetValue("operationTime", out operationTime))
{
_session.AdvanceOperationTime(operationTime.AsBsonTimestamp);
}
if (!rawDocument.GetValue("ok", false).ToBoolean())
{
var materializedDocument = rawDocument.Materialize(binaryReaderSettings);
var commandName = _command.GetElement(0).Name;
if (commandName == "$query")
{
commandName = _command["$query"].AsBsonDocument.GetElement(0).Name;
}
var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, _command, materializedDocument, "errmsg");
if (notPrimaryOrNodeIsRecoveringException != null)
{
throw notPrimaryOrNodeIsRecoveringException;
}
string message;
BsonValue errmsgBsonValue;
if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString)
{
var errmsg = errmsgBsonValue.ToString();
message = string.Format("Command {0} failed: {1}.", commandName, errmsg);
}
else
{
message = string.Format("Command {0} failed.", commandName);
}
var mappedException = ExceptionMapper.Map(connectionId, materializedDocument);
if (mappedException != null)
{
throw mappedException;
}
throw new MongoCommandException(connectionId, message, _command, materializedDocument);
}
if (rawDocument.Contains("writeConcernError"))
{
var materializedDocument = rawDocument.Materialize(binaryReaderSettings);
var writeConcernError = materializedDocument["writeConcernError"].AsBsonDocument;
var message = writeConcernError.AsBsonDocument.GetValue("errmsg", null)?.AsString;
var writeConcernResult = new WriteConcernResult(materializedDocument);
throw new MongoWriteConcernException(connectionId, message, writeConcernResult);
}
using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false))
{
var encoderFactory = new BinaryMessageEncoderFactory(stream, _messageEncoderSettings);
var encoder = (ReplyMessageBinaryEncoder <TCommandResult>)encoderFactory.GetReplyMessageEncoder <TCommandResult>(_resultSerializer);
using (var reader = encoder.CreateBinaryReader())
{
var context = BsonDeserializationContext.CreateRoot(reader);
return(_resultSerializer.Deserialize(context));
}
}
}
}
