public void BeforeSending(IEnumerable <RequestMessage> messages, ConnectionId connectionId, IByteBuffer buffer, MessageEncoderSettings encoderSettings, Stopwatch stopwatch) { using (var stream = new ByteBufferStream(buffer, ownsBuffer: false)) { var messageQueue = new Queue <RequestMessage>(messages); while (messageQueue.Count > 0) { ProcessRequestMessages(messageQueue, connectionId, stream, encoderSettings, stopwatch); } } }
public void ReaderThreadMethod(object o) { ByteBufferStream p = (ByteBufferStream)o; byte[] readBlock = new byte[8 * 1024]; int readSize; while ((readSize = p.Read(readBlock, 0, readBlock.Length)) != 0) { Interlocked.Add(ref dataRead, readSize); } }
private void CompressMessage( RequestMessage message, ByteBufferStream uncompressedMessageStream, ByteBufferStream compressedStream, MessageEncoderSettings messageEncoderSettings) { var compressedMessage = new CompressedMessage(message, uncompressedMessageStream, _sendCompressorType.Value); var compressedMessageEncoderFactory = new BinaryMessageEncoderFactory(compressedStream, messageEncoderSettings, _compressorSource); var compressedMessageEncoder = compressedMessageEncoderFactory.GetCompressedMessageEncoder(null); compressedMessageEncoder.WriteMessage(compressedMessage); }
public async Task SendMessagesAsync(IEnumerable <RequestMessage> messages, TimeSpan timeout, CancellationToken cancellationToken) { Ensure.IsNotNull(messages, "messages"); Ensure.IsInfiniteOrGreaterThanOrEqualToZero(timeout, "timeout"); ThrowIfDisposedOrNotOpen(); var messagesToSend = messages.ToList(); try { if (_listener != null) { _listener.ConnectionBeforeSendingMessages(_connectionId, messagesToSend); } using (var buffer = new MultiChunkBuffer(BsonChunkPool.Default)) { using (var stream = new ByteBufferStream(buffer, ownsByteBuffer: false)) { var writerSettings = BsonBinaryWriterSettings.Defaults; // TODO: where are writer settings supposed to come from? var binaryWriter = new BsonBinaryWriter(stream, writerSettings); var encoderFactory = new BinaryMessageEncoderFactory(binaryWriter); foreach (var message in messagesToSend) { var encoder = message.GetEncoder(encoderFactory); encoder.WriteMessage(message); } buffer.Length = (int)stream.Length; } var stopwatch = Stopwatch.StartNew(); var entry = new OutboundQueueEntry(buffer, cancellationToken); _outboundQueue.Enqueue(entry); await entry.Task; stopwatch.Stop(); if (_listener != null) { _listener.ConnectionAfterSendingMessages(_connectionId, messagesToSend, buffer.Length, stopwatch.Elapsed); } } } catch (Exception ex) { if (_listener != null) { _listener.ConnectionErrorSendingMessages(_connectionId, messagesToSend, ex); } throw; } }
public async Task <ResponseMessage> ReceiveMessageAsync( int responseTo, IMessageEncoderSelector encoderSelector, MessageEncoderSettings messageEncoderSettings, CancellationToken cancellationToken) { Ensure.IsNotNull(encoderSelector, nameof(encoderSelector)); ThrowIfDisposedOrNotOpen(); try { if (_receivingMessageEventHandler != null) { _receivingMessageEventHandler(new ConnectionReceivingMessageEvent(_connectionId, responseTo)); } ResponseMessage reply; var stopwatch = Stopwatch.StartNew(); using (var buffer = await ReceiveBufferAsync(responseTo, cancellationToken).ConfigureAwait(false)) { stopwatch.Stop(); var networkDuration = stopwatch.Elapsed; cancellationToken.ThrowIfCancellationRequested(); stopwatch.Restart(); using (var stream = new ByteBufferStream(buffer)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings); var encoder = encoderSelector.GetEncoder(encoderFactory); reply = (ResponseMessage)encoder.ReadMessage(); } stopwatch.Stop(); if (_receivedMessageEventHandler != null) { _receivedMessageEventHandler(new ConnectionReceivedMessageEvent(_connectionId, responseTo, buffer.Length, networkDuration, stopwatch.Elapsed)); } } return(reply); } catch (Exception ex) { if (_failedReceivingMessageEventHandler != null) { _failedReceivingMessageEventHandler(new ConnectionReceivingMessageFailedEvent(_connectionId, responseTo, ex)); } throw; } }
public async Task <ReplyMessage <TDocument> > ReceiveMessageAsync <TDocument>( int responseTo, IBsonSerializer <TDocument> serializer, MessageEncoderSettings messageEncoderSettings, TimeSpan timeout, CancellationToken cancellationToken) { Ensure.IsNotNull(serializer, "serializer"); Ensure.IsInfiniteOrGreaterThanOrEqualToZero(timeout, "timeout"); ThrowIfDisposedOrNotOpen(); var slidingTimeout = new SlidingTimeout(timeout); try { if (_listener != null) { _listener.ConnectionBeforeReceivingMessage(_connectionId, responseTo); } var stopwatch = Stopwatch.StartNew(); var buffer = await _inboundDropbox.ReceiveAsync(responseTo, slidingTimeout, cancellationToken).ConfigureAwait(false); int length = buffer.Length; ReplyMessage <TDocument> reply; using (var stream = new ByteBufferStream(buffer, ownsByteBuffer: true)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings); var encoder = encoderFactory.GetReplyMessageEncoder <TDocument>(serializer); reply = encoder.ReadMessage(); } stopwatch.Stop(); if (_listener != null) { _listener.ConnectionAfterReceivingMessage <TDocument>(_connectionId, reply, length, stopwatch.Elapsed); } return(reply); } catch (Exception ex) { if (_listener != null) { _listener.ConnectionErrorReceivingMessage(_connectionId, responseTo, ex); } throw; } }
private void Assert(byte[] bytes, Action <ByteBufferStream, MemoryStream> test, Action <ByteBufferStream, MemoryStream> assertResult = null) { using (var buffer = new ByteArrayBuffer(bytes)) { var memoryStream = new MemoryStream(); var byteBufferStream = new ByteBufferStream(buffer); using (new NonDisposingStream(memoryStream)) using (new NonDisposingStream(byteBufferStream)) { test(byteBufferStream, memoryStream); assertResult?.Invoke(byteBufferStream, memoryStream); } } }
public void SetContentFromStream() { // definedSize=0 TestHttpData test = new TestHttpData("test", Encoding.UTF8, 0); string contentStr = "foo_test"; var buf = Unpooled.WrappedBuffer(Encoding.UTF8.GetBytes(contentStr)); buf.MarkReaderIndex(); var bs = new ByteBufferStream(buf); try { test.SetContent(bs); Assert.False(buf.IsReadable()); Assert.Equal(test.GetString(Encoding.UTF8), contentStr); buf.ResetReaderIndex(); Assert.True(ByteBufferUtil.Equals(buf, test.GetByteBuffer())); } finally { bs.Close(); } var random = new Random(); for (int i = 0; i < 20; i++) { // Generate input data bytes. int size = random.Next(short.MaxValue); var bytes = new byte[size]; random.NextBytes(bytes); // Generate parsed HTTP data block. var httpData = new TestHttpData("name", Encoding.UTF8, 0); httpData.SetContent(new MemoryStream(bytes)); // Validate stored data. IByteBuffer buffer = httpData.GetByteBuffer(); Assert.Equal(0, buffer.ReaderIndex); Assert.Equal(bytes.Length, buffer.WriterIndex); var data = new byte[bytes.Length]; buffer.GetBytes(buffer.ReaderIndex, data); Assert.True(data.AsSpan().SequenceEqual(bytes)); } }
public async Task <ReplyMessage <TDocument> > ReceiveMessageAsync <TDocument>( int responseTo, IBsonSerializer <TDocument> serializer, MessageEncoderSettings messageEncoderSettings, CancellationToken cancellationToken) { Ensure.IsNotNull(serializer, "serializer"); ThrowIfDisposedOrNotOpen(); try { if (_listener != null) { _listener.ConnectionBeforeReceivingMessage(new ConnectionBeforeReceivingMessageEvent(_connectionId, responseTo)); } var stopwatch = Stopwatch.StartNew(); ReplyMessage <TDocument> reply; int length; using (var buffer = await ReceiveBufferAsync(responseTo, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); length = buffer.Length; using (var stream = new ByteBufferStream(buffer)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings); var encoder = encoderFactory.GetReplyMessageEncoder <TDocument>(serializer); reply = (ReplyMessage <TDocument>)encoder.ReadMessage(); } } stopwatch.Stop(); if (_listener != null) { _listener.ConnectionAfterReceivingMessage <TDocument>(new ConnectionAfterReceivingMessageEvent <TDocument>(_connectionId, reply, length, stopwatch.Elapsed)); } return(reply); } catch (Exception ex) { if (_listener != null) { _listener.ConnectionErrorReceivingMessage(new ConnectionErrorReceivingMessageEvent(_connectionId, responseTo, ex)); } throw; } }
private TResult ProcessCommandResult(ConnectionId connectionId, RawBsonDocument rawBsonDocument) { var binaryReaderSettings = new BsonBinaryReaderSettings { Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict), GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy) }; using (var stream = new ByteBufferStream(rawBsonDocument.Slice, ownsBuffer: false)) using (var reader = new BsonBinaryReader(stream, binaryReaderSettings)) { var context = BsonDeserializationContext.CreateRoot(reader); return(_resultSerializer.Deserialize(context)); } }
public async Task CopyToAsync() { var byteBuffer = Unpooled.Buffer(4096); var bufferStream = new ByteBufferStream(byteBuffer, true); var text = "庄生晓梦迷蝴蝶,望帝春心托杜鹃。"; var bytes = Encoding.UTF8.GetBytes(text); await bufferStream.WriteAsync(bytes, 0, bytes.Length); Assert.Equal(bytes.Length, bufferStream.Length); var ms = new MemoryStream(); await bufferStream.CopyToAsync(ms); ms.Position = 0; Assert.Equal(text, Encoding.UTF8.GetString(ms.ToArray())); bufferStream.Close(); }
public void CopyTo() { var byteBuffer = Unpooled.Buffer(4096); var bufferStream = new ByteBufferStream(byteBuffer, true); var text = "Hello World"; var bytes = Encoding.UTF8.GetBytes(text); bufferStream.Write(bytes, 0, bytes.Length); Assert.Equal(bytes.Length, bufferStream.Length); var ms = new MemoryStream(); bufferStream.CopyTo(ms); ms.Position = 0; Assert.Equal(text, Encoding.UTF8.GetString(ms.ToArray())); bufferStream.Close(); }
public override void CopyTo(object[] array, int arrayIndex) { ThrowIfDisposed(); using (var stream = new ByteBufferStream(_slice, ownsBuffer: false)) using (var bsonReader = new BsonBinaryReader(stream, _readerSettings)) { var context = BsonDeserializationContext.CreateRoot(bsonReader); bsonReader.ReadStartDocument(); while (bsonReader.ReadBsonType() != BsonType.EndOfDocument) { bsonReader.SkipName(); array[arrayIndex++] = DeserializeBsonValue(context).RawValue; } bsonReader.ReadEndDocument(); } }
/// <summary> /// Gets an enumerator that can enumerate the elements of the array. /// </summary> /// <returns>An enumerator.</returns> public override IEnumerator <BsonValue> GetEnumerator() { ThrowIfDisposed(); using (var stream = new ByteBufferStream(_slice, ownsBuffer: false)) using (var bsonReader = new BsonBinaryReader(stream, _readerSettings)) { var context = BsonDeserializationContext.CreateRoot(bsonReader); bsonReader.ReadStartDocument(); while (bsonReader.ReadBsonType() != BsonType.EndOfDocument) { bsonReader.SkipName(); yield return(DeserializeBsonValue(context)); } bsonReader.ReadEndDocument(); } }
private CompressedMessage GetCompressedMessage(MongoDBMessage message, CompressorType?compressorType = null) { if (!compressorType.HasValue) { compressorType = CompressorType.Zlib; } using (var memoryStream = new MemoryStream()) { var encoderFactory = new BinaryMessageEncoderFactory(memoryStream, _messageEncoderSettings); message.GetEncoder(encoderFactory).WriteMessage(message); var byteBuffer = new ByteArrayBuffer(memoryStream.ToArray()); var stream = new ByteBufferStream(byteBuffer); return(new CompressedMessage( message, stream, compressorType.Value)); } }
public ResponseMessage DecodeMessage(IByteBuffer buffer, IMessageEncoderSelector encoderSelector, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); _stopwatch.Stop(); _networkDuration = _stopwatch.Elapsed; ResponseMessage message; _stopwatch.Restart(); using (var stream = new ByteBufferStream(buffer, ownsBuffer: false)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, _messageEncoderSettings); var encoder = encoderSelector.GetEncoder(encoderFactory); message = (ResponseMessage)encoder.ReadMessage(); } _stopwatch.Stop(); _deserializationDuration = _stopwatch.Elapsed; return(message); }
/// <summary> /// Gets the index of a value in the array. /// </summary> /// <param name="value">The value to search for.</param> /// <param name="index">The zero based index at which to start the search.</param> /// <param name="count">The number of elements to search.</param> /// <returns>The zero based index of the value (or -1 if not found).</returns> public override int IndexOf(BsonValue value, int index, int count) { ThrowIfDisposed(); using (var stream = new ByteBufferStream(_slice, ownsBuffer: false)) using (var bsonReader = new BsonBinaryReader(stream, _readerSettings)) { var context = BsonDeserializationContext.CreateRoot(bsonReader); bsonReader.ReadStartDocument(); var i = 0; while (bsonReader.ReadBsonType() != BsonType.EndOfDocument) { bsonReader.SkipName(); if (i >= index) { if (count == 0) { return(-1); } if (DeserializeBsonValue(context).Equals(value)) { return(i); } count--; } else { bsonReader.SkipValue(); } i++; } bsonReader.ReadEndDocument(); return(-1); } }
/// <summary> /// Tests whether the document contains an element with the specified value. /// </summary> /// <param name="value">The value of the element to look for.</param> /// <returns> /// True if the document contains an element with the specified value. /// </returns> public override bool ContainsValue(BsonValue value) { ThrowIfDisposed(); using (var stream = new ByteBufferStream(_slice, ownsByteBuffer: false)) using (var bsonReader = new BsonBinaryReader(stream, _readerSettings)) { var context = BsonDeserializationContext.CreateRoot <RawBsonDocument>(bsonReader); bsonReader.ReadStartDocument(); while (bsonReader.ReadBsonType() != BsonType.EndOfDocument) { bsonReader.SkipName(); if (DeserializeBsonValue(context).Equals(value)) { return(true); } } bsonReader.ReadEndDocument(); return(false); } }
public void TestSerialWrites() { var str = new ByteBufferStream(16 * 1024); byte[] writeBlock = new byte[8 * 1024]; byte[] readBlock = new byte[7 * 1024]; int detritusSize = 0; for (int i = 0; i < 73; i++) { str.Write(writeBlock, 0, writeBlock.Length); detritusSize += writeBlock.Length; Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream); detritusSize -= str.Read(readBlock, 0, readBlock.Length); Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream); while (detritusSize >= readBlock.Length) { detritusSize -= str.Read(readBlock, 0, readBlock.Length); Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream); } } Assert.IsTrue(detritusSize == str.BytesCurrentlyInStream, "expect size ({0}) does not match actual size ({1})", detritusSize, str.BytesCurrentlyInStream); }
/// <summary> /// Reads the message. /// </summary> /// <returns>A message.</returns> public CompressedMessage ReadMessage() { var reader = CreateBinaryReader(); var stream = reader.BsonStream; var messageStartPosition = stream.Position; var messageLength = stream.ReadInt32(); EnsureMessageLengthIsValid(messageLength); var requestId = stream.ReadInt32(); var responseTo = stream.ReadInt32(); var opcode = (Opcode)stream.ReadInt32(); EnsureOpcodeIsValid(opcode); var originalOpcode = (Opcode)stream.ReadInt32(); var uncompressedSize = stream.ReadInt32(); var compressorType = (CompressorType)stream.ReadByte(); var compressor = _compressorSource.Get(compressorType); using (var uncompressedBuffer = new MultiChunkBuffer(new OutputBufferChunkSource(BsonChunkPool.Default))) using (var uncompressedStream = new ByteBufferStream(uncompressedBuffer, ownsBuffer: false)) { uncompressedStream.WriteInt32(uncompressedSize + MessageHeaderLength); uncompressedStream.WriteInt32(requestId); uncompressedStream.WriteInt32(responseTo); uncompressedStream.WriteInt32((int)originalOpcode); compressor.Decompress(stream, uncompressedStream); uncompressedStream.Position = 0; uncompressedBuffer.MakeReadOnly(); var originalMessageEncoderFactory = new BinaryMessageEncoderFactory(uncompressedStream, _encoderSettings, _compressorSource); var originalMessageEncoder = _originalEncoderSelector.GetEncoder(originalMessageEncoderFactory); var originalMessage = originalMessageEncoder.ReadMessage(); return(new CompressedMessage(originalMessage, null, compressorType)); } }
// public indexers /// <summary> /// Gets or sets a value by position. /// </summary> /// <param name="index">The position.</param> /// <returns>The value.</returns> public override BsonValue this[int index] { get { if (index < 0) { throw new ArgumentOutOfRangeException("index"); } ThrowIfDisposed(); using (var stream = new ByteBufferStream(_slice, ownsBuffer: false)) using (var bsonReader = new BsonBinaryReader(stream, _readerSettings)) { bsonReader.ReadStartDocument(); var i = 0; while (bsonReader.ReadBsonType() != BsonType.EndOfDocument) { bsonReader.SkipName(); if (i == index) { var context = BsonDeserializationContext.CreateRoot(bsonReader); return(DeserializeBsonValue(context)); } bsonReader.SkipValue(); i++; } bsonReader.ReadEndDocument(); throw new ArgumentOutOfRangeException("index"); } } set { throw new NotSupportedException("RawBsonArray instances are immutable."); } }
public void TwoReaderTwoWriterTest() { var str = new ByteBufferStream(1 * 1024 * 1024); Thread writerThread = new Thread(WriterThreadMethod) { Name = "WriterThread2" }; Thread writerThread2 = new Thread(WriterThreadMethod) { Name = "WriterThread2" }; Thread clientThread = new Thread(ReaderThreadMethod) { Name = "ReaderThread1" }; Thread clientThread2 = new Thread(ReaderThreadMethod) { Name = "ReaderThread2" }; clientThread.Start(str); clientThread2.Start(str); writerThread.Start(str); writerThread2.Start(str); writerThread.Join(); writerThread2.Join(); str.Dispose(); clientThread.Join(); clientThread2.Join(); Interlocked.Read(ref dataWritten); Interlocked.Read(ref dataRead); Assert.IsTrue(dataWritten == dataRead, "size comparison: data written ({0}) != data read ({1})", dataWritten, dataRead); }
public void Test20KDocument() { // manufacture an approximately 20K document using 200 strings each 100 characters long // it's enough to cause the document to straddle a chunk boundary var document = new BsonDocument(); var value = new string('x', 100); for (int i = 0; i < 200; i++) { var name = i.ToString(); document.Add(name, value); } // round trip tests var bson = document.ToBson(); var rehydrated = BsonSerializer.Deserialize <BsonDocument>(bson); Assert.IsTrue(bson.SequenceEqual(rehydrated.ToBson())); // test failure mode when 20 bytes are truncated from the buffer using (var byteBuffer = new MultiChunkBuffer(BsonChunkPool.Default)) using (var byteBufferStream = new ByteBufferStream(byteBuffer, ownsBuffer: true)) { using (var memoryStream = new MemoryStream(bson)) { memoryStream.CopyTo(byteBufferStream); } byteBufferStream.SetLength(byteBufferStream.Length - 20); byteBufferStream.Position = 0; using (var bsonReader = new BsonBinaryReader(byteBufferStream)) { Assert.Throws <EndOfStreamException>(() => BsonSerializer.Deserialize <BsonDocument>(bsonReader)); } } }
private TCommandResult ProcessReply(ConnectionId connectionId, ReplyMessage <RawBsonDocument> reply) { if (reply.NumberReturned == 0) { throw new MongoCommandException(connectionId, "Command returned no documents.", _command); } if (reply.NumberReturned > 1) { throw new MongoCommandException(connectionId, "Command returned multiple documents.", _command); } if (reply.QueryFailure) { var failureDocument = reply.QueryFailureDocument; throw ExceptionMapper.Map(connectionId, failureDocument) ?? new MongoCommandException(connectionId, "Command failed.", _command, failureDocument); } using (var rawDocument = reply.Documents[0]) { if (!rawDocument.GetValue("ok", false).ToBoolean()) { var binaryReaderSettings = new BsonBinaryReaderSettings(); if (_messageEncoderSettings != null) { binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict); binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy); } ; var materializedDocument = rawDocument.Materialize(binaryReaderSettings); var commandName = _command.GetElement(0).Name; if (commandName == "$query") { commandName = _command["$query"].AsBsonDocument.GetElement(0).Name; } var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, materializedDocument, "errmsg"); if (notPrimaryOrNodeIsRecoveringException != null) { throw notPrimaryOrNodeIsRecoveringException; } string message; BsonValue errmsgBsonValue; if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString) { var errmsg = errmsgBsonValue.ToString(); message = string.Format("Command {0} failed: {1}.", commandName, errmsg); } else { message = string.Format("Command {0} failed.", commandName); } var mappedException = ExceptionMapper.Map(connectionId, materializedDocument); if (mappedException != null) { throw mappedException; } throw new MongoCommandException(connectionId, message, _command, materializedDocument); } using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, _messageEncoderSettings); var encoder = (ReplyMessageBinaryEncoder <TCommandResult>)encoderFactory.GetReplyMessageEncoder <TCommandResult>(_resultSerializer); using (var reader = encoder.CreateBinaryReader()) { var context = BsonDeserializationContext.CreateRoot(reader); return(_resultSerializer.Deserialize(context)); } } } }
private void ProcessReplyMessage(CommandState state, ResponseMessage message, IByteBuffer buffer, ConnectionId connectionId, MessageEncoderSettings encoderSettings) { state.Stopwatch.Stop(); bool disposeOfDocuments = false; var replyMessage = message as ReplyMessage <RawBsonDocument>; if (replyMessage == null) { // ReplyMessage is generic, which means that we can't use it here, so, we need to use a different one... using (var stream = new ByteBufferStream(buffer, ownsBuffer: false)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, encoderSettings); replyMessage = (ReplyMessage <RawBsonDocument>)encoderFactory .GetReplyMessageEncoder(RawBsonDocumentSerializer.Instance) .ReadMessage(); disposeOfDocuments = true; } } try { if (replyMessage.CursorNotFound || replyMessage.QueryFailure || (state.ExpectedResponseType != ExpectedResponseType.Query && replyMessage.Documents.Count == 0)) { var queryFailureDocument = replyMessage.QueryFailureDocument; if (__securitySensitiveCommands.Contains(state.CommandName)) { queryFailureDocument = new BsonDocument(); } if (_failedEvent != null) { _failedEvent(new CommandFailedEvent( state.CommandName, new MongoCommandException( connectionId, string.Format("{0} command failed", state.CommandName), null, queryFailureDocument), state.OperationId, replyMessage.ResponseTo, connectionId, state.Stopwatch.Elapsed)); } } else { switch (state.ExpectedResponseType) { case ExpectedResponseType.Command: ProcessCommandReplyMessage(state, replyMessage, connectionId); break; case ExpectedResponseType.GLE: ProcessGLEReplyMessage(state, replyMessage, connectionId); break; case ExpectedResponseType.Query: ProcessQueryReplyMessage(state, replyMessage, connectionId); break; } } } finally { if (disposeOfDocuments && replyMessage.Documents != null) { replyMessage.Documents.ForEach(d => d.Dispose()); } } }
public async Task SendMessagesAsync(IEnumerable <RequestMessage> messages, MessageEncoderSettings messageEncoderSettings, CancellationToken cancellationToken) { Ensure.IsNotNull(messages, nameof(messages)); ThrowIfDisposedOrNotOpen(); var messagesToSend = messages.ToList(); var requestIds = messagesToSend.Select(x => x.RequestId).ToList(); try { if (_sendingMessagesEventHandler != null) { _sendingMessagesEventHandler(new ConnectionSendingMessagesEvent(_connectionId, requestIds)); } cancellationToken.ThrowIfCancellationRequested(); var stopwatch = Stopwatch.StartNew(); var outputBufferChunkSource = new OutputBufferChunkSource(BsonChunkPool.Default); using (var buffer = new MultiChunkBuffer(outputBufferChunkSource)) { using (var stream = new ByteBufferStream(buffer, ownsBuffer: false)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, messageEncoderSettings); foreach (var message in messagesToSend) { if (message.ShouldBeSent == null || message.ShouldBeSent()) { var encoder = message.GetEncoder(encoderFactory); encoder.WriteMessage(message); message.WasSent = true; } // Encoding messages includes serializing the // documents, so encoding message could be expensive // and worthy of us honoring cancellation here. cancellationToken.ThrowIfCancellationRequested(); } buffer.Length = (int)stream.Length; } stopwatch.Stop(); var serializationDuration = stopwatch.Elapsed; stopwatch.Restart(); await SendBufferAsync(buffer, cancellationToken).ConfigureAwait(false); stopwatch.Stop(); if (_sentMessagesEventHandler != null) { _sentMessagesEventHandler(new ConnectionSentMessagesEvent(_connectionId, requestIds, buffer.Length, stopwatch.Elapsed, serializationDuration)); } } } catch (Exception ex) { if (_failedSendingMessagesEvent != null) { _failedSendingMessagesEvent(new ConnectionSendingMessagesFailedEvent(_connectionId, requestIds, ex)); } throw; } }
private TCommandResult ProcessResponse(ConnectionId connectionId, CommandMessage responseMessage) { using (new CommandMessageDisposer(responseMessage)) { var rawDocument = responseMessage.Sections.OfType <Type0CommandMessageSection <RawBsonDocument> >().Single().Document; var binaryReaderSettings = new BsonBinaryReaderSettings(); if (_messageEncoderSettings != null) { binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict); binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy); } ; BsonValue clusterTime; if (rawDocument.TryGetValue("$clusterTime", out clusterTime)) { // note: we are assuming that _session is an instance of ClusterClockAdvancingClusterTime // and that calling _session.AdvanceClusterTime will have the side effect of advancing the cluster's ClusterTime also var materializedClusterTime = ((RawBsonDocument)clusterTime).Materialize(binaryReaderSettings); _session.AdvanceClusterTime(materializedClusterTime); } BsonValue operationTime; if (rawDocument.TryGetValue("operationTime", out operationTime)) { _session.AdvanceOperationTime(operationTime.AsBsonTimestamp); } if (!rawDocument.GetValue("ok", false).ToBoolean()) { var materializedDocument = rawDocument.Materialize(binaryReaderSettings); var commandName = _command.GetElement(0).Name; if (commandName == "$query") { commandName = _command["$query"].AsBsonDocument.GetElement(0).Name; } var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, materializedDocument, "errmsg"); if (notPrimaryOrNodeIsRecoveringException != null) { throw notPrimaryOrNodeIsRecoveringException; } var mappedException = ExceptionMapper.Map(connectionId, materializedDocument); if (mappedException != null) { throw mappedException; } string message; BsonValue errmsgBsonValue; if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString) { var errmsg = errmsgBsonValue.ToString(); message = string.Format("Command {0} failed: {1}.", commandName, errmsg); } else { message = string.Format("Command {0} failed.", commandName); } throw new MongoCommandException(connectionId, message, _command, materializedDocument); } using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false)) { using (var reader = new BsonBinaryReader(stream, binaryReaderSettings)) { var context = BsonDeserializationContext.CreateRoot(reader); return(_resultSerializer.Deserialize(context)); } } } }
private TCommandResult ProcessResponse(ConnectionId connectionId, CommandMessage responseMessage) { using (new CommandMessageDisposer(responseMessage)) { var rawDocument = responseMessage.Sections.OfType <Type0CommandMessageSection <RawBsonDocument> >().Single().Document; var binaryReaderSettings = new BsonBinaryReaderSettings(); if (_messageEncoderSettings != null) { binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict); #pragma warning disable 618 if (BsonDefaults.GuidRepresentationMode == GuidRepresentationMode.V2) { binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy); } #pragma warning restore 618 } ; BsonValue clusterTime; if (rawDocument.TryGetValue("$clusterTime", out clusterTime)) { // note: we are assuming that _session is an instance of ClusterClockAdvancingClusterTime // and that calling _session.AdvanceClusterTime will have the side effect of advancing the cluster's ClusterTime also var materializedClusterTime = ((RawBsonDocument)clusterTime).Materialize(binaryReaderSettings); _session.AdvanceClusterTime(materializedClusterTime); } BsonValue operationTime; if (rawDocument.TryGetValue("operationTime", out operationTime)) { _session.AdvanceOperationTime(operationTime.AsBsonTimestamp); } if (rawDocument.GetValue("ok", false).ToBoolean()) { if (rawDocument.TryGetValue("recoveryToken", out var rawRecoveryToken)) { var recoveryToken = ((RawBsonDocument)rawRecoveryToken).Materialize(binaryReaderSettings); _session.CurrentTransaction.RecoveryToken = recoveryToken; } } else { var materializedDocument = rawDocument.Materialize(binaryReaderSettings); var commandName = _command.GetElement(0).Name; if (commandName == "$query") { commandName = _command["$query"].AsBsonDocument.GetElement(0).Name; } var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, _command, materializedDocument, "errmsg"); if (notPrimaryOrNodeIsRecoveringException != null) { throw notPrimaryOrNodeIsRecoveringException; } var mappedException = ExceptionMapper.Map(connectionId, materializedDocument); if (mappedException != null) { throw mappedException; } string message; BsonValue errmsgBsonValue; if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString) { var errmsg = errmsgBsonValue.ToString(); message = string.Format("Command {0} failed: {1}.", commandName, errmsg); } else { message = string.Format("Command {0} failed.", commandName); } var exception = new MongoCommandException(connectionId, message, _command, materializedDocument); // https://jira.mongodb.org/browse/CSHARP-2678 if (IsRetryableWriteExceptionAndDeploymentDoesNotSupportRetryableWrites(exception)) { throw WrapNotSupportedRetryableWriteException(exception); } else { throw exception; } } if (rawDocument.Contains("writeConcernError")) { var materializedDocument = rawDocument.Materialize(binaryReaderSettings); var writeConcernError = materializedDocument["writeConcernError"].AsBsonDocument; var message = writeConcernError.AsBsonDocument.GetValue("errmsg", null)?.AsString; var writeConcernResult = new WriteConcernResult(materializedDocument); throw new MongoWriteConcernException(connectionId, message, writeConcernResult); } using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false)) { using (var reader = new BsonBinaryReader(stream, binaryReaderSettings)) { var context = BsonDeserializationContext.CreateRoot(reader); return(_resultSerializer.Deserialize(context)); } } } }
public async Task Execute(IJobExecutionContext context) { int jobId = int.Parse(context.JobDetail.Key.Name); int historyId = 0; IDisposableList disposableList = new IDisposableList(); try { // load the job from the database and create a history point for this scheduled execution BackupJob job = await backupJobRepository.Get(jobId); DateTime?lastRun = await backupJobRepository.GetLastRun(jobId); historyId = await backupJobRepository.AddHistory(job.ID); // sort the providers so they are executed in the correct order var providers = job.Providers.OrderBy(p => p.Order); // load and create the backup and storage providers IBackupProvider backupProvider = await providerMappingService.CreateProvider <IBackupProvider>(providers.FirstOrDefault()); IStorageProvider storageProvider = await providerMappingService.CreateProvider <IStorageProvider>(providers.LastOrDefault()); disposableList.AddRange(new IDisposable[] { backupProvider, storageProvider }); // load all the transform providers List <ITransformProvider> transformProviders = new List <ITransformProvider>(); foreach (var tp in providers.Where(p => p.Provider.Type == ProviderType.Transform)) { transformProviders.Add(await providerMappingService.CreateProvider <ITransformProvider>(tp)); } disposableList.AddRange(transformProviders); // fetch all items from the backup providers var items = await backupProvider.GetItems(lastRun); List <List <TransformBackupItem> > transformExecuteList = new List <List <TransformBackupItem> >(transformProviders.Count()); Dictionary <ITransformProvider, IEnumerable <MappedBackupItem> > transformers = new Dictionary <ITransformProvider, IEnumerable <MappedBackupItem> >(transformProviders.Count()); for (int i = 0; i < transformProviders.Count(); i++) { if (i > 0) { var mappedItems = await transformProviders[i].MapInput(transformers.Last().Value.Select(x => x.Output)); transformers.Add(transformProviders[i], mappedItems); List <TransformBackupItem> subTransformExecuteList = new List <TransformBackupItem>(); foreach (var mappedItem in mappedItems) { int currentMappedIndex = i; Func <Stream, Task> action = async(stream) => { //Dictionary<BackupItem, Stream> dictionary = new Dictionary<BackupItem, Stream>(); //foreach (var backupItem in mappedItem.Input) //{ // PassThroughStream backupItemStream = disposableList.CreateAndAdd<PassThroughStream>(); // dictionary.Add(backupItem, backupItemStream); //} //Task transformTask = transformProviders[currentMappedIndex].TransformItem(mappedItem.Output, stream, dictionary); //foreach (var kvp in dictionary) //{ // PassThroughStream passThroughStream = kvp.Value as PassThroughStream; // var transformBackupItem = transformExecuteList[currentMappedIndex - 1].FirstOrDefault(x => x.MappedBackupItem.Output == kvp.Key); // await transformBackupItem.Execute(passThroughStream); // passThroughStream.SetComplete(); //} //transformTask.Wait(); Dictionary <BackupItem, Stream> dictionary = new Dictionary <BackupItem, Stream>(); foreach (var backupItem in mappedItem.Input) { ByteBufferStream ms = disposableList.CreateAndAdd <ByteBufferStream>(); var transformBackupItem = transformExecuteList[currentMappedIndex - 1].FirstOrDefault(x => x.MappedBackupItem.Output == backupItem); await transformBackupItem.Execute(ms); dictionary.Add(backupItem, ms); } await transformProviders[currentMappedIndex].TransformItem(mappedItem.Output, stream, dictionary); }; subTransformExecuteList.Add(new TransformBackupItem { MappedBackupItem = mappedItem, Execute = action }); } transformExecuteList.Add(subTransformExecuteList); } else { var mappedItems = await transformProviders[i].MapInput(items); transformers.Add(transformProviders[i], mappedItems); List <TransformBackupItem> subTransformExecuteList = new List <TransformBackupItem>(); foreach (var mappedItem in mappedItems) { int currentMappedIndex = i; Func <Stream, Task> action = async(stream) => { Dictionary <BackupItem, Stream> dictionary = new Dictionary <BackupItem, Stream>(); foreach (var backupItem in mappedItem.Input) { Stream itemStream = await disposableList.CreateAndAdd(async() => await backupProvider.OpenRead(backupItem)); dictionary.Add(backupItem, itemStream); } await transformProviders[currentMappedIndex].TransformItem(mappedItem.Output, stream, dictionary); }; subTransformExecuteList.Add(new TransformBackupItem { MappedBackupItem = mappedItem, Execute = action }); } transformExecuteList.Add(subTransformExecuteList); } } if (transformProviders.Count() > 0) { foreach (var mappedItem in transformExecuteList.Last()) { using (WaitableByteBufferStream outputStream = new WaitableByteBufferStream()) { Task <bool> storeItem = storageProvider.StoreItem(mappedItem.MappedBackupItem.Output, outputStream); await mappedItem.Execute(outputStream); outputStream.SetComplete(); storeItem.Wait(); } } } else { foreach (var item in items) { using (Stream itemStream = await backupProvider.OpenRead(item)) { await storageProvider.StoreItem(item, itemStream); } } } await backupJobRepository.UpdateHistory(historyId, ExitCode.Success, "Backup completed successfully."); } catch (Exception ex) { logger.LogError(ex, $"Backup failed with ID {jobId} and History ID {historyId}."); if (historyId > 0) { await backupJobRepository.UpdateHistory(historyId, ExitCode.Failed, $"Backup failed with message: {ex.Message} ({ex.GetType()})"); } } finally { foreach (IDisposable item in disposableList) { try { item?.Dispose(); } catch (NotImplementedException) { // ignore this exception } catch (Exception ex) { // log every other error logger.LogError(ex, $"Failed to dispose item."); } } disposableList.Clear(); disposableList = null; } }
private TCommandResult ProcessReply(ConnectionId connectionId, ReplyMessage <RawBsonDocument> reply) { if (reply.NumberReturned == 0) { throw new MongoCommandException(connectionId, "Command returned no documents.", _command); } if (reply.NumberReturned > 1) { throw new MongoCommandException(connectionId, "Command returned multiple documents.", _command); } if (reply.QueryFailure) { var failureDocument = reply.QueryFailureDocument; throw ExceptionMapper.Map(connectionId, failureDocument) ?? new MongoCommandException(connectionId, "Command failed.", _command, failureDocument); } using (var rawDocument = reply.Documents[0]) { var binaryReaderSettings = new BsonBinaryReaderSettings(); if (_messageEncoderSettings != null) { binaryReaderSettings.Encoding = _messageEncoderSettings.GetOrDefault <UTF8Encoding>(MessageEncoderSettingsName.ReadEncoding, Utf8Encodings.Strict); #pragma warning disable 618 if (BsonDefaults.GuidRepresentationMode == GuidRepresentationMode.V2) { binaryReaderSettings.GuidRepresentation = _messageEncoderSettings.GetOrDefault <GuidRepresentation>(MessageEncoderSettingsName.GuidRepresentation, GuidRepresentation.CSharpLegacy); } #pragma warning restore 618 } ; BsonValue clusterTime; if (rawDocument.TryGetValue("$clusterTime", out clusterTime)) { // note: we are assuming that _session is an instance of ClusterClockAdvancingClusterTime // and that calling _session.AdvanceClusterTime will have the side effect of advancing the cluster's ClusterTime also var materializedClusterTime = ((RawBsonDocument)clusterTime).Materialize(binaryReaderSettings); _session.AdvanceClusterTime(materializedClusterTime); } BsonValue operationTime; if (rawDocument.TryGetValue("operationTime", out operationTime)) { _session.AdvanceOperationTime(operationTime.AsBsonTimestamp); } if (!rawDocument.GetValue("ok", false).ToBoolean()) { var materializedDocument = rawDocument.Materialize(binaryReaderSettings); var commandName = _command.GetElement(0).Name; if (commandName == "$query") { commandName = _command["$query"].AsBsonDocument.GetElement(0).Name; } var notPrimaryOrNodeIsRecoveringException = ExceptionMapper.MapNotPrimaryOrNodeIsRecovering(connectionId, _command, materializedDocument, "errmsg"); if (notPrimaryOrNodeIsRecoveringException != null) { throw notPrimaryOrNodeIsRecoveringException; } string message; BsonValue errmsgBsonValue; if (materializedDocument.TryGetValue("errmsg", out errmsgBsonValue) && errmsgBsonValue.IsString) { var errmsg = errmsgBsonValue.ToString(); message = string.Format("Command {0} failed: {1}.", commandName, errmsg); } else { message = string.Format("Command {0} failed.", commandName); } var mappedException = ExceptionMapper.Map(connectionId, materializedDocument); if (mappedException != null) { throw mappedException; } throw new MongoCommandException(connectionId, message, _command, materializedDocument); } if (rawDocument.Contains("writeConcernError")) { var materializedDocument = rawDocument.Materialize(binaryReaderSettings); var writeConcernError = materializedDocument["writeConcernError"].AsBsonDocument; var message = writeConcernError.AsBsonDocument.GetValue("errmsg", null)?.AsString; var writeConcernResult = new WriteConcernResult(materializedDocument); throw new MongoWriteConcernException(connectionId, message, writeConcernResult); } using (var stream = new ByteBufferStream(rawDocument.Slice, ownsBuffer: false)) { var encoderFactory = new BinaryMessageEncoderFactory(stream, _messageEncoderSettings); var encoder = (ReplyMessageBinaryEncoder <TCommandResult>)encoderFactory.GetReplyMessageEncoder <TCommandResult>(_resultSerializer); using (var reader = encoder.CreateBinaryReader()) { var context = BsonDeserializationContext.CreateRoot(reader); return(_resultSerializer.Deserialize(context)); } } } }