public void Store(MessageChunk chunk) => Add(new InMemoryStoredChunk { MessageId = chunk.OriginalMessageId, ChunkId = chunk.ChunkId, Content = chunk.Content });
/// <summary> /// Breaks down an insert message that may be too large into managable sizes. /// When inserting only one document there will be only one chunk. However chances /// are that when inserting thousands of documents at once there will be many. /// </summary> protected void ChunkMessage(BsonWriter writer) { var baseSize = CalculateBaseSize(writer); var chunk = new MessageChunk { Size = baseSize, Documents = new List <object>() }; foreach (var document in Documents) { var documentSize = writer.CalculateSize(document); if (documentSize + baseSize >= MaximumMessageSize) { throw new MongoException("Document is too big to fit in a message."); } if (documentSize + chunk.Size > MaximumMessageSize) { _chunks.Add(chunk); chunk = new MessageChunk { Size = baseSize, Documents = new List <object>() }; } chunk.Documents.Add(document); chunk.Size += documentSize; } _chunks.Add(chunk); }
public void JoinIfComplete_AllChunks_Joined() { var originalMessage = new BinaryMessage { MessageId = Guid.NewGuid(), Content = GetByteArray(500) }; var originalSerializedMessage = _serializer.Serialize(originalMessage); var chunks = new MessageChunk[3]; chunks[0] = new MessageChunk { MessageId = Guid.NewGuid(), ChunkId = 0, ChunksCount = 3, OriginalMessageId = originalMessage.MessageId.ToString(), Content = originalSerializedMessage.AsMemory().Slice(0, 300).ToArray() }; chunks[1] = new MessageChunk { MessageId = Guid.NewGuid(), ChunkId = 1, ChunksCount = 3, OriginalMessageId = originalMessage.MessageId.ToString(), Content = originalSerializedMessage.AsMemory().Slice(300, 300).ToArray() }; chunks[2] = new MessageChunk { MessageId = Guid.NewGuid(), ChunkId = 2, ChunksCount = 3, OriginalMessageId = originalMessage.MessageId.ToString(), Content = originalSerializedMessage.AsMemory().Slice(600).ToArray() }; var result = new ChunkConsumer(_store).JoinIfComplete(chunks[0]); result.Should().BeNull(); result = new ChunkConsumer(_store).JoinIfComplete(chunks[1]); result.Should().BeNull(); result = new ChunkConsumer(_store).JoinIfComplete(chunks[2]); result.Should().NotBeNull(); var deserializedResult = (BinaryMessage)_serializer.Deserialize(result); deserializedResult.Content.Should().BeEquivalentTo(originalMessage.Content); }
/// <summary> /// Writes out a header and the chunk of documents. /// </summary> /// <param name = "stream"></param> /// <param name = "chunk"></param> protected void WriteChunk(Stream stream, MessageChunk chunk) { WriteHeader(new BinaryWriter(stream), chunk.Size); var writer = new BsonWriter(stream, _bsonWriterSettings); writer.WriteValue(BsonType.Integer, 0); writer.Write(FullCollectionName, false); foreach (var document in chunk.Documents) { writer.WriteObject(document); } writer.Flush(); }
/// <summary> /// Breaks down an insert message that may be too large into managable sizes. /// When inserting only one document there will be only one chunk. However chances /// are that when inserting thousands of documents at once there will be many. /// </summary> protected void ChunkMessage(BsonWriter writer) { int baseSize = CalculateBaseSize(writer); MessageChunk chunk = new MessageChunk(){Size = baseSize, Documents = new List<Document>()}; foreach(Document doc in this.Documents){ int docSize = writer.CalculateSize(doc); if(docSize + baseSize >= MessageBase.MaximumMessageSize) throw new MongoException("Document is too big to fit in a message."); if(docSize + chunk.Size > MessageBase.MaximumMessageSize){ chunks.Add(chunk); chunk = new MessageChunk(){Size = baseSize, Documents = new List<Document>()}; } chunk.Documents.Add(doc); chunk.Size += docSize; } chunks.Add(chunk); }
/// <summary> /// Breaks down an insert message that may be too large into managable sizes. /// When inserting only one document there will be only one chunk. However chances /// are that when inserting thousands of documents at once there will be many. /// </summary> protected void ChunkMessage(BsonWriter writer){ var baseSize = CalculateBaseSize(writer); var chunk = new MessageChunk{Size = baseSize, Documents = new List<object>()}; foreach(var document in Documents){ var documentSize = writer.CalculateSize(document); if(documentSize + baseSize >= MaximumMessageSize) throw new MongoException("Document is too big to fit in a message."); if(documentSize + chunk.Size > MaximumMessageSize){ _chunks.Add(chunk); chunk = new MessageChunk{Size = baseSize, Documents = new List<object>()}; } chunk.Documents.Add(document); chunk.Size += documentSize; } _chunks.Add(chunk); }
private void EnsureMessageFetched() { var maxBytes = _config.FetchBytes; var maxWaitTime = _config.FetchMilliseconds; if (_messages == null) { maxWaitTime = _config.FetchMilliseconds / 2; } else if (_messages.Count == 0 || _messages.Position == _messages.Count - 1) { maxWaitTime = _config.FetchMilliseconds * 2; } else { return; } if (_coordinator.State != CoordinatorState.Stable) { Trace.TraceWarning("{0:HH:mm:ss.fff} [{1:d2}] #6 Rebalance {2}, fetch interrupted", DateTime.Now, Thread.CurrentThread.ManagedThreadId, _coordinator.State); BlockForRebalace(_config.RebalaceBlockMilliseconds); } if (_coordinator.State != CoordinatorState.Stable) { throw new InvalidOperationException("Load balancing is still not complete"); } var partition = _partitionDispatcher.SelectParition(); var offset = _offsets.GetCurrentOffset(partition); Trace.TraceInformation("{0:HH:mm:ss.fff} [{1:d2}] Fetch group '{2}', topic '{3}'[{4}], offset {5}", DateTime.Now, Thread.CurrentThread.ManagedThreadId, GroupId, Topic, partition, offset); var messages = _client.Fetch(Topic, partition, offset, maxBytes: maxBytes, maxWaitTime: maxWaitTime).ToArray(); _messages = new MessageChunk(messages, partition); }
/// <summary> /// Writes out a header and the chunk of documents. /// </summary> /// <param name="stream"></param> /// <param name="chunk"></param> protected void WriteChunk(Stream stream, MessageChunk chunk) { WriteHeader(new BinaryWriter(stream), chunk.Size); BsonWriter writer = new BsonWriter(stream); writer.WriteValue(BsonDataType.Integer,0); writer.WriteString(this.FullCollectionName); foreach(Document doc in chunk.Documents){ writer.Write(doc); } writer.Flush(); }
/// <summary> /// Writes out a header and the chunk of documents. /// </summary> /// <param name = "stream"></param> /// <param name = "chunk"></param> protected void WriteChunk(Stream stream, MessageChunk chunk){ WriteHeader(new BinaryWriter(stream), chunk.Size); var writer = new BsonWriter(stream, _bsonWriterSettings); writer.WriteValue(BsonType.Integer, 0); writer.Write(FullCollectionName, false); foreach(var document in chunk.Documents) writer.WriteObject(document); writer.Flush(); }