// Connect all the INode events. private INode ObserveNode(INode node) { node.Dead += n => OnNodeEvent(() => ProcessDeadNode(n)); node.ConnectionError += (n, e) => OnNodeEvent(() => ProcessNodeError(n, e)); node.DecodeError += (n, e) => OnNodeEvent(() => ProcessDecodeError(n, e)); node.InternalError += (n, e) => OnNodeEvent(() => ProcessNodeError(n, e)); node.RequestSent += _ => Statistics.UpdateRequestSent(); node.ResponseReceived += (n, l) => Statistics.UpdateResponseReceived(GetNodeId(n), l); node.ProduceBatchSent += (_, c, s) => { Statistics.UpdateRawProduced(c); Statistics.UpdateRawProducedBytes(s); }; node.FetchResponseReceived += (_, c, s) => { Statistics.UpdateRawReceived(c); Statistics.UpdateRawReceivedBytes(s); }; node.Connected += n => OnNodeEvent(() => Logger.LogInformation(string.Format("Connected to {0}", GetNodeName(n)))); node.ProduceAcknowledgement += (n, ack) => ProduceRouter.Acknowledge(ack); node.FetchAcknowledgement += (n, r) => ConsumeRouter.Acknowledge(r); node.OffsetAcknowledgement += (n, r) => ConsumeRouter.Acknowledge(r); node.NoMoreRequestSlot += n => NodeMaxRequestReached(n); node.RequestTimeout += n => Statistics.UpdateRequestTimeout(GetNodeId(n)); return(node); }
public void TestMessageAreSent_GlobalBatching() { var node = new Mock <INode>(); node.Setup(n => n.Post(It.IsAny <IBatchByTopicByPartition <ProduceMessage> >())).Returns(true); var cluster = new Mock <ICluster>(); cluster.Setup(c => c.RequireNewRoutingTable()) .Returns(() => Task.FromResult(new RoutingTable(new Dictionary <string, Partition[]> { { "toto", new[] { new Partition { Id = 0, Leader = node.Object } } } }))); var producer = new ProduceRouter(cluster.Object, new Configuration { BatchStrategy = BatchStrategy.Global, ProduceBatchSize = 2, ProduceBufferingTime = TimeSpan.FromDays(2), // To make sure there's no race on batch tick TaskScheduler = new CurrentThreadTaskScheduler() }, Pool); producer.Route("toto", new Message(), Partitions.Any, DateTime.UtcNow.AddMinutes(42)); // The second Route will tick a batch. All of this is done on the test thread thanks // to CurrentThreadScheduler and large ProduceBufferingTime. producer.Route("toto", new Message(), Partitions.Any, DateTime.UtcNow.AddMinutes(42)); producer.Route("toto", new Message(), Partitions.Any, DateTime.UtcNow.AddMinutes(42)); node.Verify(n => n.Post(It.Is <IBatchByTopicByPartition <ProduceMessage> >(b => b.Count == 2))); }
public void TestFilterMinInSyncEnough() { var node = new Mock <INode>(); node.Setup(n => n.Produce(It.IsAny <ProduceMessage>())).Returns(true); var cluster = new Mock <ICluster>(); cluster.Setup(c => c.RequireNewRoutingTable()) .Returns(() => Task.FromResult(new RoutingTable(new Dictionary <string, Partition[]> { { "toto", new[] { new Partition { Id = 0, Leader = node.Object, NbIsr = 2 } } } }))); var producer = new ProduceRouter(cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), SerializationConfig = new SerializationConfig { SerializeOnProduce = true }, MinInSyncReplicas = 2 }, Pool); var key = new byte[] { 65, 66 }; // 'A', 'B' var value = new byte[] { 67, 68 }; // 'C', 'D' producer.Route("toto", new Message { Key = key, Value = value }, Partitions.Any, DateTime.UtcNow.AddMinutes(42)); node.Verify(n => n.Produce(It.IsAny <ProduceMessage>()), Times.Once()); }
private void InitRouter(ProduceRouter produceRouter) { _produceRouter = produceRouter; _produceRouter.MessageEnqueued += _ => { ++MessagesEnqueued; if (_finished != null) { _finished.Signal(); } }; _produceRouter.MessageReEnqueued += _ => { ++MessagesReEnqueued; if (_finished != null) { _finished.Signal(); } }; _produceRouter.MessageExpired += (t, m) => { ++MessagesExpired; if (_finished != null) { _finished.Signal(); } }; _produceRouter.MessageRouted += _ => { ++MessagesRouted; if (_finished != null) { _finished.Signal(); } }; _produceRouter.MessagePostponed += _ => { ++MessagesPostponed; if (_finished != null) { _finished.Signal(); } }; _produceRouter.RoutingTableRequired += () => { ++RoutingTableRequired; if (_finished != null) { _finished.Signal(); } }; }
public async Task Stop() { if (!_started) { return; } _timeoutScheduler.Dispose(); _refreshMetadataTimer.Dispose(); await ConsumeRouter.Stop(); await ProduceRouter.Stop(); _agent.Complete(); await _agent.Completion; await Task.WhenAll(_nodes.Keys.Select(n => n.Stop())); _started = false; }
public void TestThrottled() { var node = new Mock <INode>(); node.Setup(n => n.Produce(It.IsAny <ProduceMessage>())).Returns(true); var cluster = new Mock <ICluster>(); cluster.Setup(c => c.RequireNewRoutingTable()) .Returns(() => Task.FromResult(new RoutingTable(new Dictionary <string, Partition[]> { { "toto", new[] { new Partition { Id = 0, Leader = node.Object, NbIsr = 1 } } } }))); var producer = new ProduceRouter(cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler() }, Pool); int throttled = -1; producer.Throttled += t => throttled = t; var key = new Mock <IDisposable>(); var value = new Mock <IDisposable>(); var acknowledgement = new ProduceAcknowledgement { OriginalBatch = new TestBatchByTopicByPartition(new[] { ProduceMessage.New("test1p", 0, new Message { Key = key.Object, Value = value.Object }, DateTime.UtcNow.AddDays(1)) }), ReceiveDate = DateTime.UtcNow, ProduceResponse = new ProduceResponse { ThrottleTime = 42, ProducePartitionResponse = new CommonResponse <ProducePartitionResponse> { TopicsResponse = new[] { new TopicData <ProducePartitionResponse> { TopicName = "test1p", PartitionsData = new[] { new ProducePartitionResponse() } }, } } } }; producer.Acknowledge(acknowledgement); Assert.AreEqual(42, throttled); }