// Connect all the INode events.
private INode ObserveNode(INode node)
{
node.Dead += n => OnNodeEvent(() => ProcessDeadNode(n));
node.ConnectionError += (n, e) => OnNodeEvent(() => ProcessNodeError(n, e));
node.DecodeError += (n, e) => OnNodeEvent(() => ProcessDecodeError(n, e));
node.InternalError += (n, e) => OnNodeEvent(() => ProcessNodeError(n, e));
node.RequestSent += _ => Statistics.UpdateRequestSent();
node.ResponseReceived += (n, l) => Statistics.UpdateResponseReceived(GetNodeId(n), l);
node.ProduceBatchSent += (_, c, s) =>
{
Statistics.UpdateRawProduced(c);
Statistics.UpdateRawProducedBytes(s);
};
node.FetchResponseReceived += (_, c, s) =>
{
Statistics.UpdateRawReceived(c);
Statistics.UpdateRawReceivedBytes(s);
};
node.Connected +=
n => OnNodeEvent(() => Logger.LogInformation(string.Format("Connected to {0}", GetNodeName(n))));
node.ProduceAcknowledgement += (n, ack) => ProduceRouter.Acknowledge(ack);
node.FetchAcknowledgement += (n, r) => ConsumeRouter.Acknowledge(r);
node.OffsetAcknowledgement += (n, r) => ConsumeRouter.Acknowledge(r);
node.NoMoreRequestSlot += n => NodeMaxRequestReached(n);
node.RequestTimeout += n => Statistics.UpdateRequestTimeout(GetNodeId(n));
return(node);
}
public void TestMessageAreSent_GlobalBatching()
{
var node = new Mock <INode>();
node.Setup(n => n.Post(It.IsAny <IBatchByTopicByPartition <ProduceMessage> >())).Returns(true);
var cluster = new Mock <ICluster>();
cluster.Setup(c => c.RequireNewRoutingTable())
.Returns(() =>
Task.FromResult(new RoutingTable(new Dictionary <string, Partition[]>
{
{ "toto", new[] { new Partition {
Id = 0, Leader = node.Object
} } }
})));
var producer = new ProduceRouter(cluster.Object,
new Configuration
{
BatchStrategy = BatchStrategy.Global,
ProduceBatchSize = 2,
ProduceBufferingTime = TimeSpan.FromDays(2), // To make sure there's no race on batch tick
TaskScheduler = new CurrentThreadTaskScheduler()
}, Pool);
producer.Route("toto", new Message(), Partitions.Any, DateTime.UtcNow.AddMinutes(42));
// The second Route will tick a batch. All of this is done on the test thread thanks
// to CurrentThreadScheduler and large ProduceBufferingTime.
producer.Route("toto", new Message(), Partitions.Any, DateTime.UtcNow.AddMinutes(42));
producer.Route("toto", new Message(), Partitions.Any, DateTime.UtcNow.AddMinutes(42));
node.Verify(n => n.Post(It.Is <IBatchByTopicByPartition <ProduceMessage> >(b => b.Count == 2)));
}
public void TestFilterMinInSyncEnough()
{
var node = new Mock <INode>();
node.Setup(n => n.Produce(It.IsAny <ProduceMessage>())).Returns(true);
var cluster = new Mock <ICluster>();
cluster.Setup(c => c.RequireNewRoutingTable())
.Returns(() =>
Task.FromResult(new RoutingTable(new Dictionary <string, Partition[]>
{
{ "toto", new[] { new Partition {
Id = 0, Leader = node.Object, NbIsr = 2
} } }
})));
var producer = new ProduceRouter(cluster.Object,
new Configuration
{
TaskScheduler = new CurrentThreadTaskScheduler(),
SerializationConfig = new SerializationConfig {
SerializeOnProduce = true
},
MinInSyncReplicas = 2
}, Pool);
var key = new byte[] { 65, 66 }; // 'A', 'B'
var value = new byte[] { 67, 68 }; // 'C', 'D'
producer.Route("toto", new Message {
Key = key, Value = value
}, Partitions.Any,
DateTime.UtcNow.AddMinutes(42));
node.Verify(n => n.Produce(It.IsAny <ProduceMessage>()), Times.Once());
}
private void InitRouter(ProduceRouter produceRouter)
{
_produceRouter = produceRouter;
_produceRouter.MessageEnqueued += _ =>
{
++MessagesEnqueued;
if (_finished != null)
{
_finished.Signal();
}
};
_produceRouter.MessageReEnqueued += _ =>
{
++MessagesReEnqueued;
if (_finished != null)
{
_finished.Signal();
}
};
_produceRouter.MessageExpired += (t, m) =>
{
++MessagesExpired;
if (_finished != null)
{
_finished.Signal();
}
};
_produceRouter.MessageRouted += _ =>
{
++MessagesRouted;
if (_finished != null)
{
_finished.Signal();
}
};
_produceRouter.MessagePostponed += _ =>
{
++MessagesPostponed;
if (_finished != null)
{
_finished.Signal();
}
};
_produceRouter.RoutingTableRequired += () =>
{
++RoutingTableRequired;
if (_finished != null)
{
_finished.Signal();
}
};
}
public async Task Stop()
{
if (!_started)
{
return;
}
_timeoutScheduler.Dispose();
_refreshMetadataTimer.Dispose();
await ConsumeRouter.Stop();
await ProduceRouter.Stop();
_agent.Complete();
await _agent.Completion;
await Task.WhenAll(_nodes.Keys.Select(n => n.Stop()));
_started = false;
}
public void TestThrottled()
{
var node = new Mock <INode>();
node.Setup(n => n.Produce(It.IsAny <ProduceMessage>())).Returns(true);
var cluster = new Mock <ICluster>();
cluster.Setup(c => c.RequireNewRoutingTable())
.Returns(() =>
Task.FromResult(new RoutingTable(new Dictionary <string, Partition[]>
{
{ "toto", new[] { new Partition {
Id = 0, Leader = node.Object, NbIsr = 1
} } }
})));
var producer = new ProduceRouter(cluster.Object,
new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler()
}, Pool);
int throttled = -1;
producer.Throttled += t => throttled = t;
var key = new Mock <IDisposable>();
var value = new Mock <IDisposable>();
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test1p", 0, new Message {
Key = key.Object, Value = value.Object
},
DateTime.UtcNow.AddDays(1))
}),
ReceiveDate = DateTime.UtcNow,
ProduceResponse =
new ProduceResponse
{
ThrottleTime = 42,
ProducePartitionResponse =
new CommonResponse <ProducePartitionResponse>
{
TopicsResponse =
new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test1p",
PartitionsData = new[] { new ProducePartitionResponse() }
},
}
}
}
};
producer.Acknowledge(acknowledgement);
Assert.AreEqual(42, throttled);
}
