public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_Partition_Blacklisted(int partitionIdBlacklisted) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, }; var blacklistedPartitions = new Dictionary <int, DateTime> { { partitionIdBlacklisted, DateTime.MaxValue } }; var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>()); var partitioner = new PartitionSelector(partitionStrategy); var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = "ThisIsMyKey" }, new DateTime()); for (var i = 0; i < 300; i++) { var partition = partitioner.GetPartition(message, partitions, blacklistedPartitions); Assert.IsTrue(partition.Id != Partition.None.Id); Assert.IsTrue(partition.Id != partitionIdBlacklisted); } }
public void TestRobinPartitionAssignWhenFiltered() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, }; var filter = new Dictionary <int, DateTime>(); int delay = partitions.Length + 2; var partitionStrategy = new RoundRobinPartitionSelection(delay); var partitioner = new PartitionSelector(partitionStrategy); var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(1, partition.Id); filter.Add(1, DateTime.UtcNow); var batch = GetPartitions(delay, partitioner, partitions, filter); Assert.AreEqual(delay, batch.Count); Assert.IsTrue(batch.All(p => p.Id == 2), "The round-robin threshold wasn't properly reset after previous partition was blacklisted"); }
public void TestAccumulatorByNodeByTopicByPartition(int batchSize, int time) { INode n1 = new NodeMock("n1"); INode n2 = new NodeMock("n2"); var accumulator = new AccumulatorByNodeByTopicByPartition <Tuple <string, int, int> >(t => t.Item1, t => t.Item2, batchSize, TimeSpan.FromMilliseconds(time)); var count = new CountdownEvent(2); var d = new Dictionary <INode, IBatchByTopicByPartition <Tuple <string, int, int> > >(); accumulator.NewBatch += (n, b) => { d[n] = b; count.Signal(); }; accumulator.Add(Tuple.Create(n1, Tuple.Create("1", 1, 1))); accumulator.Add(Tuple.Create(n1, Tuple.Create("1", 1, 2))); accumulator.Add(Tuple.Create(n2, Tuple.Create("2", 1, 1))); accumulator.Add(Tuple.Create(n2, Tuple.Create("2", 2, 2))); accumulator.Add(Tuple.Create(n2, Tuple.Create("2", 2, 3))); count.Wait(); Assert.AreEqual(2, d[n1].Count); Assert.AreEqual(3, d[n2].Count); d[n1].Dispose(); d[n2].Dispose(); }
public void Test_MessageKeyPartitionSelection_Is_Consistent() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, }; var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>()); var partitioner = new PartitionSelector(partitionStrategy); var message1 = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = "ThisIsMyKey" }, new DateTime()); var message2 = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = "ThisIsMyOtherKey" }, new DateTime()); var expectedPartition1 = partitioner.GetPartition(message1, partitions); var expectedPartition2 = partitioner.GetPartition(message2, partitions); for (var i = 0; i < 300; i++) { var currentPartition1 = partitioner.GetPartition(message1, partitions); var currentPartition2 = partitioner.GetPartition(message2, partitions); Assert.AreEqual(expectedPartition1.Id, currentPartition1.Id); Assert.AreEqual(expectedPartition2.Id, currentPartition2.Id); } }
public void TestSignalDeadNode() { var node = new NodeMock(); var routes = new Dictionary <string, Partition[]> { { "test1p", new[] { new Partition { Id = 0, Leader = node } } }, { "test2p", new[] { new Partition { Id = 1, Leader = new NodeMock() }, new Partition { Id = 2, Leader = node }, new Partition { Id = 3, Leader = new NodeMock() } } }, }; var routingTable = new RoutingTable(routes); Assert.AreEqual(1, routingTable.GetPartitions("test1p").Length); Assert.AreEqual(3, routingTable.GetPartitions("test2p").Length); routingTable = new RoutingTable(routingTable, node); Assert.AreEqual(0, routingTable.GetPartitions("test1p").Length); Assert.AreEqual(2, routingTable.GetPartitions("test2p").Length); }
public void TestFilterMinInSync() { var node = new NodeMock(); var routes = new Dictionary <string, Partition[]> { { "test1p", new[] { new Partition { Id = 0, Leader = node } } }, { "test2p", new[] { new Partition { Id = 1, Leader = new NodeMock(), NbIsr = 1 }, new Partition { Id = 2, Leader = node }, new Partition { Id = 3, Leader = new NodeMock() } } }, }; var routingTable = new RoutingTable(routes); Assert.AreEqual(1, routingTable.GetPartitions("test1p").Length); Assert.AreEqual(3, routingTable.GetPartitions("test2p").Length); routingTable = new RoutingTable(routingTable, 1); Assert.AreEqual(0, routingTable.GetPartitions("test1p").Length); Assert.AreEqual(1, routingTable.GetPartitions("test2p").Length); }
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitionStrategy = new RoundRobinPartitionSelection(delay: delay, startSeed: startSeed); var partitioner = new PartitionSelector(partitionStrategy); foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition( ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions).Id); } } }
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitioner = new PartitionSelector(delay, startSeed); foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(Partitions.Any, partitions).Id); } } }
public void TestRoundRobinPartitionAssign(int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitionStrategy = new RoundRobinPartitionSelection(delay); var partitioner = new PartitionSelector(partitionStrategy); delay = delay <= 0 ? 1 : delay; foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual(partition.Id, partitioner .GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions) .Id); } } }
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_MessageKey_Null() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, }; var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>()); var partitioner = new PartitionSelector(partitionStrategy); var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = null }, new DateTime()); var partition = partitioner.GetPartition(message, partitions); Assert.IsTrue(partition.Id != Partition.None.Id); }
public void TestRoundRobinPartitionAssign(int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitioner = new PartitionSelector(delay); delay = delay <= 0 ? 1 : delay; foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual(partition.Id, partitioner.GetPartition(Partitions.Any, partitions).Id); } } }
private void SetUp(Configuration config) { _messagesSentByTopic = new Dictionary <string, int> { { "test", 0 }, { "test2", 0 }, { "test1p", 0 }, { "test2p", 0 }, { "testallp", 0 } }; _nodes = new NodeMock[5]; for (int i = 0; i < _nodes.Length; ++i) { _nodes[i] = new NodeMock(); _nodes[i].MessageReceived += t => _messagesSentByTopic[t] += 1; // No need to use interlocked, NodeMock is synchronous } _routes = new Dictionary <string, Partition[]> { { "test1p", new[] { new Partition { Id = 0, Leader = _nodes[0] } } }, { "test2p", new[] { new Partition { Id = 0, Leader = _nodes[0] }, new Partition { Id = 1, Leader = _nodes[1] } } }, { "testallp", new[] { new Partition { Id = 0, Leader = _nodes[0] }, new Partition { Id = 1, Leader = _nodes[1] }, new Partition { Id = 2, Leader = _nodes[2] }, new Partition { Id = 3, Leader = _nodes[3] }, new Partition { Id = 4, Leader = _nodes[4] }, } } }; _cluster = new ClusterMock(_routes); MessagesEnqueued = MessagesExpired = MessagesReEnqueued = MessagesRouted = MessagesPostponed = RoutingTableRequired = 0; InitRouter(new ProduceRouter(_cluster, config, Pool)); _finished = null; }
public void TestRoutingTableReturnsEmptyForAbsentTopic() { var node = new NodeMock(); var routes = new Dictionary <string, Partition[]> { { "test1p", new[] { new Partition { Id = 0, Leader = node } } }, }; var routingTable = new RoutingTable(routes); Assert.Less(0, routingTable.GetPartitions("test1p").Length); Assert.AreEqual(0, routingTable.GetPartitions("tortemoque").Length); }
public void TestRoutingTableReturnsPartitions() { var node = new NodeMock(); var routes = new Dictionary <string, Partition[]> { { "test1p", new[] { new Partition { Id = 0, Leader = node } } }, }; var routingTable = new RoutingTable(routes); var partitions = routingTable.GetPartitions("test1p"); Assert.AreEqual(1, partitions.Length); Assert.AreEqual(0, partitions[0].Id); Assert.AreSame(node, partitions[0].Leader); }
public void TestOverflow() { var nodeMock = new NodeMock(); var partitions = Enumerable.Range(0, 10).Select(i => new Partition { Id = i, Leader = nodeMock }).ToArray(); var partitioner = new PartitionSelector(1, int.MaxValue); var batch = GetPartitions(partitions.Length, partitioner, partitions, null); var ids = batch.Select(p => p.Id).ToArray(); var expectedIds = new[] { 7, 8, 9, 0, 1, 2, 3, 4, 5, 6 }; Assert.IsTrue(expectedIds.SequenceEqual(ids)); }
public void TestFilter() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var filter = new Dictionary <int, DateTime>(); filter[0] = DateTime.UtcNow; filter[2] = DateTime.UtcNow; filter[4] = DateTime.UtcNow; var partitionStrategy = new RoundRobinPartitionSelection(); var partitioner = new PartitionSelector(partitionStrategy); var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(1, partition.Id); partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(3, partition.Id); partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(1, partition.Id); }
public void TestFilter() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var filter = new Dictionary <int, DateTime>(); filter[0] = DateTime.UtcNow; filter[2] = DateTime.UtcNow; filter[4] = DateTime.UtcNow; var partitioner = new PartitionSelector(); var partition = partitioner.GetPartition(Partitions.Any, partitions, filter); Assert.AreEqual(1, partition.Id); partition = partitioner.GetPartition(Partitions.Any, partitions, filter); Assert.AreEqual(3, partition.Id); partition = partitioner.GetPartition(Partitions.Any, partitions, filter); Assert.AreEqual(1, partition.Id); }
public void TestGetNodeForPartition() { var node = new NodeMock(); var node2 = new NodeMock(); var node3 = new NodeMock(); var routes = new Dictionary <string, Partition[]> { { "test2p", new[] { new Partition { Id = 1, Leader = node }, new Partition { Id = 2, Leader = node2 }, new Partition { Id = 3, Leader = node3 } } }, }; var rt = new RoutingTable(routes); Assert.AreSame(node, rt.GetLeaderForPartition("test2p", 1)); Assert.AreSame(node2, rt.GetLeaderForPartition("test2p", 2)); Assert.AreSame(node3, rt.GetLeaderForPartition("test2p", 3)); Assert.IsNull(rt.GetLeaderForPartition("test2p", 8)); Assert.IsNull(rt.GetLeaderForPartition("test2poulpe", 8423)); }
public void TestFilterWithHighDelay() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock } }; var filter = new Dictionary <int, DateTime> { { 2, DateTime.UtcNow } }; // Pick a delay greater than the number of partitions int delay = partitions.Length + 2; var partitionStrategy = new RoundRobinPartitionSelection(delay); var partitioner = new PartitionSelector(partitionStrategy); var firstBatch = GetPartitions(delay, partitioner, partitions, filter); Assert.AreEqual(delay, firstBatch.Count); Assert.IsTrue(firstBatch.All(p => p.Id == 1)); var secondBatch = GetPartitions(delay, partitioner, partitions, filter); Assert.AreEqual(delay, secondBatch.Count); Assert.IsTrue(secondBatch.All(p => p.Id == 3)); }