public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_Partition_Blacklisted(int partitionIdBlacklisted)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var blacklistedPartitions = new Dictionary <int, DateTime> {
{ partitionIdBlacklisted, DateTime.MaxValue }
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyKey"
}, new DateTime());
for (var i = 0; i < 300; i++)
{
var partition = partitioner.GetPartition(message, partitions, blacklistedPartitions);
Assert.IsTrue(partition.Id != Partition.None.Id);
Assert.IsTrue(partition.Id != partitionIdBlacklisted);
}
}
public void TestRobinPartitionAssignWhenFiltered()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
};
var filter = new Dictionary <int, DateTime>();
int delay = partitions.Length + 2;
var partitionStrategy = new RoundRobinPartitionSelection(delay);
var partitioner = new PartitionSelector(partitionStrategy);
var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(1, partition.Id);
filter.Add(1, DateTime.UtcNow);
var batch = GetPartitions(delay, partitioner, partitions, filter);
Assert.AreEqual(delay, batch.Count);
Assert.IsTrue(batch.All(p => p.Id == 2), "The round-robin threshold wasn't properly reset after previous partition was blacklisted");
}
public void TestAccumulatorByNodeByTopicByPartition(int batchSize, int time)
{
INode n1 = new NodeMock("n1");
INode n2 = new NodeMock("n2");
var accumulator = new AccumulatorByNodeByTopicByPartition <Tuple <string, int, int> >(t => t.Item1, t => t.Item2, batchSize, TimeSpan.FromMilliseconds(time));
var count = new CountdownEvent(2);
var d = new Dictionary <INode, IBatchByTopicByPartition <Tuple <string, int, int> > >();
accumulator.NewBatch += (n, b) =>
{
d[n] = b;
count.Signal();
};
accumulator.Add(Tuple.Create(n1, Tuple.Create("1", 1, 1)));
accumulator.Add(Tuple.Create(n1, Tuple.Create("1", 1, 2)));
accumulator.Add(Tuple.Create(n2, Tuple.Create("2", 1, 1)));
accumulator.Add(Tuple.Create(n2, Tuple.Create("2", 2, 2)));
accumulator.Add(Tuple.Create(n2, Tuple.Create("2", 2, 3)));
count.Wait();
Assert.AreEqual(2, d[n1].Count);
Assert.AreEqual(3, d[n2].Count);
d[n1].Dispose();
d[n2].Dispose();
}
public void Test_MessageKeyPartitionSelection_Is_Consistent()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message1 = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyKey"
}, new DateTime());
var message2 = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyOtherKey"
}, new DateTime());
var expectedPartition1 = partitioner.GetPartition(message1, partitions);
var expectedPartition2 = partitioner.GetPartition(message2, partitions);
for (var i = 0; i < 300; i++)
{
var currentPartition1 = partitioner.GetPartition(message1, partitions);
var currentPartition2 = partitioner.GetPartition(message2, partitions);
Assert.AreEqual(expectedPartition1.Id, currentPartition1.Id);
Assert.AreEqual(expectedPartition2.Id, currentPartition2.Id);
}
}
public void TestSignalDeadNode()
{
var node = new NodeMock();
var routes = new Dictionary <string, Partition[]>
{
{ "test1p", new[] { new Partition {
Id = 0, Leader = node
} } },
{ "test2p", new[] { new Partition {
Id = 1, Leader = new NodeMock()
}, new Partition {
Id = 2, Leader = node
}, new Partition {
Id = 3, Leader = new NodeMock()
} } },
};
var routingTable = new RoutingTable(routes);
Assert.AreEqual(1, routingTable.GetPartitions("test1p").Length);
Assert.AreEqual(3, routingTable.GetPartitions("test2p").Length);
routingTable = new RoutingTable(routingTable, node);
Assert.AreEqual(0, routingTable.GetPartitions("test1p").Length);
Assert.AreEqual(2, routingTable.GetPartitions("test2p").Length);
}
public void TestFilterMinInSync()
{
var node = new NodeMock();
var routes = new Dictionary <string, Partition[]>
{
{ "test1p", new[] { new Partition {
Id = 0, Leader = node
} } },
{ "test2p", new[] { new Partition {
Id = 1, Leader = new NodeMock(), NbIsr = 1
}, new Partition {
Id = 2, Leader = node
}, new Partition {
Id = 3, Leader = new NodeMock()
} } },
};
var routingTable = new RoutingTable(routes);
Assert.AreEqual(1, routingTable.GetPartitions("test1p").Length);
Assert.AreEqual(3, routingTable.GetPartitions("test2p").Length);
routingTable = new RoutingTable(routingTable, 1);
Assert.AreEqual(0, routingTable.GetPartitions("test1p").Length);
Assert.AreEqual(1, routingTable.GetPartitions("test2p").Length);
}
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitionStrategy = new RoundRobinPartitionSelection(delay: delay, startSeed: startSeed);
var partitioner = new PartitionSelector(partitionStrategy);
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(
ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions).Id);
}
}
}
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitioner = new PartitionSelector(delay, startSeed);
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(Partitions.Any, partitions).Id);
}
}
}
public void TestRoundRobinPartitionAssign(int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitionStrategy = new RoundRobinPartitionSelection(delay);
var partitioner = new PartitionSelector(partitionStrategy);
delay = delay <= 0 ? 1 : delay;
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual(partition.Id, partitioner
.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions)
.Id);
}
}
}
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_MessageKey_Null()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = null
}, new DateTime());
var partition = partitioner.GetPartition(message, partitions);
Assert.IsTrue(partition.Id != Partition.None.Id);
}
public void TestRoundRobinPartitionAssign(int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitioner = new PartitionSelector(delay);
delay = delay <= 0 ? 1 : delay;
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual(partition.Id, partitioner.GetPartition(Partitions.Any, partitions).Id);
}
}
}
private void SetUp(Configuration config)
{
_messagesSentByTopic = new Dictionary <string, int>
{
{ "test", 0 },
{ "test2", 0 },
{ "test1p", 0 },
{ "test2p", 0 },
{ "testallp", 0 }
};
_nodes = new NodeMock[5];
for (int i = 0; i < _nodes.Length; ++i)
{
_nodes[i] = new NodeMock();
_nodes[i].MessageReceived += t => _messagesSentByTopic[t] += 1; // No need to use interlocked, NodeMock is synchronous
}
_routes = new Dictionary <string, Partition[]>
{
{ "test1p", new[] { new Partition {
Id = 0, Leader = _nodes[0]
} } },
{ "test2p", new[] { new Partition {
Id = 0, Leader = _nodes[0]
}, new Partition {
Id = 1, Leader = _nodes[1]
} } },
{ "testallp", new[]
{
new Partition {
Id = 0, Leader = _nodes[0]
},
new Partition {
Id = 1, Leader = _nodes[1]
},
new Partition {
Id = 2, Leader = _nodes[2]
},
new Partition {
Id = 3, Leader = _nodes[3]
},
new Partition {
Id = 4, Leader = _nodes[4]
},
} }
};
_cluster = new ClusterMock(_routes);
MessagesEnqueued = MessagesExpired = MessagesReEnqueued = MessagesRouted = MessagesPostponed = RoutingTableRequired = 0;
InitRouter(new ProduceRouter(_cluster, config, Pool));
_finished = null;
}
public void TestRoutingTableReturnsEmptyForAbsentTopic()
{
var node = new NodeMock();
var routes = new Dictionary <string, Partition[]>
{
{ "test1p", new[] { new Partition {
Id = 0, Leader = node
} } },
};
var routingTable = new RoutingTable(routes);
Assert.Less(0, routingTable.GetPartitions("test1p").Length);
Assert.AreEqual(0, routingTable.GetPartitions("tortemoque").Length);
}
public void TestRoutingTableReturnsPartitions()
{
var node = new NodeMock();
var routes = new Dictionary <string, Partition[]>
{
{ "test1p", new[] { new Partition {
Id = 0, Leader = node
} } },
};
var routingTable = new RoutingTable(routes);
var partitions = routingTable.GetPartitions("test1p");
Assert.AreEqual(1, partitions.Length);
Assert.AreEqual(0, partitions[0].Id);
Assert.AreSame(node, partitions[0].Leader);
}
public void TestOverflow()
{
var nodeMock = new NodeMock();
var partitions = Enumerable.Range(0, 10).Select(i => new Partition {
Id = i, Leader = nodeMock
}).ToArray();
var partitioner = new PartitionSelector(1, int.MaxValue);
var batch = GetPartitions(partitions.Length, partitioner, partitions, null);
var ids = batch.Select(p => p.Id).ToArray();
var expectedIds = new[] { 7, 8, 9, 0, 1, 2, 3, 4, 5, 6 };
Assert.IsTrue(expectedIds.SequenceEqual(ids));
}
public void TestFilter()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var filter = new Dictionary <int, DateTime>();
filter[0] = DateTime.UtcNow;
filter[2] = DateTime.UtcNow;
filter[4] = DateTime.UtcNow;
var partitionStrategy = new RoundRobinPartitionSelection();
var partitioner = new PartitionSelector(partitionStrategy);
var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(1, partition.Id);
partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(3, partition.Id);
partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(1, partition.Id);
}
public void TestFilter()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var filter = new Dictionary <int, DateTime>();
filter[0] = DateTime.UtcNow;
filter[2] = DateTime.UtcNow;
filter[4] = DateTime.UtcNow;
var partitioner = new PartitionSelector();
var partition = partitioner.GetPartition(Partitions.Any, partitions, filter);
Assert.AreEqual(1, partition.Id);
partition = partitioner.GetPartition(Partitions.Any, partitions, filter);
Assert.AreEqual(3, partition.Id);
partition = partitioner.GetPartition(Partitions.Any, partitions, filter);
Assert.AreEqual(1, partition.Id);
}
public void TestGetNodeForPartition()
{
var node = new NodeMock();
var node2 = new NodeMock();
var node3 = new NodeMock();
var routes = new Dictionary <string, Partition[]>
{
{ "test2p", new[] { new Partition {
Id = 1, Leader = node
}, new Partition {
Id = 2, Leader = node2
}, new Partition {
Id = 3, Leader = node3
} } },
};
var rt = new RoutingTable(routes);
Assert.AreSame(node, rt.GetLeaderForPartition("test2p", 1));
Assert.AreSame(node2, rt.GetLeaderForPartition("test2p", 2));
Assert.AreSame(node3, rt.GetLeaderForPartition("test2p", 3));
Assert.IsNull(rt.GetLeaderForPartition("test2p", 8));
Assert.IsNull(rt.GetLeaderForPartition("test2poulpe", 8423));
}
public void TestFilterWithHighDelay()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
}
};
var filter = new Dictionary <int, DateTime> {
{ 2, DateTime.UtcNow }
};
// Pick a delay greater than the number of partitions
int delay = partitions.Length + 2;
var partitionStrategy = new RoundRobinPartitionSelection(delay);
var partitioner = new PartitionSelector(partitionStrategy);
var firstBatch = GetPartitions(delay, partitioner, partitions, filter);
Assert.AreEqual(delay, firstBatch.Count);
Assert.IsTrue(firstBatch.All(p => p.Id == 1));
var secondBatch = GetPartitions(delay, partitioner, partitions, filter);
Assert.AreEqual(delay, secondBatch.Count);
Assert.IsTrue(secondBatch.All(p => p.Id == 3));
}
