public void Test_MessageKeyPartitionSelection_Is_Consistent() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, }; var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>()); var partitioner = new PartitionSelector(partitionStrategy); var message1 = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = "ThisIsMyKey" }, new DateTime()); var message2 = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = "ThisIsMyOtherKey" }, new DateTime()); var expectedPartition1 = partitioner.GetPartition(message1, partitions); var expectedPartition2 = partitioner.GetPartition(message2, partitions); for (var i = 0; i < 300; i++) { var currentPartition1 = partitioner.GetPartition(message1, partitions); var currentPartition2 = partitioner.GetPartition(message2, partitions); Assert.AreEqual(expectedPartition1.Id, currentPartition1.Id); Assert.AreEqual(expectedPartition2.Id, currentPartition2.Id); } }
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitionStrategy = new RoundRobinPartitionSelection(delay: delay, startSeed: startSeed); var partitioner = new PartitionSelector(partitionStrategy); foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition( ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions).Id); } } }
public void TestRoundRobinPartitionAssign(int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitionStrategy = new RoundRobinPartitionSelection(delay); var partitioner = new PartitionSelector(partitionStrategy); delay = delay <= 0 ? 1 : delay; foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual(partition.Id, partitioner .GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions) .Id); } } }
public void TestRobinPartitionAssignWhenFiltered() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, }; var filter = new Dictionary <int, DateTime>(); int delay = partitions.Length + 2; var partitionStrategy = new RoundRobinPartitionSelection(delay); var partitioner = new PartitionSelector(partitionStrategy); var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(1, partition.Id); filter.Add(1, DateTime.UtcNow); var batch = GetPartitions(delay, partitioner, partitions, filter); Assert.AreEqual(delay, batch.Count); Assert.IsTrue(batch.All(p => p.Id == 2), "The round-robin threshold wasn't properly reset after previous partition was blacklisted"); }
public void TestRoundRobinPartitionAssignNoPartitionReturnsNone() { var partitions = new Partition[0]; var partitioner = new PartitionSelector(); Assert.AreEqual(0, Partition.None.CompareTo(partitioner.GetPartition(Partitions.Any, partitions))); }
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitioner = new PartitionSelector(delay, startSeed); foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(Partitions.Any, partitions).Id); } } }
public void TestRoundRobinPartitionAssign(int delay) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var partitioner = new PartitionSelector(delay); delay = delay <= 0 ? 1 : delay; foreach (var partition in partitions) { for (var j = 0; j < delay; ++j) { Assert.AreEqual(partition.Id, partitioner.GetPartition(Partitions.Any, partitions).Id); } } }
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_Partition_Blacklisted(int partitionIdBlacklisted) { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, }; var blacklistedPartitions = new Dictionary <int, DateTime> { { partitionIdBlacklisted, DateTime.MaxValue } }; var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>()); var partitioner = new PartitionSelector(partitionStrategy); var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = "ThisIsMyKey" }, new DateTime()); for (var i = 0; i < 300; i++) { var partition = partitioner.GetPartition(message, partitions, blacklistedPartitions); Assert.IsTrue(partition.Id != Partition.None.Id); Assert.IsTrue(partition.Id != partitionIdBlacklisted); } }
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_MessageKey_Null() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, }; var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>()); var partitioner = new PartitionSelector(partitionStrategy); var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message { Key = null }, new DateTime()); var partition = partitioner.GetPartition(message, partitions); Assert.IsTrue(partition.Id != Partition.None.Id); }
public void TestRoundRobinPartitionAssignNoPartitionReturnsNone() { var partitions = new Partition[0]; var partitionStrategy = new RoundRobinPartitionSelection(); var partitioner = new PartitionSelector(partitionStrategy); Assert.AreEqual(0, Partition.None.CompareTo(partitioner.GetPartition( ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions))); }
private static List <Partition> GetPartitions(int count, PartitionSelector partitioner, Partition[] partitions, Dictionary <int, DateTime> filter) { var result = new List <Partition>(count); for (int i = 0; i < count; i++) { result.Add(partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter)); } return(result); }
private static List <Partition> GetPartitions(int count, PartitionSelector partitioner, Partition[] partitions, Dictionary <int, DateTime> filter) { var result = new List <Partition>(count); for (int i = 0; i < count; i++) { result.Add(partitioner.GetPartition(Partitions.Any, partitions, filter)); } return(result); }
public void TestFilter() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var filter = new Dictionary <int, DateTime>(); filter[0] = DateTime.UtcNow; filter[2] = DateTime.UtcNow; filter[4] = DateTime.UtcNow; var partitionStrategy = new RoundRobinPartitionSelection(); var partitioner = new PartitionSelector(partitionStrategy); var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(1, partition.Id); partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(3, partition.Id); partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter); Assert.AreEqual(1, partition.Id); }
public void TestFilter() { var nodeMock = new NodeMock(); var partitions = new[] { new Partition { Id = 0, Leader = nodeMock }, new Partition { Id = 1, Leader = nodeMock }, new Partition { Id = 2, Leader = nodeMock }, new Partition { Id = 3, Leader = nodeMock }, new Partition { Id = 4, Leader = nodeMock }, }; var filter = new Dictionary <int, DateTime>(); filter[0] = DateTime.UtcNow; filter[2] = DateTime.UtcNow; filter[4] = DateTime.UtcNow; var partitioner = new PartitionSelector(); var partition = partitioner.GetPartition(Partitions.Any, partitions, filter); Assert.AreEqual(1, partition.Id); partition = partitioner.GetPartition(Partitions.Any, partitions, filter); Assert.AreEqual(3, partition.Id); partition = partitioner.GetPartition(Partitions.Any, partitions, filter); Assert.AreEqual(1, partition.Id); }