public void Test_MessageKeyPartitionSelection_Is_Consistent()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message1 = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyKey"
}, new DateTime());
var message2 = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyOtherKey"
}, new DateTime());
var expectedPartition1 = partitioner.GetPartition(message1, partitions);
var expectedPartition2 = partitioner.GetPartition(message2, partitions);
for (var i = 0; i < 300; i++)
{
var currentPartition1 = partitioner.GetPartition(message1, partitions);
var currentPartition2 = partitioner.GetPartition(message2, partitions);
Assert.AreEqual(expectedPartition1.Id, currentPartition1.Id);
Assert.AreEqual(expectedPartition2.Id, currentPartition2.Id);
}
}
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitionStrategy = new RoundRobinPartitionSelection(delay: delay, startSeed: startSeed);
var partitioner = new PartitionSelector(partitionStrategy);
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(
ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions).Id);
}
}
}
public void TestRoundRobinPartitionAssign(int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitionStrategy = new RoundRobinPartitionSelection(delay);
var partitioner = new PartitionSelector(partitionStrategy);
delay = delay <= 0 ? 1 : delay;
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual(partition.Id, partitioner
.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions)
.Id);
}
}
}
public void TestRobinPartitionAssignWhenFiltered()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
};
var filter = new Dictionary <int, DateTime>();
int delay = partitions.Length + 2;
var partitionStrategy = new RoundRobinPartitionSelection(delay);
var partitioner = new PartitionSelector(partitionStrategy);
var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(1, partition.Id);
filter.Add(1, DateTime.UtcNow);
var batch = GetPartitions(delay, partitioner, partitions, filter);
Assert.AreEqual(delay, batch.Count);
Assert.IsTrue(batch.All(p => p.Id == 2), "The round-robin threshold wasn't properly reset after previous partition was blacklisted");
}
public void TestRoundRobinPartitionAssignNoPartitionReturnsNone()
{
var partitions = new Partition[0];
var partitioner = new PartitionSelector();
Assert.AreEqual(0, Partition.None.CompareTo(partitioner.GetPartition(Partitions.Any, partitions)));
}
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitioner = new PartitionSelector(delay, startSeed);
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(Partitions.Any, partitions).Id);
}
}
}
public void TestRoundRobinPartitionAssign(int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitioner = new PartitionSelector(delay);
delay = delay <= 0 ? 1 : delay;
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual(partition.Id, partitioner.GetPartition(Partitions.Any, partitions).Id);
}
}
}
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_Partition_Blacklisted(int partitionIdBlacklisted)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var blacklistedPartitions = new Dictionary <int, DateTime> {
{ partitionIdBlacklisted, DateTime.MaxValue }
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyKey"
}, new DateTime());
for (var i = 0; i < 300; i++)
{
var partition = partitioner.GetPartition(message, partitions, blacklistedPartitions);
Assert.IsTrue(partition.Id != Partition.None.Id);
Assert.IsTrue(partition.Id != partitionIdBlacklisted);
}
}
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_MessageKey_Null()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = null
}, new DateTime());
var partition = partitioner.GetPartition(message, partitions);
Assert.IsTrue(partition.Id != Partition.None.Id);
}
public void TestRoundRobinPartitionAssignNoPartitionReturnsNone()
{
var partitions = new Partition[0];
var partitionStrategy = new RoundRobinPartitionSelection();
var partitioner = new PartitionSelector(partitionStrategy);
Assert.AreEqual(0, Partition.None.CompareTo(partitioner.GetPartition(
ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions)));
}
private static List <Partition> GetPartitions(int count, PartitionSelector partitioner, Partition[] partitions, Dictionary <int, DateTime> filter)
{
var result = new List <Partition>(count);
for (int i = 0; i < count; i++)
{
result.Add(partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter));
}
return(result);
}
private static List <Partition> GetPartitions(int count, PartitionSelector partitioner, Partition[] partitions, Dictionary <int, DateTime> filter)
{
var result = new List <Partition>(count);
for (int i = 0; i < count; i++)
{
result.Add(partitioner.GetPartition(Partitions.Any, partitions, filter));
}
return(result);
}
public void TestFilter()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var filter = new Dictionary <int, DateTime>();
filter[0] = DateTime.UtcNow;
filter[2] = DateTime.UtcNow;
filter[4] = DateTime.UtcNow;
var partitionStrategy = new RoundRobinPartitionSelection();
var partitioner = new PartitionSelector(partitionStrategy);
var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(1, partition.Id);
partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(3, partition.Id);
partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(1, partition.Id);
}
public void TestFilter()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var filter = new Dictionary <int, DateTime>();
filter[0] = DateTime.UtcNow;
filter[2] = DateTime.UtcNow;
filter[4] = DateTime.UtcNow;
var partitioner = new PartitionSelector();
var partition = partitioner.GetPartition(Partitions.Any, partitions, filter);
Assert.AreEqual(1, partition.Id);
partition = partitioner.GetPartition(Partitions.Any, partitions, filter);
Assert.AreEqual(3, partition.Id);
partition = partitioner.GetPartition(Partitions.Any, partitions, filter);
Assert.AreEqual(1, partition.Id);
}
