public List <string> SendMessages(
TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts)
{
var messages = new List <string>();
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000,
};
var producer = new Producer <int, string>(props);
for (var partition = 0; partition < numParts; partition++)
{
var ms =
Enumerable.Range(0, messagesPerNode)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(m => new KeyedMessage <int, string>(Topic, partition, m)).ToArray());
messages.AddRange(ms);
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
}
producer.Dispose();
return(messages);
}
public List <string> SendMessagesToBrokerPartition(
TempKafkaConfig config,
string topic,
int partition,
int numMessages,
CompressionCodecs compression = CompressionCodecs.NoCompressionCodec)
{
var header = string.Format("test-{0}-{1}", config.BrokerId, partition);
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
CompressionCodec = compression,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000, // custom: we need time to rebalance leader
};
var producer = new Producer <int, string>(props);
var ms =
Enumerable.Range(0, numMessages)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(x => new KeyedMessage <int, string>(topic, partition, x)).ToArray());
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
producer.Dispose();
return(ms);
}
public ProducerTest()
{
this.ports = TestUtils.ChoosePorts(2);
this.port1 = this.ports[0];
this.port2 = this.ports[1];
this.config1 = TestUtils.CreateBrokerConfig(
BrokerId1, this.port1, idx => new Dictionary <string, string> {
{ "num.partitons", "4" }
});
this.config2 = TestUtils.CreateBrokerConfig(
BrokerId2, this.port2, idx => new Dictionary <string, string> {
{ "num.partitons", "4" }
});
this.server1 = this.StartServer(this.config1);
this.server2 = this.StartServer(this.config2);
this.servers = new List <Process> {
this.server1, this.server2
};
this.consumer1 = new SimpleConsumer("localhost", this.port1, 1000000, 64 * 1024, string.Empty);
this.consumer2 = new SimpleConsumer("localhost", this.port2, 100, 64 * 1024, string.Empty);
this.WaitForServersToSettle();
}
public ProducerTest()
{
this.ports = TestUtils.ChoosePorts(2);
this.port1 = this.ports[0];
this.port2 = this.ports[1];
this.config1 = TestUtils.CreateBrokerConfig(
BrokerId1, this.port1, idx => new Dictionary<string, string> { { "num.partitons", "4" } });
this.config2 = TestUtils.CreateBrokerConfig(
BrokerId2, this.port2, idx => new Dictionary<string, string> { { "num.partitons", "4" } });
this.server1 = this.StartServer(this.config1);
this.server2 = this.StartServer(this.config2);
this.servers = new List<Process> { this.server1, this.server2 };
this.consumer1 = new SimpleConsumer("localhost", this.port1, 1000000, 64 * 1024, string.Empty);
this.consumer2 = new SimpleConsumer("localhost", this.port2, 100, 64 * 1024, string.Empty);
this.WaitForServersToSettle();
}
public static TempKafkaConfig CreateBrokerConfig(
int nodeId, int port, Func <int, Dictionary <string, string> > customProps = null)
{
var props = new Dictionary <string, string>
{
{ "broker.id", nodeId.ToString(CultureInfo.InvariantCulture) },
{ "host.name", "localhost" },
{ "port", port.ToString(CultureInfo.InvariantCulture) },
{ "log.dir", TempDir().FullName.Replace("\\", "\\\\") },
{ "zookeeper.connect", TestZkUtils.ZookeeperConnect },
{ "replica.socket.timeout.ms", "1500" }
};
if (customProps != null)
{
var overrides = customProps(nodeId);
foreach (var kvp in overrides)
{
props[kvp.Key] = kvp.Value;
}
}
return(TempKafkaConfig.Create(props));
}
public List<string> SendMessages(
TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts)
{
var messages = new List<string>();
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000,
};
var producer = new Producer<int, string>(props);
for (var partition = 0; partition < numParts; partition++)
{
var ms =
Enumerable.Range(0, messagesPerNode)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(m => new KeyedMessage<int, string>(Topic, partition, m)).ToArray());
messages.AddRange(ms);
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
}
producer.Dispose();
return messages;
}
public List<string> SendMessagesToBrokerPartition(
TempKafkaConfig config,
string topic,
int partition,
int numMessages,
CompressionCodecs compression = CompressionCodecs.NoCompressionCodec)
{
var header = string.Format("test-{0}-{1}", config.BrokerId, partition);
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
CompressionCodec = compression,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000, // custom: we need time to rebalance leader
};
var producer = new Producer<int, string>(props);
var ms =
Enumerable.Range(0, numMessages)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(x => new KeyedMessage<int, string>(topic, partition, x)).ToArray());
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
producer.Dispose();
return ms;
}
private Process StartServer(TempKafkaConfig config)
{
return KafkaRunClassHelper.Run(KafkaRunClassHelper.KafkaServerMainClass, config.ConfigLocation);
}
private Process StartServer(TempKafkaConfig config)
{
return(KafkaRunClassHelper.Run(KafkaRunClassHelper.KafkaServerMainClass, config.ConfigLocation));
}