public List <string> SendMessages( TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts) { var messages = new List <string>(); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, }; var producer = new Producer <int, string>(props); for (var partition = 0; partition < numParts; partition++) { var ms = Enumerable.Range(0, messagesPerNode) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(m => new KeyedMessage <int, string>(Topic, partition, m)).ToArray()); messages.AddRange(ms); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); } producer.Dispose(); return(messages); }
public List <string> SendMessagesToBrokerPartition( TempKafkaConfig config, string topic, int partition, int numMessages, CompressionCodecs compression = CompressionCodecs.NoCompressionCodec) { var header = string.Format("test-{0}-{1}", config.BrokerId, partition); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, CompressionCodec = compression, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, // custom: we need time to rebalance leader }; var producer = new Producer <int, string>(props); var ms = Enumerable.Range(0, numMessages) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(x => new KeyedMessage <int, string>(topic, partition, x)).ToArray()); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); producer.Dispose(); return(ms); }
public ProducerTest() { this.ports = TestUtils.ChoosePorts(2); this.port1 = this.ports[0]; this.port2 = this.ports[1]; this.config1 = TestUtils.CreateBrokerConfig( BrokerId1, this.port1, idx => new Dictionary <string, string> { { "num.partitons", "4" } }); this.config2 = TestUtils.CreateBrokerConfig( BrokerId2, this.port2, idx => new Dictionary <string, string> { { "num.partitons", "4" } }); this.server1 = this.StartServer(this.config1); this.server2 = this.StartServer(this.config2); this.servers = new List <Process> { this.server1, this.server2 }; this.consumer1 = new SimpleConsumer("localhost", this.port1, 1000000, 64 * 1024, string.Empty); this.consumer2 = new SimpleConsumer("localhost", this.port2, 100, 64 * 1024, string.Empty); this.WaitForServersToSettle(); }
public ProducerTest() { this.ports = TestUtils.ChoosePorts(2); this.port1 = this.ports[0]; this.port2 = this.ports[1]; this.config1 = TestUtils.CreateBrokerConfig( BrokerId1, this.port1, idx => new Dictionary<string, string> { { "num.partitons", "4" } }); this.config2 = TestUtils.CreateBrokerConfig( BrokerId2, this.port2, idx => new Dictionary<string, string> { { "num.partitons", "4" } }); this.server1 = this.StartServer(this.config1); this.server2 = this.StartServer(this.config2); this.servers = new List<Process> { this.server1, this.server2 }; this.consumer1 = new SimpleConsumer("localhost", this.port1, 1000000, 64 * 1024, string.Empty); this.consumer2 = new SimpleConsumer("localhost", this.port2, 100, 64 * 1024, string.Empty); this.WaitForServersToSettle(); }
public static TempKafkaConfig CreateBrokerConfig( int nodeId, int port, Func <int, Dictionary <string, string> > customProps = null) { var props = new Dictionary <string, string> { { "broker.id", nodeId.ToString(CultureInfo.InvariantCulture) }, { "host.name", "localhost" }, { "port", port.ToString(CultureInfo.InvariantCulture) }, { "log.dir", TempDir().FullName.Replace("\\", "\\\\") }, { "zookeeper.connect", TestZkUtils.ZookeeperConnect }, { "replica.socket.timeout.ms", "1500" } }; if (customProps != null) { var overrides = customProps(nodeId); foreach (var kvp in overrides) { props[kvp.Key] = kvp.Value; } } return(TempKafkaConfig.Create(props)); }
public List<string> SendMessages( TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts) { var messages = new List<string>(); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, }; var producer = new Producer<int, string>(props); for (var partition = 0; partition < numParts; partition++) { var ms = Enumerable.Range(0, messagesPerNode) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(m => new KeyedMessage<int, string>(Topic, partition, m)).ToArray()); messages.AddRange(ms); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); } producer.Dispose(); return messages; }
public List<string> SendMessagesToBrokerPartition( TempKafkaConfig config, string topic, int partition, int numMessages, CompressionCodecs compression = CompressionCodecs.NoCompressionCodec) { var header = string.Format("test-{0}-{1}", config.BrokerId, partition); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, CompressionCodec = compression, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, // custom: we need time to rebalance leader }; var producer = new Producer<int, string>(props); var ms = Enumerable.Range(0, numMessages) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(x => new KeyedMessage<int, string>(topic, partition, x)).ToArray()); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); producer.Dispose(); return ms; }
private Process StartServer(TempKafkaConfig config) { return KafkaRunClassHelper.Run(KafkaRunClassHelper.KafkaServerMainClass, config.ConfigLocation); }
private Process StartServer(TempKafkaConfig config) { return(KafkaRunClassHelper.Run(KafkaRunClassHelper.KafkaServerMainClass, config.ConfigLocation)); }