/// <summary> /// Initializes a new instance of the <see cref="Consumer"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> public Consumer(ConsumerConfig config) { Guard.Assert<ArgumentNullException>(() => config != null); this.config = config; this.Host = config.Host; this.Port = config.Port; }
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfig config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos) { this.name = name; this.zkClient = zkClient; this.config = config; this.broker = broker; this.partitionTopicInfos = partitionTopicInfos; this.simpleConsumer = new Consumer(this.config); }
/// <summary> /// Initializes a new instance of the <see cref="ZookeeperConsumerConnector"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. At the minimum, need to specify the group ID /// of the consumer and the ZooKeeper connection string. /// </param> /// <param name="enableFetcher"> /// Indicates whether fetchers should be enabled /// </param> public ZookeeperConsumerConnector(ConsumerConfig config, bool enableFetcher) : base(config) { this.config = config; this.enableFetcher = enableFetcher; this.ConnectZk(); this.CreateFetcher(); if (this.config.AutoCommit) { Logger.InfoFormat(CultureInfo.CurrentCulture, "starting auto committer every {0} ms", this.config.AutoCommitIntervalMs); scheduler.ScheduleWithRate(this.AutoCommit, this.config.AutoCommitIntervalMs, this.config.AutoCommitIntervalMs); } }
public void ConsumerPerformsRebalancingWhenConsumerIsRemovedAndTakesItsPartitions() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 }; IList<string> ids; IList<string> owners; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); using (var consumerConnector2 = new ZookeeperConsumerConnector(config, true)) { consumerConnector2.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); ids = client.GetChildren("/consumers/group1/ids", false).ToList(); owners = client.GetChildren("/consumers/group1/owners/test", false).ToList(); Assert.That(ids, Is.Not.Null.And.Not.Empty); Assert.That(ids.Count, Is.EqualTo(2)); Assert.That(owners, Is.Not.Null.And.Not.Empty); Assert.That(owners.Count, Is.EqualTo(2)); } WaitUntillIdle(client, 1000); ids = client.GetChildren("/consumers/group1/ids", false).ToList(); owners = client.GetChildren("/consumers/group1/owners/test", false).ToList(); Assert.That(ids, Is.Not.Null.And.Not.Empty); Assert.That(ids.Count, Is.EqualTo(1)); Assert.That(owners, Is.Not.Null.And.Not.Empty); Assert.That(owners.Count, Is.EqualTo(2)); var data1 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[0], false); var data2 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[1], false); Assert.That(data1, Is.Not.Null.And.Not.Empty); Assert.That(data2, Is.Not.Null.And.Not.Empty); Assert.That(data1, Is.EqualTo(data2)); Assert.That(data1, Is.StringStarting(ids[0])); } }
public static long GetCurrentKafkaOffset(string topic, string address, int port) { OffsetRequest request = new OffsetRequest(topic, 0, DateTime.Now.AddDays(-5).Ticks, 10); ConsumerConfig consumerConfig = new ConsumerConfig(); consumerConfig.Host = address; consumerConfig.Port = port; IConsumer consumer = new Consumers.Consumer(consumerConfig); IList<long> list = consumer.GetOffsetsBefore(request); if (list.Count > 0) { return list[0]; } else { return 0; } }
public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage() { Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); var config = new AsyncProducerConfig(clientConfig); var producer = new AsyncProducer(config); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage }); long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig); producer.Send(producerRequest); ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(consumerConfig); FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset); BufferedMessageSet response; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 0) { break; } else { totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Message resultMessage = response.Messages.First(); Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString()); }
internal ZKRebalancerListener( ConsumerConfig config, string consumerIdString, IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry, IZooKeeperClient zkClient, ZookeeperConsumerConnector zkConsumerConnector, IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues, Fetcher fetcher, object syncLock) { this.syncLock = syncLock; this.consumerIdString = consumerIdString; this.config = config; this.topicRegistry = topicRegistry; this.zkClient = zkClient; this.dirs = new ZKGroupDirs(config.GroupId); this.zkConsumerConnector = zkConsumerConnector; this.queues = queues; this.fetcher = fetcher; }
static async Task Main(string[] args) { string topicName = "order_create3"; //var config = new ProducerConfig //{ // BootstrapServers = "localhost:29092", // MessageTimeoutMs = 10000, //}; //await ProduceDataAsync(topicName, config); var consumerConfig = new ConsumerConfig { BootstrapServers = "localhost:29092", GroupId = "group-F", EnableAutoOffsetStore = false, AutoOffsetReset = AutoOffsetReset.Earliest }; ConsumeData(topicName, consumerConfig); }
public KafkaConsumer(KafkaConsumerOptions options) { MessageReceived = new MessageReceivedHandler <TMessage>(onMessageReceived); _options = options; var conf = new ConsumerConfig { GroupId = _options.GroupId, BootstrapServers = _options.BootstrapServers, // Note: The AutoOffsetReset property determines the start offset in the event // there are not yet any committed offsets for the consumer group for the // topic/partitions of interest. By default, offsets are committed // automatically, so in this example, consumption will only start from the // earliest message in the topic 'my-topic' the first time you run the program. AutoOffsetReset = (AutoOffsetReset)_options.AutoOffsetReset }; _consumer = new ConsumerBuilder <Ignore, string>(conf).Build(); cancellationToken = new CancellationTokenSource(); _consumer.Subscribe(_options.Topic); }
private IConsumer <byte[], T> InitializeConsumer() { var config = new ConsumerConfig { GroupId = _kafkaConfig.CurrentValue.ConsumerGroup, BootstrapServers = string.Join(',', _kafkaConfig.CurrentValue.Brokers), MaxPollIntervalMs = Day, EnableAutoCommit = false, EnableAutoOffsetStore = false }; var consumerBuilder = new ConsumerBuilder <byte[], T>(config); if (typeof(T) == typeof(MessageData.MessageData)) { consumerBuilder.SetValueDeserializer((IDeserializer <T>) new MessageDataDeserializer()); } var consumer = consumerBuilder.Build(); return(consumer); }
public PooledConsumer GetConsumer(string group) { var consumerObjectPool = consumerPoolDict.GetOrAdd(group, key => { var kvList = serializer.Deserialize <List <KeyValuePair <string, string> > >(serializer.SerializeToUtf8Bytes(consumerConfig)); var dict = new Dictionary <string, string>(kvList); var config = new ConsumerConfig(dict) { GroupId = group }; return(new DefaultObjectPool <PooledConsumer>(new ConsumerPooledObjectPolicy(config, logger), options.ConsumerMaxPoolSize)); }); var result = consumerObjectPool.Get(); if (result.Pool is null) { result.Pool = consumerObjectPool; result.MaxBatchSize = options.CunsumerMaxBatchSize; result.MaxMillisecondsInterval = options.CunsumerMaxMillisecondsInterval; } return(result); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { try { var config = new ConsumerConfig { BootstrapServers = "localhost:9092", GroupId = $"topic-teste-hello-world-group-0", AutoOffsetReset = AutoOffsetReset.Earliest }; var cancellationTokenSource = new CancellationTokenSource(); using var consumer = new ConsumerBuilder <Ignore, string>(config).Build(); consumer.Subscribe("topic-teste-hello-world"); try { while (true) { var consumeResult = consumer.Consume(cancellationTokenSource.Token); _logger.LogInformation(consumeResult.Message.Value); } } catch (OperationCanceledException) { consumer.Close(); } } catch (Exception ex) { _logger.LogError(ex, "Worker running at: {time}", DateTimeOffset.Now); } _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); await Task.Delay(1000, stoppingToken); } }
private void ConsumerListener() { var config = new ConsumerConfig { GroupId = _config["EventHub:GroupId"], BootstrapServers = _config["EventHub:BootstrapServers"], AutoOffsetReset = AutoOffsetReset.Earliest }; using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build()) { consumer.Subscribe(_config["EventHub:GroupId"]); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { while (true) { try { var consumeResult = consumer.Consume(cts.Token); _logger.LogInformation($"Consumed message '{consumeResult.Value}' at: '{consumeResult.TopicPartitionOffset}'."); } catch (ConsumeException e) { _logger.LogError($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. consumer.Close(); } } }
public bool ListenAndConsumeMessage <T>(string topic, Action <T> f) { var conf = new ConsumerConfig { GroupId = _messageBrokerConfigSingleton.GroupId, BootstrapServers = _messageBrokerConfigSingleton.BrokerLocation, AutoOffsetReset = AutoOffsetResetType.Earliest }; using (var c = new Consumer <string, string>(conf)) { c.Subscribe(topic); bool consuming = true; c.OnError += (_, e) => consuming = !e.IsFatal; while (consuming) { try { var cr = c.Consume(); Console.WriteLine("Received Message with Key: " + cr.Key); if (cr.Key.StartsWith(typeof(T).FullName)) { f(JsonConvert.DeserializeObject <T>(cr.Value)); } } catch (ConsumeException e) { //LOG THIS error.. } } //close the consumer c.Close(); return(true); } }
public static void OnPartitionsAssignedNotSet(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start OnPartitionsAssignedNotSet"); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000 }; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; // Producing onto the topic to make sure it exists. using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build()) { var dr = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> { Value = Serializers.Utf8.Serialize("test string", SerializationContext.Empty) }).Result; Assert.NotEqual(Offset.Invalid, dr.Offset); producer.Flush(TimeSpan.FromSeconds(10)); } using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Subscribe(singlePartitionTopic); Assert.Empty(consumer.Assignment); consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Single(consumer.Assignment); Assert.Equal(singlePartitionTopic, consumer.Assignment[0].Topic); consumer.Close(); } Assert.Equal(0, Library.HandleCount); LogToFile("end OnPartitionsAssignedNotSet"); }
static async Task Main(string[] args) { var consumerConfig = new ConsumerConfig { BootstrapServers = args[0], GroupId = "protobuf-example", AutoOffsetReset = AutoOffsetResetType.Latest }; var consumeTask = Task.Run(() => { // consume a single message then exit. using (var consumer = new Consumer <int, User>(consumerConfig, Deserializers.Int32, new ProtobufDeserializer <User>())) { consumer.Subscribe("protobuf-test-topic"); var cr = consumer.Consume(); Console.WriteLine($"User: [id: {cr.Key}, favorite color: {cr.Message.Value.FavoriteColor}]"); } }); // wait a bit so the consumer is ready to consume messages before producing one. await Task.Delay(TimeSpan.FromSeconds(10)); var producerConfig = new ProducerConfig { BootstrapServers = args[0] }; using (var producer = new Producer <int, User>(producerConfig, Serializers.Int32, new ProtobufSerializer <User>())) { await producer.ProduceAsync("protobuf-test-topic", new Message <int, User> { Key = 0, Value = new User { FavoriteColor = "green" } }); } await consumeTask; }
public static void Main(string[] args) { var configuration = new ConsumerConfig { GroupId = "test-consumer-group", BootstrapServers = "127.0.0.1:9092", AutoOffsetReset = AutoOffsetReset.Earliest }; using var builder = new ConsumerBuilder <Ignore, string>(configuration).Build(); { builder.Subscribe("test-topic"); var cancellationToken = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cancellationToken.Cancel(); }; try { while (true) { try { var consumerResult = builder.Consume(cancellationToken.Token); Console.WriteLine($"Consumed message '{consumerResult.Value}' at: '{consumerResult.TopicPartitionOffset}'."); } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { builder.Close(); } } }
static void Main(string[] args) { ConsumerConfig config = new ConsumerConfig() { BootstrapServers = "192.168.28.172:9093,192.168.28.172:9094,192.168.28.172:9095", GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Latest, EnableAutoCommit = false }; CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; using (var consumer = new ConsumerBuilder <string, object>(config).SetValueDeserializer(new KafkaConverter()).Build()) { //c.Assign(new TopicPartition("test", new Partition(1)));//从指定的Partition订阅消息使用Assign方法 consumer.Subscribe("mykafka");//订阅消息使用Subscribe方法 while (!cts.IsCancellationRequested) { var result = consumer.Consume(cts.Token); if (result.Offset % 5 == 0) { consumer.Commit(result);//手动提交,如果上面的EnableAutoCommit=true表示自动提交,则无需调用Commit方法 Console.WriteLine($"recieve message:{result.Message.Value},Committed offset: {result.Offset}"); } else { Console.WriteLine($"recieve message:{result.Message.Value},Committed offset: { result.Offset }"); } } } }
private async Task KafkaConsume() { _logger.LogDebug($"KafkaConsume is running"); var config = new ConsumerConfig { GroupId = "teslacharger", BootstrapServers = "green-kafka:9092", AutoOffsetReset = AutoOffsetResetType.Earliest, EnableAutoCommit = true }; using (var c = new Consumer <string, string>(config)) { c.Subscribe("future-consumption"); c.OnError += (_, e) => Console.WriteLine($"Error: {e.Reason}"); var ct = _cts.Token; while (!ct.IsCancellationRequested) { try { var cr = c.Consume(ct); Console.WriteLine($"Consumed message from '{cr.Topic}', partion {cr.Partition}, offset {cr.Offset}, length {cr.Value.Length}, head {cr.Value.Substring(0, 30)}"); await CheckCharging(); } catch (Exception e) { _logger.LogError(e, e.Message); } } } _logger.LogDebug($"KafkaConsume is terminating"); }
public static void startGetMessage() { configCommand = new ConsumerConfig { GroupId = consumerGroupId, BootstrapServers = brokerList, AutoOffsetReset = AutoOffsetReset.Latest }; using (consumerCommand = new ConsumerBuilder <Ignore, string>(configCommand).Build()) { consumerCommand.Subscribe(commandTopicName); CancellationTokenSource cts = new CancellationTokenSource(); //Console.CancelKeyPress += (_, e) => { // e.Cancel = true; // prevent the process from terminating. // cts.Cancel(); //}; try { while (true) { try { var cr = consumerCommand.Consume(cts.Token); OnGetMessage(cr.Value); } catch (ConsumeException e) { FileWorker.LogHelper.WriteLog($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException e) { FileWorker.LogHelper.WriteLog($"Error occured1: {e.Message}"); consumerCommand.Close(); } } }
public ConsumerConfiguration( ConsumerConfig consumerConfig, IEnumerable <string> topics, string consumerName, int workerCount, int bufferSize, Factory <IDistributionStrategy> distributionStrategyFactory, MiddlewareConfiguration middlewareConfiguration, bool autoStoreOffsets, TimeSpan autoCommitInterval, IReadOnlyList <Action <string> > statisticsHandlers, ConsumerCustomFactory customFactory) { this.consumerConfig = consumerConfig ?? throw new ArgumentNullException(nameof(consumerConfig)); if (string.IsNullOrEmpty(this.consumerConfig.GroupId)) { throw new ArgumentNullException(nameof(consumerConfig.GroupId)); } this.DistributionStrategyFactory = distributionStrategyFactory ?? throw new ArgumentNullException(nameof(distributionStrategyFactory)); this.MiddlewareConfiguration = middlewareConfiguration ?? throw new ArgumentNullException(nameof(middlewareConfiguration)); this.AutoStoreOffsets = autoStoreOffsets; this.AutoCommitInterval = autoCommitInterval; this.Topics = topics ?? throw new ArgumentNullException(nameof(topics)); this.ConsumerName = consumerName ?? Guid.NewGuid().ToString(); this.WorkerCount = workerCount; this.StatisticsHandlers = statisticsHandlers; this.CustomFactory = customFactory; this.BufferSize = bufferSize > 0 ? bufferSize : throw new ArgumentOutOfRangeException( nameof(bufferSize), bufferSize, "The value must be greater than 0"); }
public static void Main(string[] args) { DotNetEnv.Env.Load(); var conf = new ConsumerConfig { BootstrapServers = System.Environment.GetEnvironmentVariable("KAFKA_HOST"), GroupId = System.Environment.GetEnvironmentVariable("KAFKA_GROUP_ID"), AutoOffsetReset = AutoOffsetReset.Earliest }; using (var consumer = new ConsumerBuilder <Ignore, string>(conf).Build()) { consumer.Subscribe(System.Environment.GetEnvironmentVariable("KAFKA_TOPIC")); Console.WriteLine("Consumer started, press <CTRL> + C to exit"); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; try { while (true) { var message = consumer.Consume(cts.Token); Console.WriteLine(message.Value); } } catch (OperationCanceledException) { consumer.Close(); } } }
public Task StartAsync(CancellationToken cancellationToken) { var conf = new ConsumerConfig { GroupId = "st_consumer_group", BootstrapServers = _bootstrapServers, AutoOffsetReset = AutoOffsetReset.Latest }; using (var builder = new ConsumerBuilder <Ignore, string>(conf).Build()) { builder.Subscribe(topic); var cancelToken = new CancellationTokenSource(); try { while (true) { var consumer = builder.Consume(cancelToken.Token); var jObject = JObject.Parse(consumer.Message.Value); var phone = (string)jObject["phone"]; var email = (string)jObject["email"]; _sender.SendSms(new SmsMessage { From = "+19149964683", Message = $"Message: {consumer.Message.Value} received from {consumer.TopicPartitionOffset}", To = "+380638680809" }); Console.WriteLine($"Message: {consumer.Message.Value} received from {consumer.TopicPartitionOffset}"); } } catch (Exception) { builder.Close(); } } return(Task.CompletedTask); }
public void CtorCommon_SameConfigsSuccess() { var topicsConfig = new TopicsConfig() { "topic1", "topic2", "topic3" }; var consumerConfig = new ConsumerConfig() { Environment = "TestTier", ClusterId = "TestCluster", Urls = "TestUrls" }; var producerConfigs = new ProducerConfigs() { { new ProducerConfig { Environment = "TestTierProducer1", ClusterId = "TestClusterProducer1", Urls = "TestUrlsProducer1" } }, { new ProducerConfig { Environment = "TestTierProducer2", ClusterId = "TestClusterProducer2", Urls = "TestUrlsProducer2" } } }; var provider = new Mock <IServiceProvider>(); var topicsOptions = new Mock <IOptions <TopicsConfig> >(); topicsOptions.SetupGet(c => c.Value).Returns(topicsConfig); var consumerConfigOptions = new Mock <IOptions <ConsumerConfig> >(); consumerConfigOptions.SetupGet(c => c.Value).Returns(consumerConfig); var producerConfigsOptions = new Mock <IOptions <ProducerConfigs> >(); producerConfigsOptions.SetupGet(c => c.Value).Returns(producerConfigs); var runner = new FeedProxiesRunnerTestDecorator(provider.Object, topicsOptions.Object, consumerConfigOptions.Object, producerConfigsOptions.Object); runner.Topics.Should().BeEquivalentTo(topicsConfig); runner.ConsumerConfig.Should().BeEquivalentTo(consumerConfig); runner.ProducerConfigs.Should().BeEquivalentTo(producerConfigs); }
static void Consume(string topic, ClientConfig config) { var consumerConfig = new ConsumerConfig(config); consumerConfig.GroupId = "Group-Consumer"; consumerConfig.AutoOffsetReset = AutoOffsetReset.Earliest; consumerConfig.EnableAutoCommit = false; CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build()) { consumer.Subscribe(topic); //var totalCount = 0; try { while (true) { var cr = consumer.Consume(cts.Token); // totalCount += JObject.Parse(cr.Value).Value<int>("count"); Console.WriteLine($"Consumed record with key {cr.Key} and value {cr.Value}"); } } catch (OperationCanceledException) { // Ctrl-C was pressed. } finally { consumer.Close(); } } }
static IConsumer <Ignore, string> ConfigureConsumer(IServiceProvider provider) { var options = provider.GetRequiredService <IOptions <AppConfig> >().Value; var config = new ConsumerConfig { BootstrapServers = options.BrokerList, GroupId = "consumer", EnableAutoCommit = false, StatisticsIntervalMs = 5000, SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; var logging = provider.GetRequiredService <ILoggerFactory>().CreateLogger <IConsumer <Ignore, string> >(); // Note: If a key or value deserializer is not set (as is the case below), the // deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers // will be used automatically (where available). The default deserializer for string // is UTF8. The default deserializer for Ignore returns null for all input data // (including non-null data). return(new ConsumerBuilder <Ignore, string>(config) // Note: All handlers are called on the main .Consume thread. .SetErrorHandler((_, e) => logging.LogError($"Error: {e.Reason}")) .SetStatisticsHandler((_, json) => logging.LogInformation($"Statistics: {json}")) .SetPartitionsAssignedHandler((c, partitions) => { logging.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]"); // possibly manually specify start offsets or override the partition assignment provided by // the consumer group by returning a list of topic/partition/offsets to assign to, e.g.: // // return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp])); }) .SetPartitionsRevokedHandler((c, partitions) => { logging.LogInformation($"Revoking assignment: [{string.Join(", ", partitions)}]"); }) .Build()); }
static void runConsumerManual() { var config = new ConsumerConfig { BootstrapServers = brokerList, GroupId = groupId, EnableAutoCommit = false }; bool cancelled = false; int noRecordsCount = 0; using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build()) { consumer.Subscribe(topicName); var cancelToken = new CancellationTokenSource(); ConsumeResult <Ignore, string> consumeResult = null; while (!cancelled) { consumeResult = consumer.Consume(cancelToken.Token); noRecordsCount++; // handle message Console.WriteLine($"Consumer Record:(Key: {consumeResult.Message.Key}, Value: {consumeResult.Message.Value} Partition: {consumeResult.TopicPartition.Partition} Offset: {consumeResult.TopicPartitionOffset.Offset}"); if (consumeResult.Offset % 50 == 0) { consumer.Commit(consumeResult); } } // commit the rest consumer.Commit(consumeResult); consumer.Close(); } }
public KafkaQueueService( IConfiguration configuration, ISerializer serializer, IOptions <QueueOptions> options ) : base(serializer, options) { _producer = new Lazy <IProducer <Null, string> >(() => { var config = new ProducerConfig { BootstrapServers = configuration.GetSection(Constants.OptionNameBootstrapServer).Value, BatchNumMessages = 1, MessageTimeoutMs = 1, SocketNagleDisable = true, Acks = Acks.Leader }; return(new ProducerBuilder <Null, string>(config).Build()); }); _consumer = new Lazy <IConsumer <Ignore, string> >(() => { var groupIdSection = configuration.GetSection(Constants.OptionNameConsumerGroupId); var groupId = groupIdSection.Exists() && !string.IsNullOrEmpty(groupIdSection.Value) ? groupIdSection.Value : Constants.DefaultGroupId; var config = new ConsumerConfig { BootstrapServers = configuration.GetSection(Constants.OptionNameBootstrapServer).Value, GroupId = groupId, AutoOffsetReset = AutoOffsetReset.Earliest, StatisticsIntervalMs = 5000, SessionTimeoutMs = 6000 }; var consumer = new ConsumerBuilder <Ignore, string>(config) .SetErrorHandler((_, e) => StaticLog.Error($"Receiving Error: {e.Code} - {e.Reason}")) .Build(); return(consumer); }); }
/// <summary>Creates new instance <see cref="KafkaConsumer"/>.</summary> public KafkaConsumer(ILogger logger, string brokerEndpoints, string groupId = null) { _logger = logger; var config = new ConsumerConfig { BootstrapServers = brokerEndpoints, ApiVersionRequest = true, GroupId = !string.IsNullOrEmpty(groupId) ? groupId : Guid.NewGuid().ToString(), EnableAutoCommit = false, FetchWaitMaxMs = 5, FetchErrorBackoffMs = 5, QueuedMinMessages = 1000, SessionTimeoutMs = 6000, //StatisticsIntervalMs = 5000, #if DEBUG Debug = "msg", #endif AutoOffsetReset = AutoOffsetReset.Latest, EnablePartitionEof = true }; config.Set("fetch.message.max.bytes", "10240"); // Note: If a key or value deserializer is not set (as is the case below), the // deserializer corresponding to the appropriate type from Confluent.Kafka.Serdes // will be used automatically (where available). The default deserializer for string // is UTF8. The default deserializer for Ignore returns null for all input data // (including non-null data). _consumer = new ConsumerBuilder <Null, string>(config) .SetKeyDeserializer(Deserializers.Null) .SetValueDeserializer(Deserializers.Utf8) .SetLogHandler(OnLog) .SetErrorHandler(OnError) .SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}")) .SetRebalanceHandler(OnRebalance) .Build(); }
public ActionResult Init() { var config = new ConsumerConfig { BootstrapServers = "10.104.51.12:9092,10.104.51.13:9092,10.104.51.14:9092", GroupId = "group", EnablePartitionEof = true }; // Note: If a key or value deserializer is not set (as is the case below), the // deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers // will be used automatically (where available). The default deserializer for string // is UTF8. The default deserializer for Ignore returns null for all input data // (including non-null data). _consumer = new ConsumerBuilder <Ignore, byte[]>(config) // Note: All handlers are called on the main .Consume thread. .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) // .SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}")) .SetPartitionsAssignedHandler((c, partitions) => { Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}]"); // possibly manually specify start offsets or override the partition assignment provided by // the consumer group by returning a list of topic/partition/offsets to assign to, e.g.: // // return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp])); }) .SetPartitionsRevokedHandler((c, partitions) => { Console.WriteLine($"Revoking assignment: [{string.Join(", ", partitions)}]"); }) .Build(); _consumer.Subscribe("video"); var topic = _consumer.Assignment.FirstOrDefault()?.Topic; return(Content("init - " + topic)); }
static void Main(string[] args) { var consumerConfig = new ConsumerConfig { GroupId = "test-consumer-group", BootstrapServers = "192.168.6.23:9092", AutoOffsetReset = AutoOffsetReset.Earliest, //AutoCommitIntervalMs = 5000, EnableAutoCommit = true }; using (var c = new ConsumerBuilder <Null, string>(consumerConfig).Build()) { var cts = new CancellationTokenSource(); c.Subscribe("daily_event", cts.Token, message => { var userInfo = UserContext.GetUserInfo(); Console.WriteLine($"{userInfo.UserName} send message:{message}"); }); } Console.ReadKey(); }
public void Work(System.Threading.CancellationToken token) { var config = new ConsumerConfig { BootstrapServers = KafkaSettings.BootstrapServers, GroupId = "foo", AutoOffsetReset = AutoOffsetReset.Earliest }; using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build()) { consumer.Subscribe(KafkaSettings.SearchTopic); while (!token.IsCancellationRequested) { var consumeResult = consumer.Consume(token); Console.WriteLine(consumeResult.Message); } consumer.Close(); } }
public IConsumer <string, string> CreateConsumer(string topicName, IEnumerable <string> topics) { var configConsumer = new ConsumerConfig { BootstrapServers = brokerList, GroupId = topicName + "Consumer", EnableAutoCommit = true, SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Latest, AutoCommitIntervalMs = 100 }; var consumer = new ConsumerBuilder <string, string>(configConsumer) .SetErrorHandler((_, e) => { logger.LogError($"ConsumerGroupId{topicName + "Consumer"} KfkConsumerError: {e.Reason}"); }) .Build(); consumer.Subscribe(topics); return(consumer); }
/// <summary> /// 消息队列 客户端消费者 /// </summary> /// <param name="client">TCP 客户端</param> /// <param name="config">队列数据 读取配置</param> /// <param name="log">日志处理</param> /// <param name="readerIndexNode"></param> protected Consumer(MasterServer.TcpInternalClient client, ConsumerConfig config, AutoCSer.ILog log, DataStructure.Abstract.Node readerIndexNode) { if (client == null) { throw new InvalidOperationException(); } this.client = client; if (config == null) { Config = defaultConfig; } else { config.Format(); Config = config; } Log = log ?? client._TcpClient_.Log; getDequeueIdentityNode = new DataStructure.Parameter.Value(readerIndexNode, OperationParameter.OperationType.MessageQueueGetDequeueIdentity); getDequeueIdentityNode.Parameter.SetJson((Cache.MessageQueue.ReaderConfig)Config); getMessageNode = new DataStructure.Parameter.Value(readerIndexNode, OperationParameter.OperationType.MessageQueueDequeue); setDequeueIdentityNode = new DataStructure.Parameter.Value(readerIndexNode, OperationParameter.OperationType.MessageQueueSetDequeueIdentity); }
private void InitializeConsumer() { var consumerConfig = new ConsumerConfig { BootstrapServers = Environment.GetEnvironmentVariable("Producer"), GroupId = Environment.GetEnvironmentVariable("GroupId"), AllowAutoCreateTopics = true, EnableAutoCommit = false, AutoOffsetReset = AutoOffsetReset.Earliest }; try { _consumer = new ConsumerBuilder <Null, string>(consumerConfig).Build(); _consumer.Subscribe(Environment.GetEnvironmentVariable("CardEdited")); } catch (Exception ex) { var scope = _serviceScopeFactory.CreateScope(); var logger = scope.ServiceProvider.GetRequiredService <ILogger <CardEventListener> >(); logger.LogInformation("Subcription to event broker failed {0}", ex.Message); } }
private static ConsumerConfig GetDefaultKafkaConsumerConfiguration(Consumer consumer) { var result = new ConsumerConfig { BootstrapServers = consumer.Servers, ClientId = consumer.ClientId, GroupId = consumer.GroupId, EnableAutoCommit = consumer.AutoCommit, StatisticsIntervalMs = consumer.StatisticsIntervalMs ?? KafkaDefaultValues.StatisticsIntervalMs, SessionTimeoutMs = consumer.SessionTimeoutMs ?? KafkaDefaultValues.SessionTimeoutMs, AutoOffsetReset = GetAutoOffsetReset(consumer.AutoOffsetReset), EnablePartitionEof = consumer.EnablePartitionEof, IsolationLevel = GetIsolationLevel(consumer.IsolationLevel), EnableSslCertificateVerification = consumer.EnableSslCertificateVerification, }; if (!string.IsNullOrEmpty(consumer.Debug)) { result.Debug = consumer.Debug; } return(result); }
private IConsumer <Ignore, string> GetConsumer(ConsumerConfig consumerConfig) { var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig) .SetErrorHandler((_, e) => { _logger.LogInformation($"Error: {e.Reason}"); }) .SetStatisticsHandler((_, json) => { _logger.LogInformation($"Statistics: {json}"); }) .SetPartitionsAssignedHandler((c, partitions) => { _logger.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]"); }) .SetPartitionsRevokedHandler((c, partitions) => { _logger.LogInformation($"Revoking assignment: [{string.Join(", ", partitions)}]"); }) .Build(); return(consumer); }
/// <summary> /// Initializes a new instance of the <see cref="Fetcher"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> /// <param name="zkClient"> /// The wrapper above ZooKeeper client. /// </param> public Fetcher(ConsumerConfig config, IZooKeeperClient zkClient) { this.config = config; this.zkClient = zkClient; }
public void ConsumerPorformsRebalancingWhenBrokerIsRemovedFromTopic() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 }; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); client.CreateEphemeral(brokerTopicPath, 1); WaitUntillIdle(client, 1000); client.DeleteRecursive(brokerTopicPath); WaitUntillIdle(client, 1000); IList<string> children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children.Count, Is.EqualTo(2)); Assert.That(children, Has.None.EqualTo("2345-0")); var topicRegistry = ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>("topicRegistry", consumerConnector); Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty); Assert.That(topicRegistry.Count, Is.EqualTo(1)); var item = topicRegistry["test"]; Assert.That(item.Count, Is.EqualTo(2)); Assert.That(item.Where(x => x.Value.BrokerId == 2345).Count(), Is.EqualTo(0)); } }
public void ProducerSends1Message() { int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets(); var producerConfig = new ProducerConfig(clientConfig); var mockPartitioner = new MockAlwaysZeroPartitioner(); using (var producer = new Producer<string, Message>(producerConfig, mockPartitioner, new DefaultEncoder(), null)) { var producerData = new ProducerData<string, Message>( CurrentTestTopic, "somekey", new List<Message> { originalMessage }); producer.Send(producerData); Thread.Sleep(waitSingle); while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfig(clientConfig) { Host = multipleBrokersHelper.BrokerThatHasChanged.Address, Port = multipleBrokersHelper.BrokerThatHasChanged.Port }; IConsumer consumer = new Consumers.Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 0) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); } }
public void ConsumerPorformsRebalancingWhenNewBrokerIsAddedToTopic() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 }; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>( "zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); IList<string> children = client.GetChildren("/consumers/group1/ids", false); string consumerId = children[0]; client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); client.CreateEphemeral(brokerTopicPath, 1); WaitUntillIdle(client, 500); children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children.Count, Is.EqualTo(3)); Assert.That(children, Contains.Item("2345-0")); string data = client.ReadData<string>("/consumers/group1/owners/test/2345-0"); Assert.That(data, Is.Not.Null); Assert.That(data, Contains.Substring(consumerId)); var topicRegistry = ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>( "topicRegistry", consumerConnector); Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty); Assert.That(topicRegistry.Count, Is.EqualTo(1)); var item = topicRegistry["test"]; Assert.That(item.Count, Is.EqualTo(3)); var broker = topicRegistry["test"].SingleOrDefault(x => x.Key.BrokerId == 2345); Assert.That(broker, Is.Not.Null); } }
public void ConsumerConnectorIsCreatedConnectsDisconnectsAndShutsDown() { var config = new ConsumerConfig(clientConfig); using (new ZookeeperConsumerConnector(config, true)) { } }
public void ConsumerConnectorReceivesAShutdownSignal() { // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); // putting the shutdown command into the queue FieldInfo fi = typeof(ZookeeperConsumerConnector).GetField( "queues", BindingFlags.NonPublic | BindingFlags.Instance); var value = (IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>>) fi.GetValue(consumerConnector); foreach (var topicConsumerQueueMap in value) { topicConsumerQueueMap.Value.Add(ZookeeperConsumerConnector.ShutdownCommand); } var sets = messages[CurrentTestTopic]; var resultMessages = new List<Message>(); foreach (var set in sets) { foreach (var message in set) { resultMessages.Add(message); } } Assert.AreEqual(0, resultMessages.Count); } }
public void ConsumerMultiFetchGetsMessage() { ProducerSendMultiRequest(); ConsumerConfig config = new ConsumerConfig(clientConfig); IConsumer cons = new Consumers.Consumer(config); MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest> { new FetchRequest(CurrentTestTopic, 0, 0), new FetchRequest(CurrentTestTopic, 0, 0), new FetchRequest(CurrentTestTopic + "2", 0, 0) }); IList<BufferedMessageSet> response = cons.MultiFetch(request); for (int ix = 0; ix < response.Count; ix++) { IEnumerable<Message> messageSet = response[ix].Messages; Console.WriteLine(string.Format("Request #{0}-->", ix)); foreach (Message msg in messageSet) { Console.WriteLine(msg); } } }
public void ConsumerPorformsRebalancingOnStart() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1" }; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { ZooKeeperClient client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); IList<string> children = client.GetChildren("/consumers", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("group1")); children = client.GetChildren("/consumers/group1", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("ids")); Assert.That(children, Contains.Item("owners")); children = client.GetChildren("/consumers/group1/ids", false); Assert.That(children, Is.Not.Null.And.Not.Empty); string consumerId = children[0]; children = client.GetChildren("/consumers/group1/owners", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(1)); Assert.That(children, Contains.Item("test")); children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(2)); string partId = children[0]; string data = client.ReadData<string>("/consumers/group1/owners/test/" + partId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Contains.Substring(consumerId)); data = client.ReadData<string>("/consumers/group1/ids/" + consumerId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Is.EqualTo("{ \"test\": 1 }")); } using (var client = new ZooKeeperClient(config.ZkConnect, config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); //// Should be created as ephemeral IList<string> children = client.GetChildren("/consumers/group1/ids"); Assert.That(children, Is.Null.Or.Empty); //// Should be created as ephemeral children = client.GetChildren("/consumers/group1/owners/test"); Assert.That(children, Is.Null.Or.Empty); } }
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 }; using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; KafkaMessageStream myStream = sets[0]; var enumerator = myStream.GetEnumerator(); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString()); Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext()); Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state enumerator.Reset(); // producing again string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); Thread.Sleep(3000); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString()); } }
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; try { foreach (var set in sets) { foreach (var message in set) { resultMessages.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(2, resultMessages.Count); Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString()); Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString()); }
public void ConsumerConnectorConsumesTwoDifferentTopics() { string topic1 = CurrentTestTopic + "1"; string topic2 = CurrentTestTopic + "2"; // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 }); producer.Send(producerRequest1); var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages1 = new List<Message>(); var resultMessages2 = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); Assert.IsTrue(messages.ContainsKey(topic1)); Assert.IsTrue(messages.ContainsKey(topic2)); var sets1 = messages[topic1]; try { foreach (var set in sets1) { foreach (var message in set) { resultMessages1.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } var sets2 = messages[topic2]; try { foreach (var set in sets2) { foreach (var message in set) { resultMessages2.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(1, resultMessages1.Count); Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString()); Assert.AreEqual(1, resultMessages2.Count); Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString()); }
public void ConsumerGetsOffsets() { OffsetRequest request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10); ConsumerConfig config = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(config); IList<long> list = consumer.GetOffsetsBefore(request); foreach (long l in list) { Console.Out.WriteLine(l); } }
public void ConsumerFetchMessage() { ProducerSendsMessage(); ConsumerConfig config = new ConsumerConfig(clientConfig); IConsumer consumer = new Kafka.Client.Consumers.Consumer(config); FetchRequest request = new FetchRequest(CurrentTestTopic, 0, 0); BufferedMessageSet response = consumer.Fetch(request); Assert.NotNull(response); foreach (var message in response.Messages) { Console.WriteLine(message); } }
public void ProducerSendsAndConsumerReceivesMultiRequest() { string testTopic1 = CurrentTestTopic + "1"; string testTopic2 = CurrentTestTopic + "2"; string testTopic3 = CurrentTestTopic + "3"; Message sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage")); Message sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage")); Message sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage")); Message sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage")); List<ProducerRequest> requests = new List<ProducerRequest> { new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }), new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }), new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }), new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 }) }; var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, clientConfig); long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, clientConfig); long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, clientConfig); producer.MultiSend(requests); ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(consumerConfig); MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest> { new FetchRequest(testTopic1, 0, currentOffset1), new FetchRequest(testTopic2, 0, currentOffset2), new FetchRequest(testTopic3, 0, currentOffset3) }); IList<BufferedMessageSet> messageSets; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; while (true) { Thread.Sleep(waitSingle); messageSets = consumer.MultiFetch(request); if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0) { break; } else { totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } } Assert.AreEqual(3, messageSets.Count); Assert.AreEqual(2, messageSets[0].Messages.Count()); Assert.AreEqual(1, messageSets[1].Messages.Count()); Assert.AreEqual(1, messageSets[2].Messages.Count()); Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString()); Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString()); Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString()); Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString()); }