/// <summary>
/// Initializes a new instance of the <see cref="Consumer"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
public Consumer(ConsumerConfig config)
{
Guard.Assert<ArgumentNullException>(() => config != null);
this.config = config;
this.Host = config.Host;
this.Port = config.Port;
}
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfig config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos)
{
this.name = name;
this.zkClient = zkClient;
this.config = config;
this.broker = broker;
this.partitionTopicInfos = partitionTopicInfos;
this.simpleConsumer = new Consumer(this.config);
}
/// <summary>
/// Initializes a new instance of the <see cref="ZookeeperConsumerConnector"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration. At the minimum, need to specify the group ID
/// of the consumer and the ZooKeeper connection string.
/// </param>
/// <param name="enableFetcher">
/// Indicates whether fetchers should be enabled
/// </param>
public ZookeeperConsumerConnector(ConsumerConfig config, bool enableFetcher)
: base(config)
{
this.config = config;
this.enableFetcher = enableFetcher;
this.ConnectZk();
this.CreateFetcher();
if (this.config.AutoCommit)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "starting auto committer every {0} ms", this.config.AutoCommitIntervalMs);
scheduler.ScheduleWithRate(this.AutoCommit, this.config.AutoCommitIntervalMs, this.config.AutoCommitIntervalMs);
}
}
public void ConsumerPerformsRebalancingWhenConsumerIsRemovedAndTakesItsPartitions()
{
var config = new ConsumerConfig(clientConfig)
{
AutoCommit = false,
GroupId = "group1",
ZkSessionTimeoutMs = 60000,
ZkConnectionTimeoutMs = 60000
};
IList<string> ids;
IList<string> owners;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
using (var consumerConnector2 = new ZookeeperConsumerConnector(config, true))
{
consumerConnector2.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
ids = client.GetChildren("/consumers/group1/ids", false).ToList();
owners = client.GetChildren("/consumers/group1/owners/test", false).ToList();
Assert.That(ids, Is.Not.Null.And.Not.Empty);
Assert.That(ids.Count, Is.EqualTo(2));
Assert.That(owners, Is.Not.Null.And.Not.Empty);
Assert.That(owners.Count, Is.EqualTo(2));
}
WaitUntillIdle(client, 1000);
ids = client.GetChildren("/consumers/group1/ids", false).ToList();
owners = client.GetChildren("/consumers/group1/owners/test", false).ToList();
Assert.That(ids, Is.Not.Null.And.Not.Empty);
Assert.That(ids.Count, Is.EqualTo(1));
Assert.That(owners, Is.Not.Null.And.Not.Empty);
Assert.That(owners.Count, Is.EqualTo(2));
var data1 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[0], false);
var data2 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[1], false);
Assert.That(data1, Is.Not.Null.And.Not.Empty);
Assert.That(data2, Is.Not.Null.And.Not.Empty);
Assert.That(data1, Is.EqualTo(data2));
Assert.That(data1, Is.StringStarting(ids[0]));
}
}
public static long GetCurrentKafkaOffset(string topic, string address, int port)
{
OffsetRequest request = new OffsetRequest(topic, 0, DateTime.Now.AddDays(-5).Ticks, 10);
ConsumerConfig consumerConfig = new ConsumerConfig();
consumerConfig.Host = address;
consumerConfig.Port = port;
IConsumer consumer = new Consumers.Consumer(consumerConfig);
IList<long> list = consumer.GetOffsetsBefore(request);
if (list.Count > 0)
{
return list[0];
}
else
{
return 0;
}
}
public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage()
{
Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message"));
var config = new AsyncProducerConfig(clientConfig);
var producer = new AsyncProducer(config);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage });
long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig);
producer.Send(producerRequest);
ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(consumerConfig);
FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset);
BufferedMessageSet response;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 0)
{
break;
}
else
{
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Message resultMessage = response.Messages.First();
Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString());
}
internal ZKRebalancerListener(
ConsumerConfig config,
string consumerIdString,
IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry,
IZooKeeperClient zkClient,
ZookeeperConsumerConnector zkConsumerConnector,
IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues,
Fetcher fetcher,
object syncLock)
{
this.syncLock = syncLock;
this.consumerIdString = consumerIdString;
this.config = config;
this.topicRegistry = topicRegistry;
this.zkClient = zkClient;
this.dirs = new ZKGroupDirs(config.GroupId);
this.zkConsumerConnector = zkConsumerConnector;
this.queues = queues;
this.fetcher = fetcher;
}
static async Task Main(string[] args)
{
string topicName = "order_create3";
//var config = new ProducerConfig
//{
// BootstrapServers = "localhost:29092",
// MessageTimeoutMs = 10000,
//};
//await ProduceDataAsync(topicName, config);
var consumerConfig = new ConsumerConfig
{
BootstrapServers = "localhost:29092",
GroupId = "group-F",
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Earliest
};
ConsumeData(topicName, consumerConfig);
}
public KafkaConsumer(KafkaConsumerOptions options)
{
MessageReceived = new MessageReceivedHandler <TMessage>(onMessageReceived);
_options = options;
var conf = new ConsumerConfig
{
GroupId = _options.GroupId,
BootstrapServers = _options.BootstrapServers,
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = (AutoOffsetReset)_options.AutoOffsetReset
};
_consumer = new ConsumerBuilder <Ignore, string>(conf).Build();
cancellationToken = new CancellationTokenSource();
_consumer.Subscribe(_options.Topic);
}
private IConsumer <byte[], T> InitializeConsumer()
{
var config = new ConsumerConfig
{
GroupId = _kafkaConfig.CurrentValue.ConsumerGroup,
BootstrapServers = string.Join(',', _kafkaConfig.CurrentValue.Brokers),
MaxPollIntervalMs = Day,
EnableAutoCommit = false,
EnableAutoOffsetStore = false
};
var consumerBuilder = new ConsumerBuilder <byte[], T>(config);
if (typeof(T) == typeof(MessageData.MessageData))
{
consumerBuilder.SetValueDeserializer((IDeserializer <T>) new MessageDataDeserializer());
}
var consumer = consumerBuilder.Build();
return(consumer);
}
public PooledConsumer GetConsumer(string group)
{
var consumerObjectPool = consumerPoolDict.GetOrAdd(group, key =>
{
var kvList = serializer.Deserialize <List <KeyValuePair <string, string> > >(serializer.SerializeToUtf8Bytes(consumerConfig));
var dict = new Dictionary <string, string>(kvList);
var config = new ConsumerConfig(dict)
{
GroupId = group
};
return(new DefaultObjectPool <PooledConsumer>(new ConsumerPooledObjectPolicy(config, logger), options.ConsumerMaxPoolSize));
});
var result = consumerObjectPool.Get();
if (result.Pool is null)
{
result.Pool = consumerObjectPool;
result.MaxBatchSize = options.CunsumerMaxBatchSize;
result.MaxMillisecondsInterval = options.CunsumerMaxMillisecondsInterval;
}
return(result);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
var config = new ConsumerConfig
{
BootstrapServers = "localhost:9092",
GroupId = $"topic-teste-hello-world-group-0",
AutoOffsetReset = AutoOffsetReset.Earliest
};
var cancellationTokenSource = new CancellationTokenSource();
using var consumer = new ConsumerBuilder <Ignore, string>(config).Build();
consumer.Subscribe("topic-teste-hello-world");
try
{
while (true)
{
var consumeResult = consumer.Consume(cancellationTokenSource.Token);
_logger.LogInformation(consumeResult.Message.Value);
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Worker running at: {time}", DateTimeOffset.Now);
}
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
await Task.Delay(1000, stoppingToken);
}
}
private void ConsumerListener()
{
var config = new ConsumerConfig
{
GroupId = _config["EventHub:GroupId"],
BootstrapServers = _config["EventHub:BootstrapServers"],
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build())
{
consumer.Subscribe(_config["EventHub:GroupId"]);
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
_logger.LogInformation($"Consumed message '{consumeResult.Value}' at: '{consumeResult.TopicPartitionOffset}'.");
}
catch (ConsumeException e)
{
_logger.LogError($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
consumer.Close();
}
}
}
public bool ListenAndConsumeMessage <T>(string topic, Action <T> f)
{
var conf = new ConsumerConfig
{
GroupId = _messageBrokerConfigSingleton.GroupId,
BootstrapServers = _messageBrokerConfigSingleton.BrokerLocation,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
using (var c = new Consumer <string, string>(conf))
{
c.Subscribe(topic);
bool consuming = true;
c.OnError += (_, e) => consuming = !e.IsFatal;
while (consuming)
{
try
{
var cr = c.Consume();
Console.WriteLine("Received Message with Key: " + cr.Key);
if (cr.Key.StartsWith(typeof(T).FullName))
{
f(JsonConvert.DeserializeObject <T>(cr.Value));
}
}
catch (ConsumeException e)
{
//LOG THIS error..
}
}
//close the consumer
c.Close();
return(true);
}
}
public static void OnPartitionsAssignedNotSet(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start OnPartitionsAssignedNotSet");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
// Producing onto the topic to make sure it exists.
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
{
var dr = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("test string", SerializationContext.Empty)
}).Result;
Assert.NotEqual(Offset.Invalid, dr.Offset);
producer.Flush(TimeSpan.FromSeconds(10));
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Subscribe(singlePartitionTopic);
Assert.Empty(consumer.Assignment);
consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Single(consumer.Assignment);
Assert.Equal(singlePartitionTopic, consumer.Assignment[0].Topic);
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end OnPartitionsAssignedNotSet");
}
static async Task Main(string[] args)
{
var consumerConfig = new ConsumerConfig
{
BootstrapServers = args[0],
GroupId = "protobuf-example",
AutoOffsetReset = AutoOffsetResetType.Latest
};
var consumeTask = Task.Run(() =>
{
// consume a single message then exit.
using (var consumer = new Consumer <int, User>(consumerConfig, Deserializers.Int32, new ProtobufDeserializer <User>()))
{
consumer.Subscribe("protobuf-test-topic");
var cr = consumer.Consume();
Console.WriteLine($"User: [id: {cr.Key}, favorite color: {cr.Message.Value.FavoriteColor}]");
}
});
// wait a bit so the consumer is ready to consume messages before producing one.
await Task.Delay(TimeSpan.FromSeconds(10));
var producerConfig = new ProducerConfig {
BootstrapServers = args[0]
};
using (var producer = new Producer <int, User>(producerConfig, Serializers.Int32, new ProtobufSerializer <User>()))
{
await producer.ProduceAsync("protobuf-test-topic", new Message <int, User> {
Key = 0, Value = new User {
FavoriteColor = "green"
}
});
}
await consumeTask;
}
public static void Main(string[] args)
{
var configuration = new ConsumerConfig
{
GroupId = "test-consumer-group",
BootstrapServers = "127.0.0.1:9092",
AutoOffsetReset = AutoOffsetReset.Earliest
};
using var builder = new ConsumerBuilder <Ignore, string>(configuration).Build();
{
builder.Subscribe("test-topic");
var cancellationToken = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cancellationToken.Cancel(); };
try
{
while (true)
{
try
{
var consumerResult = builder.Consume(cancellationToken.Token);
Console.WriteLine($"Consumed message '{consumerResult.Value}' at: '{consumerResult.TopicPartitionOffset}'.");
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
builder.Close();
}
}
}
static void Main(string[] args)
{
ConsumerConfig config = new ConsumerConfig()
{
BootstrapServers = "192.168.28.172:9093,192.168.28.172:9094,192.168.28.172:9095",
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Latest,
EnableAutoCommit = false
};
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
using (var consumer = new ConsumerBuilder <string, object>(config).SetValueDeserializer(new KafkaConverter()).Build())
{
//c.Assign(new TopicPartition("test", new Partition(1)));//从指定的Partition订阅消息使用Assign方法
consumer.Subscribe("mykafka");//订阅消息使用Subscribe方法
while (!cts.IsCancellationRequested)
{
var result = consumer.Consume(cts.Token);
if (result.Offset % 5 == 0)
{
consumer.Commit(result);//手动提交，如果上面的EnableAutoCommit=true表示自动提交，则无需调用Commit方法
Console.WriteLine($"recieve message:{result.Message.Value},Committed offset: {result.Offset}");
}
else
{
Console.WriteLine($"recieve message:{result.Message.Value}，Committed offset: { result.Offset }");
}
}
}
}
private async Task KafkaConsume()
{
_logger.LogDebug($"KafkaConsume is running");
var config = new ConsumerConfig
{
GroupId = "teslacharger",
BootstrapServers = "green-kafka:9092",
AutoOffsetReset = AutoOffsetResetType.Earliest,
EnableAutoCommit = true
};
using (var c = new Consumer <string, string>(config))
{
c.Subscribe("future-consumption");
c.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
var ct = _cts.Token;
while (!ct.IsCancellationRequested)
{
try
{
var cr = c.Consume(ct);
Console.WriteLine($"Consumed message from '{cr.Topic}', partion {cr.Partition}, offset {cr.Offset}, length {cr.Value.Length}, head {cr.Value.Substring(0, 30)}");
await CheckCharging();
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
}
}
_logger.LogDebug($"KafkaConsume is terminating");
}
public static void startGetMessage()
{
configCommand = new ConsumerConfig
{
GroupId = consumerGroupId,
BootstrapServers = brokerList,
AutoOffsetReset = AutoOffsetReset.Latest
};
using (consumerCommand = new ConsumerBuilder <Ignore, string>(configCommand).Build())
{
consumerCommand.Subscribe(commandTopicName);
CancellationTokenSource cts = new CancellationTokenSource();
//Console.CancelKeyPress += (_, e) => {
// e.Cancel = true; // prevent the process from terminating.
// cts.Cancel();
//};
try
{
while (true)
{
try
{
var cr = consumerCommand.Consume(cts.Token);
OnGetMessage(cr.Value);
}
catch (ConsumeException e)
{
FileWorker.LogHelper.WriteLog($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException e)
{
FileWorker.LogHelper.WriteLog($"Error occured1: {e.Message}");
consumerCommand.Close();
}
}
}
public ConsumerConfiguration(
ConsumerConfig consumerConfig,
IEnumerable <string> topics,
string consumerName,
int workerCount,
int bufferSize,
Factory <IDistributionStrategy> distributionStrategyFactory,
MiddlewareConfiguration middlewareConfiguration,
bool autoStoreOffsets,
TimeSpan autoCommitInterval,
IReadOnlyList <Action <string> > statisticsHandlers,
ConsumerCustomFactory customFactory)
{
this.consumerConfig = consumerConfig ?? throw new ArgumentNullException(nameof(consumerConfig));
if (string.IsNullOrEmpty(this.consumerConfig.GroupId))
{
throw new ArgumentNullException(nameof(consumerConfig.GroupId));
}
this.DistributionStrategyFactory =
distributionStrategyFactory ?? throw new ArgumentNullException(nameof(distributionStrategyFactory));
this.MiddlewareConfiguration = middlewareConfiguration ?? throw new ArgumentNullException(nameof(middlewareConfiguration));
this.AutoStoreOffsets = autoStoreOffsets;
this.AutoCommitInterval = autoCommitInterval;
this.Topics = topics ?? throw new ArgumentNullException(nameof(topics));
this.ConsumerName = consumerName ?? Guid.NewGuid().ToString();
this.WorkerCount = workerCount;
this.StatisticsHandlers = statisticsHandlers;
this.CustomFactory = customFactory;
this.BufferSize = bufferSize > 0 ?
bufferSize :
throw new ArgumentOutOfRangeException(
nameof(bufferSize),
bufferSize,
"The value must be greater than 0");
}
public static void Main(string[] args)
{
DotNetEnv.Env.Load();
var conf = new ConsumerConfig
{
BootstrapServers = System.Environment.GetEnvironmentVariable("KAFKA_HOST"),
GroupId = System.Environment.GetEnvironmentVariable("KAFKA_GROUP_ID"),
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var consumer = new ConsumerBuilder <Ignore, string>(conf).Build())
{
consumer.Subscribe(System.Environment.GetEnvironmentVariable("KAFKA_TOPIC"));
Console.WriteLine("Consumer started, press <CTRL> + C to exit");
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true;
cts.Cancel();
};
try
{
while (true)
{
var message = consumer.Consume(cts.Token);
Console.WriteLine(message.Value);
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
var conf = new ConsumerConfig
{
GroupId = "st_consumer_group",
BootstrapServers = _bootstrapServers,
AutoOffsetReset = AutoOffsetReset.Latest
};
using (var builder = new ConsumerBuilder <Ignore,
string>(conf).Build())
{
builder.Subscribe(topic);
var cancelToken = new CancellationTokenSource();
try
{
while (true)
{
var consumer = builder.Consume(cancelToken.Token);
var jObject = JObject.Parse(consumer.Message.Value);
var phone = (string)jObject["phone"];
var email = (string)jObject["email"];
_sender.SendSms(new SmsMessage
{
From = "+19149964683",
Message = $"Message: {consumer.Message.Value} received from {consumer.TopicPartitionOffset}",
To = "+380638680809"
});
Console.WriteLine($"Message: {consumer.Message.Value} received from {consumer.TopicPartitionOffset}");
}
}
catch (Exception)
{
builder.Close();
}
}
return(Task.CompletedTask);
}
public void CtorCommon_SameConfigsSuccess()
{
var topicsConfig = new TopicsConfig()
{
"topic1", "topic2", "topic3"
};
var consumerConfig = new ConsumerConfig()
{
Environment = "TestTier", ClusterId = "TestCluster", Urls = "TestUrls"
};
var producerConfigs = new ProducerConfigs()
{
{ new ProducerConfig {
Environment = "TestTierProducer1", ClusterId = "TestClusterProducer1", Urls = "TestUrlsProducer1"
} },
{ new ProducerConfig {
Environment = "TestTierProducer2", ClusterId = "TestClusterProducer2", Urls = "TestUrlsProducer2"
} }
};
var provider = new Mock <IServiceProvider>();
var topicsOptions = new Mock <IOptions <TopicsConfig> >();
topicsOptions.SetupGet(c => c.Value).Returns(topicsConfig);
var consumerConfigOptions = new Mock <IOptions <ConsumerConfig> >();
consumerConfigOptions.SetupGet(c => c.Value).Returns(consumerConfig);
var producerConfigsOptions = new Mock <IOptions <ProducerConfigs> >();
producerConfigsOptions.SetupGet(c => c.Value).Returns(producerConfigs);
var runner = new FeedProxiesRunnerTestDecorator(provider.Object, topicsOptions.Object,
consumerConfigOptions.Object, producerConfigsOptions.Object);
runner.Topics.Should().BeEquivalentTo(topicsConfig);
runner.ConsumerConfig.Should().BeEquivalentTo(consumerConfig);
runner.ProducerConfigs.Should().BeEquivalentTo(producerConfigs);
}
static void Consume(string topic, ClientConfig config)
{
var consumerConfig = new ConsumerConfig(config);
consumerConfig.GroupId = "Group-Consumer";
consumerConfig.AutoOffsetReset = AutoOffsetReset.Earliest;
consumerConfig.EnableAutoCommit = false;
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build())
{
consumer.Subscribe(topic);
//var totalCount = 0;
try
{
while (true)
{
var cr = consumer.Consume(cts.Token);
// totalCount += JObject.Parse(cr.Value).Value<int>("count");
Console.WriteLine($"Consumed record with key {cr.Key} and value {cr.Value}");
}
}
catch (OperationCanceledException)
{
// Ctrl-C was pressed.
}
finally
{
consumer.Close();
}
}
}
static IConsumer <Ignore, string> ConfigureConsumer(IServiceProvider provider)
{
var options = provider.GetRequiredService <IOptions <AppConfig> >().Value;
var config = new ConsumerConfig
{
BootstrapServers = options.BrokerList,
GroupId = "consumer",
EnableAutoCommit = false,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var logging = provider.GetRequiredService <ILoggerFactory>().CreateLogger <IConsumer <Ignore, string> >();
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
return(new ConsumerBuilder <Ignore, string>(config)
// Note: All handlers are called on the main .Consume thread.
.SetErrorHandler((_, e) => logging.LogError($"Error: {e.Reason}"))
.SetStatisticsHandler((_, json) => logging.LogInformation($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
logging.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]");
// possibly manually specify start offsets or override the partition assignment provided by
// the consumer group by returning a list of topic/partition/offsets to assign to, e.g.:
//
// return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp]));
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
logging.LogInformation($"Revoking assignment: [{string.Join(", ", partitions)}]");
})
.Build());
}
static void runConsumerManual()
{
var config = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = groupId,
EnableAutoCommit = false
};
bool cancelled = false;
int noRecordsCount = 0;
using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build())
{
consumer.Subscribe(topicName);
var cancelToken = new CancellationTokenSource();
ConsumeResult <Ignore, string> consumeResult = null;
while (!cancelled)
{
consumeResult = consumer.Consume(cancelToken.Token);
noRecordsCount++;
// handle message
Console.WriteLine($"Consumer Record:(Key: {consumeResult.Message.Key}, Value: {consumeResult.Message.Value} Partition: {consumeResult.TopicPartition.Partition} Offset: {consumeResult.TopicPartitionOffset.Offset}");
if (consumeResult.Offset % 50 == 0)
{
consumer.Commit(consumeResult);
}
}
// commit the rest
consumer.Commit(consumeResult);
consumer.Close();
}
}
public KafkaQueueService(
IConfiguration configuration,
ISerializer serializer,
IOptions <QueueOptions> options
) : base(serializer, options)
{
_producer = new Lazy <IProducer <Null, string> >(() => {
var config = new ProducerConfig
{
BootstrapServers = configuration.GetSection(Constants.OptionNameBootstrapServer).Value,
BatchNumMessages = 1,
MessageTimeoutMs = 1,
SocketNagleDisable = true,
Acks = Acks.Leader
};
return(new ProducerBuilder <Null, string>(config).Build());
});
_consumer = new Lazy <IConsumer <Ignore, string> >(() => {
var groupIdSection = configuration.GetSection(Constants.OptionNameConsumerGroupId);
var groupId = groupIdSection.Exists() && !string.IsNullOrEmpty(groupIdSection.Value)
? groupIdSection.Value : Constants.DefaultGroupId;
var config = new ConsumerConfig
{
BootstrapServers = configuration.GetSection(Constants.OptionNameBootstrapServer).Value,
GroupId = groupId,
AutoOffsetReset = AutoOffsetReset.Earliest,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000
};
var consumer = new ConsumerBuilder <Ignore, string>(config)
.SetErrorHandler((_, e) => StaticLog.Error($"Receiving Error: {e.Code} - {e.Reason}"))
.Build();
return(consumer);
});
}
/// <summary>Creates new instance <see cref="KafkaConsumer"/>.</summary>
public KafkaConsumer(ILogger logger, string brokerEndpoints, string groupId = null)
{
_logger = logger;
var config = new ConsumerConfig
{
BootstrapServers = brokerEndpoints,
ApiVersionRequest = true,
GroupId = !string.IsNullOrEmpty(groupId) ? groupId : Guid.NewGuid().ToString(),
EnableAutoCommit = false,
FetchWaitMaxMs = 5,
FetchErrorBackoffMs = 5,
QueuedMinMessages = 1000,
SessionTimeoutMs = 6000,
//StatisticsIntervalMs = 5000,
#if DEBUG
Debug = "msg",
#endif
AutoOffsetReset = AutoOffsetReset.Latest,
EnablePartitionEof = true
};
config.Set("fetch.message.max.bytes", "10240");
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Serdes
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
_consumer = new ConsumerBuilder <Null, string>(config)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(Deserializers.Utf8)
.SetLogHandler(OnLog)
.SetErrorHandler(OnError)
.SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}"))
.SetRebalanceHandler(OnRebalance)
.Build();
}
public ActionResult Init()
{
var config = new ConsumerConfig
{
BootstrapServers = "10.104.51.12:9092,10.104.51.13:9092,10.104.51.14:9092",
GroupId = "group",
EnablePartitionEof = true
};
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
_consumer = new ConsumerBuilder <Ignore, byte[]>(config)
// Note: All handlers are called on the main .Consume thread.
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
// .SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}]");
// possibly manually specify start offsets or override the partition assignment provided by
// the consumer group by returning a list of topic/partition/offsets to assign to, e.g.:
//
// return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp]));
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
Console.WriteLine($"Revoking assignment: [{string.Join(", ", partitions)}]");
})
.Build();
_consumer.Subscribe("video");
var topic = _consumer.Assignment.FirstOrDefault()?.Topic;
return(Content("init - " + topic));
}
static void Main(string[] args)
{
var consumerConfig = new ConsumerConfig
{
GroupId = "test-consumer-group",
BootstrapServers = "192.168.6.23:9092",
AutoOffsetReset = AutoOffsetReset.Earliest,
//AutoCommitIntervalMs = 5000,
EnableAutoCommit = true
};
using (var c = new ConsumerBuilder <Null, string>(consumerConfig).Build())
{
var cts = new CancellationTokenSource();
c.Subscribe("daily_event", cts.Token, message =>
{
var userInfo = UserContext.GetUserInfo();
Console.WriteLine($"{userInfo.UserName} send message:{message}");
});
}
Console.ReadKey();
}
public void Work(System.Threading.CancellationToken token)
{
var config = new ConsumerConfig
{
BootstrapServers = KafkaSettings.BootstrapServers,
GroupId = "foo",
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build())
{
consumer.Subscribe(KafkaSettings.SearchTopic);
while (!token.IsCancellationRequested)
{
var consumeResult = consumer.Consume(token);
Console.WriteLine(consumeResult.Message);
}
consumer.Close();
}
}
public IConsumer <string, string> CreateConsumer(string topicName, IEnumerable <string> topics)
{
var configConsumer = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = topicName + "Consumer",
EnableAutoCommit = true,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Latest,
AutoCommitIntervalMs = 100
};
var consumer = new ConsumerBuilder <string, string>(configConsumer)
.SetErrorHandler((_, e) =>
{
logger.LogError($"ConsumerGroupId{topicName + "Consumer"} KfkConsumerError: {e.Reason}");
})
.Build();
consumer.Subscribe(topics);
return(consumer);
}
/// <summary>
/// 消息队列 客户端消费者
/// </summary>
/// <param name="client">TCP 客户端</param>
/// <param name="config">队列数据 读取配置</param>
/// <param name="log">日志处理</param>
/// <param name="readerIndexNode"></param>
protected Consumer(MasterServer.TcpInternalClient client, ConsumerConfig config, AutoCSer.ILog log, DataStructure.Abstract.Node readerIndexNode)
{
if (client == null)
{
throw new InvalidOperationException();
}
this.client = client;
if (config == null)
{
Config = defaultConfig;
}
else
{
config.Format();
Config = config;
}
Log = log ?? client._TcpClient_.Log;
getDequeueIdentityNode = new DataStructure.Parameter.Value(readerIndexNode, OperationParameter.OperationType.MessageQueueGetDequeueIdentity);
getDequeueIdentityNode.Parameter.SetJson((Cache.MessageQueue.ReaderConfig)Config);
getMessageNode = new DataStructure.Parameter.Value(readerIndexNode, OperationParameter.OperationType.MessageQueueDequeue);
setDequeueIdentityNode = new DataStructure.Parameter.Value(readerIndexNode, OperationParameter.OperationType.MessageQueueSetDequeueIdentity);
}
private void InitializeConsumer()
{
var consumerConfig = new ConsumerConfig
{
BootstrapServers = Environment.GetEnvironmentVariable("Producer"),
GroupId = Environment.GetEnvironmentVariable("GroupId"),
AllowAutoCreateTopics = true,
EnableAutoCommit = false,
AutoOffsetReset = AutoOffsetReset.Earliest
};
try
{
_consumer = new ConsumerBuilder <Null, string>(consumerConfig).Build();
_consumer.Subscribe(Environment.GetEnvironmentVariable("CardEdited"));
}
catch (Exception ex)
{
var scope = _serviceScopeFactory.CreateScope();
var logger = scope.ServiceProvider.GetRequiredService <ILogger <CardEventListener> >();
logger.LogInformation("Subcription to event broker failed {0}", ex.Message);
}
}
private static ConsumerConfig GetDefaultKafkaConsumerConfiguration(Consumer consumer)
{
var result = new ConsumerConfig
{
BootstrapServers = consumer.Servers,
ClientId = consumer.ClientId,
GroupId = consumer.GroupId,
EnableAutoCommit = consumer.AutoCommit,
StatisticsIntervalMs = consumer.StatisticsIntervalMs ?? KafkaDefaultValues.StatisticsIntervalMs,
SessionTimeoutMs = consumer.SessionTimeoutMs ?? KafkaDefaultValues.SessionTimeoutMs,
AutoOffsetReset = GetAutoOffsetReset(consumer.AutoOffsetReset),
EnablePartitionEof = consumer.EnablePartitionEof,
IsolationLevel = GetIsolationLevel(consumer.IsolationLevel),
EnableSslCertificateVerification = consumer.EnableSslCertificateVerification,
};
if (!string.IsNullOrEmpty(consumer.Debug))
{
result.Debug = consumer.Debug;
}
return(result);
}
private IConsumer <Ignore, string> GetConsumer(ConsumerConfig consumerConfig)
{
var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig)
.SetErrorHandler((_, e) =>
{
_logger.LogInformation($"Error: {e.Reason}");
})
.SetStatisticsHandler((_, json) =>
{
_logger.LogInformation($"Statistics: {json}");
})
.SetPartitionsAssignedHandler((c, partitions) =>
{
_logger.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]");
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
_logger.LogInformation($"Revoking assignment: [{string.Join(", ", partitions)}]");
})
.Build();
return(consumer);
}
/// <summary>
/// Initializes a new instance of the <see cref="Fetcher"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
/// <param name="zkClient">
/// The wrapper above ZooKeeper client.
/// </param>
public Fetcher(ConsumerConfig config, IZooKeeperClient zkClient)
{
this.config = config;
this.zkClient = zkClient;
}
public void ConsumerPorformsRebalancingWhenBrokerIsRemovedFromTopic()
{
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 };
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
client.CreateEphemeral(brokerTopicPath, 1);
WaitUntillIdle(client, 1000);
client.DeleteRecursive(brokerTopicPath);
WaitUntillIdle(client, 1000);
IList<string> children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children.Count, Is.EqualTo(2));
Assert.That(children, Has.None.EqualTo("2345-0"));
var topicRegistry = ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>("topicRegistry", consumerConnector);
Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty);
Assert.That(topicRegistry.Count, Is.EqualTo(1));
var item = topicRegistry["test"];
Assert.That(item.Count, Is.EqualTo(2));
Assert.That(item.Where(x => x.Value.BrokerId == 2345).Count(), Is.EqualTo(0));
}
}
public void ProducerSends1Message()
{
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData"));
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets();
var producerConfig = new ProducerConfig(clientConfig);
var mockPartitioner = new MockAlwaysZeroPartitioner();
using (var producer = new Producer<string, Message>(producerConfig, mockPartitioner, new DefaultEncoder(), null))
{
var producerData = new ProducerData<string, Message>(
CurrentTestTopic, "somekey", new List<Message> { originalMessage });
producer.Send(producerData);
Thread.Sleep(waitSingle);
while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged())
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfig(clientConfig)
{
Host = multipleBrokersHelper.BrokerThatHasChanged.Address,
Port = multipleBrokersHelper.BrokerThatHasChanged.Port
};
IConsumer consumer = new Consumers.Consumer(consumerConfig);
var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 0)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString());
}
}
public void ConsumerPorformsRebalancingWhenNewBrokerIsAddedToTopic()
{
var config = new ConsumerConfig(clientConfig)
{
AutoCommit = false,
GroupId = "group1",
ZkSessionTimeoutMs = 60000,
ZkConnectionTimeoutMs = 60000
};
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>(
"zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
IList<string> children = client.GetChildren("/consumers/group1/ids", false);
string consumerId = children[0];
client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
client.CreateEphemeral(brokerTopicPath, 1);
WaitUntillIdle(client, 500);
children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children.Count, Is.EqualTo(3));
Assert.That(children, Contains.Item("2345-0"));
string data = client.ReadData<string>("/consumers/group1/owners/test/2345-0");
Assert.That(data, Is.Not.Null);
Assert.That(data, Contains.Substring(consumerId));
var topicRegistry =
ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>(
"topicRegistry", consumerConnector);
Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty);
Assert.That(topicRegistry.Count, Is.EqualTo(1));
var item = topicRegistry["test"];
Assert.That(item.Count, Is.EqualTo(3));
var broker = topicRegistry["test"].SingleOrDefault(x => x.Key.BrokerId == 2345);
Assert.That(broker, Is.Not.Null);
}
}
public void ConsumerConnectorIsCreatedConnectsDisconnectsAndShutsDown()
{
var config = new ConsumerConfig(clientConfig);
using (new ZookeeperConsumerConnector(config, true))
{
}
}
public void ConsumerConnectorReceivesAShutdownSignal()
{
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
// putting the shutdown command into the queue
FieldInfo fi = typeof(ZookeeperConsumerConnector).GetField(
"queues", BindingFlags.NonPublic | BindingFlags.Instance);
var value =
(IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>>)
fi.GetValue(consumerConnector);
foreach (var topicConsumerQueueMap in value)
{
topicConsumerQueueMap.Value.Add(ZookeeperConsumerConnector.ShutdownCommand);
}
var sets = messages[CurrentTestTopic];
var resultMessages = new List<Message>();
foreach (var set in sets)
{
foreach (var message in set)
{
resultMessages.Add(message);
}
}
Assert.AreEqual(0, resultMessages.Count);
}
}
public void ConsumerMultiFetchGetsMessage()
{
ProducerSendMultiRequest();
ConsumerConfig config = new ConsumerConfig(clientConfig);
IConsumer cons = new Consumers.Consumer(config);
MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest>
{
new FetchRequest(CurrentTestTopic, 0, 0),
new FetchRequest(CurrentTestTopic, 0, 0),
new FetchRequest(CurrentTestTopic + "2", 0, 0)
});
IList<BufferedMessageSet> response = cons.MultiFetch(request);
for (int ix = 0; ix < response.Count; ix++)
{
IEnumerable<Message> messageSet = response[ix].Messages;
Console.WriteLine(string.Format("Request #{0}-->", ix));
foreach (Message msg in messageSet)
{
Console.WriteLine(msg);
}
}
}
public void ConsumerPorformsRebalancingOnStart()
{
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1" };
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
ZooKeeperClient client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
IList<string> children = client.GetChildren("/consumers", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("group1"));
children = client.GetChildren("/consumers/group1", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("ids"));
Assert.That(children, Contains.Item("owners"));
children = client.GetChildren("/consumers/group1/ids", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
string consumerId = children[0];
children = client.GetChildren("/consumers/group1/owners", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(1));
Assert.That(children, Contains.Item("test"));
children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(2));
string partId = children[0];
string data = client.ReadData<string>("/consumers/group1/owners/test/" + partId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Contains.Substring(consumerId));
data = client.ReadData<string>("/consumers/group1/ids/" + consumerId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Is.EqualTo("{ \"test\": 1 }"));
}
using (var client = new ZooKeeperClient(config.ZkConnect, config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
{
client.Connect();
//// Should be created as ephemeral
IList<string> children = client.GetChildren("/consumers/group1/ids");
Assert.That(children, Is.Null.Or.Empty);
//// Should be created as ephemeral
children = client.GetChildren("/consumers/group1/owners/test");
Assert.That(children, Is.Null.Or.Empty);
}
}
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 };
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
KafkaMessageStream myStream = sets[0];
var enumerator = myStream.GetEnumerator();
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString());
Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext());
Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state
enumerator.Reset();
// producing again
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
Thread.Sleep(3000);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString());
}
}
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
try
{
foreach (var set in sets)
{
foreach (var message in set)
{
resultMessages.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(2, resultMessages.Count);
Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString());
Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString());
}
public void ConsumerConnectorConsumesTwoDifferentTopics()
{
string topic1 = CurrentTestTopic + "1";
string topic2 = CurrentTestTopic + "2";
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 });
producer.Send(producerRequest1);
var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages1 = new List<Message>();
var resultMessages2 = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
Assert.IsTrue(messages.ContainsKey(topic1));
Assert.IsTrue(messages.ContainsKey(topic2));
var sets1 = messages[topic1];
try
{
foreach (var set in sets1)
{
foreach (var message in set)
{
resultMessages1.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
var sets2 = messages[topic2];
try
{
foreach (var set in sets2)
{
foreach (var message in set)
{
resultMessages2.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(1, resultMessages1.Count);
Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString());
Assert.AreEqual(1, resultMessages2.Count);
Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString());
}
public void ConsumerGetsOffsets()
{
OffsetRequest request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10);
ConsumerConfig config = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(config);
IList<long> list = consumer.GetOffsetsBefore(request);
foreach (long l in list)
{
Console.Out.WriteLine(l);
}
}
public void ConsumerFetchMessage()
{
ProducerSendsMessage();
ConsumerConfig config = new ConsumerConfig(clientConfig);
IConsumer consumer = new Kafka.Client.Consumers.Consumer(config);
FetchRequest request = new FetchRequest(CurrentTestTopic, 0, 0);
BufferedMessageSet response = consumer.Fetch(request);
Assert.NotNull(response);
foreach (var message in response.Messages)
{
Console.WriteLine(message);
}
}
public void ProducerSendsAndConsumerReceivesMultiRequest()
{
string testTopic1 = CurrentTestTopic + "1";
string testTopic2 = CurrentTestTopic + "2";
string testTopic3 = CurrentTestTopic + "3";
Message sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage"));
Message sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage"));
Message sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage"));
Message sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage"));
List<ProducerRequest> requests = new List<ProducerRequest>
{
new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }),
new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }),
new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }),
new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 })
};
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, clientConfig);
long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, clientConfig);
long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, clientConfig);
producer.MultiSend(requests);
ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(consumerConfig);
MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest>
{
new FetchRequest(testTopic1, 0, currentOffset1),
new FetchRequest(testTopic2, 0, currentOffset2),
new FetchRequest(testTopic3, 0, currentOffset3)
});
IList<BufferedMessageSet> messageSets;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
while (true)
{
Thread.Sleep(waitSingle);
messageSets = consumer.MultiFetch(request);
if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0)
{
break;
}
else
{
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
}
Assert.AreEqual(3, messageSets.Count);
Assert.AreEqual(2, messageSets[0].Messages.Count());
Assert.AreEqual(1, messageSets[1].Messages.Count());
Assert.AreEqual(1, messageSets[2].Messages.Count());
Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString());
Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString());
Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString());
Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString());
}