public KafkaMessageConsumer(string groupId, string topic,
KafkaMessagingGatewayConfiguration globalConfiguration,
KafkaMessagingConsumerConfiguration consumerConfiguration)
{
_topic = topic;
_autoCommitEnabled = consumerConfiguration.EnableAutoCommit;
_consumerConfig = new ConsumerConfig()
{
GroupId = groupId,
ClientId = globalConfiguration.Name,
BootstrapServers = string.Join(",", globalConfiguration.BootStrapServers),
MaxInFlight = globalConfiguration.MaxInFlightRequestsPerConnection,
SessionTimeoutMs = 6000,
EnablePartitionEof = true,
/*/
* By default, we always call acknowledge after processing a handler and commit then.
* This has the potential to cause a lot of traffic for the Kafka cluster as every commit is a new message on the consumer_offsets topic.
* To lower the load, you can enable AutoCommit and the AutoCommitIntervalMs. The downside being that if the consumer dies, you may process a message more than once when a new consumer resumes reading a partition.
* /*/
AutoCommitIntervalMs = consumerConfiguration.AutoCommitIntervalMs,
EnableAutoCommit = consumerConfiguration.EnableAutoCommit,
AllowAutoCreateTopics = true,
AutoOffsetReset = consumerConfiguration.OffsetDefault
};
_consumer = new ConsumerBuilder <Null, string>(_consumerConfig)
.SetPartitionsAssignedHandler((consumer, list) =>
{
_logger.Value.InfoFormat($"Assigned partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}");
})
.SetPartitionsRevokedHandler((consumer, list) =>
{
_logger.Value.InfoFormat($"Revoked partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}");
})
.SetErrorHandler((consumer, error) =>
{
if (error.IsBrokerError)
{
_logger.Value.Error($"BrokerError: Member id: {consumer.MemberId}, error: {error}");
}
else
{
_logger.Value.Error($"ConsumeError: Member Id: {consumer.MemberId}, error: {error}");
}
})
.Build();
_logger.Value.InfoFormat($"Kakfa consumer subscribing to {_topic}");
_consumer.Subscribe(new [] { _topic });
_creator = new KafkaMessageCreator();
}
public KafkaMessageConsumer(string groupId, string topic,
KafkaMessagingGatewayConfiguration globalConfiguration,
KafkaMessagingConsumerConfiguration consumerConfiguration)
{
_topic = topic;
_consumerConfig = new ConsumerConfig()
{
GroupId = groupId,
ClientId = globalConfiguration.Name,
BootstrapServers = string.Join(",", globalConfiguration.BootStrapServers),
MaxInFlight = globalConfiguration.MaxInFlightRequestsPerConnection,
SessionTimeoutMs = 6000,
EnablePartitionEof = true,
//We always call acknowledge after processing a handler and commit then.
AutoCommitIntervalMs = 0,
EnableAutoCommit = false,
AutoOffsetReset = consumerConfiguration.OffsetDefault
};
_consumer = new ConsumerBuilder <Null, string>(_consumerConfig)
.SetPartitionsAssignedHandler((consumer, list) =>
{
_logger.Value.InfoFormat($"Assigned partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}");
})
.SetPartitionsRevokedHandler((consumer, list) =>
{
_logger.Value.InfoFormat($"Revoked partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}");
})
.SetErrorHandler((consumer, error) =>
{
if (error.IsBrokerError)
{
_logger.Value.Error($"BrokerError: Member id: {consumer.MemberId}, error: {error}");
}
else
{
_logger.Value.Error($"ConsumeError: Member Id: {consumer.MemberId}, error: {error}");
}
})
.Build();
_logger.Value.InfoFormat($"Kakfa consumer subscribing to {_topic}");
_consumer.Subscribe(new [] { _topic });
}
public KafkaMessageConsumer(string groupId, string topic,
KafkaMessagingGatewayConfiguration globalConfiguration,
KafkaMessagingConsumerConfiguration consumerConfiguration)
{
var consumerConfig = new ConsumerConfig()
{
GroupId = groupId,
ClientId = globalConfiguration.Name,
BootstrapServers = string.Join(",", globalConfiguration.BootStrapServers),
MaxInFlight = globalConfiguration.MaxInFlightRequestsPerConnection,
SessionTimeoutMs = 6000,
EnablePartitionEof = true,
AutoCommitIntervalMs = consumerConfiguration.AutoCommitInterval.Milliseconds,
EnableAutoCommit = consumerConfiguration.EnableAutoCommit,
AutoOffsetReset = consumerConfiguration.AutoResetOffset
};
_consumer = new ConsumerBuilder <Null, string>(consumerConfig)
.SetPartitionsAssignedHandler((consumer, list) =>
{
_logger.Value.InfoFormat($"Assigned partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}");
})
.SetPartitionsRevokedHandler((consumer, list) =>
{
_logger.Value.InfoFormat($"Revoked partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}");
})
.SetErrorHandler((consumer, error) =>
{
if (error.IsBrokerError)
{
_logger.Value.Error($"BrokerError: Member id: {consumer.MemberId}, error: {error}");
}
else
{
_logger.Value.Error($"ConsumeError: Member Id: {consumer.MemberId}, error: {error}");
}
})
.Build();
_consumer.Subscribe(new [] { topic });
}
public KafkaMessageConsumer(string groupId, string topic,
KafkaMessagingGatewayConfiguration globalConfiguration,
KafkaMessagingConsumerConfiguration consumerConfiguration)
{
var config = globalConfiguration.ToConfig();
config = config.Concat(consumerConfiguration.ToConfig());
config = config.Concat(new[] { new KeyValuePair <string, object>("group.id", groupId) });
_consumer = new Consumer <Null, string>(config, null, new StringDeserializer(Encoding.UTF8));
_consumer.OnPartitionsAssigned += (_, partitions) => OnPartionsAssigned(partitions);
_consumer.OnPartitionsRevoked += (_, partitions) => OnPartionsRevoked(partitions);
if (_logger.Value.IsErrorEnabled())
{
_consumer.OnError += (_, error) =>
_logger.Value.Error($"BrokerError: Member id: {_consumer.MemberId}, error: {error}");
_consumer.OnConsumeError += (_, error) =>
_logger.Value.Error($"ConsumeError: Member Id: {_consumer.MemberId}, error: {error}");
}
_consumer.Subscribe(new [] { topic });
}
