public KafkaMessageProducerFactory( KafkaMessagingGatewayConfiguration globalConfiguration ) : this(globalConfiguration, new KafkaPublication { MakeChannels = OnMissingChannel.Create }) { }
public KafkaMessageProducerFactory( KafkaMessagingGatewayConfiguration globalConfiguration, KafkaPublication publication) { _globalConfiguration = globalConfiguration; _publication = publication; _configHook = null; }
/// <summary> /// This constructs a <see cref="KafkaProducerRegistryFactory"/> which can be used to create a <see cref="KafkaMessageProducer"/>. /// It takes a dependency on a <see cref="KafkaMessagingGatewayConfiguration"/> to connect to the broker, and a <see cref="KafkaPublication"/> /// that determines how we publish to Kafka and the parameters of any topic if required. /// </summary> /// <param name="globalConfiguration">Configures how we connect to the broker</param> /// <param name="publication">How do we publish, both producer parameters and topic configuration</param> public KafkaProducerRegistryFactory( KafkaMessagingGatewayConfiguration globalConfiguration, IEnumerable <KafkaPublication> publications) { _globalConfiguration = globalConfiguration; _publications = publications; _configHook = null; }
public KafkaMessageProducer(KafkaMessagingGatewayConfiguration globalConfiguration, KafkaMessagingProducerConfiguration producerConfiguration) { var serialiser = new StringSerializer(Encoding.UTF8); var config = globalConfiguration.ToConfig(); config = config.Concat(producerConfiguration.ToConfig()); _producer = new Producer <Null, string>(config, null, serialiser); }
public KafkaMessageConsumer(string groupId, string topic, KafkaMessagingGatewayConfiguration globalConfiguration, KafkaMessagingConsumerConfiguration consumerConfiguration) { _topic = topic; _autoCommitEnabled = consumerConfiguration.EnableAutoCommit; _consumerConfig = new ConsumerConfig() { GroupId = groupId, ClientId = globalConfiguration.Name, BootstrapServers = string.Join(",", globalConfiguration.BootStrapServers), MaxInFlight = globalConfiguration.MaxInFlightRequestsPerConnection, SessionTimeoutMs = 6000, EnablePartitionEof = true, /*/ * By default, we always call acknowledge after processing a handler and commit then. * This has the potential to cause a lot of traffic for the Kafka cluster as every commit is a new message on the consumer_offsets topic. * To lower the load, you can enable AutoCommit and the AutoCommitIntervalMs. The downside being that if the consumer dies, you may process a message more than once when a new consumer resumes reading a partition. * /*/ AutoCommitIntervalMs = consumerConfiguration.AutoCommitIntervalMs, EnableAutoCommit = consumerConfiguration.EnableAutoCommit, AllowAutoCreateTopics = true, AutoOffsetReset = consumerConfiguration.OffsetDefault }; _consumer = new ConsumerBuilder <Null, string>(_consumerConfig) .SetPartitionsAssignedHandler((consumer, list) => { _logger.Value.InfoFormat($"Assigned partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}"); }) .SetPartitionsRevokedHandler((consumer, list) => { _logger.Value.InfoFormat($"Revoked partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}"); }) .SetErrorHandler((consumer, error) => { if (error.IsBrokerError) { _logger.Value.Error($"BrokerError: Member id: {consumer.MemberId}, error: {error}"); } else { _logger.Value.Error($"ConsumeError: Member Id: {consumer.MemberId}, error: {error}"); } }) .Build(); _logger.Value.InfoFormat($"Kakfa consumer subscribing to {_topic}"); _consumer.Subscribe(new [] { _topic }); _creator = new KafkaMessageCreator(); }
public KafkaMessageProducer( KafkaMessagingGatewayConfiguration configuration, KafkaPublication publication) { if (string.IsNullOrEmpty(publication.Topic)) { throw new ConfigurationException("Topic is required for a publication"); } _clientConfig = new ClientConfig { Acks = (Confluent.Kafka.Acks)((int)publication.Replication), BootstrapServers = string.Join(",", configuration.BootStrapServers), ClientId = configuration.Name, Debug = configuration.Debug, SaslMechanism = configuration.SaslMechanisms.HasValue ? (Confluent.Kafka.SaslMechanism?)((int)configuration.SaslMechanisms.Value) : null, SaslKerberosPrincipal = configuration.SaslKerberosPrincipal, SaslUsername = configuration.SaslUsername, SaslPassword = configuration.SaslPassword, SecurityProtocol = configuration.SecurityProtocol.HasValue ? (Confluent.Kafka.SecurityProtocol?)((int)configuration.SecurityProtocol.Value) : null, SslCaLocation = configuration.SslCaLocation, SslKeyLocation = configuration.SslKeystoreLocation, }; _producerConfig = new ProducerConfig(_clientConfig) { BatchNumMessages = publication.BatchNumberMessages, EnableIdempotence = publication.EnableIdempotence, MaxInFlight = publication.MaxInFlightRequestsPerConnection, LingerMs = publication.LingerMs, MessageTimeoutMs = publication.MessageTimeoutMs, MessageSendMaxRetries = publication.MessageSendMaxRetries, Partitioner = (Confluent.Kafka.Partitioner)((int)publication.Partitioner), QueueBufferingMaxMessages = publication.QueueBufferingMaxMessages, QueueBufferingMaxKbytes = publication.QueueBufferingMaxKbytes, RequestTimeoutMs = publication.RequestTimeoutMs, RetryBackoffMs = publication.RetryBackoff, TransactionalId = publication.TransactionalId, }; MakeChannels = publication.MakeChannels; Topic = publication.Topic; NumPartitions = publication.NumPartitions; ReplicationFactor = publication.ReplicationFactor; TopicFindTimeoutMs = publication.TopicFindTimeoutMs; MaxOutStandingMessages = publication.MaxOutStandingMessages; MaxOutStandingCheckIntervalMilliSeconds = publication.MaxOutStandingCheckIntervalMilliSeconds; }
public KafkaMessageConsumer(string groupId, string topic, KafkaMessagingGatewayConfiguration globalConfiguration, KafkaMessagingConsumerConfiguration consumerConfiguration) { _topic = topic; _consumerConfig = new ConsumerConfig() { GroupId = groupId, ClientId = globalConfiguration.Name, BootstrapServers = string.Join(",", globalConfiguration.BootStrapServers), MaxInFlight = globalConfiguration.MaxInFlightRequestsPerConnection, SessionTimeoutMs = 6000, EnablePartitionEof = true, //We always call acknowledge after processing a handler and commit then. AutoCommitIntervalMs = 0, EnableAutoCommit = false, AutoOffsetReset = consumerConfiguration.OffsetDefault }; _consumer = new ConsumerBuilder <Null, string>(_consumerConfig) .SetPartitionsAssignedHandler((consumer, list) => { _logger.Value.InfoFormat($"Assigned partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}"); }) .SetPartitionsRevokedHandler((consumer, list) => { _logger.Value.InfoFormat($"Revoked partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}"); }) .SetErrorHandler((consumer, error) => { if (error.IsBrokerError) { _logger.Value.Error($"BrokerError: Member id: {consumer.MemberId}, error: {error}"); } else { _logger.Value.Error($"ConsumeError: Member Id: {consumer.MemberId}, error: {error}"); } }) .Build(); _logger.Value.InfoFormat($"Kakfa consumer subscribing to {_topic}"); _consumer.Subscribe(new [] { _topic }); }
public KafkaMessageConsumer(string groupId, string topic, KafkaMessagingGatewayConfiguration globalConfiguration, KafkaMessagingConsumerConfiguration consumerConfiguration) { var consumerConfig = new ConsumerConfig() { GroupId = groupId, ClientId = globalConfiguration.Name, BootstrapServers = string.Join(",", globalConfiguration.BootStrapServers), MaxInFlight = globalConfiguration.MaxInFlightRequestsPerConnection, SessionTimeoutMs = 6000, EnablePartitionEof = true, AutoCommitIntervalMs = consumerConfiguration.AutoCommitInterval.Milliseconds, EnableAutoCommit = consumerConfiguration.EnableAutoCommit, AutoOffsetReset = consumerConfiguration.AutoResetOffset }; _consumer = new ConsumerBuilder <Null, string>(consumerConfig) .SetPartitionsAssignedHandler((consumer, list) => { _logger.Value.InfoFormat($"Assigned partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}"); }) .SetPartitionsRevokedHandler((consumer, list) => { _logger.Value.InfoFormat($"Revoked partitions: [{string.Join(", ", list)}], member id: {consumer.MemberId}"); }) .SetErrorHandler((consumer, error) => { if (error.IsBrokerError) { _logger.Value.Error($"BrokerError: Member id: {consumer.MemberId}, error: {error}"); } else { _logger.Value.Error($"ConsumeError: Member Id: {consumer.MemberId}, error: {error}"); } }) .Build(); _consumer.Subscribe(new [] { topic }); }
public KafkaMessageProducer(KafkaMessagingGatewayConfiguration globalConfiguration, KafkaMessagingProducerConfiguration producerConfiguration) { _producerConfig = new ProducerConfig { BootstrapServers = string.Join(",", globalConfiguration.BootStrapServers), ClientId = globalConfiguration.Name, MaxInFlight = globalConfiguration.MaxInFlightRequestsPerConnection, QueueBufferingMaxMessages = producerConfiguration.QueueBufferingMaxMessages, Acks = producerConfiguration.Acks, QueueBufferingMaxKbytes = producerConfiguration.QueueBufferingMaxKbytes, MessageSendMaxRetries = producerConfiguration.MessageSendMaxRetries, BatchNumMessages = producerConfiguration.BatchNumberMessages, LingerMs = producerConfiguration.QueueBufferingMax, RequestTimeoutMs = producerConfiguration.RequestTimeout, MessageTimeoutMs = producerConfiguration.MessageTimeout, RetryBackoffMs = producerConfiguration.RetryBackoff }; _producer = new ProducerBuilder <Null, string>(_producerConfig).Build(); }
public KafkaMessageConsumer(string groupId, string topic, KafkaMessagingGatewayConfiguration globalConfiguration, KafkaMessagingConsumerConfiguration consumerConfiguration) { var config = globalConfiguration.ToConfig(); config = config.Concat(consumerConfiguration.ToConfig()); config = config.Concat(new[] { new KeyValuePair <string, object>("group.id", groupId) }); _consumer = new Consumer <Null, string>(config, null, new StringDeserializer(Encoding.UTF8)); _consumer.OnPartitionsAssigned += (_, partitions) => OnPartionsAssigned(partitions); _consumer.OnPartitionsRevoked += (_, partitions) => OnPartionsRevoked(partitions); if (_logger.Value.IsErrorEnabled()) { _consumer.OnError += (_, error) => _logger.Value.Error($"BrokerError: Member id: {_consumer.MemberId}, error: {error}"); _consumer.OnConsumeError += (_, error) => _logger.Value.Error($"ConsumeError: Member Id: {_consumer.MemberId}, error: {error}"); } _consumer.Subscribe(new [] { topic }); }
public KafkaMessageConsumer( KafkaMessagingGatewayConfiguration configuration, RoutingKey routingKey, string groupId, AutoOffsetReset offsetDefault = AutoOffsetReset.Earliest, int sessionTimeoutMs = 10000, int maxPollIntervalMs = 10000, IsolationLevel isolationLevel = IsolationLevel.ReadCommitted, long commitBatchSize = 10, int sweepUncommittedOffsetsIntervalMs = 30000, int readCommittedOffsetsTimeoutMs = 5000, int numPartitions = 1, short replicationFactor = 1, int topicFindTimeoutMs = 10000, OnMissingChannel makeChannels = OnMissingChannel.Create ) { if (configuration is null) { throw new ConfigurationException("You must set a KafkaMessaginGatewayConfiguration to connect to a broker"); } if (routingKey is null) { throw new ConfigurationException("You must set a RoutingKey as the Topic for the consumer"); } if (groupId is null) { throw new ConfigurationException("You must set a GroupId for the consumer"); } Topic = routingKey; _clientConfig = new ClientConfig { BootstrapServers = string.Join(",", configuration.BootStrapServers), ClientId = configuration.Name, Debug = configuration.Debug, SaslMechanism = configuration.SaslMechanisms.HasValue ? (Confluent.Kafka.SaslMechanism?)((int)configuration.SaslMechanisms.Value) : null, SaslKerberosPrincipal = configuration.SaslKerberosPrincipal, SaslUsername = configuration.SaslUsername, SaslPassword = configuration.SaslPassword, SecurityProtocol = configuration.SecurityProtocol.HasValue ? (Confluent.Kafka.SecurityProtocol?)((int)configuration.SecurityProtocol.Value) : null, SslCaLocation = configuration.SslCaLocation }; _consumerConfig = new ConsumerConfig(_clientConfig) { GroupId = groupId, ClientId = configuration.Name, AutoOffsetReset = offsetDefault, BootstrapServers = string.Join(",", configuration.BootStrapServers), SessionTimeoutMs = sessionTimeoutMs, MaxPollIntervalMs = maxPollIntervalMs, EnablePartitionEof = true, AllowAutoCreateTopics = false, //We will do this explicit always so as to allow us to set parameters for the topic IsolationLevel = isolationLevel, //We commit the last offset for acknowledged requests when a batch of records has been processed. EnableAutoOffsetStore = false, EnableAutoCommit = false, // https://www.confluent.io/blog/cooperative-rebalancing-in-kafka-streams-consumer-ksqldb/ PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky, }; _maxBatchSize = commitBatchSize; _sweepUncommittedInterval = TimeSpan.FromMilliseconds(sweepUncommittedOffsetsIntervalMs); _readCommittedOffsetsTimeoutMs = readCommittedOffsetsTimeoutMs; _consumer = new ConsumerBuilder <string, string>(_consumerConfig) .SetPartitionsAssignedHandler((consumer, list) => { var partitions = list.Select(p => $"{p.Topic} : {p.Partition.Value}"); _logger.Value.InfoFormat("Parition Added {0}", String.Join(",", partitions)); _partitions.AddRange(list); }) .SetPartitionsRevokedHandler((consumer, list) => { _consumer.Commit(list); var revokedPartitions = list.Select(tpo => $"{tpo.Topic} : {tpo.Partition}").ToList(); _logger.Value.InfoFormat("Partitions for consumer revoked {0}", string.Join(",", revokedPartitions)); _partitions = _partitions.Where(tp => list.All(tpo => tpo.TopicPartition != tp)).ToList(); }) .SetPartitionsLostHandler((consumer, list) => { var lostPartitions = list.Select(tpo => $"{tpo.Topic} : {tpo.Partition}").ToList(); _logger.Value.InfoFormat("Partitions for consumer lost {0}", string.Join(",", lostPartitions)); _partitions = _partitions.Where(tp => list.All(tpo => tpo.TopicPartition != tp)).ToList(); }) .SetErrorHandler((consumer, error) => { _logger.Value.Error($"Code: {error.Code}, Reason: {error.Reason}, Fatal: {error.IsFatal}"); }) .Build(); _logger.Value.InfoFormat($"Kakfa consumer subscribing to {Topic}"); _consumer.Subscribe(new [] { Topic.Value }); _creator = new KafkaMessageCreator(); MakeChannels = makeChannels; Topic = routingKey; NumPartitions = numPartitions; ReplicationFactor = replicationFactor; TopicFindTimeoutMs = topicFindTimeoutMs; EnsureTopic(); }