private static void UseKafka(IConfiguration configuration) { lifeTime.ApplicationStarted.Register(() => { try { var conf = new Confluent.Kafka.ConsumerConfig { GroupId = configuration["TomatoLog:Flow:Kafka:Group"], BootstrapServers = configuration["TomatoLog:Flow:Kafka:BootstrapServers"], AutoOffsetReset = Confluent.Kafka.AutoOffsetReset.Earliest }; using (var consumer = new Confluent.Kafka.ConsumerBuilder <Confluent.Kafka.Ignore, string>(conf).Build()) { var topic = configuration["TomatoLog:Flow:Kafka:Topic"]; consumer.Subscribe(topic); logger.LogInformation($"Kafka Consume started."); var cancellationToken = cts_kafka.Token; while (!cancellationToken.IsCancellationRequested) { Confluent.Kafka.ConsumeResult <Confluent.Kafka.Ignore, string> result = null; try { result = consumer.Consume(cancellationToken); LogMessage log = JsonConvert.DeserializeObject <LogMessage>(result.Value); logWriter.Write(log); filterService.Filter(log); } catch (Exception e) { logger.LogError($"Error occured: {e.Message}{e.StackTrace}"); } finally { consumer.Commit(result); } } } } catch (OperationCanceledException ex) { logger.LogError("{0}/{1}", ex.Message, ex.StackTrace); } }); lifeTime.ApplicationStopping.Register(() => { cts_kafka.Cancel(); logger.LogInformation($"Kafka Consume stoped."); }); }
public void Compare_SameInstance_ReturnsTrue() { var config1 = new Confluent.Kafka.ConsumerConfig { BootstrapServers = "myserver", PartitionAssignmentStrategy = Confluent.Kafka.PartitionAssignmentStrategy.Range, EnableAutoCommit = false }; var config2 = config1; _dictionary.TryAdd(config1, null); _dictionary.Should().ContainKey(config2); }
private Confluent.Kafka.IConsumer <byte[], byte[]> BuildConfluentConsumer(Confluent.Kafka.ConsumerConfig config) { return(new Confluent.Kafka.ConsumerBuilder <byte[], byte[]>(config) .SetPartitionsAssignedHandler((_, partitions) => { partitions.ForEach(partition => _logger.LogTrace("Assigned topic {topic} partition {partition}, member id: {memberId}", partition.Topic, partition.Partition, _innerConsumer.MemberId)); }) .SetPartitionsRevokedHandler((_, partitions) => { partitions.ForEach(partition => _logger.LogTrace("Revoked topic {topic} partition {partition}, member id: {memberId}", partition.Topic, partition.Partition, _innerConsumer.MemberId)); }) .SetOffsetsCommittedHandler((_, offsets) => { foreach (var offset in offsets.Offsets) { if (offset.Offset == Confluent.Kafka.Offset.Unset) { continue; } if (offset.Error != null && offset.Error.Code != Confluent.Kafka.ErrorCode.NoError) { _logger.LogError( "Error occurred committing the offset {topic} {partition} @{offset}: {errorCode} - {errorReason} ", offset.Topic, offset.Partition, offset.Offset, offset.Error.Code, offset.Error.Reason); } else { _logger.LogTrace("Successfully committed offset {topic} {partition} @{offset}.", offset.Topic, offset.Partition, offset.Offset); } } }) .SetErrorHandler((_, e) => { _logger.Log(e.IsFatal ? LogLevel.Critical : LogLevel.Warning, "Error in Kafka consumer: {reason}.", e.Reason); }) .SetStatisticsHandler((_, json) => { _logger.LogInformation($"Statistics: {json}"); }) .Build()); }
public void Compare_DifferentParameters_ReturnsFalse() { var config1 = new Confluent.Kafka.ConsumerConfig { BootstrapServers = "myserver", PartitionAssignmentStrategy = Confluent.Kafka.PartitionAssignmentStrategy.Range, EnableAutoCommit = false }; var config2 = new Confluent.Kafka.ConsumerConfig { PartitionAssignmentStrategy = Confluent.Kafka.PartitionAssignmentStrategy.Range, BootstrapServers = "myserver" }; _dictionary.TryAdd(config1, null); _dictionary.Should().NotContainKey(config2); }
public void WithReceiverConfig(Confluent.Kafka.ConsumerConfig config) { ConsumerConfig = config; }
public InnerConsumerWrapper(Confluent.Kafka.ConsumerConfig config, CancellationToken cancellationToken, ILogger logger) { _innerConsumer = BuildConfluentConsumer(config); _cancellationToken = cancellationToken; _logger = logger; }
public InnerConsumerWrapper(Confluent.Kafka.ConsumerConfig config, bool enableAutoRecovery, ILogger logger) { _config = config; _enableAutoRecovery = enableAutoRecovery; _logger = logger; }