/// <summary>
/// Materialize the count state in the Topic_Counts change log topic for the
/// specified partitions into a Dictionary<string, int>
/// </summary>
public static void LoadCountState(RocksDb db, string brokerList, IEnumerable <Partition> partitions, ColumnFamilyHandle columnFamily, CancellationToken ct)
{
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = ConsumerGroup_LoadState,
EnablePartitionEof = true
};
int msgCount = 0;
using (var consumer = new ConsumerBuilder <string, int>(cConfig).Build())
{
consumer.Assign(partitions.Select(p => new TopicPartitionOffset(Topic_Counts, p, Offset.Beginning)));
int eofCount = 0;
while (true)
{
var cr = consumer.Consume();
if (cr.IsPartitionEOF)
{
eofCount += 1;
if (eofCount == partitions.Count())
{
break;
}
}
else
{
msgCount += 1;
db.Put(Encoding.UTF8.GetBytes(cr.Message.Key), BitConverter.GetBytes(cr.Message.Value), columnFamily);
}
}
}
Console.WriteLine($"Finished materializing word counts state. Backed by {msgCount} messages in Kafka topic '{Topic_Counts}'");
}
static void Main(string[] args)
{
var config = new ConsumerConfig
{
GroupId = "ORDER",
BootstrapServers = "localhost:9092",
EnableAutoCommit = false
};
using var c = new ConsumerBuilder <Ignore, string>(config).Build();
c.Subscribe("orders");
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true;
cts.Cancel();
};
try
{
while (true)
{
var cr = c.Consume(cts.Token);
Order order = JsonConvert.DeserializeObject <Order>(cr.Message.Value);
//Console.WriteLine($"Consumed message '{order.Id}' from topic {cr.Topic}, partition {cr.Partition}, offset {cr.Offset}");
Console.WriteLine($"Kafka => Order '{order.Id}' is accepted from topic '{cr.Topic}'");
}
}
catch (OperationCanceledException)
{
}
finally
{
c.Close();
}
}
public long GetStashedCount()
{
var conf = new ConsumerConfig
{
GroupId = this._topic,
BootstrapServers = this._bootstrapServers,
EnableAutoCommit = true,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
using var consumer = new ConsumerBuilder <string, string>(conf).Build();
consumer.Subscribe(this._topic);
var i = 0;
while (true)
{
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
var result = consumer.Consume(cts.Token);
if (result.IsPartitionEOF)
{
break;
}
i += 1;
}
consumer.Close();
return(i);
}
static void Main(string[] args)
{
if (args.Length == 0)
{
Console.WriteLine("You must input a consumer group id");
return;
}
string groupId = args[0];
Console.WriteLine($"Consumer in group id: {groupId}");
Console.WriteLine("Running consumer");
var consumerConfig = new ConsumerConfig
{
BootstrapServers = "localhost:9092",
GroupId = groupId
};
using var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build();
consumer.Subscribe("Student");
while (true)
{
var result = consumer.Consume(5);
if (string.IsNullOrEmpty(result?.Message?.Value))
{
continue;
}
var obj = JsonSerializer.Deserialize <StudentRecordUpdate>(result.Message.Value);
Console.WriteLine("Received a message");
Console.WriteLine($"Student Id: {obj.StudentId} updated state to {obj.State.ToString()}");
}
}
public void SimpleProduceConsume(string bootstrapServers)
{
LogToFile("start SimpleProduceConsume");
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
string testString1 = "hello world";
string testString2 = null;
DeliveryResult <Null, string> produceResult1;
DeliveryResult <Null, string> produceResult2;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
produceResult1 = ProduceMessage(singlePartitionTopic, producer, testString1);
produceResult2 = ProduceMessage(singlePartitionTopic, producer, testString2);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
ConsumeMessage(consumer, produceResult1, testString1);
ConsumeMessage(consumer, produceResult2, testString2);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end SimpleProduceConsume");
}
static void Main(string[] args)
{
var conf = new ConsumerConfig
{
GroupId = "test-consumer-group",
BootstrapServers = "PLAINTEXT://kafka:9092",
AutoOffsetReset = AutoOffsetReset.Earliest
};
using var c = new ConsumerBuilder <Ignore, string>(conf).Build();
c.Subscribe("quickstart-events");
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true;
cts.Cancel();
};
try
{
Console.WriteLine("Ready to Consume events:");
while (true)
{
var cr = c.Consume(cts.Token);
Console.WriteLine($"Consumed message from topic {cr.Topic}, partition {cr.Partition}, offset {cr.Offset}. Message:");
Console.WriteLine(cr.Message.Value);
}
}
catch (OperationCanceledException)
{
}
finally
{
c.Close();
}
}
public async Task <List <string> > ExecuteAsync(CancellationToken stopingToken, string topicName)
{
var result = new List <string>();
var config = new ConsumerConfig
{
BootstrapServers = _kafcaConnection,
GroupId = $"{topicName}-group-0",
AutoOffsetReset = AutoOffsetReset.Earliest
};
try
{
using var consumer = new ConsumerBuilder <Ignore, string>(config).Build();
consumer.Subscribe(topicName);
try
{
var message = StartProcess;
while (!string.IsNullOrEmpty(message))
{
var cr = consumer.Consume(stopingToken);
message = cr.Message.Value;
result.Add(message);
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
catch (Exception ex)
{
}
return(result);
}
public KafkaConsumer(IServiceScopeFactory serviceScopeFactory, IHostApplicationLifetime application,
ILogger <KafkaConsumer <TMessage, THandler> > logger, IOptions <ConsumerConfig> config,
IOptions <KafkaConsumerConfig <TMessage> > consumerConfig)
{
_logger = logger;
_application = application;
_serviceScopeFactory = serviceScopeFactory;
_config = consumerConfig.Value;
_builder = new ConsumerBuilder <Null, TMessage>(config.Value)
.SetValueDeserializer(KafkaDataConverter <TMessage> .Instance);
_policy = Policy
.Handle <Exception>()
.WaitAndRetryAsync(_config.RetryCount,
i => _config.RetryInterval * i,
(ex, _, retry, _) =>
{
var level = retry > _config.RetryCount * 0.7
? LogLevel.Critical
: LogLevel.Warning;
_logger.Log(level, ex, "Error occured in kafka handler for '{N}'", _config.MessageType.Name);
});
}
/// <inheritdoc cref="IConfluentConsumerBuilder.Build" />
public IConsumer <byte[]?, byte[]?> Build()
{
if (_config == null)
{
throw new InvalidOperationException("SetConfig must be called to provide the consumer configuration.");
}
var builder = new ConsumerBuilder <byte[]?, byte[]?>(_config);
if (_statisticsHandler != null)
{
builder.SetStatisticsHandler(_statisticsHandler);
}
if (_errorHandler != null)
{
builder.SetErrorHandler(_errorHandler);
}
if (_partitionsAssignedHandler != null)
{
builder.SetPartitionsAssignedHandler(_partitionsAssignedHandler);
}
if (_partitionsRevokedHandler != null)
{
builder.SetPartitionsRevokedHandler(_partitionsRevokedHandler);
}
if (_offsetsCommittedHandler != null)
{
builder.SetOffsetsCommittedHandler(_offsetsCommittedHandler);
}
return(builder.Build());
}
public static async Task ConsumeMessage(MainConfig cfg, CancellationToken cancellationToken, ILogger logger)
{
var config = new ConsumerConfig
{
BootstrapServers = cfg.Kafka.BootstrapServers,
GroupId = cfg.Kafka.GroupId,
ClientId = cfg.Kafka.ClientId
};
using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build())
{
consumer.Subscribe(cfg.Kafka.Topic);
while (!cancellationToken.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cancellationToken);
if (consumeResult.Message != null)
{
//todo warning! "Application maximum poll interval (300000ms) exceeded by X ms"
//https://github.com/confluentinc/confluent-kafka-dotnet/issues/785
var message = consumeResult.Message.Value;
await KafkaMessageHandler.NotifyFromKafkaMessage(message, logger, cfg);
}
}
catch (ConsumeException e)
{
logger.Information($"Error occured while consuming: {e.Error.Reason}", e);
//throw;
}
}
consumer.Close();
}
}
public static IConsumer <int, string> CreateConsumer(string brokerList,
List <string> topics, ILogger logger)
{
var config = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = "sample-consumer",
EnableAutoCommit = false,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true,
ReconnectBackoffMs = 30000, // 30 seconds
ReconnectBackoffMaxMs = 180000 // 3 minutes
};
var consumer = new ConsumerBuilder <int, string>(config)
// Note: All handlers are called on the main .Consume thread.
.SetErrorHandler((_, e) => logger.LogInformation($"Error: {e.Reason}"))
// .SetStatisticsHandler((_, json) => logger.LogInformation($"Statistics: {json}"))
.SetPartitionsAssignedHandler((c, partitions) =>
{
logger.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]");
// possibly manually specify start offsets or override the partition assignment provided by
// the consumer group by returning a list of topic/partition/offsets to assign to, e.g.:
// return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp]));
})
.SetPartitionsRevokedHandler((c, partitions) =>
{
logger.LogInformation($"Revoking assignment: [{string.Join(", ", partitions)}]");
})
.Build();
consumer.Subscribe(topics);
return(consumer);
}
public static void Subscribe <T>(string urlServer,
string topic,
string groupId,
Action <string> action)
{
var config = KafkaConfigManagement.Instance;
using (var consumer = new ConsumerBuilder <Ignore, string>(config.GetConsumerConfig(urlServer,
groupId)).Build())
{
consumer.Subscribe(topic);
try
{
while (true)
{
try
{
var cr = consumer.Consume();
var msg = JsonConvert.DeserializeObject <T>(cr.Value);
if (msg != null)
{
action(cr.Value);
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
}
public void Listen()
{
#region MyRegion New Style
var config = new ConsumerConfig
{
GroupId = "booking",
BootstrapServers = "localhost:9092",
EnableAutoCommit = false
};
using var consumer = new ConsumerBuilder <Ignore, string>(config).Build();
consumer.Subscribe("booking");
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
try
{
while (true)
{
var cr = consumer.Consume(cts.Token);
bookingStream.Publish(new Model.BookingMessage {
Message = cr.Message.Value
});
}
}
catch (OperationCanceledException)
{
}
finally
{
consumer.Close();
}
#endregion
}
private static void MonitorGasTankIsEmpty()
{
using (var c = new ConsumerBuilder <Ignore, string>(ConsumerConfig).Build())
{
c.Subscribe(nameof(GasTankIsEmptyEvent));
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
try
{
while (true)
{
try
{
var cr = c.Consume(cts.Token);
var ev = JsonConvert.DeserializeObject <GasTankIsEmptyEvent>(cr.Value);
HandleGasTankIsEmptyEvent(ev);
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
c.Close();
}
}
}
public void Subscribe()
{
using (var c = new ConsumerBuilder <Ignore, string>(_consumerConfig).Build())
{
c.Subscribe("my-topic");
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
var cr = c.Consume(cts.Token);
Console.WriteLine(
$"Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'.");
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}
public async Task invokes_expected_handler_when_consuming()
{
var handlerMock = new Mock <IMessageHandler <FooMessage> >();
var handlerStub = handlerMock.Object;
var messageRegistrationStub = new MessageRegistrationBuilder()
.WithHandlerInstanceType(handlerStub.GetType())
.WithMessageInstanceType(typeof(FooMessage))
.WithMessageType("foo")
.Build();
var registry = new MessageHandlerRegistry();
registry.Register(messageRegistrationStub);
var sut = new ConsumerBuilder()
.WithUnitOfWork(new UnitOfWorkStub(handlerStub))
.WithMessageHandlerRegistry(registry)
.Build();
await sut.ConsumeSingle(CancellationToken.None);
handlerMock.Verify(x => x.Handle(It.IsAny <FooMessage>(), It.IsAny <MessageHandlerContext>()), Times.Once);
}
public void Listen()
{
using (var consumer = new ConsumerBuilder <Ignore, string>(_kafkaConfig).Build())
{
consumer.Subscribe(_kafkaTopics);
var cts = new CancellationTokenSource();
while (true)
{
try
{
var message = consumer.Consume(cts.Token);
_alertStream.Publish(new Alert {
Message = message.Value
});
Console.WriteLine($"Consumed message: {message.Value}");
} catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
}
public void Run()
{
ConsumerConfig configs = new ConsumerConfig {
GroupId = groupId,
BootstrapServers = bootstrapServer,
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var c = new ConsumerBuilder <String, String>(configs).Build()) {
c.Subscribe("twitter");
CancellationTokenSource cancellationToken = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true;
cancellationToken.Cancel();
};
try
{
while (true)
{
try
{
ConsumeResult <string, string> result = c.Consume(cancellationToken.Token);
Console.WriteLine($"Consumed message '{result.Message.Value}' at: '{result.TopicPartitionOffset}'.");
}
catch (ConsumeException e)
{
Console.WriteLine($"Error ocurried: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
c.Close();
}
}
}
static void Main(string[] args)
{
IEnumerable <KeyValuePair <string, string> > config = BuildConsumerConfig();
using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build())
{
consumer.Subscribe("logistics.instruction.job-state-change-dev");
while (true)
{
var consumeResult = consumer.Consume(5000);
if (consumeResult != null)
{
consumer.Commit(consumeResult);
Console.WriteLine($"message consumed: \n {consumeResult.Message.Value}");
}
else
{
Console.WriteLine("No messages. Trying again");
}
}
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
MetricsRegistry _metricsRegistry = new MetricsRegistry(_configuration);
Logger _receivedMessageounter = _metricsRegistry._receivedMessage();
var counter = _receivedMessageounter.CountOperation("counter", "operation(s)", true, LogEventLevel.Information);
counter.Increment();
try
{
var config = new ConsumerConfig
{
GroupId = _groupId,
BootstrapServers = _bootstrapServers,
};
using (var consumer = new ConsumerBuilder <Null, string>(config).Build())
{
consumer.Subscribe(_topic);
while (!stoppingToken.IsCancellationRequested)
{
var cr = consumer.Consume();
Item item = JsonConvert.DeserializeObject <Item>(cr.Message.Value);
Console.WriteLine("Product Name " + item.Name);
await Task.Delay(1000, stoppingToken);
}
}
}
catch (Exception ex)
{
_logger.LogError("Exception Occured: {ex}", ex.ToString());
}
}
public void ExecuteCore(CancellationToken stoppingToken)
{
ConsumerBuilder <TK, TV> builder = new ConsumerBuilder <TK, TV>(_config).SetValueDeserializer(new KafkaDeserializer <TV>());
using (IConsumer <TK, TV> consumer = builder.Build())
{
if (_config.Active)
{
consumer.Subscribe(_config.Topic);
while (!stoppingToken.IsCancellationRequested)
{
ConsumeResult <TK, TV> result = consumer.Consume(TimeSpan.FromMilliseconds(1000));
if (result != null)
{
_handler.HandleAsync(result.Key, result.Value).GetAwaiter().GetResult();
consumer.Commit(result);
consumer.StoreOffset(result);
}
}
}
}
}
public async Task will_not_call_commit_when_auto_commit_is_enabled()
{
var handlerStub = Dummy.Of <IMessageHandler <FooMessage> >();
var messageRegistrationStub = new MessageRegistrationBuilder()
.WithHandlerInstanceType(handlerStub.GetType())
.WithMessageInstanceType(typeof(FooMessage))
.WithMessageType("foo")
.Build();
var wasCalled = false;
var resultSpy = new MessageResultBuilder()
.WithOnCommit(() =>
{
wasCalled = true;
return(Task.CompletedTask);
})
.Build();
var consumerScopeFactoryStub = new ConsumerScopeFactoryStub(new ConsumerScopeStub(resultSpy));
var registry = new MessageHandlerRegistry();
registry.Register(messageRegistrationStub);
var consumer = new ConsumerBuilder()
.WithConsumerScopeFactory(consumerScopeFactoryStub)
.WithUnitOfWork(new UnitOfWorkStub(handlerStub))
.WithMessageHandlerRegistry(registry)
.WithEnableAutoCommit(true)
.Build();
await consumer.ConsumeSingle(CancellationToken.None);
Assert.False(wasCalled);
}
IConsumer <TInKey, TInValue> constructConsumer(string instanceId, CancellationTokenSource errorCts)
{
var cConfig = new ConsumerConfig
{
ClientId = $"{Name}-consumer-{instanceId}",
GroupId = $"{Name}-group",
BootstrapServers = BootstrapServers,
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Latest,
Debug = DebugContext
};
var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig)
.SetKeyDeserializer(InKeyDeserializer)
.SetValueDeserializer(InValueDeserializer)
.SetLogHandler((_, m) => Logger(m))
.SetErrorHandler((c, e) =>
{
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
return(cBuilder.Build());
}
/// <summary>
/// 单笔消费模式-单行消费
/// </summary>
/// <param name="Topic">主题</param>
/// <param name="TimeOut">持续监听时间,单位ms 默认值:300ms</param>
/// <returns>待消费数据</returns>
public ConsumeResult <TKey, TValue> ConsumeOneRow(string Topic, int TimeOut = 300)
{
var builder = new ConsumerBuilder <TKey, TValue>(ConsumerConfig);
//设置反序列化方式
builder.SetValueDeserializer(new KafkaDConverter <TValue>());
using var consumer = builder.Build();
consumer.Subscribe(Topic);
try
{
var result = consumer.Consume(TimeSpan.FromMilliseconds(TimeOut));
if (result != null)
{
//手动提交，如果上面的EnableAutoCommit=true表示自动提交，则无需调用Commit方法
consumer.Commit();
}
return(result);
}
catch (Exception ex)
{
Logger.Error(LoggerType.KafkaException, $"Topic:{Topic}", null, ex.Message + ex.StackTrace);
return(null);
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// TODO: Move this to a service and inject it using DI
using var consumer = new ConsumerBuilder <Null, string>(_config).Build();
var topic = "mihai";
try
{
consumer.Subscribe(topic);
_logger.LogInformation($"Subscribed to topic {topic}.");
}
catch (Exception ex)
{
_logger.LogError(ex, "A Kafka error occurred.");
}
while (!stoppingToken.IsCancellationRequested)
{
try
{
var cr = consumer.Consume(stoppingToken);
var message = JsonSerializer.Deserialize <DockerHubPayload>(cr.Message.Value);
_logger.LogInformation("Received following message from Kafka: {@message}", message);
await _kubernetesService.PatchAllDeploymentAsync(message?.Repository?.RepoName, message?.PushData?.Tag, stoppingToken);
}
catch (ConsumeException ex)
{
_logger.LogError(ex, "Could not consume the specified Kafka topic.");
}
catch (Exception ex)
{
_logger.LogCritical(ex, "An error occurred while calling K8s API.");
}
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_config.GroupId = Constants.GroupId;
using var c = new ConsumerBuilder <Ignore, AnimalInfo>(_config)
.SetValueDeserializer(new MyDeserializer <AnimalInfo>())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")).Build();
c.Subscribe(Constants.Topic);
try
{
var workerBlock = new ActionBlock <ConsumeResult <Ignore, AnimalInfo> >(ProcessMessage, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 6
});
while (!stoppingToken.IsCancellationRequested)
{
try
{
var cr = c.Consume(stoppingToken);
workerBlock.Post(cr);
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occurred: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
/// <inheritdoc />
public void Initialize()
{
// Note: If a key or value deserializer is not set (as is the case below), the
// deserializer corresponding to the appropriate type from Confluent.Kafka.Serdes
// will be used automatically (where available). The default deserializer for string
// is UTF8. The default deserializer for Ignore returns null for all input data
// (including non-null data).
_consumer = new ConsumerBuilder <Ignore, TransportMessage>(_config)
.SetKeyDeserializer(Deserializers.Ignore)
.SetValueDeserializer(new TransportMessageDeserializer())
.SetLogHandler(ConsumerOnLog)
.SetErrorHandler(ConsumerOnError)
.SetStatisticsHandler(ConsumerOnStatistics)
.SetPartitionsAssignedHandler(ConsumerOnPartitionsAssigned)
.SetPartitionsRevokedHandler(ConsumerOnPartitionsRevoked)
.Build();
var topics = _subscriptions.SelectMany(a => a.Value).ToArray();
var tcs = new TaskCompletionSource <bool>();
_waitAssigned.TryAdd(topics, new KeyValuePair <string, TaskCompletionSource <bool> >(topics.First(), tcs));
_consumer.Subscribe(topics);
_initializationTask = tcs.Task;
}
public Task StartAsync(CancellationToken cancellationToken)
{
var conf = new ConsumerConfig
{
GroupId = "helloword",
BootstrapServers = "localhost:9092"
};
using (var c = new ConsumerBuilder <Null, string>(conf).Build())
{
c.Subscribe("messagesHelloWord");
var cts = new CancellationTokenSource();
try
{
while (true)
{
try
{
var cr = c.Consume(cts.Token);
_logger.LogInformation($"Mensagem: {cr.Message.Value} recebida de {cr.TopicPartitionOffset}");
}
catch (ConsumeException e)
{
_logger.LogError($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
c.Close();
}
}
return(Task.CompletedTask);
}
public void CommandThread()
{
Console.WriteLine("CommandThread Start");
Active = true;
var consumer = new ConsumerBuilder <string, string>(config).Build();
var topicp = new TopicPartition(topic, 0);
consumer.Assign(topicp);
while (Active)
{
try
{
var consumeresult = consumer.Consume(canceltoken);
if (!consumeresult.IsPartitionEOF)
{
var input = consumeresult.Value;
string command;
string parameter;
if (input.Contains(" "))
{
command = input.Substring(0, input.IndexOf(" ")).Trim();
parameter = input.Substring(input.IndexOf(" "), input.Length - command.Length).Trim();
}
else
{
command = input;
parameter = "";
}
Console.WriteLine("COMMAND----------> " + input);
if (command.Equals("Add-Channel"))
{
controller.addThread(parameter);
}
else if (command.Equals("Drop-Channel"))
{
controller.dropThread(parameter);
}
else if (command.Equals("SCPList-Channels"))
{
Console.WriteLine("The active threads are:");
controller.listThreads();
}
else if (command.Equals("SCPCount"))
{
Console.WriteLine("The message queue has " + controller.queueSize() + " messages in it right now");
}
else if (command.Equals("SCPExit") || command.Equals("System-Shutdown"))
{
controller.exit();
Active = false;
source.Cancel();
}
else if (command.Equals("SCPBlacklist"))
{
blacklist.Add(parameter);
}
else if (command.Equals("SCPUnblacklist"))
{
blacklist.Remove(parameter);
}
}
else
{
Thread.Sleep(100);
}
}
catch (System.OperationCanceledException e)
{
}
}
}
public void Consumer_Exiting(string bootstrapServers)
{
LogToFile("start Consumer_Exiting");
int N = 2;
var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N);
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000,
Debug = "all"
};
for (int i = 0; i < 4; ++i)
{
consumerConfig.Set("group.id", Guid.NewGuid().ToString());
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetPartitionsAssignedHandler((c, partitions) =>
{
return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
int tryCount = 10;
while (tryCount-- > 0)
{
var record = consumer.Consume(TimeSpan.FromSeconds(10));
if (record != null)
{
break;
}
}
Assert.True(tryCount > 0);
// there should be no ill effect doing any of this before disposing a consumer.
switch (i)
{
case 0:
LogToFile(" -- Unsubscribe");
consumer.Unsubscribe();
break;
case 1:
LogToFile(" -- Commit");
consumer.Commit();
break;
case 3:
LogToFile(" -- Close");
consumer.Close();
break;
case 4:
break;
}
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Exiting");
}
