protected override async Task <DataItem <TConsumerValue> > FetchNextDataAsync(CancellationToken cancellationToken)
{
var entity = new DataItem <TConsumerValue>();
ConsumeResult <TConsumerKey, TConsumerValue> consumerResult;
try
{
consumerResult = await Task.Run(() =>
{
var r = Consumer.Consume(cancellationToken);
while (r != null && r.IsPartitionEOF)
{
r = Consumer.Consume(cancellationToken);
}
return(r);
}, cancellationToken)
.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return(null);
}
if (consumerResult == null)
{
return(null);
}
entity.Data = consumerResult.Message.Value;
entity.Metadata = new PipelineMetadata {
Metadata = consumerResult.Message, Timestamp = DateTime.Now
};
return(entity);
}
private async Task RunConsumer(CancellationToken token = default)
{
var count = 0;
try
{
while (!token.IsCancellationRequested)
{
try
{
var message = _kafkaConsumer.Consume(token);
await HandleMessageAsync(message);
if (count++ % 10 == 0)
{
_kafkaConsumer.Commit(message);
count = 0;
}
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
}
}
catch (OperationCanceledException)
{
_kafkaConsumer.Dispose();
}
}
private async Task RunConsumerAsync(CancellationToken token)
{
try
{
_consumer.Subscribe();
while (true)
{
try
{
var message = _consumer.Consume(token);
var handlerCompletionResponse = await HandleMessage(message);
if (handlerCompletionResponse)
{
_consumer.Commit();
}
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
}
}
catch (OperationCanceledException)
{
_consumer.Dispose();
}
}
public async Task KafkaActivity_ProduceConsumeMessage_CorrectActivityReceived()
{
// Arrange
var baggageKey = "testKey";
_activityContextAccessor.CurrentActivity.AddBaggage(baggageKey, "testing");
var mockMessage = new MockMessage();
var messageId = Guid.NewGuid().ToString();
var message = new KafkaMessage <string, MockMessage>
{
Key = "mock-message",
Value = mockMessage,
MessageId = messageId,
MessageType = "mockMessage.Created"
};
var cts = new CancellationTokenSource();
var consumedMessage = new KafkaMessage <string, MockMessage>();
// Act
#pragma warning disable 4014
Task.Factory.StartNew(() =>
#pragma warning restore 4014
{
while (!cts.IsCancellationRequested)
{
consumedMessage = (KafkaMessage <string, MockMessage>)_kafkaConsumer.Consume(cts.Token);
_semaphore.Release();
}
}, TaskCreationOptions.LongRunning);
await _kafkaProducer.ProduceAsync(message);
await _semaphore.WaitAsync(30000);
// Assert
Assert.NotEmpty(consumedMessage.Headers[Constants.CorrelationIdHeaderName]);
Assert.NotEmpty(consumedMessage.Headers[baggageKey]);
}
public Task StartAsync(CancellationToken cancellationToken)
{
Task.Run(() =>
{
using (IKafkaConsumer <Ignore, string> c = this._kafkaConsumerProvider.GetKafkaConsumer <Ignore, string>("consumer-selection-count"))
{
c.Subscribe();
try
{
while (true)
{
try
{
var cr = c.Consume(cancellationToken);
_logger.LogInformation($"1. Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'.");
SelectionSummaryKafkaMessage kafkaSocketMessage = null;
try
{
kafkaSocketMessage = cr.Message?.Value?.FromJSON <SelectionSummaryKafkaMessage>();
}
catch (Exception ex)
{
_logger.LogError($"Exception occured in KafkaSocketMessage deserialization: {ex.Message}");
}
if (kafkaSocketMessage != null)
{
SocketMessage calculationResultMessage = new SocketMessage()
{
Code = "selection_count_calculated",
Data = kafkaSocketMessage
};
Task.Run(async() => { await _sampleSocketMessageHandler.SendMessageAsync(kafkaSocketMessage.UserId, calculationResultMessage.ToJSON()); });
}
}
catch (ConsumeException e)
{
_logger.LogError($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException e)
{
_logger.LogError($"OperationCanceledException occured: {e.Message}");
c.Close();
}
catch (Exception ex)
{
_logger.LogError($"Exception occured: {ex.Message}");
}
}
}, cancellationToken);
Task.Run(() =>
{
using (IKafkaConsumer <Ignore, string> c = this._kafkaConsumerProvider.GetKafkaConsumer <Ignore, string>("consumer-selection-synchronization"))
{
c.Subscribe();
try
{
while (true)
{
try
{
var cr = c.Consume(cancellationToken);
_logger.LogInformation($"2. Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'.");
SelectionSummaryKafkaMessage[] kafkaSocketMessage = null;
try
{
kafkaSocketMessage = cr.Message?.Value?.FromJSON <SelectionSummaryKafkaMessage[]>();
}
catch (Exception ex)
{
_logger.LogError($"Exception occured in KafkaSocketMessage deserialization: {ex.Message}");
}
if (kafkaSocketMessage != null)
{
SocketMessage calculationResultMessage = new SocketMessage()
{
Code = "selection_synchronization_calculated",
Data = kafkaSocketMessage
};
Task.Run(async() => { await _sampleSocketMessageHandler.SendMessageAsync(kafkaSocketMessage.First().UserId, calculationResultMessage.ToJSON()); });
}
}
catch (ConsumeException e)
{
_logger.LogError($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException e)
{
_logger.LogError($"OperationCanceledException occured: {e.Message}");
c.Close();
}
catch (Exception ex)
{
_logger.LogError($"Exception occured: {ex.Message}");
}
}
}, cancellationToken);
return(Task.CompletedTask);
}
