protected override async Task <DataItem <TConsumerValue> > FetchNextDataAsync(CancellationToken cancellationToken) { var entity = new DataItem <TConsumerValue>(); ConsumeResult <TConsumerKey, TConsumerValue> consumerResult; try { consumerResult = await Task.Run(() => { var r = Consumer.Consume(cancellationToken); while (r != null && r.IsPartitionEOF) { r = Consumer.Consume(cancellationToken); } return(r); }, cancellationToken) .ConfigureAwait(false); } catch (OperationCanceledException) { return(null); } if (consumerResult == null) { return(null); } entity.Data = consumerResult.Message.Value; entity.Metadata = new PipelineMetadata { Metadata = consumerResult.Message, Timestamp = DateTime.Now }; return(entity); }
private async Task RunConsumer(CancellationToken token = default) { var count = 0; try { while (!token.IsCancellationRequested) { try { var message = _kafkaConsumer.Consume(token); await HandleMessageAsync(message); if (count++ % 10 == 0) { _kafkaConsumer.Commit(message); count = 0; } } catch (Exception e) { _logger.LogError(e, e.Message); } } } catch (OperationCanceledException) { _kafkaConsumer.Dispose(); } }
private async Task RunConsumerAsync(CancellationToken token) { try { _consumer.Subscribe(); while (true) { try { var message = _consumer.Consume(token); var handlerCompletionResponse = await HandleMessage(message); if (handlerCompletionResponse) { _consumer.Commit(); } } catch (Exception e) { _logger.LogError(e, e.Message); } } } catch (OperationCanceledException) { _consumer.Dispose(); } }
public async Task KafkaActivity_ProduceConsumeMessage_CorrectActivityReceived() { // Arrange var baggageKey = "testKey"; _activityContextAccessor.CurrentActivity.AddBaggage(baggageKey, "testing"); var mockMessage = new MockMessage(); var messageId = Guid.NewGuid().ToString(); var message = new KafkaMessage <string, MockMessage> { Key = "mock-message", Value = mockMessage, MessageId = messageId, MessageType = "mockMessage.Created" }; var cts = new CancellationTokenSource(); var consumedMessage = new KafkaMessage <string, MockMessage>(); // Act #pragma warning disable 4014 Task.Factory.StartNew(() => #pragma warning restore 4014 { while (!cts.IsCancellationRequested) { consumedMessage = (KafkaMessage <string, MockMessage>)_kafkaConsumer.Consume(cts.Token); _semaphore.Release(); } }, TaskCreationOptions.LongRunning); await _kafkaProducer.ProduceAsync(message); await _semaphore.WaitAsync(30000); // Assert Assert.NotEmpty(consumedMessage.Headers[Constants.CorrelationIdHeaderName]); Assert.NotEmpty(consumedMessage.Headers[baggageKey]); }
public Task StartAsync(CancellationToken cancellationToken) { Task.Run(() => { using (IKafkaConsumer <Ignore, string> c = this._kafkaConsumerProvider.GetKafkaConsumer <Ignore, string>("consumer-selection-count")) { c.Subscribe(); try { while (true) { try { var cr = c.Consume(cancellationToken); _logger.LogInformation($"1. Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'."); SelectionSummaryKafkaMessage kafkaSocketMessage = null; try { kafkaSocketMessage = cr.Message?.Value?.FromJSON <SelectionSummaryKafkaMessage>(); } catch (Exception ex) { _logger.LogError($"Exception occured in KafkaSocketMessage deserialization: {ex.Message}"); } if (kafkaSocketMessage != null) { SocketMessage calculationResultMessage = new SocketMessage() { Code = "selection_count_calculated", Data = kafkaSocketMessage }; Task.Run(async() => { await _sampleSocketMessageHandler.SendMessageAsync(kafkaSocketMessage.UserId, calculationResultMessage.ToJSON()); }); } } catch (ConsumeException e) { _logger.LogError($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException e) { _logger.LogError($"OperationCanceledException occured: {e.Message}"); c.Close(); } catch (Exception ex) { _logger.LogError($"Exception occured: {ex.Message}"); } } }, cancellationToken); Task.Run(() => { using (IKafkaConsumer <Ignore, string> c = this._kafkaConsumerProvider.GetKafkaConsumer <Ignore, string>("consumer-selection-synchronization")) { c.Subscribe(); try { while (true) { try { var cr = c.Consume(cancellationToken); _logger.LogInformation($"2. Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'."); SelectionSummaryKafkaMessage[] kafkaSocketMessage = null; try { kafkaSocketMessage = cr.Message?.Value?.FromJSON <SelectionSummaryKafkaMessage[]>(); } catch (Exception ex) { _logger.LogError($"Exception occured in KafkaSocketMessage deserialization: {ex.Message}"); } if (kafkaSocketMessage != null) { SocketMessage calculationResultMessage = new SocketMessage() { Code = "selection_synchronization_calculated", Data = kafkaSocketMessage }; Task.Run(async() => { await _sampleSocketMessageHandler.SendMessageAsync(kafkaSocketMessage.First().UserId, calculationResultMessage.ToJSON()); }); } } catch (ConsumeException e) { _logger.LogError($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException e) { _logger.LogError($"OperationCanceledException occured: {e.Message}"); c.Close(); } catch (Exception ex) { _logger.LogError($"Exception occured: {ex.Message}"); } } }, cancellationToken); return(Task.CompletedTask); }