public async Task PlainSource_should_resume_on_deserialization_errors() { Directive Decider(Exception cause) => cause is SerializationException ? Directive.Resume : Directive.Stop; int elementsCount = 10; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); await Produce(topic1, Enumerable.Range(1, elementsCount), ProducerSettings); var settings = ConsumerSettings <Null, int> .Create(Sys, null, new IntDeserializer()) .WithBootstrapServers(KafkaUrl) .WithProperty("auto.offset.reset", "earliest") .WithGroupId(group1); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)) .Select(c => c.Value) .RunWith(this.SinkProbe <int>(), _materializer); probe.Request(elementsCount); probe.ExpectNoMsg(TimeSpan.FromSeconds(10)); probe.Cancel(); }
public void WhenMessageFails_OnMessageFault_IsCalled() { // arrange var onMessageFaultMock = new Mock <Action <ConsumerSettings, object, Exception> >(); var consumerSettings = new ConsumerSettings { Instances = 1, Topic = "topic1", ConsumerMode = ConsumerMode.Subscriber, ConsumerType = typeof(IConsumer <SomeMessage>), MessageType = typeof(SomeMessage), OnMessageFault = onMessageFaultMock.Object }; var p = new ConsumerInstancePool <SomeMessage>(consumerSettings, _busMock.Object, x => new byte[0]); var message = new SomeMessage(); _serializerMock.Setup(x => x.Deserialize(typeof(SomeMessage), It.IsAny <byte[]>())).Returns(message); var ex = new Exception("Something went bad"); _consumerMock.Setup(x => x.OnHandle(message, consumerSettings.Topic)).Returns(Task.FromException <SomeResponse>(ex)); // act p.Submit(message); var commitedMsg = p.Commit(message); // assert commitedMsg.Should().BeSameAs(message); // it should commit the failed message _consumerMock.Verify(x => x.OnHandle(message, consumerSettings.Topic), Times.Once); // handler called once onMessageFaultMock.Verify(x => x(consumerSettings, message, ex), Times.Once); // callback called once }
public static void Main(string[] args) { Config fallbackConfig = ConfigurationFactory.ParseString(@" akka.suppress-json-serializer-warning=true akka.loglevel = DEBUG ").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf")); var system = ActorSystem.Create("TestKafka", fallbackConfig); var materializer = system.Materializer(); var consumerSettings = ConsumerSettings <Null, string> .Create(system, null, null) .WithBootstrapServers("localhost:29092") .WithGroupId("group1"); var subscription = Subscriptions.Topics("akka100"); KafkaConsumer.PlainSource(consumerSettings, subscription) .RunForeach(result => { Console.WriteLine($"Consumer: {result.Topic}/{result.Partition} {result.Offset}: {result.Value}"); }, materializer); Console.ReadLine(); }
protected async Task ProcessMessagesAsync(Message message, CancellationToken token) { // Process the message. var mf = ConsumerSettings.FormatIf(message, _log.IsDebugEnabled); _log.DebugFormat(CultureInfo.InvariantCulture, "Received message - {0}", mf); await MessageProcessor.ProcessMessage(message).ConfigureAwait(false); if (token.IsCancellationRequested) { // Note: Use the cancellationToken passed as necessary to determine if the subscriptionClient has already been closed. // If subscriptionClient has already been closed, you can choose to not call CompleteAsync() or AbandonAsync() etc. // to avoid unnecessary exceptions. _log.DebugFormat(CultureInfo.InvariantCulture, "Abandon message - {0}", mf); await Client.AbandonAsync(message.SystemProperties.LockToken).ConfigureAwait(false); } else { // Complete the message so that it is not received again. // This can be done only if the subscriptionClient is created in ReceiveMode.PeekLock mode (which is the default). _log.DebugFormat(CultureInfo.InvariantCulture, "Complete message - {0}", mf); await Client.CompleteAsync(message.SystemProperties.LockToken).ConfigureAwait(false); } }
public async Task PlainSource_should_resume_stage_if_broker_unavailable() { var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var topicPartition1 = new TopicPartition(topic1, 0); await GivenInitializedTopic(topicPartition1); var config = ConsumerSettings <Null, string> .Create(Sys, null, null) .WithBootstrapServers("localhost:10092") .WithGroupId(group1); var regex = new Regex("\\[localhost:10092\\/bootstrap: Connect to [a-zA-Z0-9#:.*]* failed:"); var logProbe = CreateTestProbe(); Sys.EventStream.Subscribe <Info>(logProbe.Ref); var(control, probe) = CreateProbe(config, Subscriptions.Assignment(topicPartition1)); probe.Request(1); AwaitAssert(() => { var info = logProbe.ExpectMsg <Info>(); regex.IsMatch(info.Message.ToString() ?? "").Should().BeTrue(); info.Message.ToString().Should().Contain("[Resume]"); }); //AwaitCondition(() => control.IsShutdown.IsCompleted, TimeSpan.FromSeconds(10)); }
public KafkaConsumerProcessorTest() { _topicPartition = new TopicPartition("topic-a", 0); var consumerSettings = new ConsumerSettings { MessageType = typeof(SomeMessage), Topic = _topicPartition.Topic, ConsumerType = typeof(SomeMessageConsumer), ConsumerMode = ConsumerMode.Consumer }; consumerSettings.SetGroup("group-a"); var massageBusMock = new MessageBusMock(); massageBusMock.BusSettings.Consumers.Add(consumerSettings); massageBusMock.DependencyResolverMock.Setup(x => x.Resolve(typeof(SomeMessageConsumer))).Returns(_consumer); byte[] MessageValueProvider(Message m) => m.Value; var consumerInstancePoolMock = new Mock <ConsumerInstancePoolMessageProcessor <Message> >(consumerSettings, massageBusMock.Bus, (Func <Message, byte[]>)MessageValueProvider, null); _messageQueueWorkerMock = new Mock <MessageQueueWorker <Message> >(consumerInstancePoolMock.Object, _checkpointTrigger.Object); _subject = new KafkaConsumerProcessor(consumerSettings, _topicPartition, _commitControllerMock.Object, massageBusMock.Bus, _messageQueueWorkerMock.Object); }
private void ApplySettings(ConsumerSettings <K, V> updatedSettings) { _settings = updatedSettings; _pollTimeout = _settings.PollTimeout; _positionTimeout = _settings.PositionTimeout; _commitRefreshing = CommitRefreshing.Create <K, V>(_settings.CommitRefreshInterval); try { if (_log.IsDebugEnabled) { _log.Debug($"Creating Kafka consumer with settings: {JsonConvert.SerializeObject(_settings)}"); } var localSelf = Self; _consumer = _settings.CreateKafkaConsumer( consumeErrorHandler: (c, e) => localSelf.Tell(new Status.Failure(new KafkaException(e))), partitionAssignedHandler: (c, tp) => localSelf.Tell(new PartitionAssigned(tp.ToImmutableHashSet())), partitionRevokedHandler: (c, tp) => localSelf.Tell(new PartitionRevoked(tp.ToImmutableHashSet())), statisticHandler: (c, json) => _statisticsHandler.OnStatistics(c, json)); if (_settings.ConnectionCheckerSettings.Enabled) { _connectionCheckerActor = Context.ActorOf(ConnectionChecker.Props(_settings.ConnectionCheckerSettings)); } } catch (Exception e) { ProcessError(e); throw; } }
static void Consumer() { string topic = "insidetech-5"; int countMessage = 0; var consumerSettings = new ConsumerSettings( "127.0.0.1:9092"); var consumer = new Consumer(consumerSettings); Task.Run(() => { consumer.Subscriber <Model.User>(topic, (receive) => { if (receive != null) { Console.WriteLine($"Message: {++countMessage}, Date: {receive.CreatedAt}, Email: {receive.Email}, Name: {receive.Name}, Key: {receive.Key}"); } return(true); }, (error) => { Console.WriteLine(((Exception)error).Message); }); }); }
private ConsumerSettings <Null, string> CreateConsumerSettings(string group) { return(ConsumerSettings <Null, string> .Create(Sys, null, new StringDeserializer(Encoding.UTF8)) .WithBootstrapServers(KafkaUrl) .WithProperty("auto.offset.reset", "earliest") .WithGroupId(group)); }
public ConsumerRuntimeInfo(ConsumerSettings consumerSettings) { ConsumerSettings = consumerSettings; if (consumerSettings.ConsumerType == null) { throw new ConfigurationMessageBusException($"{nameof(consumerSettings.ConsumerType)} is not set on the {consumerSettings}"); } if (consumerSettings.MessageType == null) { throw new ConfigurationMessageBusException($"{nameof(consumerSettings.MessageType)} is not set on the {consumerSettings}"); } ConsumerOnHandleMethod = consumerSettings.ConsumerType.GetMethod(nameof(IConsumer <object> .OnHandle), new[] { consumerSettings.MessageType, typeof(string) }); if (consumerSettings.ConsumerMode == ConsumerMode.RequestResponse) { if (consumerSettings.ResponseType == null) { throw new ConfigurationMessageBusException($"{nameof(consumerSettings.ResponseType)} is not set on the {consumerSettings}"); } var taskType = typeof(Task <>).MakeGenericType(consumerSettings.ResponseType); TaskResultProperty = taskType.GetProperty(nameof(Task <object> .Result)); } }
public PartitionConsumerForConsumers(EventHubMessageBus messageBus, ConsumerSettings consumerSettings) : base(messageBus) { _logger = messageBus.LoggerFactory.CreateLogger <PartitionConsumerForConsumers>(); _instancePool = new ConsumerInstancePoolMessageProcessor <EventData>(consumerSettings, messageBus, e => e.Body.Array); _queueWorker = new MessageQueueWorker <EventData>(_instancePool, new CheckpointTrigger(consumerSettings), messageBus.LoggerFactory); }
public static void InitializeKafkaConsumer(IHandler handler) { CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); ConsumerSettings settings = null; settings = new ConsumerSettings() { GroupName = "CustomerServiceAutomation", TopicName = CustomerServiceConfig.CustomerServiceTopic, KafkaUri = KafkaUri, AutoCommit = true, BatchRead = false, ReadAsync = true, ReadFromEnd = false, MaxQueueSize = 50000, MaxBatchSize = 2000 }; if (KafkaDriver.Equals(RPL)) { CommonConsumer = new RPLConsumer(); } else if (KafkaDriver.Equals(MISAKAI)) { CommonConsumer = new MisakaiConsumer(); } else { CommonConsumer = new JavaConsumer(); } LogResult.Report(Log, "log_ForInfo", "Kafka Consumer Initialized For " + "Topic: " + CustomerServiceConfig.CustomerServiceTopic); KafkaConsumer = Task.Factory.StartNew(() => CommonConsumer.StartConsuming(handler, settings, cancellationTokenSource.Token)); }
public KafkaSourceStageLogic(KafkaSourceStage <K, V> stage, Attributes attributes, TaskCompletionSource <NotUsed> completion) : base(stage.Shape) { _settings = stage.Settings; _subscription = stage.Subscription; _out = stage.Out; _completion = completion; _buffer = new Queue <ConsumerRecord <K, V> >(stage.Settings.BufferSize); var supervisionStrategy = attributes.GetAttribute <ActorAttributes.SupervisionStrategy>(null); _decider = supervisionStrategy != null ? supervisionStrategy.Decider : Deciders.ResumingDecider; SetHandler(_out, onPull: () => { if (_buffer.Count > 0) { Push(_out, _buffer.Dequeue()); } else { if (_isPaused) { _consumer.Resume(_assignedPartitions); _isPaused = false; Log.Debug("Polling resumed, buffer is empty"); } PullQueue(); } }); }
public void WhenRequestExpired_OnMessageExpired_IsCalled() { // arrange var onMessageExpiredMock = new Mock <Action <ConsumerSettings, object> >(); var consumerSettings = new ConsumerSettings { Instances = 1, Topic = "topic1", ConsumerMode = ConsumerMode.RequestResponse, ConsumerType = typeof(IRequestHandler <SomeRequest, SomeResponse>), MessageType = typeof(SomeRequest), OnMessageExpired = onMessageExpiredMock.Object }; var p = new ConsumerInstancePool <SomeRequest>(consumerSettings, _busMock.Object, x => new byte[0]); var request = new SomeRequest(); string requestId; string replyTo; DateTimeOffset?expires = _currentTime.AddSeconds(-10); _busMock.Setup(x => x.DeserializeRequest(typeof(SomeRequest), It.IsAny <byte[]>(), out requestId, out replyTo, out expires)).Returns(request); // act p.Submit(request); var commitedMsg = p.Commit(request); // assert commitedMsg.Should().BeSameAs(request); // it should commit the request message _handlerMock.Verify(x => x.OnHandle(It.IsAny <SomeRequest>(), It.IsAny <string>()), Times.Never); // the handler should not be called onMessageExpiredMock.Verify(x => x(consumerSettings, request), Times.Once); // callback called once }
protected virtual void AssertConsumerSettings(ConsumerSettings consumerSettings) { if (consumerSettings == null) { throw new ArgumentNullException(nameof(consumerSettings)); } Assert.IsNotNull(consumerSettings.Topic, () => new ConfigurationMessageBusException($"The {nameof(ConsumerSettings)}.{nameof(consumerSettings.Topic)} is not set")); Assert.IsNotNull(consumerSettings.MessageType, () => new ConfigurationMessageBusException($"The {nameof(ConsumerSettings)}.{nameof(consumerSettings.MessageType)} is not set")); Assert.IsNotNull(consumerSettings.ConsumerType, () => new ConfigurationMessageBusException($"The {nameof(ConsumerSettings)}.{nameof(consumerSettings.ConsumerType)} is not set")); Assert.IsNotNull(consumerSettings.ConsumerMethod, () => new ConfigurationMessageBusException($"The {nameof(ConsumerSettings)}.{nameof(consumerSettings.ConsumerMethod)} is not set")); if (consumerSettings.ConsumerMode == ConsumerMode.RequestResponse) { Assert.IsNotNull(consumerSettings.ResponseType, () => new ConfigurationMessageBusException($"The {nameof(ConsumerSettings)}.{nameof(consumerSettings.ResponseType)} is not set")); Assert.IsNotNull(consumerSettings.ConsumerMethodResult, () => new ConfigurationMessageBusException($"The {nameof(ConsumerSettings)}.{nameof(consumerSettings.ConsumerMethodResult)} is not set")); } }
protected void Setup() { testTopic = "akka100"; subscription = Subscriptions.Topics(testTopic); probe = this.CreateTestProbe(); string configText = File.ReadAllText("akka.test.conf"); var config = ConfigurationFactory.ParseString(configText); var system_producer = ActorSystem.Create("TestKafka", config); materializer_producer = system_producer.Materializer(); var system_consumer = ActorSystem.Create("TestKafka", config); materializer_consumer = system_producer.Materializer(); this.Sys.Settings.Config.WithFallback(config); producerSettings = ProducerSettings <Null, string> .Create(system_producer, null, null) .WithBootstrapServers("kafka:9092"); consumerSettings = ConsumerSettings <Null, string> .Create(system_consumer, null, null) .WithBootstrapServers("kafka:9092") .WithGroupId("group1"); }
public void WhenMessageFailsThenOnMessageFaultIsCalled() { // arrange var onMessageFaultMock = new Mock <Action <AbstractConsumerSettings, object, Exception> >(); var consumerSettings = new ConsumerSettings { Instances = 1, Topic = "topic1", ConsumerMode = ConsumerMode.Consumer, ConsumerType = typeof(IConsumer <SomeMessage>), MessageType = typeof(SomeMessage), OnMessageFault = onMessageFaultMock.Object }; var p = new ConsumerInstancePool <SomeMessage>(consumerSettings, _busMock.Bus, x => Array.Empty <byte>()); var message = new SomeMessage(); _busMock.SerializerMock.Setup(x => x.Deserialize(typeof(SomeMessage), It.IsAny <byte[]>())).Returns(message); var ex = new Exception("Something went bad"); _busMock.ConsumerMock.Setup(x => x.OnHandle(message, consumerSettings.Topic)) .Returns(Task.FromException <SomeResponse>(ex)); // act p.ProcessMessage(message).Wait(); // assert _busMock.ConsumerMock.Verify(x => x.OnHandle(message, consumerSettings.Topic), Times.Once); // handler called once onMessageFaultMock.Verify(x => x(consumerSettings, message, ex), Times.Once); // callback called once }
public void WhenRequestExpiredThenOnMessageExpiredIsCalled() { // arrange var onMessageExpiredMock = new Mock <Action <AbstractConsumerSettings, object> >(); var consumerSettings = new ConsumerSettings { Instances = 1, Topic = "topic1", ConsumerMode = ConsumerMode.RequestResponse, ConsumerType = typeof(IRequestHandler <SomeRequest, SomeResponse>), MessageType = typeof(SomeRequest), OnMessageExpired = onMessageExpiredMock.Object }; var p = new ConsumerInstancePool <SomeRequest>(consumerSettings, _busMock.Bus, x => Array.Empty <byte>()); var request = new SomeRequest(); var requestMessage = new MessageWithHeaders(); requestMessage.SetHeader(ReqRespMessageHeaders.Expires, _busMock.CurrentTime.AddSeconds(-10)); _busMock.BusMock.Setup(x => x.DeserializeRequest(typeof(SomeRequest), It.IsAny <byte[]>(), out requestMessage)) .Returns(request); // act p.ProcessMessage(request).Wait(); // assert _busMock.HandlerMock.Verify(x => x.OnHandle(It.IsAny <SomeRequest>(), It.IsAny <string>()), Times.Never); // the handler should not be called onMessageExpiredMock.Verify(x => x(consumerSettings, request), Times.Once); // callback called once }
protected async Task ProcessMessagesAsync(Message message, CancellationToken token) { if (message is null) { throw new ArgumentNullException(nameof(message)); } // Process the message. var mf = ConsumerSettings.FormatIf(message, _logger.IsEnabled(LogLevel.Debug)); _logger.LogDebug("Received message - {0}", mf); if (token.IsCancellationRequested) { // Note: Use the cancellationToken passed as necessary to determine if the subscriptionClient has already been closed. // If subscriptionClient has already been closed, you can choose to not call CompleteAsync() or AbandonAsync() etc. // to avoid unnecessary exceptions. _logger.LogDebug("Abandon message - {0}", mf); await Client.AbandonAsync(message.SystemProperties.LockToken).ConfigureAwait(false); return; } var exception = await MessageProcessor.ProcessMessage(message).ConfigureAwait(false); if (exception != null) { if (mf == null) { mf = ConsumerSettings.FormatIf(message, true); } _logger.LogError(exception, "Abandon message (exception occured while processing) - {0}", mf); try { // Execute the event hook ConsumerSettings.OnMessageFault?.Invoke(MessageBus, ConsumerSettings, null, exception, message); MessageBus.Settings.OnMessageFault?.Invoke(MessageBus, ConsumerSettings, null, exception, message); } catch (Exception eh) { MessageBusBase.HookFailed(_logger, eh, nameof(IConsumerEvents.OnMessageFault)); } var messageProperties = new Dictionary <string, object> { // Set the exception message ["SMB.Exception"] = exception.Message }; await Client.AbandonAsync(message.SystemProperties.LockToken, messageProperties).ConfigureAwait(false); return; } // Complete the message so that it is not received again. // This can be done only if the subscriptionClient is created in ReceiveMode.PeekLock mode (which is the default). _logger.LogDebug("Complete message - {0}", mf); await Client.CompleteAsync(message.SystemProperties.LockToken).ConfigureAwait(false); }
private (IControl, TestSubscriber.Probe <string>) CreateProbe(ConsumerSettings <Null, string> consumerSettings, ISubscription sub) { return(KafkaConsumer .PlainSource(consumerSettings, sub) .Select(c => c.Value) .ToMaterialized(this.SinkProbe <string>(), Keep.Both) .Run(Materializer)); }
public KafkaSourceStage(ConsumerSettings <K, V> settings, ISubscription subscription) { Settings = settings; Subscription = subscription; Shape = new SourceShape <ConsumerRecord <K, V> >(Out); Settings = settings; Subscription = subscription; }
/// <summary> /// CommittableSourceStage /// </summary> /// <param name="settings">Consumer settings</param> /// <param name="subscription">Subscription to be used</param> /// <param name="metadataFromMessage">Function to extract string metadata from consumed message</param> public CommittableSourceStage(ConsumerSettings <K, V> settings, ISubscription subscription, Func <ConsumeResult <K, V>, string> metadataFromMessage = null) : base("CommittableSource") { _metadataFromMessage = metadataFromMessage ?? (msg => string.Empty); Settings = settings; Subscription = subscription; }
private TestSubscriber.Probe <string> CreateProbe(ConsumerSettings <Null, string> consumerSettings, string topic, ISubscription sub) { return(KafkaConsumer .PlainSource(consumerSettings, sub) .Where(c => !c.Value.Equals(InitialMsg)) .Select(c => c.Value) .RunWith(this.SinkProbe <string>(), _materializer)); }
public KafkaConsumerProcessor(ConsumerSettings consumerSettings, TopicPartition topicPartition, IKafkaCommitController commitController, MessageBusBase messageBus, MessageQueueWorker <Message> messageQueueWorker) { Log.InfoFormat("Creating for Group: {0}, Topic: {1}, Partition: {2}, MessageType: {3}", consumerSettings.Group, consumerSettings.Topic, topicPartition, consumerSettings.MessageType); _consumerSettings = consumerSettings; TopicPartition = topicPartition; _commitController = commitController; _messageQueueWorker = messageQueueWorker; }
public AnaliseVendasRelatorioWorker( ILogger <AnaliseVendasRelatorioWorker> logger, IProcessaArquivoService gerarRelatorioService, IOptions <ConsumerSettings> config) { this.logger = logger; this.processaArquivoService = gerarRelatorioService; this.config = config.Value; }
public SingleSourceStageLogic(SourceShape <TMessage> shape, ConsumerSettings <K, V> settings, ISubscription subscription, Attributes attributes, Func <BaseSingleSourceLogic <K, V, TMessage>, IMessageBuilder <K, V, TMessage> > messageBuilderFactory) : base(shape, attributes, messageBuilderFactory) { _shape = shape; _settings = settings; _subscription = subscription; }
public static string FormatIf(this ConsumerSettings consumerSettings, bool logLevel) { if (!logLevel) { return(string.Empty); } return($"Topic: {consumerSettings.Topic}, MessageType: {consumerSettings.MessageType}"); }
public void Initialize(ConsumerSettings consumerSettings, string topic, int queueCount, bool continueWhenHandleFail, int retryIntervalSeconds, IInterceptor[] interceptors, params Assembly[] assemblies) { _consumer = new Consumer(consumerSettings); _consumer.Subscribe(topic, queueCount); _commandHandlerProvider = new CommandHandlerProvider(); _commandHandlerProvider.Initialize(interceptors, assemblies); _commandProcessor = new DefaultMessageProcessor(string.Empty, _commandHandlerProvider, _mailboxProvider, continueWhenHandleFail, retryIntervalSeconds); }
public KafkaConsumerProcessor(ConsumerSettings consumerSettings, TopicPartition topicPartition, IKafkaCommitController commitController, MessageBusBase messageBus) : this(consumerSettings, topicPartition, commitController, messageBus, new MessageQueueWorker <Message>( new ConsumerInstancePoolMessageProcessor <Message>(consumerSettings, messageBus, m => m.Value, (m, ctx) => ctx.SetTransportMessage(m)), new CheckpointTrigger(consumerSettings))) { }
public void AfterCreation_DefaultInstances_ShouldBe1() { // arrange // act var cs = new ConsumerSettings(); // assert cs.Instances.ShouldBeEquivalentTo(1); }
public void Setup() { queue.Clear(); consumerSettings = ConsumerSettings.CreateDefault(); }