Esempio n. 1
0
        public void GivenIfSomeMessageFailsWhenCommitThenReturnsFirstNonFailedMessage()
        {
            // arrange
            var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object, NullLoggerFactory.Instance);

            var taskQueue = new Queue <Task <Exception> >();

            taskQueue.Enqueue(Task.FromResult <Exception>(null));
            taskQueue.Enqueue(Task.Delay(3000).ContinueWith(x => (Exception)null));
            taskQueue.Enqueue(Task.FromException <Exception>(new Exception()));
            taskQueue.Enqueue(Task.FromResult <Exception>(null));
            taskQueue.Enqueue(Task.FromException <Exception>(new Exception()));
            taskQueue.Enqueue(Task.FromResult <Exception>(null));

            var messages = taskQueue.Select(x => new SomeMessage()).ToArray();

            _consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => taskQueue.Dequeue());

            foreach (var t in messages)
            {
                w.Submit(t);
            }

            // act
            var result = w.WaitAll().Result;

            // assert
            result.Success.Should().BeFalse();
            result.LastSuccessMessage.Should().BeSameAs(messages[1]);
        }
        public void Commit_IfSomeMessageFails_ReturnsFirstNonFailedMessage()
        {
            // arrange
            var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object);

            var taskQueue = new Queue <Task>();

            taskQueue.Enqueue(Task.CompletedTask);
            taskQueue.Enqueue(Task.Delay(3000));
            taskQueue.Enqueue(Task.FromException(new Exception()));
            taskQueue.Enqueue(Task.CompletedTask);
            taskQueue.Enqueue(Task.FromException(new Exception()));
            taskQueue.Enqueue(Task.CompletedTask);

            var messages = taskQueue.ToList().Select(x => new SomeMessage()).ToArray();

            _consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => taskQueue.Dequeue());

            foreach (var t in messages)
            {
                w.Submit(t);
            }

            // act
            var success = w.WaitAll(out SomeMessage lastGoodMessage);

            // assert
            success.Should().BeFalse();
            lastGoodMessage.Should().BeSameAs(messages[1]);
        }
 public PartitionConsumerForConsumers(EventHubMessageBus messageBus, ConsumerSettings consumerSettings)
     : base(messageBus)
 {
     _logger       = messageBus.LoggerFactory.CreateLogger <PartitionConsumerForConsumers>();
     _instancePool = new ConsumerInstancePoolMessageProcessor <EventData>(consumerSettings, messageBus, e => e.Body.Array);
     _queueWorker  = new MessageQueueWorker <EventData>(_instancePool, new CheckpointTrigger(consumerSettings), messageBus.LoggerFactory);
 }
Esempio n. 4
0
        public KafkaConsumerProcessor(ConsumerSettings consumerSettings, TopicPartition topicPartition, IKafkaCommitController commitController, MessageBusBase messageBus, MessageQueueWorker <Message> messageQueueWorker)
        {
            Log.InfoFormat("Creating for Group: {0}, Topic: {1}, Partition: {2}, MessageType: {3}", consumerSettings.Group, consumerSettings.Topic, topicPartition, consumerSettings.MessageType);

            _consumerSettings   = consumerSettings;
            TopicPartition      = topicPartition;
            _commitController   = commitController;
            _messageQueueWorker = messageQueueWorker;
        }
Esempio n. 5
0
        public KafkaConsumerProcessor(ConsumerSettings consumerSettings, TopicPartition topicPartition, IKafkaCommitController commitController, MessageBusBase messageBus, MessageQueueWorker <ConsumeResult> messageQueueWorker)
        {
            _messageBus = messageBus ?? throw new ArgumentNullException(nameof(messageBus));

            _logger = _messageBus.LoggerFactory.CreateLogger <KafkaConsumerProcessor>();
            _logger.LogInformation("Creating for Group: {0}, Topic: {1}, Partition: {2}, MessageType: {3}", consumerSettings.GetGroup(), consumerSettings.Topic, topicPartition, consumerSettings.MessageType);

            _consumerSettings   = consumerSettings;
            TopicPartition      = topicPartition;
            _commitController   = commitController;
            _messageQueueWorker = messageQueueWorker;
        }
Esempio n. 6
0
        public void ShouldStartAndStop()
        {
            // Given
            var queueConfig = GetQueueConfig();
            var target      = new MessageQueueWorker(queueConfig, GetConsumer, GetCommandExecutor(), GetMessageQueue);

            // When
            target.Start();
            target.Stop();
            target.Start();
            target.Stop();

            // Then
        }
Esempio n. 7
0
        public MessageQueue(IMessageSerializer messageSerializer, IMessageFeedFactory messageFeedFactory)
        {
            _outputQueue = new ConcurrentQueue <IdentifiedMessage>();
            _clients     = new ConcurrentDictionary <ClientId, IDataClient>();
            _pipes       = new ConcurrentDictionary <ClientId, Pipe>();

            _cancellationTokenSource = new CancellationTokenSource();

            var worker = new MessageQueueWorker(this, _cancellationTokenSource.Token, messageSerializer, messageFeedFactory);

            _workerThread = new Thread(worker.DoWork)
            {
                Name = "MessageQueueWorker"
            };
        }
Esempio n. 8
0
        public void WhenCommitThenWaitsOnAllMessagesToComplete()
        {
            // arrange
            var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object, NullLoggerFactory.Instance);

            var numFinishedMessages = 0;

            _consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => Task.Delay(50).ContinueWith(t => { Interlocked.Increment(ref numFinishedMessages); return((Exception)null); }, TaskScheduler.Current));

            const int numMessages = 100;

            for (var i = 0; i < numMessages; i++)
            {
                w.Submit(new SomeMessage());
            }

            // act
            var result = w.WaitAll().Result;

            // assert
            result.Success.Should().BeTrue();
            numFinishedMessages.Should().Be(numMessages);
        }
        public void Commit_WaitsOnAllMessagesToComplete()
        {
            // arrange
            //_checkpointTriggerMock.SetupGet(x => x.IsEnabled).Returns(false);
            var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object);

            var numFinishedMessages = 0;

            _consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => Task.Delay(50).ContinueWith(t => Interlocked.Increment(ref numFinishedMessages)));

            const int numMessages = 100;

            for (var i = 0; i < numMessages; i++)
            {
                w.Submit(new SomeMessage());
            }

            // act
            var success = w.WaitAll(out SomeMessage lastGoodMessage);

            // assert
            success.Should().BeTrue();
            numFinishedMessages.ShouldBeEquivalentTo(numMessages);
        }
Esempio n. 10
0
 public PartitionConsumerForConsumers(EventHubMessageBus messageBus, ConsumerSettings consumerSettings)
     : base(messageBus)
 {
     _instancePool = new ConsumerInstancePool <EventData>(consumerSettings, messageBus, e => e.Body.Array);
     _queueWorker  = new MessageQueueWorker <EventData>(_instancePool, new CheckpointTrigger(consumerSettings));
 }
        private void Initialize(MessageQueueWorkerInfo[] workerInfos,
            KeyValuePair<string, CreateQueueMessageWrapperHandler>[] createMessageWrappers)
        {
            ModuleProc PROC = new ModuleProc(DYN_MODULE_NAME, "Initialize");

            try
            {
                _uniqueKey = KEY_PREFIX + Guid.NewGuid().ToString();
                this.ExecutorService.AddExecutor(this);
                this.RegisterForShutdown();

                _queueWorkers = new SortedDictionary<string, MessageQueueWorker>(StringComparer.InvariantCultureIgnoreCase);
                foreach (MessageQueueWorkerInfo workerInfo in workerInfos)
                {
                    MessageQueueWorker worker = new MessageQueueWorker(this, this.ExecutorService, _poolType, workerInfo, createMessageWrappers);
                    string key = worker.UniqueKey;
                    _queueWorkers.Add(key, worker);

                    if (workerInfo.DependencyWorker != null)
                    {
                        string workerKey = workerInfo.DependencyWorker.WorkerKey;
                        if (_queueWorkers.ContainsKey(workerKey))
                        {
                            worker.DependencyWorker = _queueWorkers[workerKey];
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Log.Exception(PROC, ex);
            }
        }