public void GivenIfSomeMessageFailsWhenCommitThenReturnsFirstNonFailedMessage()
{
// arrange
var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object, NullLoggerFactory.Instance);
var taskQueue = new Queue <Task <Exception> >();
taskQueue.Enqueue(Task.FromResult <Exception>(null));
taskQueue.Enqueue(Task.Delay(3000).ContinueWith(x => (Exception)null));
taskQueue.Enqueue(Task.FromException <Exception>(new Exception()));
taskQueue.Enqueue(Task.FromResult <Exception>(null));
taskQueue.Enqueue(Task.FromException <Exception>(new Exception()));
taskQueue.Enqueue(Task.FromResult <Exception>(null));
var messages = taskQueue.Select(x => new SomeMessage()).ToArray();
_consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => taskQueue.Dequeue());
foreach (var t in messages)
{
w.Submit(t);
}
// act
var result = w.WaitAll().Result;
// assert
result.Success.Should().BeFalse();
result.LastSuccessMessage.Should().BeSameAs(messages[1]);
}
public void Commit_IfSomeMessageFails_ReturnsFirstNonFailedMessage()
{
// arrange
var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object);
var taskQueue = new Queue <Task>();
taskQueue.Enqueue(Task.CompletedTask);
taskQueue.Enqueue(Task.Delay(3000));
taskQueue.Enqueue(Task.FromException(new Exception()));
taskQueue.Enqueue(Task.CompletedTask);
taskQueue.Enqueue(Task.FromException(new Exception()));
taskQueue.Enqueue(Task.CompletedTask);
var messages = taskQueue.ToList().Select(x => new SomeMessage()).ToArray();
_consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => taskQueue.Dequeue());
foreach (var t in messages)
{
w.Submit(t);
}
// act
var success = w.WaitAll(out SomeMessage lastGoodMessage);
// assert
success.Should().BeFalse();
lastGoodMessage.Should().BeSameAs(messages[1]);
}
public PartitionConsumerForConsumers(EventHubMessageBus messageBus, ConsumerSettings consumerSettings)
: base(messageBus)
{
_logger = messageBus.LoggerFactory.CreateLogger <PartitionConsumerForConsumers>();
_instancePool = new ConsumerInstancePoolMessageProcessor <EventData>(consumerSettings, messageBus, e => e.Body.Array);
_queueWorker = new MessageQueueWorker <EventData>(_instancePool, new CheckpointTrigger(consumerSettings), messageBus.LoggerFactory);
}
public KafkaConsumerProcessor(ConsumerSettings consumerSettings, TopicPartition topicPartition, IKafkaCommitController commitController, MessageBusBase messageBus, MessageQueueWorker <Message> messageQueueWorker)
{
Log.InfoFormat("Creating for Group: {0}, Topic: {1}, Partition: {2}, MessageType: {3}", consumerSettings.Group, consumerSettings.Topic, topicPartition, consumerSettings.MessageType);
_consumerSettings = consumerSettings;
TopicPartition = topicPartition;
_commitController = commitController;
_messageQueueWorker = messageQueueWorker;
}
public KafkaConsumerProcessor(ConsumerSettings consumerSettings, TopicPartition topicPartition, IKafkaCommitController commitController, MessageBusBase messageBus, MessageQueueWorker <ConsumeResult> messageQueueWorker)
{
_messageBus = messageBus ?? throw new ArgumentNullException(nameof(messageBus));
_logger = _messageBus.LoggerFactory.CreateLogger <KafkaConsumerProcessor>();
_logger.LogInformation("Creating for Group: {0}, Topic: {1}, Partition: {2}, MessageType: {3}", consumerSettings.GetGroup(), consumerSettings.Topic, topicPartition, consumerSettings.MessageType);
_consumerSettings = consumerSettings;
TopicPartition = topicPartition;
_commitController = commitController;
_messageQueueWorker = messageQueueWorker;
}
public void ShouldStartAndStop()
{
// Given
var queueConfig = GetQueueConfig();
var target = new MessageQueueWorker(queueConfig, GetConsumer, GetCommandExecutor(), GetMessageQueue);
// When
target.Start();
target.Stop();
target.Start();
target.Stop();
// Then
}
public MessageQueue(IMessageSerializer messageSerializer, IMessageFeedFactory messageFeedFactory)
{
_outputQueue = new ConcurrentQueue <IdentifiedMessage>();
_clients = new ConcurrentDictionary <ClientId, IDataClient>();
_pipes = new ConcurrentDictionary <ClientId, Pipe>();
_cancellationTokenSource = new CancellationTokenSource();
var worker = new MessageQueueWorker(this, _cancellationTokenSource.Token, messageSerializer, messageFeedFactory);
_workerThread = new Thread(worker.DoWork)
{
Name = "MessageQueueWorker"
};
}
public void WhenCommitThenWaitsOnAllMessagesToComplete()
{
// arrange
var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object, NullLoggerFactory.Instance);
var numFinishedMessages = 0;
_consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => Task.Delay(50).ContinueWith(t => { Interlocked.Increment(ref numFinishedMessages); return((Exception)null); }, TaskScheduler.Current));
const int numMessages = 100;
for (var i = 0; i < numMessages; i++)
{
w.Submit(new SomeMessage());
}
// act
var result = w.WaitAll().Result;
// assert
result.Success.Should().BeTrue();
numFinishedMessages.Should().Be(numMessages);
}
public void Commit_WaitsOnAllMessagesToComplete()
{
// arrange
//_checkpointTriggerMock.SetupGet(x => x.IsEnabled).Returns(false);
var w = new MessageQueueWorker <SomeMessage>(_consumerInstancePoolMock.Object, _checkpointTriggerMock.Object);
var numFinishedMessages = 0;
_consumerInstancePoolMock.Setup(x => x.ProcessMessage(It.IsAny <SomeMessage>())).Returns(() => Task.Delay(50).ContinueWith(t => Interlocked.Increment(ref numFinishedMessages)));
const int numMessages = 100;
for (var i = 0; i < numMessages; i++)
{
w.Submit(new SomeMessage());
}
// act
var success = w.WaitAll(out SomeMessage lastGoodMessage);
// assert
success.Should().BeTrue();
numFinishedMessages.ShouldBeEquivalentTo(numMessages);
}
public PartitionConsumerForConsumers(EventHubMessageBus messageBus, ConsumerSettings consumerSettings)
: base(messageBus)
{
_instancePool = new ConsumerInstancePool <EventData>(consumerSettings, messageBus, e => e.Body.Array);
_queueWorker = new MessageQueueWorker <EventData>(_instancePool, new CheckpointTrigger(consumerSettings));
}
private void Initialize(MessageQueueWorkerInfo[] workerInfos,
KeyValuePair<string, CreateQueueMessageWrapperHandler>[] createMessageWrappers)
{
ModuleProc PROC = new ModuleProc(DYN_MODULE_NAME, "Initialize");
try
{
_uniqueKey = KEY_PREFIX + Guid.NewGuid().ToString();
this.ExecutorService.AddExecutor(this);
this.RegisterForShutdown();
_queueWorkers = new SortedDictionary<string, MessageQueueWorker>(StringComparer.InvariantCultureIgnoreCase);
foreach (MessageQueueWorkerInfo workerInfo in workerInfos)
{
MessageQueueWorker worker = new MessageQueueWorker(this, this.ExecutorService, _poolType, workerInfo, createMessageWrappers);
string key = worker.UniqueKey;
_queueWorkers.Add(key, worker);
if (workerInfo.DependencyWorker != null)
{
string workerKey = workerInfo.DependencyWorker.WorkerKey;
if (_queueWorkers.ContainsKey(workerKey))
{
worker.DependencyWorker = _queueWorkers[workerKey];
}
}
}
}
catch (Exception ex)
{
Log.Exception(PROC, ex);
}
}
