public bool Produce(ProduceMessage message)
{
MessageReceived(message.Topic);
var ack = new ProduceAcknowledgement
{
OriginalBatch = new TestBatchByTopicByPartition(new[] { message }),
ProduceResponse =
new CommonResponse <ProducePartitionResponse>()
{
TopicsResponse =
new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = message.Topic,
PartitionsData =
new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = message.Partition
}
}
}
}
},
ReceiveDate = DateTime.UtcNow
};
ProduceAcknowledgement(this, ack);
return(true);
}
public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitionStrategy = new RoundRobinPartitionSelection(delay: delay, startSeed: startSeed);
var partitioner = new PartitionSelector(partitionStrategy);
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(
ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions).Id);
}
}
}
public void TestRoundRobinPartitionAssign(int delay)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
new Partition {
Id = 4, Leader = nodeMock
},
};
var partitionStrategy = new RoundRobinPartitionSelection(delay);
var partitioner = new PartitionSelector(partitionStrategy);
delay = delay <= 0 ? 1 : delay;
foreach (var partition in partitions)
{
for (var j = 0; j < delay; ++j)
{
Assert.AreEqual(partition.Id, partitioner
.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions)
.Id);
}
}
}
public void TestNodeFailToConnectReRouteMessages()
{
int rec = 0;
var ev = new ManualResetEvent(false);
var config = new Configuration {
ProduceBatchSize = 1, ProduceBufferingTime = TimeSpan.FromMilliseconds(15)
};
var node =
new Node("[Failing node]", () => new ConnectFailingConnectionMock(), new DummySerialization(), config, 1);
bool dead = false;
node.Dead += _ =>
{
Assert.AreEqual(node, _);
dead = true;
};
node.ProduceAcknowledgement += (n, ack) =>
{
Assert.AreSame(node, n);
if (Interlocked.Add(ref rec, ack.OriginalBatch.SelectMany(g => g).Count()) == 2)
{
ev.Set();
}
};
node.Produce(ProduceMessage.New("poulpe", 1, new Message(), DateTime.UtcNow.AddMinutes(5)));
node.Produce(ProduceMessage.New("poulpe", 2, new Message(), DateTime.UtcNow.AddMinutes(5)));
ev.WaitOne();
Assert.IsTrue(dead);
}
public async Task TestAcknowledgementResponseNoneProduceWasSentDiscard()
{
_finished = new AsyncCountdownEvent(1);
int discarded = 0;
_produceRouter.MessageDiscarded += (t, m) =>
{
discarded += 1;
_finished.Signal();
};
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch = new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test1p", 0, new Message(),
DateTime.UtcNow.AddDays(1))
}),
ReceiveDate = DateTime.UtcNow
};
_produceRouter.Acknowledge(acknowledgement);
await _finished.WaitAsync();
Assert.AreEqual(1, discarded);
}
public void TestProduceWithNoErrorsNoAck()
{
var node = new Node("Node", () => new EchoConnectionMock(), new ProduceSerialization(new CommonResponse <ProducePartitionResponse>()),
new Configuration {
ProduceBufferingTime = TimeSpan.FromMilliseconds(15), RequiredAcks = RequiredAcks.None
}, 1);
var count = new CountdownEvent(2);
bool batch = false;
ProduceAcknowledgement acknowledgement = new ProduceAcknowledgement();
node.ProduceBatchSent += (n, c, s) =>
{
count.Signal();
};
node.ProduceAcknowledgement += (n, ack) =>
{
acknowledgement = ack;
count.Signal();
};
node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
count.Wait();
Assert.AreEqual(ErrorCode.NoError, acknowledgement.ProduceResponse.TopicsResponse[0].PartitionsData.First().ErrorCode);
}
public void TestProduceReplicaNotAvailableIsNotAnError()
{
var acknowledgement = new ProduceAcknowledgement
{
ProduceResponse = new CommonResponse <ProducePartitionResponse>
{
TopicsResponse = new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test",
PartitionsData = new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.ReplicaNotAvailable,
Offset = 0,
Partition = 0
}
}
}
}
},
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test", 0, new Message(),
DateTime.UtcNow.AddDays(1))
})
};
_TestAcknowledgementNoError(acknowledgement, 1);
}
public void TestProduceWithNoErrors()
{
double requestLatency = 0;
var expectedLatency = 3;
var node = new Node("Node", () => new EchoConnectionMock(false, expectedLatency), new ProduceSerialization(new CommonResponse <ProducePartitionResponse>()),
new Configuration {
ProduceBufferingTime = TimeSpan.FromMilliseconds(15)
}, 1);
var count = new CountdownEvent(2);
node.ResponseReceived += (n, l) =>
{
Assert.AreSame(node, n);
requestLatency = l;
count.Signal();
};
bool batch = false;
node.ProduceBatchSent += (n, c, s) =>
{
Assert.That(n, Is.EqualTo(node));
Assert.That(c, Is.EqualTo(1));
Assert.That(s, Is.EqualTo(0));
batch = true;
count.Signal();
};
node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
count.Wait();
Assert.IsTrue(batch);
Assert.GreaterOrEqual(requestLatency, expectedLatency);
}
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_Partition_Blacklisted(int partitionIdBlacklisted)
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var blacklistedPartitions = new Dictionary <int, DateTime> {
{ partitionIdBlacklisted, DateTime.MaxValue }
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyKey"
}, new DateTime());
for (var i = 0; i < 300; i++)
{
var partition = partitioner.GetPartition(message, partitions, blacklistedPartitions);
Assert.IsTrue(partition.Id != Partition.None.Id);
Assert.IsTrue(partition.Id != partitionIdBlacklisted);
}
}
public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_MessageKey_Null()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = null
}, new DateTime());
var partition = partitioner.GetPartition(message, partitions);
Assert.IsTrue(partition.Id != Partition.None.Id);
}
public void Test_MessageKeyPartitionSelection_Is_Consistent()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 0, Leader = nodeMock
},
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
};
var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
var partitioner = new PartitionSelector(partitionStrategy);
var message1 = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyKey"
}, new DateTime());
var message2 = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
Key = "ThisIsMyOtherKey"
}, new DateTime());
var expectedPartition1 = partitioner.GetPartition(message1, partitions);
var expectedPartition2 = partitioner.GetPartition(message2, partitions);
for (var i = 0; i < 300; i++)
{
var currentPartition1 = partitioner.GetPartition(message1, partitions);
var currentPartition2 = partitioner.GetPartition(message2, partitions);
Assert.AreEqual(expectedPartition1.Id, currentPartition1.Id);
Assert.AreEqual(expectedPartition2.Id, currentPartition2.Id);
}
}
public void TestRobinPartitionAssignWhenFiltered()
{
var nodeMock = new NodeMock();
var partitions = new[]
{
new Partition {
Id = 1, Leader = nodeMock
},
new Partition {
Id = 2, Leader = nodeMock
},
new Partition {
Id = 3, Leader = nodeMock
},
};
var filter = new Dictionary <int, DateTime>();
int delay = partitions.Length + 2;
var partitionStrategy = new RoundRobinPartitionSelection(delay);
var partitioner = new PartitionSelector(partitionStrategy);
var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);
Assert.AreEqual(1, partition.Id);
filter.Add(1, DateTime.UtcNow);
var batch = GetPartitions(delay, partitioner, partitions, filter);
Assert.AreEqual(delay, batch.Count);
Assert.IsTrue(batch.All(p => p.Id == 2), "The round-robin threshold wasn't properly reset after previous partition was blacklisted");
}
private RabbitMQRequest(long chatId, string message)
{
IProduceMessage produceMessage = new ProduceMessage();
produceMessage.ProduceMessageString(Environment.GetEnvironmentVariable("queueName"), JsonConvert.SerializeObject(new Payload()
{
ChatId = chatId, Message = message
}));
}
// Raise the MessageExpired event and release a message.
private void OnMessageExpired(ProduceMessage message)
{
_cluster.Logger.LogError(string.Format(
"[Producer] Not able to send message before reaching TTL for [topic: {0} / partition: {1}], message expired.",
message.Topic, message.RequiredPartition));
message.Message = CheckReleaseMessage(message.Message);
MessageExpired(message.Topic, message.Message);
}
// Raise the MessageExpired event and release a message.
private void OnMessageExpired(ProduceMessage message)
{
_cluster.Logger.LogError(string.Format(
"[Producer] Not able to send message before reaching TTL for [topic: {0} / partition: {1}], message expired.",
message.Topic, message.RequiredPartition));
ClearMessage(message.Message, shouldClearKeyValue: false);
MessageExpired(message.Topic, message.Message);
}
public void TestRoundRobinPartitionAssignNoPartitionReturnsNone()
{
var partitions = new Partition[0];
var partitionStrategy = new RoundRobinPartitionSelection();
var partitioner = new PartitionSelector(partitionStrategy);
Assert.AreEqual(0, Partition.None.CompareTo(partitioner.GetPartition(
ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions)));
}
/// <summary>
/// Prepare a new message for routing.
/// </summary>
/// <param name="topic"></param>
/// <param name="message"></param>
/// <param name="partition"></param>
/// <param name="expirationDate"></param>
public void Route(string topic, Message message, int partition, DateTime expirationDate)
{
if (_configuration.SerializationConfig.SerializeOnProduce)
{
var serializers = _configuration.SerializationConfig.GetSerializersForTopic(topic);
message.SerializeKeyValue(_pool.Reserve(), serializers);
}
Route(ProduceMessage.New(topic, partition, message, expirationDate));
}
public void TestAcknowledgementMultipleNoError()
{
var acknowledgement = new ProduceAcknowledgement
{
ProduceResponse = new ProduceResponse {
ProducePartitionResponse = new CommonResponse <ProducePartitionResponse>()
{
TopicsResponse = new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test",
PartitionsData = new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 0
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 1
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 2
}
}
}
}
}
},
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test", 0, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 1, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 2, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test2", 0, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test2", 1, new Message(),
DateTime.UtcNow.AddDays(1))
})
};
_TestAcknowledgementNoError(acknowledgement, 5);
}
private static List <Partition> GetPartitions(int count, PartitionSelector partitioner, Partition[] partitions, Dictionary <int, DateTime> filter)
{
var result = new List <Partition>(count);
for (int i = 0; i < count; i++)
{
result.Add(partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter));
}
return(result);
}
public void TestMaxRetry()
{
int discarded = 0;
_produceRouter.MessageDiscarded += (t, m) => discarded += 1;
var pm = ProduceMessage.New("test1p", Partitions.Any, new Message(), DateTime.UtcNow.AddMinutes(5));
pm.Retried = 42;
_produceRouter.ReEnqueue(pm);
Assert.AreEqual(1, discarded);
}
public Partition GetPartition(ProduceMessage produceMessage, Partition[] partitions, IReadOnlyDictionary <int, DateTime> blacklist)
{
var partitionId = GetPartitionIdFromKey(produceMessage.Message.Key, partitions.Length);
if (partitionId != Partition.None.Id && !blacklist.ContainsKey(partitionId))
{
return(partitions[partitionId]);
}
return(_roundRobinSelection.GetPartition(produceMessage, partitions, blacklist));
}
/// <summary>
/// Inner Post produce implementation. Post a ProduceEvent message
/// to the inner actor.
/// </summary>
/// <param name="produceMessage"></param>
/// <returns>True if effectiveley posted, false otherwise</returns>
private bool Post(ProduceMessage produceMessage)
{
if (_messages.Completion.IsCompleted)
{
return(false);
}
_produceMessages.Enqueue(produceMessage);
// There is a race here, since Stop can happen in between the
// previous and next lines. However this only happen when Stop
// is required which means we can safely let a few messages in
// the queue, they will be signaled as discarded to the outer
// world (Stop is permanent).
return(_messages.Post(RouterMessageType.ProduceEvent));
}
/// <summary>
/// Try to send again a message following an error.
/// Check for retry if needed.
/// </summary>
internal void ReEnqueueAfterError(ProduceMessage message)
{
if (_configuration.MaxRetry < 0 || ++message.Retried <= _configuration.MaxRetry)
{
ReEnqueueMessage(message);
}
else
{
_cluster.Logger.LogError(string.Format(
"[Producer] Not able to reenqueue: too many retry ({1}). Discarding message for topic '{0}'",
message.Topic, message.Retried));
OnMessageDiscarded(message);
}
}
public async Task TestAcknowledgementResponseNoneProduceWasNotSent()
{
_finished = new AsyncCountdownEvent(3);
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{ ProduceMessage.New("test1p", 0, new Message(), DateTime.UtcNow.AddDays(1)) })
};
_produceRouter.Acknowledge(acknowledgement);
await _finished.WaitAsync();
CheckCounters(expectedMessagesReEnqueued: 1, expectedMessagesRouted: 1, expectedRoutingTableRequired: 1);
}
public async Task <object> ProduceMessage([FromBody] ProduceMessage produceMessage)
{
var producer = GetProducer(produceMessage.Broker);
var result = await producer.ProduceAsync(produceMessage.Topic, new Message <string, string>
{
Key = produceMessage.Key,
Value = new
{
Headers = produceMessage.MessageHeaders,
Payload = produceMessage.MessagePayload.ToJson()
}.ToJson()
});
return(result);
}
/// <summary>
/// Prepare a message for routing. If we're not in Stop state
/// the MessageEnqueued event will be raised.
/// </summary>
/// <param name="message"></param>
private void Route(ProduceMessage message)
{
if (Post(message))
{
MessageEnqueued(message.Topic);
}
else
{
_cluster.Logger.LogError(
string.Format(
"[Producer] Failed to route message, discarding message for [topic: {0} / partition: {1}]",
message.Topic, message.Partition));
OnMessageDiscarded(message);
}
}
public void TestProduceTimeout()
{
// Prepare
var connection = new Mock <IConnection>();
var node = new Node("Node", () => connection.Object,
new Node.Serialization(null, Compatibility.V0_8_2, Pool, new byte[0], RequiredAcks.Leader, 1, CompressionCodec.None, 0, 100),
new Configuration
{
TaskScheduler = new CurrentThreadTaskScheduler(),
ErrorStrategy = ErrorStrategy.Discard,
ProduceBufferingTime = TimeSpan.FromMilliseconds(15),
ProduceBatchSize = 1,
ClientRequestTimeoutMs = 2,
MaxInFlightRequests = -1
},
new TimeoutScheduler(3), 1);
var ackEvent = new ManualResetEvent(false);
var timeoutEvent = new ManualResetEvent(false);
connection.Setup(c => c.SendAsync(It.IsAny <int>(), It.IsAny <ReusableMemoryStream>(), It.IsAny <bool>()))
.Returns(Success);
connection.Setup(c => c.ConnectAsync()).Returns(Success);
node.ProduceAcknowledgement += (n, ack) =>
{
Assert.AreSame(node, n);
Assert.AreEqual(default(ProduceResponse), ack.ProduceResponse);
Assert.AreNotEqual(default(DateTime), ack.ReceiveDate);
ackEvent.Set();
};
node.RequestTimeout += n =>
{
Assert.AreSame(node, n);
timeoutEvent.Set();
};
var exception = new Exception();
node.ConnectionError += (n, e) =>
{
exception = e;
};
node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
WaitHandle.WaitAll(new [] { ackEvent, timeoutEvent });
Assert.IsInstanceOf <TimeoutException>(exception);
}
public void TestMaxInFlightRequests()
{
// Prepare
var connection = new Mock <IConnection>();
var node = new Node("Node", () => connection.Object,
new Node.Serialization(null, Compatibility.V0_8_2, Pool, new byte[0], RequiredAcks.Leader, 1, CompressionCodec.None, 0, 100),
new Configuration
{
TaskScheduler = new CurrentThreadTaskScheduler(),
ErrorStrategy = ErrorStrategy.Discard,
ProduceBufferingTime = TimeSpan.FromMilliseconds(15),
ProduceBatchSize = 1,
ClientRequestTimeoutMs = 60000,
MaxInFlightRequests = 1
},
new TimeoutScheduler(3), 1);
var ev = new ManualResetEvent(false);
var corrs = new Queue <int>();
connection.Setup(c => c.SendAsync(It.IsAny <int>(), It.IsAny <ReusableMemoryStream>(), It.IsAny <bool>()))
.Returns((int c, ReusableMemoryStream d, bool a) =>
{
corrs.Enqueue(c);
return(Success);
});
connection.Setup(c => c.ConnectAsync()).Returns(Success);
node.RequestSent += n =>
{
ev.Set();
};
node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
ev.WaitOne();
ev.Reset();
node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
Assert.IsFalse(ev.WaitOne(50));
var corr = corrs.Dequeue();
connection.Raise(c => c.Response += null, connection.Object, corr, new ReusableMemoryStream(null));
node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
ev.WaitOne();
}
public void TestUnlimitedRetry()
{
SetUp(new Configuration
{
MaxRetry = -1
});
int discarded = 0;
_produceRouter.MessageDiscarded += (t, m) => discarded += 1;
var pm = ProduceMessage.New("test1p", Partitions.Any, new Message(), DateTime.UtcNow.AddMinutes(5));
_produceRouter.ReEnqueue(pm);
_produceRouter.ReEnqueue(pm);
pm.Retried = int.MaxValue;
_produceRouter.ReEnqueue(pm);
Assert.AreEqual(0, discarded);
}
/// <summary>
/// Postpone a message for later check and start the check timer if needed.
/// The MessagePostponed event is raised.
/// </summary>
/// <param name="produceMessage"></param>
private void PostponeMessage(ProduceMessage produceMessage)
{
if (_numberOfPostponedMessages >= _configuration.MaxPostponedMessages)
{
_cluster.Logger.LogError(
string.Format(
"[Producer] Too many postponed messages, discarding message for [topic: {0} / partition: {1}]",
produceMessage.Topic, produceMessage.RequiredPartition));
OnMessageDiscarded(produceMessage);
return;
}
Queue <ProduceMessage> postponedQueue;
if (!_postponedMessages.TryGetValue(produceMessage.Topic, out postponedQueue))
{
postponedQueue = new Queue <ProduceMessage>();
_postponedMessages.Add(produceMessage.Topic, postponedQueue);
}
postponedQueue.Enqueue(produceMessage);
if (postponedQueue.Count == 1)
{
if (produceMessage.RequiredPartition >= 0)
{
_cluster.Logger.LogError(
string.Format("[Producer] No node available for [topic: {0} / partition: {1}], postponing messages.",
produceMessage.Topic, produceMessage.RequiredPartition));
}
else
{
_cluster.Logger.LogError(string.Format(
"[Producer] No partition available for topic {0}, postponing messages.", produceMessage.Topic));
}
}
++_numberOfPostponedMessages;
MessagePostponed(produceMessage.Topic);
if (_checkPostponedMessages == null)
{
_checkPostponedMessages =
new Timer(_ => _messages.Post(RouterMessageType.CheckPostponed),
null, _configuration.MessageTtl, _configuration.MessageTtl);
}
}