public bool Produce(ProduceMessage message)
{
MessageReceived(message.Topic);
var ack = new ProduceAcknowledgement
{
OriginalBatch = new TestBatchByTopicByPartition(new[] { message }),
ProduceResponse =
new CommonResponse <ProducePartitionResponse>()
{
TopicsResponse =
new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = message.Topic,
PartitionsData =
new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = message.Partition
}
}
}
}
},
ReceiveDate = DateTime.UtcNow
};
ProduceAcknowledgement(this, ack);
return(true);
}
public void TestProduceReplicaNotAvailableIsNotAnError()
{
var acknowledgement = new ProduceAcknowledgement
{
ProduceResponse = new CommonResponse <ProducePartitionResponse>
{
TopicsResponse = new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test",
PartitionsData = new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.ReplicaNotAvailable,
Offset = 0,
Partition = 0
}
}
}
}
},
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test", 0, new Message(),
DateTime.UtcNow.AddDays(1))
})
};
_TestAcknowledgementNoError(acknowledgement, 1);
}
public async Task TestAcknowledgementResponseNoneProduceWasSentDiscard()
{
_finished = new AsyncCountdownEvent(1);
int discarded = 0;
_produceRouter.MessageDiscarded += (t, m) =>
{
discarded += 1;
_finished.Signal();
};
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch = new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test1p", 0, new Message(),
DateTime.UtcNow.AddDays(1))
}),
ReceiveDate = DateTime.UtcNow
};
_produceRouter.Acknowledge(acknowledgement);
await _finished.WaitAsync();
Assert.AreEqual(1, discarded);
}
public void _TestAcknowledgementNoError(ProduceAcknowledgement acknowledgement, int expected)
{
var ev = new ManualResetEvent(false);
int rec = 0;
int success = 0;
int discarded = 0;
_produceRouter.MessagesAcknowledged += (t, i) =>
{
Interlocked.Add(ref success, i);
if (Interlocked.Add(ref rec, i) == expected)
{
ev.Set();
}
};
_produceRouter.MessageDiscarded += (t, m) =>
{
Interlocked.Increment(ref discarded);
if (Interlocked.Increment(ref rec) == expected)
{
ev.Set();
}
};
_produceRouter.Acknowledge(acknowledgement);
ev.WaitOne();
Assert.AreEqual(expected, rec);
Assert.AreEqual(expected, success);
Assert.AreEqual(0, discarded);
}
public void TestProduceWithNoErrorsNoAck()
{
var node = new Node("Node", () => new EchoConnectionMock(), new ProduceSerialization(new CommonResponse <ProducePartitionResponse>()),
new Configuration {
ProduceBufferingTime = TimeSpan.FromMilliseconds(15), RequiredAcks = RequiredAcks.None
}, 1);
var count = new CountdownEvent(2);
bool batch = false;
ProduceAcknowledgement acknowledgement = new ProduceAcknowledgement();
node.ProduceBatchSent += (n, c, s) =>
{
count.Signal();
};
node.ProduceAcknowledgement += (n, ack) =>
{
acknowledgement = ack;
count.Signal();
};
node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
count.Wait();
Assert.AreEqual(ErrorCode.NoError, acknowledgement.ProduceResponse.TopicsResponse[0].PartitionsData.First().ErrorCode);
}
public void TestAcknowledgementMultipleNoError()
{
var acknowledgement = new ProduceAcknowledgement
{
ProduceResponse = new ProduceResponse {
ProducePartitionResponse = new CommonResponse <ProducePartitionResponse>()
{
TopicsResponse = new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test",
PartitionsData = new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 0
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 1
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 2
}
}
}
}
}
},
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test", 0, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 1, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 2, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test2", 0, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test2", 1, new Message(),
DateTime.UtcNow.AddDays(1))
})
};
_TestAcknowledgementNoError(acknowledgement, 5);
}
public async Task TestProduceAcknowledgement()
{
_cluster.Start();
var pa = new ProduceAcknowledgement();
_nodeMocks[0].Raise(n => n.ProduceAcknowledgement += null, _nodeMocks[0].Object, pa);
await _cluster.Stop();
_routerMock.Verify(r => r.Acknowledge(pa));
Assert.AreEqual(0, _internalErrors);
}
public async Task TestAcknowledgementResponseNoneProduceWasNotSent()
{
_finished = new AsyncCountdownEvent(3);
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{ ProduceMessage.New("test1p", 0, new Message(), DateTime.UtcNow.AddDays(1)) })
};
_produceRouter.Acknowledge(acknowledgement);
await _finished.WaitAsync();
CheckCounters(expectedMessagesReEnqueued: 1, expectedMessagesRouted: 1, expectedRoutingTableRequired: 1);
}
public void TestDisposableMessagesAreDisposed()
{
var key = new Mock <IDisposable>();
var value = new Mock <IDisposable>();
var acknowledgement = new ProduceAcknowledgement
{
ProduceResponse = new ProduceResponse {
ProducePartitionResponse = new CommonResponse <ProducePartitionResponse>()
{
TopicsResponse = new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test",
PartitionsData = new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 0
}
}
}
}
}
},
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test", 0, new Message {
Key = key.Object, Value = value.Object
},
DateTime.UtcNow.AddDays(1))
})
};
_produceRouter.Acknowledge(acknowledgement);
key.Verify(k => k.Dispose(), Times.Once());
value.Verify(v => v.Dispose(), Times.Once());
}
public async Task TestAcknowledgementResponseNoneProduceWasSentRetry()
{
InitRouter(new ProduceRouter(_cluster, new Configuration {
ErrorStrategy = ErrorStrategy.Retry
}, Pool));
_finished = new AsyncCountdownEvent(3);
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch = new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test1p", 0, new Message(),
DateTime.UtcNow.AddDays(1))
}),
ReceiveDate = DateTime.UtcNow
};
_produceRouter.Acknowledge(acknowledgement);
await _finished.WaitAsync();
CheckCounters(expectedMessagesReEnqueued: 1, expectedMessagesRouted: 1, expectedRoutingTableRequired: 1);
}
public void TestDisposableMessagesDiscardedAreNotDisposed()
{
var key = new Mock <IDisposable>();
var value = new Mock <IDisposable>();
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch = new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test1p", 0, new Message {
Key = key.Object, Value = value.Object
},
DateTime.UtcNow.AddDays(1))
}),
ReceiveDate = DateTime.UtcNow
};
_produceRouter.Acknowledge(acknowledgement);
key.Verify(k => k.Dispose(), Times.Never());
value.Verify(v => v.Dispose(), Times.Never());
}
public void Acknowledge(ProduceAcknowledgement acknowledgement)
{
// do nothing
}
public void TestProduceRecoverableErrorsAreRerouted()
{
SetUp(new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler(), RetryIfNotEnoughReplicasAfterAppend = true
});
var acknowledgement = new ProduceAcknowledgement
{
ProduceResponse = new CommonResponse <ProducePartitionResponse>()
{
TopicsResponse = new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test",
PartitionsData = new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 0
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NotLeaderForPartition,
Offset = 0,
Partition = 1
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.LeaderNotAvailable,
Offset = 0,
Partition = 2
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.RequestTimedOut,
Offset = 0,
Partition = 3
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.UnknownTopicOrPartition,
Offset = 0,
Partition = 4
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NotEnoughReplicasAfterAppend,
Offset = 0,
Partition = 5
},
}
}
}
},
OriginalBatch = new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test", 0, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 1, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 2, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 3, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 4, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 5, new Message(),
DateTime.UtcNow.AddDays(1))
})
};
const int exp = 6;
var ev = new ManualResetEvent(false);
int rec = 0;
int success = 0;
int discarded = 0;
int rerouted = 0;
_produceRouter.MessageReEnqueued += t =>
{
Interlocked.Increment(ref rerouted);
if (Interlocked.Increment(ref rec) == exp)
{
ev.Set();
}
};
_produceRouter.MessagesAcknowledged += (t, i) =>
{
Interlocked.Add(ref success, i);
if (Interlocked.Add(ref rec, i) == exp)
{
ev.Set();
}
};
_produceRouter.MessageDiscarded += (t, m) =>
{
Interlocked.Increment(ref discarded);
if (Interlocked.Increment(ref rec) == exp)
{
ev.Set();
}
};
_produceRouter.Acknowledge(acknowledgement);
ev.WaitOne();
// Check counts
Assert.AreEqual(6, rec);
Assert.AreEqual(1, success);
Assert.AreEqual(0, discarded);
Assert.AreEqual(5, rerouted);
// Check logs
var warnings = _cluster.Logger as TestLogger;
var recov = warnings.WarningLog.Where(s => s.Contains("Recoverable")).ToArray();
var ignore = warnings.WarningLog.Where(s => s.Contains("Will ignore")).ToArray();
Assert.AreEqual(5, recov.Length);
Assert.AreEqual(5, ignore.Length);
for (int i = 1; i <= 5; ++i)
{
Assert.IsTrue(recov[i - 1].Contains("partition: " + i));
Assert.IsTrue(ignore[i - 1].Contains("partition: " + i));
}
}
public void TestNonRecoverableErrorsAreDiscarded()
{
var acknowledgement = new ProduceAcknowledgement
{
ProduceResponse = new CommonResponse <ProducePartitionResponse>()
{
TopicsResponse = new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test",
PartitionsData = new[]
{
new ProducePartitionResponse
{
ErrorCode = ErrorCode.NoError,
Offset = 0,
Partition = 0
},
new ProducePartitionResponse
{
ErrorCode = ErrorCode.MessageSizeTooLarge,
Offset = 0,
Partition = 1
}
}
}
}
},
OriginalBatch = new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test", 0, new Message(),
DateTime.UtcNow.AddDays(1)),
ProduceMessage.New("test", 1, new Message(),
DateTime.UtcNow.AddDays(1))
})
};
var ev = new ManualResetEvent(false);
int rec = 0;
int success = 0;
int discarded = 0;
_produceRouter.MessagesAcknowledged += (t, i) =>
{
Interlocked.Add(ref success, i);
if (Interlocked.Add(ref rec, i) == 2)
{
ev.Set();
}
};
_produceRouter.MessageDiscarded += (t, m) =>
{
Interlocked.Increment(ref discarded);
if (Interlocked.Increment(ref rec) == 2)
{
ev.Set();
}
};
_produceRouter.Acknowledge(acknowledgement);
ev.WaitOne();
Assert.AreEqual(2, rec);
Assert.AreEqual(1, success);
Assert.AreEqual(1, discarded);
}
public void TestThrottled()
{
var node = new Mock <INode>();
node.Setup(n => n.Produce(It.IsAny <ProduceMessage>())).Returns(true);
var cluster = new Mock <ICluster>();
cluster.Setup(c => c.RequireNewRoutingTable())
.Returns(() =>
Task.FromResult(new RoutingTable(new Dictionary <string, Partition[]>
{
{ "toto", new[] { new Partition {
Id = 0, Leader = node.Object, NbIsr = 1
} } }
})));
var producer = new ProduceRouter(cluster.Object,
new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler()
}, Pool);
int throttled = -1;
producer.Throttled += t => throttled = t;
var key = new Mock <IDisposable>();
var value = new Mock <IDisposable>();
var acknowledgement = new ProduceAcknowledgement
{
OriginalBatch =
new TestBatchByTopicByPartition(new[]
{
ProduceMessage.New("test1p", 0, new Message {
Key = key.Object, Value = value.Object
},
DateTime.UtcNow.AddDays(1))
}),
ReceiveDate = DateTime.UtcNow,
ProduceResponse =
new ProduceResponse
{
ThrottleTime = 42,
ProducePartitionResponse =
new CommonResponse <ProducePartitionResponse>
{
TopicsResponse =
new[]
{
new TopicData <ProducePartitionResponse>
{
TopicName = "test1p",
PartitionsData = new[] { new ProducePartitionResponse() }
},
}
}
}
};
producer.Acknowledge(acknowledgement);
Assert.AreEqual(42, throttled);
}
