public async Task ShouldUpdateMetadataOnes()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
await Task.Delay(routerProxy._cacheExpiration);
await Task.Delay(1);
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
}
public async Task FetchMessagesCacheContainsNoneOfRequestTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List <Message> messages = CreateTestMessages(10, 4096);
await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(7, offset)).ToList();
CheckMessages(messages.Take(7).ToList(), result);
// Now let's consume again
result = (await consumer.FetchMessages(2, offset + 8)).ToList();
CheckMessages(messages.Skip(8).ToList(), result);
}
public async Task FetchMessagesCacheContainsAllRequestTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List<Message> messages = CreateTestMessages(10, 1);
await producer.SendMessageAsync(_topic, messages, partition: _partitionId, timeout: TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
// Now let's consume again
result = (await consumer.FetchMessages(5, offset + 5)).ToList();
CheckMessages(messages.Skip(5).ToList(), result);
}
public async Task ProtocolGateway()
{
int partitionId = 0;
var router = new BrokerRouter(Options);
var producer = new Producer(router);
string messge1 = Guid.NewGuid().ToString();
var respose = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(messge1) }, 1, null, MessageCodec.CodecNone, partitionId);
var offset = respose.FirstOrDefault().Offset;
ProtocolGateway protocolGateway = new ProtocolGateway(IntegrationConfig.IntegrationUri);
var fetch = new Fetch
{
Topic = IntegrationConfig.IntegrationTopic,
PartitionId = partitionId,
Offset = offset,
MaxBytes = 32000,
};
var fetches = new List<Fetch> { fetch };
var fetchRequest = new FetchRequest
{
MaxWaitTime = 1000,
MinBytes = 10,
Fetches = fetches
};
var r = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
// var r1 = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
Assert.IsTrue(r.Messages.FirstOrDefault().Value.ToUtf8String() == messge1);
}
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = new TimeSpan(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool sendExOnFirstTime = true;
Func<Task<FetchResponse>> ShouldReturnErrorAndThenNoError = async () =>
{
Task.Delay(routerProxy._cacheExpiration).Wait();
Task.Delay(1).Wait();
if (sendExOnFirstTime)
{
sendExOnFirstTime = false;
return new FetchResponse() { Error = (short)code };
}
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnErrorAndThenNoError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldRecoverFromFailerByUpdateMetadataOnce() //Do not debug this test !!
{
var log = new DefaultTraceLog();
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(1000);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
int numberOfCall = 100;
long numberOfErrorSend = 0;
TaskCompletionSource <int> x = new TaskCompletionSource <int>();
Func <Task <FetchResponse> > ShouldReturnNotLeaderForPartitionAndThenNoError = async() =>
{
log.DebugFormat("FetchResponse Start ");
if (!x.Task.IsCompleted)
{
if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall)
{
await Task.Delay(routerProxy._cacheExpiration);
await Task.Delay(1);
x.TrySetResult(1);
log.DebugFormat("all is complete ");
}
await x.Task;
log.DebugFormat("SocketException ");
throw new BrokerConnectionException("", new KafkaEndpoint());
}
log.DebugFormat("Completed ");
return(new FetchResponse()
{
Error = (short)ErrorResponseCode.NoError
});
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnNotLeaderForPartitionAndThenNoError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
}
await Task.WhenAll(tasks);
Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend");
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend),
"FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount");
}
public async Task ShouldThrowFormatExceptionWhenTopicIsInvalid()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
string invalidTopic = " ";
var fetchRequest = new FetchRequest();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
await protocolGateway.SendProtocolRequest(fetchRequest, invalidTopic, 0);
}
public async Task ShouldThrowFormatExceptionWhenTopicIsInvalid()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
string invalidTopic = " ";
var fetchRequest = new FetchRequest();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
await protocolGateway.SendProtocolRequest(fetchRequest, invalidTopic, 0);
}
public async Task FetchOffsetConsumerGroupDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = Guid.NewGuid().ToString();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
await consumer.FetchOffset(consumerGroup);
}
public async Task FetchLastOffsetPartitionDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 100;
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
await consumer.FetchLastOffset();
}
public async Task FetchMessagesInvalidOffsetTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = -1;
// Now let's consume
await consumer.FetchMessages(5, offset);
}
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Now let's consume throw KafkaApplicationException
await consumer.FetchMessages(5, offset + 1);
}
public async Task UpdateOrCreateOffsetConsumerGroupEmptyTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(string.Empty, offest);
}
public async Task FetchLastOffsetTopicDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var res = await consumer.FetchLastOffset();
Assert.AreEqual(0, res);
}
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(
ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
public ManualConsumer(int partitionId, string topic, ProtocolGateway gateway, string clientId, int maxSizeOfMessageSet)
{
if (string.IsNullOrEmpty(topic)) throw new ArgumentNullException("topic");
if (gateway == null) throw new ArgumentNullException("gateway");
if (maxSizeOfMessageSet <= 0) throw new ArgumentOutOfRangeException("maxSizeOfMessageSet", "argument must be larger than zero");
_gateway = gateway;
_partitionId = partitionId;
_topic = topic;
_clientId = clientId;
_maxSizeOfMessageSet = maxSizeOfMessageSet;
}
public async Task FetchLastOffsetSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var offset = await consumer.FetchLastOffset();
Assert.AreNotEqual(-1, offset);
}
public async Task FetchMessagesNoNewMessagesInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
Assert.AreEqual(0, result.Count, "Should not get any messages");
}
public async Task FetchOffsetConsumerGroupIsNullTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
await consumer.FetchOffset(null);
}
public async Task ShouldTryToRefreshMataDataIfOnExceptions(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = new TimeSpan(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = new TimeSpan(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldTryToRefreshMataDataIfSocketException()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(typeof(BrokerConnectionException), routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public KafkaQueueAdapter(HashRingBasedStreamQueueMapper queueMapper, KafkaStreamProviderOptions options,
string providerName, IKafkaBatchFactory batchFactory, ILogger logger)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (batchFactory == null)
{
throw new ArgumentNullException(nameof(batchFactory));
}
if (queueMapper == null)
{
throw new ArgumentNullException(nameof(queueMapper));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_options = options;
_streamQueueMapper = queueMapper;
Name = providerName;
_batchFactory = batchFactory;
_logger = logger;
// Creating a producer
KafkaOptions kafkaOptions = new KafkaOptions(_options.ConnectionStrings.ToArray())
{
Log = new KafkaLogBridge(logger)
};
var broker = new BrokerRouter(kafkaOptions);
_producer = new Producer(broker)
{
BatchDelayTime = TimeSpan.FromMilliseconds(_options.TimeToWaitForBatchInMs),
BatchSize = _options.ProduceBatchSize
};
_gateway = new ProtocolGateway(kafkaOptions);
_logger.Info("KafkaQueueAdapter - Created a KafkaQueueAdapter");
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_protocolGateway = new ProtocolGateway(BrokerRouter);
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_asyncCollection = new AsyncCollection<TopicMessage>();
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(() =>
{
BatchSendAsync();
BrokerRouter.Log.InfoFormat("ending the sending thread");
});
}
public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
try
{
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.IsTrue(false, "Should throw exception");
}
catch (Exception ex)
{
Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
}
}
public async Task ShouldRecoverFromFailerByUpdateMetadataOnceFullScenario1()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(0);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
CreateSuccessfulSendMock(routerProxy);
//Send Successful Message
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(ErrorResponseCode.LeaderNotAvailable, TimeSpan.Zero);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
routerProxy.BrokerConn1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
//Reset variables
routerProxy.BrokerConn0.FetchRequestCallCount = 0;
routerProxy.BrokerConn1.FetchRequestCallCount = 0;
routerProxy.BrokerConn0.MetadataRequestCallCount = 0;
routerProxy.BrokerConn1.MetadataRequestCallCount = 0;
//Send Successful Message that was recover from exception
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
}
public async Task ShouldRecoverUpdateMetadataForNewTopic()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, _partitionId);
}
routerProxy.BrokerConn0.MetadataResponseFunction = async() =>
{
var response = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
response.Topics[0].Name = "test2";
return(response);
};
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", _partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}
public async Task FetchMessagesSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 1;
var topic = "ManualConsumerTestTopic";
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List<Message> messages = CreateTestMessages(5, 1);
await producer.SendMessageAsync(topic, messages, partition: partitionId, timeout: TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages, result);
}
public async Task FetchMessagesBufferUnderRunTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var smallMessageSet = 4096 / 2;
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List <Message> messages = CreateTestMessages(10, 4096);
await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
}
public async Task FetchMessagesSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = "ManualConsumerTestTopic";
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List <Message> messages = CreateTestMessages(5, 1);
await producer.SendMessageAsync(topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages, result);
}
public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
var newOffset = 10;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
var res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(offest, res);
await consumer.UpdateOrCreateOffset(consumerGroup, newOffset);
res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(newOffset, res);
}
public async Task ProtocolGateway()
{
int partitionId = 0;
var router = new BrokerRouter(Options);
var producer = new Producer(router);
string messge1 = Guid.NewGuid().ToString();
var respose = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(messge1) }, 1, null, MessageCodec.CodecNone, partitionId);
var offset = respose.FirstOrDefault().Offset;
ProtocolGateway protocolGateway = new ProtocolGateway(IntegrationConfig.IntegrationUri);
var fetch = new Fetch
{
Topic = IntegrationConfig.IntegrationTopic,
PartitionId = partitionId,
Offset = offset,
MaxBytes = 32000,
};
var fetches = new List <Fetch> {
fetch
};
var fetchRequest = new FetchRequest
{
MaxWaitTime = 1000,
MinBytes = 10,
Fetches = fetches
};
var r = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
// var r1 = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
Assert.IsTrue(r.Messages.FirstOrDefault().Value.ToUtf8String() == messge1);
}
public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
var newOffset = 10;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
var res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(offest, res);
await consumer.UpdateOrCreateOffset(consumerGroup, newOffset);
res = await consumer.FetchOffset(consumerGroup);
Assert.AreEqual(newOffset, res);
}
public async Task UpdateOrCreateOffsetPartitionDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 100;
var consumerGroup = Guid.NewGuid().ToString();
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
}
public async Task FetchOffsetTopicDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic + Guid.NewGuid();
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
await consumer.FetchOffset(consumerGroup);
}
public async Task FetchOffsetConsumerGroupIsEmptyTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = 5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
await consumer.FetchOffset(string.Empty);
}
public async Task FetchMessagesTopicDoesntExist()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 1;
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var offset = await consumer.FetchLastOffset();
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
Assert.AreEqual(0, result.Count);
}
public async Task ShouldRecoverFromFailerByUpdateMetadataOnce() //Do not debug this test !!
{
var log = new DefaultTraceLog();
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(1000);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
int numberOfCall = 100;
long numberOfErrorSend = 0;
TaskCompletionSource<int> x = new TaskCompletionSource<int>();
Func<Task<FetchResponse>> ShouldReturnNotLeaderForPartitionAndThenNoError = async () =>
{
log.DebugFormat("FetchResponse Start ");
if (!x.Task.IsCompleted)
{
if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall)
{
await Task.Delay(routerProxy._cacheExpiration);
await Task.Delay(1);
x.TrySetResult(1);
log.DebugFormat("all is complete ");
}
await x.Task;
log.DebugFormat("SocketException ");
throw new BrokerConnectionException("",new KafkaEndpoint());
}
log.DebugFormat("Completed ");
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnNotLeaderForPartitionAndThenNoError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
}
await Task.WhenAll(tasks);
Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend");
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend),
"FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount");
}
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool firstTime = true;
Func<Task<FetchResponse>> ShouldReturnError = async () =>
{
if (firstTime)
{
firstTime = !firstTime;
return new FetchResponse() { Error = (short)code };
}
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
}
public async Task FetchLastOffsetPartitionDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 100;
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
await consumer.FetchLastOffset();
}
public async Task ShouldRecoverUpdateMetadataForNewTopic()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool firstTime = true;
Func<Task<FetchResponse>> ShouldReturnError = async () =>
{
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
}
//change MetadataResponseFunction
routerProxy.BrokerConn0.MetadataResponseFunction = async () =>
{
var response = await BrokerRouterProxy.DefaultMetadataResponse();
response.Topics[0].Name = "test2";
return response;
};
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}
public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
try
{
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.IsTrue(false, "Should throw exception");
}
catch (Exception ex)
{
Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
}
}
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(
ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
public async Task ShouldUpdateMetadataOnes()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
await Task.Delay(routerProxy._cacheExpiration);
await Task.Delay(1);
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
}
public async Task ShouldRecoverFromFailerByUpdateMetadataOnceFullScenario1()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(0);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
CreateSuccessfulSendMock(routerProxy);
//Send Successful Message
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(ErrorResponseCode.LeaderNotAvailable, TimeSpan.Zero);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
routerProxy.BrokerConn1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
//Reset variables
routerProxy.BrokerConn0.FetchRequestCallCount = 0;
routerProxy.BrokerConn1.FetchRequestCallCount = 0;
routerProxy.BrokerConn0.MetadataRequestCallCount = 0;
routerProxy.BrokerConn1.MetadataRequestCallCount = 0;
//Send Successful Message that was recover from exception
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
}
public async Task UpdateOrCreateOffsetNegativeOffsetTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var partitionId = 0;
var topic = _topic + Guid.NewGuid();
var consumerGroup = _defaultConsumerGroup;
ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offest = -5;
await consumer.UpdateOrCreateOffset(consumerGroup, offest);
}
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Now let's consume
var result = (await consumer.FetchMessages(5, offset + 1)).ToList();
Assert.AreEqual(0, result.Count, "Should not get any messages");
}
public async Task FetchLastOffsetSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic;
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var offset = await consumer.FetchLastOffset();
Assert.AreNotEqual(-1, offset);
}
public async Task FetchMessagesInvalidOffsetTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = -1;
// Now let's consume
await consumer.FetchMessages(5, offset);
}
public async Task FetchLastOffsetTopicDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var protocolGateway = new ProtocolGateway(_kafkaUri);
var topic = _topic + Guid.NewGuid();
ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);
var res = await consumer.FetchLastOffset();
Assert.AreEqual(0, res);
}
public async Task FetchMessagesBufferUnderRunTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
var smallMessageSet = 4096 / 2;
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List<Message> messages = CreateTestMessages(10, 4096);
await producer.SendMessageAsync(_topic, messages, partition: _partitionId, timeout: TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
}
public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool firstTime = true;
Func<Task<FetchResponse>> ShouldReturnError = async () =>
{
if (firstTime)
{
firstTime = !firstTime;
object[] args = new object[1];
args[0] = "error Test";
throw (Exception)Activator.CreateInstance(exceptionType, args);
}
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
try
{
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.IsTrue(false, "Should throw exception");
}
catch (Exception ex)
{
Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
}
}
public async Task ShouldRecoverUpdateMetadataForNewTopic()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, _partitionId);
}
routerProxy.BrokerConn0.MetadataResponseFunction = async () =>
{
var response = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
response.Topics[0].Name = "test2";
return response;
};
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", _partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}