public void CancellationShouldInterruptConsumption()
{
var routerProxy = new BrokerRouterProxy(new MoqMockingKernel());
routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); };
var router = routerProxy.Create();
var options = CreateOptions(router);
using (var consumer = new Consumer(options))
{
var tokenSrc = new CancellationTokenSource();
var consumeTask = Task.Run(() => consumer.Consume(tokenSrc.Token).FirstOrDefault());
//wait until the fake broker is running and requesting fetches
TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 10);
tokenSrc.Cancel();
Assert.That(
Assert.Throws<AggregateException>(consumeTask.Wait).InnerException,
Is.TypeOf<OperationCanceledException>());
}
}
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = new TimeSpan(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool sendExOnFirstTime = true;
Func<Task<FetchResponse>> ShouldReturnErrorAndThenNoError = async () =>
{
Task.Delay(routerProxy._cacheExpiration).Wait();
Task.Delay(1).Wait();
if (sendExOnFirstTime)
{
sendExOnFirstTime = false;
return new FetchResponse() { Error = (short)code };
}
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnErrorAndThenNoError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public void EmptyTopicMetadataShouldThrowException()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
var common = new CommonQueries(router);
common.GetTopic("MissingTopic");
}
public void GetTopicShouldReturnTopic()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
var common = new CommonQueries(router);
var result = common.GetTopic(BrokerRouterProxy.TestTopic);
Assert.That(result.Name, Is.EqualTo(BrokerRouterProxy.TestTopic));
}
public async Task ShouldThrowFormatExceptionWhenTopicIsInvalid()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
string invalidTopic = " ";
var fetchRequest = new FetchRequest();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
await protocolGateway.SendProtocolRequest(fetchRequest, invalidTopic, 0);
}
public void GetTopicOffsetShouldQueryEachBroker()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
var common = new CommonQueries(router);
var result = common.GetTopicOffsetAsync(BrokerRouterProxy.TestTopic).Result;
Assert.That(routerProxy.BrokerConn0.OffsetRequestCallCount, Is.EqualTo(1));
Assert.That(routerProxy.BrokerConn1.OffsetRequestCallCount, Is.EqualTo(1));
}
public void RefreshTopicMetadataShouldIgnoreCacheAndAlwayCauseMetadataRequest()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
router.RefreshTopicMetadata(TestTopic);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
router.RefreshTopicMetadata(TestTopic);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}
public void BrokerRouteShouldCycleThroughEachBrokerUntilOneIsFound()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.MetadataResponseFunction = () => { throw new Exception("some error"); };
var router = routerProxy.Create();
var result = router.GetTopicMetadata(TestTopic);
Assert.That(result, Is.Not.Null);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(1));
}
public async Task RefreshAllTopicMetadataShouldAlwaysDoRequest()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
await router.RefreshAllTopicMetadata();
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
await router.RefreshAllTopicMetadata();
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}
public async Task BrokerRouteShouldThrowNoLeaderElectedForPartition()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.MetadataResponse = BrokerRouterProxy.CreateMetadataResponseWithNotEndToElectLeader;
var router = routerProxy.Create();
Assert.Throws <NoLeaderElectedForPartition>(async() => await router.RefreshMissingTopicMetadata(TestTopic));
Assert.AreEqual(0, router.GetAllTopicMetadataFromLocalCache().Count);
}
public void GetTopicOffsetShouldQueryEachBroker()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
var common = new MetadataQueries(router);
var result = common.GetTopicOffsetAsync(BrokerRouterProxy.TestTopic).Result;
Assert.That(routerProxy.BrokerConn0.OffsetRequestCallCount, Is.EqualTo(1));
Assert.That(routerProxy.BrokerConn1.OffsetRequestCallCount, Is.EqualTo(1));
}
public async Task GetTopicShouldReturnTopic()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
await router.RefreshMissingTopicMetadata(BrokerRouterProxy.TestTopic);
var common = new MetadataQueries(router);
var result = common.GetTopicFromCache(BrokerRouterProxy.TestTopic);
Assert.That(result.Name, Is.EqualTo(BrokerRouterProxy.TestTopic));
}
public async Task SendProtocolRequestShouldNotTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(
ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy();
routerProxy.CacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
routerProxy.Connection1.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy.CacheExpiration);
routerProxy.Connection1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
Assert.ThrowsAsync <RequestException>(async() => await router.SendAsync(new FetchRequest(), BrokerRouterProxy.TestTopic, PartitionId, CancellationToken.None));
}
public async Task SelectExactPartitionShouldReturnRequestedPartition()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
await router.RefreshMissingTopicMetadata(TestTopic);
var p0 = router.SelectBrokerRouteFromLocalCache(TestTopic, 0);
var p1 = router.SelectBrokerRouteFromLocalCache(TestTopic, 1);
Assert.That(p0.PartitionId, Is.EqualTo(0));
Assert.That(p1.PartitionId, Is.EqualTo(1));
}
public async Task SelectPartitionShouldThrowWhenTopicsCollectionIsEmpty()
{
var metadataResponse = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
metadataResponse.Topics.Clear();
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.MetadataResponseFunction = async() => metadataResponse;
routerProxy.Create().SelectBrokerRouteFromLocalCache(TestTopic);
}
public async Task SelectExactPartitionShouldReturnRequestedPartition()
{
var routerProxy = new BrokerRouterProxy();
var router = routerProxy.Create();
await router.GetTopicMetadataAsync(TestTopic, CancellationToken.None);
var p0 = router.GetBrokerRoute(TestTopic, 0);
var p1 = router.GetBrokerRoute(TestTopic, 1);
Assert.That(p0.PartitionId, Is.EqualTo(0));
Assert.That(p1.PartitionId, Is.EqualTo(1));
}
public void SelectPartitionShouldThrowWhenBrokerCollectionIsEmpty()
{
var metadataResponse = CreateMetaResponse();
metadataResponse.Brokers.Clear();
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.MetadataResponseFunction = () => metadataResponse;
routerProxy.Create().SelectBrokerRoute(TestTopic);
}
public void GetTopicOffsetShouldThrowAnyException()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.OffsetResponseFunction = () => { throw new ApplicationException("test 99"); };
var router = routerProxy.Create();
var common = new CommonQueries(router);
common.GetTopicOffsetAsync(BrokerRouterProxy.TestTopic).ContinueWith(t =>
{
Assert.That(t.IsFaulted, Is.True);
Assert.That(t.Exception.Flatten().ToString(), Is.StringContaining("test 99"));
}).Wait();
}
public async Task BrokerRouteShouldThrowIfCycleCouldNotConnectToAnyServer()
{
var routerProxy = new BrokerRouterProxy();
routerProxy.Connection1.MetadataResponseFunction = () => { throw new Exception("some error"); };
routerProxy.Connection2.MetadataResponseFunction = () => { throw new Exception("some error"); };
var router = routerProxy.Create();
Assert.ThrowsAsync <RequestException>(async() => await router.GetTopicMetadataAsync(TestTopic, CancellationToken.None));
Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(routerProxy.Connection2.MetadataRequestCallCount, Is.EqualTo(1));
}
public void GetTopicOffsetShouldThrowAnyException()
{
var routerProxy = new BrokerRouterProxy();
routerProxy.Connection1.OffsetResponseFunction = () => { throw new Exception("test 99"); };
var router = routerProxy.Create();
router.GetTopicOffsetsAsync(BrokerRouterProxy.TestTopic, 2, -1, CancellationToken.None).ContinueWith(t =>
{
Assert.That(t.IsFaulted, Is.True);
Assert.That(t.Exception.Flatten().ToString(), Does.Contain("test 99"));
}).Wait();
}
public void BrokerRouteShouldCycleThroughEachBrokerUntilOneIsFound()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.MetadataResponseFunction = () => { throw new Exception("some error"); };
var router = routerProxy.Create();
var result = router.GetTopicMetadata(TestTopic);
Assert.That(result, Is.Not.Null);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(1));
}
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(
ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
public async Task SimultaneouslyRefreshMissingTopicMetadataShouldGetDataFromCacheOnSameRequest()
{
var routerProxy = new BrokerRouterProxy();
var router = routerProxy.Create();
List <Task> x = new List <Task>();
x.Add(router.GetTopicMetadataAsync(TestTopic, CancellationToken.None)); //do not debug
x.Add(router.GetTopicMetadataAsync(TestTopic, CancellationToken.None)); //do not debug
await Task.WhenAll(x.ToArray());
Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(1));
}
public async Task SimultaneouslyRefreshTopicMetadataShouldGetDataFromCacheOnSameRequest()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
List <Task> x = new List <Task>();
x.Add(router.RefreshTopicMetadata(TestTopic)); //do not debug
x.Add(router.RefreshTopicMetadata(TestTopic)); //do not debug
await Task.WhenAll(x.ToArray());
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
}
public async Task BrokerRouteShouldReturnTopicFromCache()
{
var routerProxy = new BrokerRouterProxy();
var router = routerProxy.Create();
await router.GetTopicMetadataAsync(TestTopic, CancellationToken.None);
var result1 = router.GetTopicMetadata(TestTopic);
var result2 = router.GetTopicMetadata(TestTopic);
Assert.AreEqual(1, router.GetTopicMetadata().Count);
Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(result1.TopicName, Is.EqualTo(TestTopic));
Assert.That(result2.TopicName, Is.EqualTo(TestTopic));
}
public void GetTopicOffsetShouldThrowAnyException()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.OffsetResponseFunction = () => { throw new ApplicationException("test 99"); };
var router = routerProxy.Create();
var common = new MetadataQueries(router);
common.GetTopicOffsetAsync(BrokerRouterProxy.TestTopic).ContinueWith(t =>
{
Assert.That(t.IsFaulted, Is.True);
Assert.That(t.Exception.Flatten().ToString(), Is.StringContaining("test 99"));
}).Wait();
}
public void BrokerRouteShouldReturnTopicFromCache()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
var result1 = router.GetTopicMetadata(TestTopic);
var result2 = router.GetTopicMetadata(TestTopic);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(result1.Count, Is.EqualTo(1));
Assert.That(result1[0].Name, Is.EqualTo(TestTopic));
Assert.That(result2.Count, Is.EqualTo(1));
Assert.That(result2[0].Name, Is.EqualTo(TestTopic));
}
public async Task SelectPartitionShouldThrowWhenTopicsCollectionIsEmpty()
{
var metadataResponse = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
metadataResponse.Topics.Clear();
var routerProxy = new BrokerRouterProxy();
#pragma warning disable 1998
routerProxy.Connection1.MetadataResponseFunction = async() => metadataResponse;
#pragma warning restore 1998
Assert.Throws <CachedMetadataException>(() => routerProxy.Create().GetBrokerRoute(TestTopic));
}
public void BrokerRouteShouldReturnTopicFromCache()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
var result1 = router.GetTopicMetadata(TestTopic);
var result2 = router.GetTopicMetadata(TestTopic);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(result1.Count, Is.EqualTo(1));
Assert.That(result1[0].Name, Is.EqualTo(TestTopic));
Assert.That(result2.Count, Is.EqualTo(1));
Assert.That(result2[0].Name, Is.EqualTo(TestTopic));
}
public async Task BrokerRouteShouldCycleThroughEachBrokerUntilOneIsFound()
{
var routerProxy = new BrokerRouterProxy();
routerProxy.Connection1.MetadataResponseFunction = () => { throw new Exception("some error"); };
var router = routerProxy.Create();
await router.GetTopicMetadataAsync(TestTopic, CancellationToken.None);
var result = router.GetTopicMetadata(TestTopic);
Assert.That(result, Is.Not.Null);
Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(routerProxy.Connection2.MetadataRequestCallCount, Is.EqualTo(1));
}
public async Task SelectExactPartitionShouldThrowWhenBrokerCollectionIsEmpty()
{
var metadataResponse = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
metadataResponse.Brokers.Clear();
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.MetadataResponseFunction = async() => metadataResponse;
var router = routerProxy.Create();
await router.RefreshMissingTopicMetadata(TestTopic);
router.SelectBrokerRouteFromLocalCache(TestTopic, 1);
}
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy();
routerProxy.CacheExpiration = new TimeSpan(10);
var router = routerProxy.Create();
routerProxy.Connection1.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy.CacheExpiration);
routerProxy.Connection1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await router.SendAsync(new FetchRequest(), BrokerRouterProxy.TestTopic, PartitionId, CancellationToken.None);
Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.Connection1.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = new TimeSpan(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldTryToRefreshMataDataIfOnExceptions(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task BrokerRouteShouldReturnAllTopicsFromCache()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
await router.RefreshAllTopicMetadata();
var result1 = router.GetAllTopicMetadataFromLocalCache();
var result2 = router.GetAllTopicMetadataFromLocalCache();
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
Assert.That(result1.Count, Is.EqualTo(1));
Assert.That(result1[0].Name, Is.EqualTo(TestTopic));
Assert.That(result2.Count, Is.EqualTo(1));
Assert.That(result2[0].Name, Is.EqualTo(TestTopic));
}
public async Task ShouldTryToRefreshMataDataIfOnExceptions(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy();
routerProxy.CacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
routerProxy.Connection1.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy.CacheExpiration);
routerProxy.Connection1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await router.SendAsync(new FetchRequest(), BrokerRouterProxy.TestTopic, PartitionId, CancellationToken.None);
Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.Connection1.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldTryToRefreshMataDataIfSocketException()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(typeof(BrokerConnectionException), routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task RefreshTopicMetadataShouldIgnoreCacheAndAlwaysCauseMetadataRequestAfterExpertionDate()
{
var routerProxy = new BrokerRouterProxy(_kernel);
var router = routerProxy.Create();
TimeSpan cacheExpiration = TimeSpan.FromMilliseconds(100);
await router.RefreshTopicMetadata(TestTopic);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
await Task.Delay(routerProxy._cacheExpiration);
await Task.Delay(1);//After cache is expair
await router.RefreshTopicMetadata(TestTopic);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = new TimeSpan(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
}
public async Task SelectExactPartitionShouldThrowWhenBrokerCollectionIsEmpty()
{
var metadataResponse = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
metadataResponse = new MetadataResponse(topics: metadataResponse.Topics);
var routerProxy = new BrokerRouterProxy();
#pragma warning disable 1998
routerProxy.Connection1.MetadataResponseFunction = async() => metadataResponse;
#pragma warning restore 1998
var router = routerProxy.Create();
await router.GetTopicMetadataAsync(TestTopic, CancellationToken.None);
Assert.Throws <CachedMetadataException>(() => router.GetBrokerRoute(TestTopic, 1));
}
public void ConsumerShouldCreateTaskForEachBroker()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.FetchResponseFunction = () => { return(new FetchResponse()); };
var router = routerProxy.Create();
var options = CreateOptions(router);
options.PartitionWhitelist = new List <int>();
var consumer = new Consumer(options);
var test = consumer.Consume().Take(1);
TaskTest.WaitFor(() => consumer.ConsumerTaskCount >= 2);
Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2));
}
public void ConsumerShouldReturnOffset()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); };
var router = routerProxy.Create();
var options = CreateOptions(router);
options.PartitionWhitelist = new List<int>();
var consumer = new Consumer(options);
var test = consumer.Consume().Take(1);
while (consumer.ConsumerTaskCount <= 0)
{
Thread.Sleep(100);
}
Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2));
}
public void ConsumerWhitelistShouldOnlyConsumeSpecifiedPartition()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); };
var router = routerProxy.Create();
var options = CreateOptions(router);
options.PartitionWhitelist = new List<int> { 0 };
var consumer = new Consumer(options);
var test = consumer.Consume().Take(1);
while (consumer.ConsumerTaskCount <= 0)
{
Thread.Sleep(100);
}
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1));
Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(0));
}
public void ConsumerWhitelistShouldOnlyConsumeSpecifiedPartition()
{
var routerProxy = new BrokerRouterProxy(new MoqMockingKernel());
routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); };
var router = routerProxy.Create();
var options = CreateOptions(router);
options.PartitionWhitelist = new List<int> { 0 };
using (var consumer = new Consumer(options))
{
var test = consumer.Consume();
TaskTest.WaitFor(() => consumer.ConsumerTaskCount > 0);
TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 0);
Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(1), "Consumer should only create one consuming thread for partition 0.");
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1));
Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(0));
}
}
public void ConnectionExceptionOnOneShouldCommunicateBackWhichMessagesFailed()
{
//TODO is there a way to communicate back which client failed and which succeeded.
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn1.ProduceResponseFunction = () => { throw new ApplicationException("some exception"); };
var router = routerProxy.Create();
var producer = new Producer(router);
var messages = new List<Message>
{
new Message{Value = "1"}, new Message{Value = "2"}
};
//this will produce an exception, but message 1 succeeded and message 2 did not.
//should we return a ProduceResponse with an error and no error for the other messages?
//at this point though the client does not know which message is routed to which server.
//the whole batch of messages would need to be returned.
var test = producer.SendMessageAsync("UnitTest", messages).Result;
}
public void ConsumerWithEmptyWhitelistShouldConsumeAllPartition()
{
var routerProxy = new BrokerRouterProxy(new MoqMockingKernel());
var router = routerProxy.Create();
var options = CreateOptions(router);
options.PartitionWhitelist = new List<int>();
using (var consumer = new Consumer(options))
{
var test = consumer.Consume();
TaskTest.WaitFor(() => consumer.ConsumerTaskCount > 0);
TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 0);
TaskTest.WaitFor(() => routerProxy.BrokerConn1.FetchRequestCallCount > 0);
Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2), "Consumer should create one consuming thread for each partition.");
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1), "BrokerConn0 not sent FetchRequest");
Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1), "BrokerConn1 not sent FetchRequest");
}
}
public void CancellationShouldInterruptConsumption()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.FetchResponseFunction = () => { while (true) Thread.Yield(); };
var router = routerProxy.Create();
var options = CreateOptions(router);
var consumer = new Consumer(options);
var tokenSrc = new CancellationTokenSource();
var consumeTask = Task.Run(() => consumer.Consume(tokenSrc.Token).FirstOrDefault());
if (consumeTask.Wait(TimeSpan.FromSeconds(3)))
Assert.Fail();
tokenSrc.Cancel();
Assert.That(
Assert.Throws<AggregateException>(consumeTask.Wait).InnerException,
Is.TypeOf<OperationCanceledException>());
}
public async Task ShouldRecoverFromFailerByUpdateMetadataOnceFullScenario1()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(0);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
CreateSuccessfulSendMock(routerProxy);
//Send Successful Message
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(ErrorResponseCode.LeaderNotAvailable, TimeSpan.Zero);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
routerProxy.BrokerConn1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
//Reset variables
routerProxy.BrokerConn0.FetchRequestCallCount = 0;
routerProxy.BrokerConn1.FetchRequestCallCount = 0;
routerProxy.BrokerConn0.MetadataRequestCallCount = 0;
routerProxy.BrokerConn1.MetadataRequestCallCount = 0;
//Send Successful Message that was recover from exception
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
}
public async Task ShouldUpdateMetadataOnes()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
await Task.Delay(routerProxy._cacheExpiration);
await Task.Delay(1);
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
}
public void SelectPartitionShouldThrowWhenBrokerCollectionIsEmpty()
{
var metadataResponse = CreateMetaResponse();
metadataResponse.Brokers.Clear();
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.BrokerConn0.MetadataResponseFunction = () => metadataResponse;
routerProxy.Create().SelectBrokerRoute(TestTopic);
}
public void ConsumerShouldCreateTaskForEachBroker()
{
var routerProxy = new BrokerRouterProxy(new MoqMockingKernel());
routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); };
var router = routerProxy.Create();
var options = CreateOptions(router);
options.PartitionWhitelist = new List<int>();
using (var consumer = new Consumer(options))
{
var test = consumer.Consume();
TaskTest.WaitFor(() => consumer.ConsumerTaskCount >= 2);
Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2));
}
}
public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
try
{
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
Assert.IsTrue(false, "Should throw exception");
}
catch (Exception ex)
{
Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
}
}
public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool firstTime = true;
Func<Task<FetchResponse>> ShouldReturnError = async () =>
{
if (firstTime)
{
firstTime = !firstTime;
object[] args = new object[1];
args[0] = "error Test";
throw (Exception)Activator.CreateInstance(exceptionType, args);
}
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
try
{
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
Assert.IsTrue(false, "Should throw exception");
}
catch (Exception ex)
{
Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
}
}
public void SelectExactPartitionShouldThrowWhenPartitionDoesNotExist()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy.Create().SelectBrokerRoute(TestTopic, 3);
}
public async Task ShouldRecoverFromFailerByUpdateMetadataOnce() //Do not debug this test !!
{
var log = new DefaultTraceLog();
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(1000);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
int numberOfCall = 100;
long numberOfErrorSend = 0;
TaskCompletionSource<int> x = new TaskCompletionSource<int>();
Func<Task<FetchResponse>> ShouldReturnNotLeaderForPartitionAndThenNoError = async () =>
{
log.DebugFormat("FetchResponse Start ");
if (!x.Task.IsCompleted)
{
if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall)
{
await Task.Delay(routerProxy._cacheExpiration);
await Task.Delay(1);
x.TrySetResult(1);
log.DebugFormat("all is complete ");
}
await x.Task;
log.DebugFormat("SocketException ");
throw new BrokerConnectionException("",new KafkaEndpoint());
}
log.DebugFormat("Completed ");
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnNotLeaderForPartitionAndThenNoError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
}
await Task.WhenAll(tasks);
Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend");
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend),
"FetchRequestCallCount");
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount");
}
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(
ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
}
public void SelectPartitionShouldUsePartitionSelector(string testCase)
{
var key = testCase.ToIntSizedBytes();
var routerProxy = new BrokerRouterProxy(_kernel);
_mockPartitionSelector.Setup(x => x.Select(It.IsAny<Topic>(), key))
.Returns(() => new Partition
{
ErrorCode = 0,
Isrs = new List<int> { 1 },
PartitionId = 0,
LeaderId = 0,
Replicas = new List<int> { 1 },
});
routerProxy.PartitionSelector = _mockPartitionSelector.Object;
var result = routerProxy.Create().SelectBrokerRoute(TestTopic, key);
_mockPartitionSelector.Verify(f => f.Select(It.Is<Topic>(x => x.Name == TestTopic), key), Times.Once());
}
public async Task ShouldRecoverUpdateMetadataForNewTopic()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool firstTime = true;
Func<Task<FetchResponse>> ShouldReturnError = async () =>
{
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
}
//change MetadataResponseFunction
routerProxy.BrokerConn0.MetadataResponseFunction = async () =>
{
var response = await BrokerRouterProxy.DefaultMetadataResponse();
response.Topics[0].Name = "test2";
return response;
};
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(ErrorResponseCode code)
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
int partitionId = 0;
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
bool firstTime = true;
Func<Task<FetchResponse>> ShouldReturnError = async () =>
{
if (firstTime)
{
firstTime = !firstTime;
return new FetchResponse() { Error = (short)code };
}
return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
};
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
}
public async Task ShouldRecoverUpdateMetadataForNewTopic()
{
var routerProxy = new BrokerRouterProxy(_kernel);
routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
var router = routerProxy.Create();
ProtocolGateway protocolGateway = new ProtocolGateway(router);
var fetchRequest = new FetchRequest();
routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
int numberOfCall = 1000;
Task[] tasks = new Task[numberOfCall];
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, _partitionId);
}
routerProxy.BrokerConn0.MetadataResponseFunction = async () =>
{
var response = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
response.Topics[0].Name = "test2";
return response;
};
for (int i = 0; i < numberOfCall / 2; i++)
{
tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", _partitionId);
}
await Task.WhenAll(tasks);
Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
}