public void SendAsyncShouldBlockWhenMaximumAsyncQueueReached() { int count = 0; var semaphore = new SemaphoreSlim(0); var routerProxy = new BrokerRouterProxy(_kernel); //block the second call returning from send message async routerProxy.BrokerConn0.ProduceResponseFunction = () => { semaphore.Wait(); return(new ProduceResponse()); }; var router = routerProxy.Create(); using (var producer = new Producer(router, 1)) { var messages = new List <Message> { new Message("1"), new Message("2") }; Task.Factory.StartNew(() => { producer.SendMessageAsync(BrokerRouterProxy.TestTopic, messages); count++; producer.SendMessageAsync(BrokerRouterProxy.TestTopic, messages); count++; }); TaskTest.WaitFor(() => count > 0); Assert.That(count, Is.EqualTo(1), "Only one SendMessageAsync should continue."); Assert.That(producer.ActiveCount, Is.EqualTo(1), "One async call shoud be active."); semaphore.Release(); TaskTest.WaitFor(() => count > 1); Assert.That(count, Is.EqualTo(2), "The second SendMessageAsync should continue after semaphore is released."); } }
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code) { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = new TimeSpan(10); var router = routerProxy.Create(); int partitionId = 0; ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); bool sendExOnFirstTime = true; Func<Task<FetchResponse>> ShouldReturnErrorAndThenNoError = async () => { Task.Delay(routerProxy._cacheExpiration).Wait(); Task.Delay(1).Wait(); if (sendExOnFirstTime) { sendExOnFirstTime = false; return new FetchResponse() { Error = (short)code }; } return new FetchResponse() { Error = (short)ErrorResponseCode.NoError }; }; routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnErrorAndThenNoError; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse; await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2)); }
private void CreateSuccessfulSendMock(BrokerRouterProxy routerProxy) { routerProxy.Connection1.FetchResponseFunction = ShouldReturnValidMessage; routerProxy.Connection1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; routerProxy.Connection2.FetchResponseFunction = ShouldReturnValidMessage; routerProxy.Connection2.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; }
public void CancellationShouldInterruptConsumption() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.FetchResponseFunction = () => { return(new FetchResponse()); }; var router = routerProxy.Create(); var options = CreateOptions(router); var consumer = new Consumer(options); var tokenSrc = new CancellationTokenSource(); var consumeTask = Task.Run(() => consumer.Consume(tokenSrc.Token).FirstOrDefault()); //wait until the fake broker is running and requesting fetches TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 10); tokenSrc.Cancel(); Assert.That( Assert.Throws <AggregateException>(consumeTask.Wait).InnerException, Is.TypeOf <OperationCanceledException>()); }
public void CancellationShouldInterruptConsumption() { var routerProxy = new BrokerRouterProxy(new MoqMockingKernel()); routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); }; var router = routerProxy.Create(); var options = CreateOptions(router); using (var consumer = new Consumer(options)) { var tokenSrc = new CancellationTokenSource(); var consumeTask = Task.Run(() => consumer.Consume(tokenSrc.Token).FirstOrDefault()); //wait until the fake broker is running and requesting fetches TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 10); tokenSrc.Cancel(); Assert.That( Assert.Throws<AggregateException>(consumeTask.Wait).InnerException, Is.TypeOf<OperationCanceledException>()); } }
public async Task ConsumerWithEmptyWhitelistShouldConsumeAllPartition() { var routerProxy = new BrokerRouterProxy(new MoqMockingKernel()); var router = routerProxy.Create(); var options = CreateOptions(router); options.PartitionWhitelist = new List <int>(); using (var consumer = new Consumer(options)) { var test = consumer.Consume(); await TaskTest.WaitFor(() => consumer.ConsumerTaskCount > 0); await TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 0); await TaskTest.WaitFor(() => routerProxy.BrokerConn1.FetchRequestCallCount > 0); Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2), "Consumer should create one consuming thread for each partition."); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1), "BrokerConn0 not sent FetchRequest"); Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1), "BrokerConn1 not sent FetchRequest"); } }
public async Task ShouldUpdateMetadataOnes() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); ProtocolGateway protocolGateway = new ProtocolGateway(router); routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; int numberOfCall = 1000; Task[] tasks = new Task[numberOfCall]; for (int i = 0; i < numberOfCall / 2; i++) { tasks[i] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId); } await Task.Delay(routerProxy._cacheExpiration); await Task.Delay(1); for (int i = 0; i < numberOfCall / 2; i++) { tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId); } await Task.WhenAll(tasks); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall)); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1)); }
public async Task SelectPartitionShouldUsePartitionSelector(string testCase) { var key = testCase.ToIntSizedBytes(); var routerProxy = new BrokerRouterProxy(_kernel); _mockPartitionSelector.Setup(x => x.Select(It.IsAny <Topic>(), key)) .Returns(() => new Partition { ErrorCode = 0, Isrs = new List <int> { 1 }, PartitionId = 0, LeaderId = 0, Replicas = new List <int> { 1 }, }); routerProxy.PartitionSelector = _mockPartitionSelector.Object; var router = routerProxy.Create(); await router.RefreshMissingTopicMetadata(TestTopic); var result = router.SelectBrokerRouteFromLocalCache(TestTopic, key); _mockPartitionSelector.Verify(f => f.Select(It.Is <Topic>(x => x.Name == TestTopic), key), Times.Once()); }
public async Task ConsumerWhitelistShouldOnlyConsumeSpecifiedPartition() { var routerProxy = new BrokerRouterProxy(new MoqMockingKernel()); routerProxy.BrokerConn0.FetchResponseFunction = async() => { return(new FetchResponse()); }; var router = routerProxy.Create(); var options = CreateOptions(router); options.PartitionWhitelist = new List <int> { 0 }; using (var consumer = new Consumer(options)) { var test = consumer.Consume(); await TaskTest.WaitFor(() => consumer.ConsumerTaskCount > 0); await TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 0); Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(1), "Consumer should only create one consuming thread for partition 0."); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1)); Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(0)); } }
public void CancellationShouldInterruptConsumption() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.FetchResponseFunction = () => { while (true) { Thread.Yield(); } }; var router = routerProxy.Create(); var options = CreateOptions(router); var consumer = new Consumer(options); var tokenSrc = new CancellationTokenSource(); var consumeTask = Task.Run(() => consumer.Consume(tokenSrc.Token).FirstOrDefault()); if (consumeTask.Wait(TimeSpan.FromSeconds(3))) { Assert.Fail(); } tokenSrc.Cancel(); Assert.That( Assert.Throws <AggregateException>(consumeTask.Wait).InnerException, Is.TypeOf <OperationCanceledException>()); }
public void EmptyTopicMetadataShouldThrowException() { var routerProxy = new BrokerRouterProxy(); var router = routerProxy.Create(); Assert.Throws <CachedMetadataException>(() => router.GetTopicMetadata("MissingTopic")); }
public async Task SelectBrokerRouteShouldChange() { var routerProxy = new BrokerRouterProxy(); var router = routerProxy.Create(); routerProxy.MetadataResponse = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; await router.RefreshTopicMetadataAsync(TestTopic, true, CancellationToken.None); var router1 = router.GetBrokerRoute(TestTopic, 0); Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(1)); await Task.Delay(routerProxy.CacheExpiration); await Task.Delay(1);//After cache is expair routerProxy.MetadataResponse = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker; await router.RefreshTopicMetadataAsync(TestTopic, true, CancellationToken.None); var router2 = router.GetBrokerRoute(TestTopic, 0); Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(2)); Assert.That(router1.Connection.Endpoint, Is.EqualTo(routerProxy.Connection1.Endpoint)); Assert.That(router2.Connection.Endpoint, Is.EqualTo(routerProxy.Connection2.Endpoint)); Assert.That(router1.Connection.Endpoint, Is.Not.EqualTo(router2.Connection.Endpoint)); }
public async Task ShouldUpdateMetadataOnce() { var routerProxy = new BrokerRouterProxy(); routerProxy.CacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); routerProxy.Connection1.FetchResponseFunction = ShouldReturnValidMessage; routerProxy.Connection1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; int numberOfCall = 1000; Task[] tasks = new Task[numberOfCall]; for (int i = 0; i < numberOfCall / 2; i++) { tasks[i] = router.SendAsync(new FetchRequest(), BrokerRouterProxy.TestTopic, PartitionId, CancellationToken.None); } await Task.Delay(routerProxy.CacheExpiration); await Task.Delay(1); for (int i = 0; i < numberOfCall / 2; i++) { tasks[i + numberOfCall / 2] = router.SendAsync(new FetchRequest(), BrokerRouterProxy.TestTopic, PartitionId, CancellationToken.None); } await Task.WhenAll(tasks); Assert.That(routerProxy.Connection1.FetchRequestCallCount, Is.EqualTo(numberOfCall)); Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(1)); }
public void ConnectionExceptionOnOneShouldCommunicateBackWhichMessagesFailed() { //TODO is there a way to communicate back which client failed and which succeeded. var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn1.ProduceResponseFunction = () => { throw new ApplicationException("some exception"); }; var router = routerProxy.Create(); var producer = new Producer(router); var messages = new List <Message> { new Message { Value = "1" }, new Message { Value = "2" } }; //this will produce an exception, but message 1 succeeded and message 2 did not. //should we return a ProduceResponse with an error and no error for the other messages? //at this point though the client does not know which message is routed to which server. //the whole batch of messages would need to be returned. var test = producer.SendMessageAsync("UnitTest", messages).Result; }
public async Task SelectBrokerRouteShouldChange() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); routerProxy.MetadataResponse = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; await router.RefreshTopicMetadata(TestTopic); var router1 = router.SelectBrokerRouteFromLocalCache(TestTopic, 0); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1)); await Task.Delay(routerProxy._cacheExpiration); await Task.Delay(1);//After cache is expair routerProxy.MetadataResponse = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker; await router.RefreshTopicMetadata(TestTopic); var router2 = router.SelectBrokerRouteFromLocalCache(TestTopic, 0); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); Assert.That(router1.Connection.Endpoint, Is.EqualTo(routerProxy.BrokerConn0.Endpoint)); Assert.That(router2.Connection.Endpoint, Is.EqualTo(routerProxy.BrokerConn1.Endpoint)); Assert.That(router1.Connection.Endpoint, Is.Not.EqualTo(router2.Connection.Endpoint)); }
public void ShouldSendAsyncToAllConnectionsEvenWhenExceptionOccursOnOne() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn1.ProduceResponseFunction = () => { throw new ApplicationException("some exception"); }; var router = routerProxy.Create(); var producer = new Producer(router); var messages = new List <Message> { new Message { Value = "1" }, new Message { Value = "2" } }; producer.SendMessageAsync("UnitTest", messages).ContinueWith(t => { Assert.That(t.IsFaulted, Is.True); Assert.That(t.Exception, Is.Not.Null); Assert.That(t.Exception.ToString(), Is.StringContaining("ApplicationException")); Assert.That(routerProxy.BrokerConn0.ProduceRequestCallCount, Is.EqualTo(1)); Assert.That(routerProxy.BrokerConn1.ProduceRequestCallCount, Is.EqualTo(1)); }).Wait(); }
public async Task ShouldRecoverFromFailerByUpdateMetadataOnce() //Do not debug this test !! { var log = new DefaultTraceLog(); var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(1000); var router = routerProxy.Create(); int partitionId = 0; ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); int numberOfCall = 100; long numberOfErrorSend = 0; TaskCompletionSource <int> x = new TaskCompletionSource <int>(); Func <Task <FetchResponse> > ShouldReturnNotLeaderForPartitionAndThenNoError = async() => { log.DebugFormat("FetchResponse Start "); if (!x.Task.IsCompleted) { if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall) { await Task.Delay(routerProxy._cacheExpiration); await Task.Delay(1); x.TrySetResult(1); log.DebugFormat("all is complete "); } await x.Task; log.DebugFormat("SocketException "); throw new BrokerConnectionException("", new KafkaEndpoint()); } log.DebugFormat("Completed "); return(new FetchResponse() { Error = (short)ErrorResponseCode.NoError }); }; routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnNotLeaderForPartitionAndThenNoError; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; Task[] tasks = new Task[numberOfCall]; for (int i = 0; i < numberOfCall; i++) { tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); } await Task.WhenAll(tasks); Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend"); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend), "FetchRequestCallCount"); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount"); }
public void EmptyTopicMetadataShouldThrowException() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); var common = new CommonQueries(router); common.GetTopic("MissingTopic"); }
public async Task SelectExactPartitionShouldThrowWhenPartitionDoesNotExist() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); await router.RefreshMissingTopicMetadata(TestTopic); router.SelectBrokerRouteFromLocalCache(TestTopic, 3); }
public void EmptyTopicMetadataShouldThrowException() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); var common = new MetadataQueries(router); common.GetTopicFromCache("MissingTopic"); }
public async Task SelectExactPartitionShouldThrowWhenPartitionDoesNotExist() { var routerProxy = new BrokerRouterProxy(); var router = routerProxy.Create(); await router.GetTopicMetadataAsync(TestTopic, CancellationToken.None); Assert.Throws <CachedMetadataException>(() => router.GetBrokerRoute(TestTopic, 3)); }
public async Task ShouldThrowFormatExceptionWhenTopicIsInvalid() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); string invalidTopic = " "; var fetchRequest = new FetchRequest(); ProtocolGateway protocolGateway = new ProtocolGateway(router); await protocolGateway.SendProtocolRequest(fetchRequest, invalidTopic, 0); }
public void GetTopicShouldReturnTopic() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); var common = new CommonQueries(router); var result = common.GetTopic(BrokerRouterProxy.TestTopic); Assert.That(result.Name, Is.EqualTo(BrokerRouterProxy.TestTopic)); }
public void GetTopicOffsetShouldQueryEachBroker() { var routerProxy = new BrokerRouterProxy(); var router = routerProxy.Create(); var result = router.GetTopicOffsetsAsync(BrokerRouterProxy.TestTopic, 2, -1, CancellationToken.None).Result; Assert.That(routerProxy.Connection1.OffsetRequestCallCount, Is.EqualTo(1)); Assert.That(routerProxy.Connection2.OffsetRequestCallCount, Is.EqualTo(1)); }
public async Task GetTopicShouldReturnTopic() { var routerProxy = new BrokerRouterProxy(); var router = routerProxy.Create(); await router.GetTopicMetadataAsync(BrokerRouterProxy.TestTopic, CancellationToken.None); var result = router.GetTopicMetadata(BrokerRouterProxy.TestTopic); Assert.That(result.TopicName, Is.EqualTo(BrokerRouterProxy.TestTopic)); }
public void GetTopicOffsetShouldQueryEachBroker() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); var common = new CommonQueries(router); var result = common.GetTopicOffsetAsync(BrokerRouterProxy.TestTopic).Result; Assert.That(routerProxy.BrokerConn0.OffsetRequestCallCount, Is.EqualTo(1)); Assert.That(routerProxy.BrokerConn1.OffsetRequestCallCount, Is.EqualTo(1)); }
public async Task ShouldRecoverFromFailureByUpdateMetadataOnce() //Do not debug this test !! { var routerProxy = new BrokerRouterProxy(); routerProxy.CacheExpiration = TimeSpan.FromMilliseconds(1000); var router = routerProxy.Create(); int partitionId = 0; var fetchRequest = new FetchRequest(); int numberOfCall = 100; long numberOfErrorSend = 0; TaskCompletionSource <int> x = new TaskCompletionSource <int>(); Func <Task <FetchResponse> > ShouldReturnNotLeaderForPartitionAndThenNoError = async() => { var log = TestConfig.DebugLog; log.Debug(() => LogEvent.Create("FetchResponse Start ")); if (!x.Task.IsCompleted) { if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall) { await Task.Delay(routerProxy.CacheExpiration); await Task.Delay(1); x.TrySetResult(1); log.Debug(() => LogEvent.Create("all is complete ")); } await x.Task; log.Debug(() => LogEvent.Create("SocketException ")); throw new ConnectionException(""); } log.Debug(() => LogEvent.Create("Completed ")); return(new FetchResponse()); }; routerProxy.Connection1.FetchResponseFunction = ShouldReturnNotLeaderForPartitionAndThenNoError; routerProxy.Connection1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; Task[] tasks = new Task[numberOfCall]; for (int i = 0; i < numberOfCall; i++) { tasks[i] = router.SendAsync(fetchRequest, BrokerRouterProxy.TestTopic, partitionId, CancellationToken.None); } await Task.WhenAll(tasks); Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend"); Assert.That(routerProxy.Connection1.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend), "FetchRequestCallCount"); Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount"); }
public void GetTopicOffsetShouldQueryEachBroker() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); var common = new MetadataQueries(router); var result = common.GetTopicOffsetAsync(BrokerRouterProxy.TestTopic).Result; Assert.That(routerProxy.BrokerConn0.OffsetRequestCallCount, Is.EqualTo(1)); Assert.That(routerProxy.BrokerConn1.OffsetRequestCallCount, Is.EqualTo(1)); }
public void BrokerRouteShouldCycleThroughEachBrokerUntilOneIsFound() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.MetadataResponseFunction = () => { throw new Exception("some error"); }; var router = routerProxy.Create(); var result = router.GetTopicMetadata(TestTopic); Assert.That(result, Is.Not.Null); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1)); Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(1)); }
public async Task SendProtocolRequestShouldThrowException(Type exceptionType) { var routerProxy = new BrokerRouterProxy(); routerProxy.CacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); routerProxy.Connection1.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy.CacheExpiration); routerProxy.Connection1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; Assert.ThrowsAsync(exceptionType, async() => await router.SendAsync(new FetchRequest(), BrokerRouterProxy.TestTopic, PartitionId, CancellationToken.None)); }
public void SelectExactPartitionShouldReturnRequestedPartition() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); var p0 = router.SelectBrokerRoute(TestTopic, 0); var p1 = router.SelectBrokerRoute(TestTopic, 1); Assert.That(p0.PartitionId, Is.EqualTo(0)); Assert.That(p1.PartitionId, Is.EqualTo(1)); }
public async Task BrokerRouteShouldThrowNoLeaderElectedForPartition() { var routerProxy = new BrokerRouterProxy { MetadataResponse = BrokerRouterProxy.CreateMetadataResponseWithNotEndToElectLeader }; var router = routerProxy.Create(); Assert.ThrowsAsync <CachedMetadataException>(async() => await router.GetTopicMetadataAsync(TestTopic, CancellationToken.None)); Assert.AreEqual(0, router.GetTopicMetadata().Count); }
public async Task RefreshAllTopicMetadataShouldAlwaysDoRequest() { var routerProxy = new BrokerRouterProxy(); var router = routerProxy.Create(); await router.RefreshTopicMetadataAsync(CancellationToken.None); Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(1)); await router.RefreshTopicMetadataAsync(CancellationToken.None); Assert.That(routerProxy.Connection1.MetadataRequestCallCount, Is.EqualTo(2)); }
public async Task RefreshAllTopicMetadataShouldAlwaysDoRequest() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); await router.RefreshAllTopicMetadata(); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1)); await router.RefreshAllTopicMetadata(); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); }
public void RefreshTopicMetadataShouldIgnoreCacheAndAlwayCauseMetadataRequest() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); router.RefreshTopicMetadata(TestTopic); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1)); router.RefreshTopicMetadata(TestTopic); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); }
public void GetTopicOffsetShouldThrowAnyException() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.OffsetResponseFunction = () => { throw new ApplicationException("test 99"); }; var router = routerProxy.Create(); var common = new CommonQueries(router); common.GetTopicOffsetAsync(BrokerRouterProxy.TestTopic).ContinueWith(t => { Assert.That(t.IsFaulted, Is.True); Assert.That(t.Exception.Flatten().ToString(), Is.StringContaining("test 99")); }).Wait(); }
public void BrokerRouteShouldReturnTopicFromCache() { var routerProxy = new BrokerRouterProxy(_kernel); var router = routerProxy.Create(); var result1 = router.GetTopicMetadata(TestTopic); var result2 = router.GetTopicMetadata(TestTopic); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1)); Assert.That(result1.Count, Is.EqualTo(1)); Assert.That(result1[0].Name, Is.EqualTo(TestTopic)); Assert.That(result2.Count, Is.EqualTo(1)); Assert.That(result2[0].Name, Is.EqualTo(TestTopic)); }
public async Task ShouldTryToRefreshMataDataIfOnExceptions(Type exceptionType) { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); ProtocolGateway protocolGateway = new ProtocolGateway(router); routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration); routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2)); }
public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code) { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = new TimeSpan(10); var router = routerProxy.Create(); ProtocolGateway protocolGateway = new ProtocolGateway(router); routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration); routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2)); }
public void ConsumerShouldReturnOffset() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); }; var router = routerProxy.Create(); var options = CreateOptions(router); options.PartitionWhitelist = new List<int>(); var consumer = new Consumer(options); var test = consumer.Consume().Take(1); while (consumer.ConsumerTaskCount <= 0) { Thread.Sleep(100); } Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2)); }
public void ConsumerWhitelistShouldOnlyConsumeSpecifiedPartition() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); }; var router = routerProxy.Create(); var options = CreateOptions(router); options.PartitionWhitelist = new List<int> { 0 }; var consumer = new Consumer(options); var test = consumer.Consume().Take(1); while (consumer.ConsumerTaskCount <= 0) { Thread.Sleep(100); } Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1)); Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(0)); }
public void ConsumerWhitelistShouldOnlyConsumeSpecifiedPartition() { var routerProxy = new BrokerRouterProxy(new MoqMockingKernel()); routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); }; var router = routerProxy.Create(); var options = CreateOptions(router); options.PartitionWhitelist = new List<int> { 0 }; using (var consumer = new Consumer(options)) { var test = consumer.Consume(); TaskTest.WaitFor(() => consumer.ConsumerTaskCount > 0); TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 0); Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(1), "Consumer should only create one consuming thread for partition 0."); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1)); Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(0)); } }
public void ConnectionExceptionOnOneShouldCommunicateBackWhichMessagesFailed() { //TODO is there a way to communicate back which client failed and which succeeded. var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn1.ProduceResponseFunction = () => { throw new ApplicationException("some exception"); }; var router = routerProxy.Create(); var producer = new Producer(router); var messages = new List<Message> { new Message{Value = "1"}, new Message{Value = "2"} }; //this will produce an exception, but message 1 succeeded and message 2 did not. //should we return a ProduceResponse with an error and no error for the other messages? //at this point though the client does not know which message is routed to which server. //the whole batch of messages would need to be returned. var test = producer.SendMessageAsync("UnitTest", messages).Result; }
public void ConsumerWithEmptyWhitelistShouldConsumeAllPartition() { var routerProxy = new BrokerRouterProxy(new MoqMockingKernel()); var router = routerProxy.Create(); var options = CreateOptions(router); options.PartitionWhitelist = new List<int>(); using (var consumer = new Consumer(options)) { var test = consumer.Consume(); TaskTest.WaitFor(() => consumer.ConsumerTaskCount > 0); TaskTest.WaitFor(() => routerProxy.BrokerConn0.FetchRequestCallCount > 0); TaskTest.WaitFor(() => routerProxy.BrokerConn1.FetchRequestCallCount > 0); Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2), "Consumer should create one consuming thread for each partition."); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1), "BrokerConn0 not sent FetchRequest"); Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.GreaterThanOrEqualTo(1), "BrokerConn1 not sent FetchRequest"); } }
public void CancellationShouldInterruptConsumption() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.FetchResponseFunction = () => { while (true) Thread.Yield(); }; var router = routerProxy.Create(); var options = CreateOptions(router); var consumer = new Consumer(options); var tokenSrc = new CancellationTokenSource(); var consumeTask = Task.Run(() => consumer.Consume(tokenSrc.Token).FirstOrDefault()); if (consumeTask.Wait(TimeSpan.FromSeconds(3))) Assert.Fail(); tokenSrc.Cancel(); Assert.That( Assert.Throws<AggregateException>(consumeTask.Wait).InnerException, Is.TypeOf<OperationCanceledException>()); }
public async Task ShouldRecoverFromFailerByUpdateMetadataOnce() //Do not debug this test !! { var log = new DefaultTraceLog(); var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(1000); var router = routerProxy.Create(); int partitionId = 0; ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); int numberOfCall = 100; long numberOfErrorSend = 0; TaskCompletionSource<int> x = new TaskCompletionSource<int>(); Func<Task<FetchResponse>> ShouldReturnNotLeaderForPartitionAndThenNoError = async () => { log.DebugFormat("FetchResponse Start "); if (!x.Task.IsCompleted) { if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall) { await Task.Delay(routerProxy._cacheExpiration); await Task.Delay(1); x.TrySetResult(1); log.DebugFormat("all is complete "); } await x.Task; log.DebugFormat("SocketException "); throw new BrokerConnectionException("",new KafkaEndpoint()); } log.DebugFormat("Completed "); return new FetchResponse() { Error = (short)ErrorResponseCode.NoError }; }; routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnNotLeaderForPartitionAndThenNoError; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; Task[] tasks = new Task[numberOfCall]; for (int i = 0; i < numberOfCall; i++) { tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); } await Task.WhenAll(tasks); Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend"); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend), "FetchRequestCallCount"); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount"); }
public void SelectPartitionShouldUsePartitionSelector(string testCase) { var key = testCase.ToIntSizedBytes(); var routerProxy = new BrokerRouterProxy(_kernel); _mockPartitionSelector.Setup(x => x.Select(It.IsAny<Topic>(), key)) .Returns(() => new Partition { ErrorCode = 0, Isrs = new List<int> { 1 }, PartitionId = 0, LeaderId = 0, Replicas = new List<int> { 1 }, }); routerProxy.PartitionSelector = _mockPartitionSelector.Object; var result = routerProxy.Create().SelectBrokerRoute(TestTopic, key); _mockPartitionSelector.Verify(f => f.Select(It.Is<Topic>(x => x.Name == TestTopic), key), Times.Once()); }
public async Task ShouldRecoverFromFailerByUpdateMetadataOnceFullScenario1() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(0); var router = routerProxy.Create(); int partitionId = 0; ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); CreateSuccessfulSendMock(routerProxy); //Send Successful Message await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount"); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount"); Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount"); routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(ErrorResponseCode.LeaderNotAvailable, TimeSpan.Zero); routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker; routerProxy.BrokerConn1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker; //Reset variables routerProxy.BrokerConn0.FetchRequestCallCount = 0; routerProxy.BrokerConn1.FetchRequestCallCount = 0; routerProxy.BrokerConn0.MetadataRequestCallCount = 0; routerProxy.BrokerConn1.MetadataRequestCallCount = 0; //Send Successful Message that was recover from exception await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount"); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount"); Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount"); Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount"); }
private void CreateSuccessfulSendMock(BrokerRouterProxy routerProxy) { routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; routerProxy.BrokerConn1.FetchResponseFunction = ShouldReturnValidMessage; routerProxy.BrokerConn1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; }
public void SelectPartitionShouldThrowWhenBrokerCollectionIsEmpty() { var metadataResponse = CreateMetaResponse(); metadataResponse.Brokers.Clear(); var routerProxy = new BrokerRouterProxy(_kernel); routerProxy.BrokerConn0.MetadataResponseFunction = () => metadataResponse; routerProxy.Create().SelectBrokerRoute(TestTopic); }
public async Task SendProtocolRequestShouldThrowException(Type exceptionType) { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); int partitionId = 0; ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); bool firstTime = true; Func<Task<FetchResponse>> ShouldReturnError = async () => { if (firstTime) { firstTime = !firstTime; object[] args = new object[1]; args[0] = "error Test"; throw (Exception)Activator.CreateInstance(exceptionType, args); } return new FetchResponse() { Error = (short)ErrorResponseCode.NoError }; }; routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse; try { await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); Assert.IsTrue(false, "Should throw exception"); } catch (Exception ex) { Assert.That(ex.GetType(), Is.EqualTo(exceptionType)); } }
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata( ErrorResponseCode code) { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); ProtocolGateway protocolGateway = new ProtocolGateway(router); routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration); routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId); }
public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(ErrorResponseCode code) { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); int partitionId = 0; ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); bool firstTime = true; Func<Task<FetchResponse>> ShouldReturnError = async () => { if (firstTime) { firstTime = !firstTime; return new FetchResponse() { Error = (short)code }; } return new FetchResponse() { Error = (short)ErrorResponseCode.NoError }; }; routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse; await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); }
public async Task ShouldRecoverUpdateMetadataForNewTopic() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); int partitionId = 0; ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); bool firstTime = true; Func<Task<FetchResponse>> ShouldReturnError = async () => { return new FetchResponse() { Error = (short)ErrorResponseCode.NoError }; }; routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse; int numberOfCall = 1000; Task[] tasks = new Task[numberOfCall]; for (int i = 0; i < numberOfCall / 2; i++) { tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId); } //change MetadataResponseFunction routerProxy.BrokerConn0.MetadataResponseFunction = async () => { var response = await BrokerRouterProxy.DefaultMetadataResponse(); response.Topics[0].Name = "test2"; return response; }; for (int i = 0; i < numberOfCall / 2; i++) { tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", partitionId); } await Task.WhenAll(tasks); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall)); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); }
public async Task SendProtocolRequestShouldThrowException(Type exceptionType) { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); ProtocolGateway protocolGateway = new ProtocolGateway(router); routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration); routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; try { await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId); Assert.IsTrue(false, "Should throw exception"); } catch (Exception ex) { Assert.That(ex.GetType(), Is.EqualTo(exceptionType)); } }
public void ConsumerShouldCreateTaskForEachBroker() { var routerProxy = new BrokerRouterProxy(new MoqMockingKernel()); routerProxy.BrokerConn0.FetchResponseFunction = () => { return new FetchResponse(); }; var router = routerProxy.Create(); var options = CreateOptions(router); options.PartitionWhitelist = new List<int>(); using (var consumer = new Consumer(options)) { var test = consumer.Consume(); TaskTest.WaitFor(() => consumer.ConsumerTaskCount >= 2); Assert.That(consumer.ConsumerTaskCount, Is.EqualTo(2)); } }
public async Task ShouldRecoverUpdateMetadataForNewTopic() { var routerProxy = new BrokerRouterProxy(_kernel); routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10); var router = routerProxy.Create(); ProtocolGateway protocolGateway = new ProtocolGateway(router); var fetchRequest = new FetchRequest(); routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage; routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers; int numberOfCall = 1000; Task[] tasks = new Task[numberOfCall]; for (int i = 0; i < numberOfCall / 2; i++) { tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, _partitionId); } routerProxy.BrokerConn0.MetadataResponseFunction = async () => { var response = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers(); response.Topics[0].Name = "test2"; return response; }; for (int i = 0; i < numberOfCall / 2; i++) { tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", _partitionId); } await Task.WhenAll(tasks); Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall)); Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2)); }