public void WatermarkOffsets(string bootstrapServers) { LogToFile("start WatermarkOffsets"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var testString = "hello world"; DeliveryResult <Null, string> dr; using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> { Value = testString }).Result; Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary. } var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000 }; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new List <TopicPartitionOffset>() { dr.TopicPartitionOffset }); var record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition); Assert.Equal(getOffsets.Low, Offset.Unset); // the offset of the next message to be read. Assert.Equal(getOffsets.High, dr.Offset + 1); var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20)); Assert.NotEqual(queryOffsets.Low, Offset.Unset); Assert.Equal(getOffsets.High, queryOffsets.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end WatermarkOffsets"); }
private Metadata.IResponse HandleMetadataRequest(Metadata.IRequest req) { switch (req) { case Metadata.ListTopics _: return(new Metadata.Topics(Try <List <TopicMetadata> > .From(() => { using (var adminClient = new DependentAdminClientBuilder(_consumer.Handle).Build()) { return adminClient.GetMetadata(_settings.MetadataRequestTimeout).Topics; } }))); default: throw new InvalidOperationException($"Unknown metadata request: {req}"); } }
public void AddBrokers(string bootstrapServers) { var producerConfig = new ProducerConfig { BootstrapServers = "localhost:65533" }; using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { try { var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3)); Assert.True(false, "Broker should not be reached here"); } catch (KafkaException e) { Assert.Equal(ErrorCode.Local_Transport, e.Error.Code); } // test is > 0 note == 1 since bootstrapServers could include more than one broker. int brokersAdded = adminClient.AddBrokers(bootstrapServers); Assert.True(brokersAdded > 0, "Should have added one broker or more"); brokersAdded = adminClient.AddBrokers(bootstrapServers); Assert.True(brokersAdded > 0, "Should have added one broker or more (duplicates considered added)"); var newMetadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3)); Assert.True(newMetadata.Brokers.Count >= 1); brokersAdded = adminClient.AddBrokers(""); Assert.True(brokersAdded == 0, "Should not have added brokers"); newMetadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3)); Assert.True(newMetadata.Brokers.Count > 0); } }
public void Producer_Handles(string bootstrapServers) { LogToFile("start Producer_Handles"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) { using (var producer1 = new ProducerBuilder <byte[], byte[]>(producerConfig).Build()) using (var producer2 = new DependentProducerBuilder <string, string>(producer1.Handle).Build()) using (var producer3 = new DependentProducerBuilder <byte[], byte[]>(producer1.Handle).Build()) using (var producer4 = new DependentProducerBuilder <int, string>(producer2.Handle).Build()) using (var producer5 = new DependentProducerBuilder <int, int>(producer3.Handle).Build()) using (var producer6 = new DependentProducerBuilder <string, byte[]>(producer4.Handle).Build()) using (var producer7 = new ProducerBuilder <double, double>(producerConfig).Build()) using (var adminClient = new DependentAdminClientBuilder(producer7.Handle).Build()) { var r1 = producer1.ProduceAsync(topic.Name, new Message <byte[], byte[]> { Key = new byte[] { 42 }, Value = new byte[] { 33 } }).Result; Assert.Equal(new byte[] { 42 }, r1.Key); Assert.Equal(new byte[] { 33 }, r1.Value); Assert.Equal(0, r1.Offset); var r2 = producer2.ProduceAsync(topic.Name, new Message <string, string> { Key = "hello", Value = "world" }).Result; Assert.Equal("hello", r2.Key); Assert.Equal("world", r2.Value); var r3 = producer3.ProduceAsync(topic.Name, new Message <byte[], byte[]> { Key = new byte[] { 40 }, Value = new byte[] { 31 } }).Result; Assert.Equal(new byte[] { 40 }, r3.Key); Assert.Equal(new byte[] { 31 }, r3.Value); var r4 = producer4.ProduceAsync(topic.Name, new Message <int, string> { Key = 42, Value = "mellow world" }).Result; Assert.Equal(42, r4.Key); Assert.Equal("mellow world", r4.Value); var r5 = producer5.ProduceAsync(topic.Name, new Message <int, int> { Key = int.MaxValue, Value = int.MinValue }).Result; Assert.Equal(int.MaxValue, r5.Key); Assert.Equal(int.MinValue, r5.Value); var r6 = producer6.ProduceAsync(topic.Name, new Message <string, byte[]> { Key = "yellow mould", Value = new byte[] { 69 } }).Result; Assert.Equal("yellow mould", r6.Key); Assert.Equal(new byte[] { 69 }, r6.Value); var r7 = producer7.ProduceAsync(topic.Name, new Message <double, double> { Key = 44.0, Value = 234.4 }).Result; Assert.Equal(44.0, r7.Key); Assert.Equal(234.4, r7.Value); var topicMetadata = adminClient.GetMetadata(singlePartitionTopic, TimeSpan.FromSeconds(10)); Assert.Single(topicMetadata.Topics); // implicitly check this does not throw. } var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString() }; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0)); var r1 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(new byte[] { 42 }, r1.Key); Assert.Equal(new byte[] { 33 }, r1.Value); Assert.Equal(0, r1.Offset); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 1)); var r2 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal("hello", r2.Key); Assert.Equal("world", r2.Value); Assert.Equal(1, r2.Offset); } using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 2)); var r3 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(new byte[] { 40 }, r3.Key); Assert.Equal(new byte[] { 31 }, r3.Value); Assert.Equal(2, r3.Offset); } using (var consumer = new ConsumerBuilder <int, string>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 3)); var r4 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(42, r4.Key); Assert.Equal("mellow world", r4.Value); Assert.Equal(3, r4.Offset); } using (var consumer = new ConsumerBuilder <int, int>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 4)); var r5 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(int.MaxValue, r5.Key); Assert.Equal(int.MinValue, r5.Value); Assert.Equal(4, r5.Offset); } using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 5)); var r6 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal("yellow mould", r6.Key); Assert.Equal(new byte[] { 69 }, r6.Value); Assert.Equal(5, r6.Offset); } using (var consumer = new ConsumerBuilder <double, double>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 6)); var r7 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(44.0, r7.Key); Assert.Equal(234.4, r7.Value); Assert.Equal(6, r7.Offset); } } Assert.Equal(0, Library.HandleCount); LogToFile("end Producer_Handles"); }
public async void AdminClient_AclOperations(string bootstrapServers) { LogToFile("start AdminClient_AclOperations"); var topicName = Guid.NewGuid().ToString(); var groupName = Guid.NewGuid().ToString(); var maxDuration = TimeSpan.FromSeconds(30); var noError = new Error(ErrorCode.NoError, "", false); var unknownError = new Error(ErrorCode.Unknown, "Unknown broker error", false); var noErrorCreateResult = new CreateAclReport { Error = noError }; var unknownErrorCreateResult = new CreateAclReport { Error = unknownError }; var newACLs = new List <AclBinding> { new AclBinding() { Pattern = new ResourcePattern { Type = ResourceType.Topic, Name = topicName, ResourcePatternType = ResourcePatternType.Literal }, Entry = new AccessControlEntry { Principal = "User:test-user-1", Host = "*", Operation = AclOperation.Read, PermissionType = AclPermissionType.Allow } }, new AclBinding() { Pattern = new ResourcePattern { Type = ResourceType.Topic, Name = topicName, ResourcePatternType = ResourcePatternType.Prefixed }, Entry = new AccessControlEntry { Principal = "User:test-user-2", Host = "*", Operation = AclOperation.Write, PermissionType = AclPermissionType.Deny } }, new AclBinding() { Pattern = new ResourcePattern { Type = ResourceType.Group, Name = groupName, ResourcePatternType = ResourcePatternType.Prefixed }, Entry = new AccessControlEntry { Principal = "User:test-user-2", Host = "some-host", Operation = AclOperation.All, PermissionType = AclPermissionType.Allow } }, }; var invalidACLs = new List <AclBinding> { new AclBinding() { Pattern = new ResourcePattern { Type = ResourceType.Topic, Name = topicName, ResourcePatternType = ResourcePatternType.Literal }, Entry = new AccessControlEntry { // Principal must be in the form "{principalType}:{principalName}" // Broker returns ErrUnknown in this case Principal = "wrong-principal", Host = "*", Operation = AclOperation.Read, PermissionType = AclPermissionType.Allow } } }; var validAndInvalidACLs = new List <AclBinding> { new AclBinding() { Pattern = new ResourcePattern { Type = ResourceType.Topic, Name = topicName, ResourcePatternType = ResourcePatternType.Literal }, Entry = new AccessControlEntry { // Principal must be in the form "{principalType}:{principalName}" // Broker returns ErrUnknown in this case Principal = "wrong-principal", Host = "*", Operation = AclOperation.Read, PermissionType = AclPermissionType.Allow } }, new AclBinding() { Pattern = new ResourcePattern { Type = ResourceType.Group, Name = groupName, ResourcePatternType = ResourcePatternType.Prefixed }, Entry = new AccessControlEntry { Principal = "User:test-user-2", Host = "some-host", Operation = AclOperation.All, PermissionType = AclPermissionType.Allow } }, }; var aclBindingFilters = new List <AclBindingFilter> { new AclBindingFilter() { PatternFilter = new ResourcePatternFilter { Type = ResourceType.Any, ResourcePatternType = ResourcePatternType.Any }, EntryFilter = new AccessControlEntryFilter { Operation = AclOperation.Any, PermissionType = AclPermissionType.Any } }, new AclBindingFilter() { PatternFilter = new ResourcePatternFilter { Type = ResourceType.Any, ResourcePatternType = ResourcePatternType.Prefixed }, EntryFilter = new AccessControlEntryFilter { Operation = AclOperation.Any, PermissionType = AclPermissionType.Any } }, new AclBindingFilter() { PatternFilter = new ResourcePatternFilter { Type = ResourceType.Topic, ResourcePatternType = ResourcePatternType.Any }, EntryFilter = new AccessControlEntryFilter { Operation = AclOperation.Any, PermissionType = AclPermissionType.Any } }, new AclBindingFilter() { PatternFilter = new ResourcePatternFilter { Type = ResourceType.Group, ResourcePatternType = ResourcePatternType.Any }, EntryFilter = new AccessControlEntryFilter { Operation = AclOperation.Any, PermissionType = AclPermissionType.Any } } }; var createAclsOptions = new CreateAclsOptions { RequestTimeout = maxDuration }; var describeAclsOptions = new DescribeAclsOptions { RequestTimeout = maxDuration }; var deleteAclsOptions = new DeleteAclsOptions { RequestTimeout = maxDuration }; // - construction of admin client from configuration. // - creation of multiple ACL. // - CreateAcls should be idempotent using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build()) { for (int i = 0; i < 2; ++i) { await adminClient.CreateAclsAsync( newACLs, createAclsOptions ); } } // - construction of admin client from a producer handle // - CreateACLs with server side validation errors using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { var createAclsException = await Assert.ThrowsAsync <CreateAclsException>(() => adminClient.CreateAclsAsync( invalidACLs, createAclsOptions ) ); Assert.Equal(new CreateAclsException( new List <CreateAclReport> { unknownErrorCreateResult } ), createAclsException); } // - construction of admin client from a producer handle // - CreateACLs with errors and succeeded items using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { var createAclsException = await Assert.ThrowsAsync <CreateAclsException>(() => adminClient.CreateAclsAsync( validAndInvalidACLs, createAclsOptions ) ); Assert.Equal(new CreateAclsException( new List <CreateAclReport> { unknownErrorCreateResult, noErrorCreateResult } ), createAclsException); // DescribeAcls must return the three ACLs var describeAclsResult = await adminClient.DescribeAclsAsync(aclBindingFilters[0], describeAclsOptions); var newACLsShuffled = new List <AclBinding> { newACLs[0], newACLs[2], newACLs[1], }; Assert.Equal(new DescribeAclsResult { AclBindings = newACLs }, describeAclsResult); Assert.Equal(new DescribeAclsResult { AclBindings = newACLsShuffled }, describeAclsResult); } // - construction of admin client from configuration. using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build()) { // DeleteAcls with ResourcePatternType Prefixed var resultDeleteAcls = await adminClient.DeleteAclsAsync( new List <AclBindingFilter> { aclBindingFilters[1] }, deleteAclsOptions ); Assert.Single(resultDeleteAcls); Assert.Equal(2, resultDeleteAcls[0].AclBindings.Count); Assert.Equal(new DeleteAclsResult { AclBindings = new List <AclBinding> { newACLs[1], newACLs[2] } }, resultDeleteAcls[0]); // DeleteAcls with ResourceType Topic and Group resultDeleteAcls = await adminClient.DeleteAclsAsync( new List <AclBindingFilter> { aclBindingFilters[2], aclBindingFilters[3], }, deleteAclsOptions ); Assert.Equal(2, resultDeleteAcls.Count); Assert.Equal(new DeleteAclsResult { AclBindings = new List <AclBinding> { newACLs[0] } }, resultDeleteAcls[0]); Assert.Equal(new DeleteAclsResult { AclBindings = new List <AclBinding>() }, resultDeleteAcls[1]); // All the ACLs should have been deleted var describeAclsResult = await adminClient.DescribeAclsAsync(aclBindingFilters[0], describeAclsOptions); Assert.Equal(new DescribeAclsResult { AclBindings = new List <AclBinding> { } }, describeAclsResult); } Assert.Equal(0, Library.HandleCount); LogToFile("end AdminClient_AclOperations"); }
public void AdminClient_CreateTopics(string bootstrapServers) { LogToFile("start AdminClient_CreateTopics"); var topicName1 = Guid.NewGuid().ToString(); var topicName2 = Guid.NewGuid().ToString(); var topicName3 = Guid.NewGuid().ToString(); var topicName4 = Guid.NewGuid().ToString(); var topicName5 = Guid.NewGuid().ToString(); // test // - construction of admin client from configuration. // - creation of more than one topic. using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build()) { adminClient.CreateTopicsAsync( new TopicSpecification[] { new TopicSpecification { Name = topicName1, NumPartitions = 2, ReplicationFactor = 1 }, new TopicSpecification { Name = topicName2, NumPartitions = 12, ReplicationFactor = 1 } } ).Wait(); } // test // - construction of admin client from a producer handle // - creation of topic // - producing to created topics works. using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient2 = new DependentAdminClientBuilder(producer.Handle).Build()) { adminClient2.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topicName3, NumPartitions = 24, ReplicationFactor = 1 } }).Wait(); var deliveryReport1 = producer.ProduceAsync(topicName1, new Message <Null, Null>()).Result; var deliveryReport2 = producer.ProduceAsync(topicName2, new Message <Null, Null>()).Result; var deliveryReport3 = producer.ProduceAsync(topicName3, new Message <Null, Null>()).Result; Assert.Equal(topicName1, deliveryReport1.Topic); Assert.Equal(topicName2, deliveryReport2.Topic); Assert.Equal(topicName3, deliveryReport3.Topic); } // test // - create topic with same name as existing topic // - as well as another topic that does exist (and for which create should succeed). using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build()) { try { adminClient.CreateTopicsAsync(new List <TopicSpecification> { new TopicSpecification { Name = topicName3, NumPartitions = 1, ReplicationFactor = 1 }, new TopicSpecification { Name = topicName4, NumPartitions = 1, ReplicationFactor = 1 } } ).Wait(); Assert.True(false, "Expect CreateTopics request to throw an exception."); } // if awaited, the CreateTopicsException is not wrapped. // this is an annoyance if used synchronously, but not atypical. catch (AggregateException ex) { Assert.True(ex.InnerException.GetType() == typeof(CreateTopicsException)); var cte = (CreateTopicsException)ex.InnerException; Assert.Equal(2, cte.Results.Count); Assert.Single(cte.Results.Where(r => r.Error.IsError)); Assert.Single(cte.Results.Where(r => !r.Error.IsError)); Assert.Equal(topicName3, cte.Results.Where(r => r.Error.IsError).First().Topic); Assert.Equal(topicName4, cte.Results.Where(r => !r.Error.IsError).First().Topic); } } // test // - validate only using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build()) { adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topicName5, NumPartitions = 1, ReplicationFactor = 1 } }, new CreateTopicsOptions { ValidateOnly = true, RequestTimeout = TimeSpan.FromSeconds(30) } ).Wait(); // creating for real shouldn't throw exception. adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topicName5, NumPartitions = 1, ReplicationFactor = 1 } } ).Wait(); } Assert.Equal(0, Library.HandleCount); LogToFile("end AdminClient_CreateTopics"); }
public void Start() { var logContext = $"{nameof(Start)} for SubscriberId={_subscriberId}"; try { IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(_consumerProperties).Build(); var processor = new KafkaMessageProcessor(_subscriberId, _handler, _loggerFactory.CreateLogger <KafkaMessageProcessor>()); using (IAdminClient adminClient = new DependentAdminClientBuilder(consumer.Handle).Build()) { foreach (string topic in _topics) { VerifyTopicExistsBeforeSubscribing(adminClient, topic); } } List <string> topicsList = new List <string>(_topics); _logger.LogDebug($"{logContext}: Subscribing to topics='{String.Join(",", topicsList)}'"); consumer.Subscribe(topicsList); // Set state to started before starting the processing thread instead of after as in the Java code // (prevent setting it to it started after it has potentially already been set to stopped) _state = EventuateKafkaConsumerState.Started; Task.Run(() => { try { while (!_cancellationTokenSource.IsCancellationRequested) { try { ConsumeResult <string, string> record = consumer.Consume(TimeSpan.FromMilliseconds(ConsumePollMilliseconds)); if (record != null) { _logger.LogDebug( $"{logContext}: process record at offset='{record.Offset}', key='{record.Key}', value='{record.Value}'"); processor.Process(record); } else { processor.ThrowExceptionIfHandlerFailed(); } MaybeCommitOffsets(consumer, processor); } catch (ConsumeException e) { _logger.LogError($"{logContext}: ConsumeException - {e.Error}. Continuing."); } } _state = EventuateKafkaConsumerState.Stopped; } catch (TaskCanceledException) { _logger.LogInformation($"{logContext}: Shutdown by cancel"); _state = EventuateKafkaConsumerState.Stopped; } catch (KafkaMessageProcessorFailedException e) { _logger.LogError($"{logContext}: Terminating due to KafkaMessageProcessorFailedException - {e}"); _state = EventuateKafkaConsumerState.MessageHandlingFailed; } catch (Exception e) { _logger.LogError($"{logContext}: Exception - {e}"); _state = EventuateKafkaConsumerState.Failed; // Java throws here, but seems like it isn't necessary } finally { // Try to put the last of the offsets away. Note that the // callbacks are done asynchronously so there is no guarantee // that all the offsets are ready. Worst case is that there // are messages processed more than once. MaybeCommitOffsets(consumer, processor); consumer.Dispose(); _logger.LogDebug($"{logContext}: Stopped in state {_state.ToString()}"); } }, _cancellationTokenSource.Token); } catch (Exception e) { _logger.LogError(e, $"{logContext}: Error subscribing"); _state = EventuateKafkaConsumerState.FailedToStart; throw; } }
public void AdminClient_CreatePartitions(string bootstrapServers) { LogToFile("start AdminClient_CreatePartitions"); var topicName1 = Guid.NewGuid().ToString(); var topicName2 = Guid.NewGuid().ToString(); var topicName3 = Guid.NewGuid().ToString(); var topicName4 = Guid.NewGuid().ToString(); var topicName5 = Guid.NewGuid().ToString(); var topicName6 = Guid.NewGuid().ToString(); // test creating a new partition works. using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topicName1, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> { new PartitionsSpecification { Topic = topicName1, IncreaseTo = 2 } }).Wait(); var dr1 = producer.ProduceAsync(new TopicPartition(topicName1, 0), new Message <Null, Null>()).Result; var dr2 = producer.ProduceAsync(new TopicPartition(topicName1, 1), new Message <Null, Null>()).Result; try { producer.ProduceAsync(new TopicPartition(topicName1, 2), new Message <Null, Null>()).Wait(); Assert.True(false, "expecting exception"); } catch (AggregateException ex) { Assert.IsType <ProduceException <Null, Null> >(ex.InnerException); Assert.True(((ProduceException <Null, Null>)ex.InnerException).Error.IsError); } } // check validate only works. using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topicName2, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> { new PartitionsSpecification { Topic = topicName2, IncreaseTo = 10 } }, new CreatePartitionsOptions { ValidateOnly = true }).Wait(); // forces a metadata request. var dr1 = producer.ProduceAsync(new TopicPartition(topicName2, 0), new Message <Null, Null>()).Result; try { // since we have metadata, this throws immediately (i.e. not wrapped in AggregateException) var dr2 = producer.ProduceAsync(new TopicPartition(topicName2, 1), new Message <Null, Null>()).Result; Assert.True(false, "expecting exception"); } catch (AggregateException ex) { Assert.IsType <ProduceException <Null, Null> >(ex.InnerException); Assert.True(((ProduceException <Null, Null>)ex.InnerException).Error.IsError); } } // check valid Assignments property value works. using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topicName3, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); adminClient.CreatePartitionsAsync( new List <PartitionsSpecification> { new PartitionsSpecification { Topic = topicName2, IncreaseTo = 2, ReplicaAssignments = new List <List <int> > { new List <int> { 0 } } } }, new CreatePartitionsOptions { ValidateOnly = true } ).Wait(); } // check invalid Assignments property value works. using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topicName4, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); try { adminClient.CreatePartitionsAsync( new List <PartitionsSpecification> { new PartitionsSpecification { Topic = topicName2, IncreaseTo = 2, ReplicaAssignments = new List <List <int> > { new List <int> { 42 } } } }, new CreatePartitionsOptions { ValidateOnly = true } ).Wait(); Assert.True(false, "Expecting exception"); } catch (AggregateException ex) { Assert.True(ex.InnerException.GetType() == typeof(CreatePartitionsException)); var cpe = (CreatePartitionsException)ex.InnerException; Assert.Single(cpe.Results); Assert.True(cpe.Results.First().Error.IsError); } } // more than one. using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build()) { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topicName5, NumPartitions = 1, ReplicationFactor = 1 }, new TopicSpecification { Name = topicName6, NumPartitions = 1, ReplicationFactor = 1 } } ).Wait(); Thread.Sleep(TimeSpan.FromSeconds(1)); // just a simple check there wasn't an exception. adminClient.CreatePartitionsAsync( new List <PartitionsSpecification> { new PartitionsSpecification { Topic = topicName5, IncreaseTo = 2 }, new PartitionsSpecification { Topic = topicName6, IncreaseTo = 3 } } ).Wait(); } Assert.Equal(0, Library.HandleCount); LogToFile("end AdminClient_CreatePartitions"); }
public void AdminClient_NullReferenceChecks(string bootstrapServers) { LogToFile("start AdminClient_NullReferenceChecks"); var topicName1 = Guid.NewGuid().ToString(); string nullTopic = null; Exception createTopicsException = null; Exception createPartitionsException = null; // test creating a null topic throws a related exception using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { try { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = nullTopic, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); Assert.True(false, "Expected exception."); } catch (ArgumentException ex) { Assert.Contains("topic", ex.Message.ToLower()); createTopicsException = ex; } } // test creating a partition with null topic throws exception using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { try { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = topicName1, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> { new PartitionsSpecification { Topic = nullTopic, IncreaseTo = 2 } }).Wait(); Assert.True(false, "Expected exception."); } catch (ArgumentException ex) { Assert.Contains("topic", ex.Message.ToLower()); createPartitionsException = ex; } } Assert.True(createTopicsException != null && createPartitionsException != null); Assert.True(createTopicsException.GetType() == createPartitionsException.GetType(), ".CreateTopic and .CreatePartition should have consistent interface for null-related exceptions."); // test adding a null list of brokers throws null reference exception. using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { try { adminClient.AddBrokers(null); Assert.True(false, "Expected exception."); } catch (ArgumentNullException ex) { Assert.Contains("broker", ex.Message.ToLower()); } } // test retrieving metadata for a null topic using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { try { adminClient.GetMetadata(null, TimeSpan.FromSeconds(10)); Assert.True(false, "Expected exception."); } catch (ArgumentNullException ex) { Assert.Contains("value cannot be null", ex.Message.ToLower()); } } // Deleting null topic throws exception using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { try { adminClient.DeleteTopicsAsync(new List <string> { topicName1, nullTopic }); Assert.True(false, "Expected exception."); } catch (ArgumentException ex) { Assert.Contains("topic", ex.Message); } } // ListGroup throws exception if group is null using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new DependentAdminClientBuilder(producer.Handle).Build()) { try { adminClient.ListGroup(null, TimeSpan.FromSeconds(10)); Assert.True(false, "Expected exception."); } catch (ArgumentNullException ex) { Assert.Contains("group", ex.Message); } } Assert.Equal(0, Library.HandleCount); LogToFile("end AdminClient_NullReferenceChecks"); }