private static bool TopicExists(string topicName) { var config = new AdminClientConfig() { BootstrapServers = _server }; using (var client = new AdminClient(config)) { // Which call auto-creates the topic? var metaBroker = client.GetMetadata(TimeSpan.FromSeconds(3)); var metaTopic = client.GetMetadata(_topicName, TimeSpan.FromSeconds(3)); // Auto-creates topic. Broker setting? var resources = new List <ConfigResource>() { new ConfigResource() { Name = _topicName, Type = ResourceType.Topic } }; var options = new DescribeConfigsOptions() { RequestTimeout = TimeSpan.FromSeconds(3) }; var configTask = client.DescribeConfigsAsync(resources, options); var configResults = configTask.Result; } return(false); }
public static void AdminClient_DescribeConfigs(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start AdminClient_DescribeConfigs"); using (var adminClient = new AdminClient(new AdminClientConfig { BootstrapServers = bootstrapServers })) { // broker configs // --- var configResource = new ConfigResource { Name = "0", Type = ResourceType.Broker }; var results = adminClient.DescribeConfigsAsync(new List <ConfigResource> { configResource }).Result; Assert.Single(results); Assert.True(results[0].Entries.Count > 50); // note: unlike other parts of the api, Entries is kept as a dictionary since it's convenient for // the most typical use case. Assert.Single(results[0].Entries.Where(e => e.Key == "advertised.listeners")); Assert.Single(results[0].Entries.Where(e => e.Key == "num.network.threads")); var a = results.Select(aa => aa.Entries.Where(b => b.Value.Synonyms.Count > 0).ToList()).ToList(); // topic configs, more than one. // --- results = adminClient.DescribeConfigsAsync(new List <ConfigResource> { new ConfigResource { Name = singlePartitionTopic, Type = ResourceType.Topic }, new ConfigResource { Name = partitionedTopic, Type = ResourceType.Topic } }).Result; Assert.Equal(2, results.Count); Assert.True(results[0].Entries.Count > 20); Assert.True(results[1].Entries.Count > 20); Assert.Single(results[0].Entries.Where(e => e.Key == "compression.type")); Assert.Single(results[0].Entries.Where(e => e.Key == "flush.ms")); // options are specified. // --- results = adminClient.DescribeConfigsAsync(new List <ConfigResource> { configResource }, new DescribeConfigsOptions { RequestTimeout = TimeSpan.FromSeconds(10) }).Result; Assert.Single(results); Assert.True(results[0].Entries.Count > 20); // empty config resource // --- try { results = adminClient.DescribeConfigsAsync(new List <ConfigResource> { new ConfigResource() }).Result; Assert.True(false); } catch (ArgumentException) { // expected. } // invalid config resource // --- try { results = adminClient.DescribeConfigsAsync( new List <ConfigResource> { new ConfigResource { Name = "invalid.name.for.resource", Type = ResourceType.Broker } } ).Result; Assert.True(false); } catch (AggregateException ex) { Assert.True(ex.InnerException.GetType() == typeof(KafkaException)); var ace = (KafkaException)ex.InnerException; Assert.Contains("Expected an int32", ace.Message); } // invalid topic. // --- // // TODO: this creates the topic, then describes what it just created. what we want? does java explicitly not do this? // // results = adminClient.DescribeConfigsAsync(new List<ConfigResource> { // new ConfigResource { Name = "my-nonsense-topic", ResourceType = ConfigType.Topic } // }).Result; } Assert.Equal(0, Library.HandleCount); LogToFile("end AdminClient_DescribeConfigs"); }
public static void CancellationDelayMax(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start CancellationDelayMax"); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000, EnablePartitionEof = false, CancellationDelayMaxMs = 2 }; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers, CancellationDelayMaxMs = 2 }; var adminClientConfig = new AdminClientConfig { BootstrapServers = bootstrapServers, CancellationDelayMaxMs = 2 }; using (var topic = new TemporaryTopic(bootstrapServers, 3)) using (var consumer = new Consumer(consumerConfig)) using (var producer = new Producer(producerConfig)) using (var adminClient = new AdminClient(adminClientConfig)) { consumer.Subscribe(topic.Name); // for the consumer, check that the cancellation token is honored. for (int i = 0; i < 20; ++i) { var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(2)); var sw = Stopwatch.StartNew(); try { var record = consumer.Consume(cts.Token); } catch (OperationCanceledException) { // expected. } // 2ms + 2ms + quite a bit of leeway (but still much less than the default of 50). // in practice this should 4 almost all of the time. var elapsed = sw.ElapsedMilliseconds; Skip.If(elapsed > 8); } consumer.Close(); // for the producer, make do with just a simple check that this does not throw or hang. var dr = producer.ProduceAsync(topic.Name, new Message { Key = new byte[] { 42 }, Value = new byte[] { 255 } }).Result; // for the admin client, make do with just simple check that this does not throw or hang. var cr = new Confluent.Kafka.Admin.ConfigResource { Type = ResourceType.Topic, Name = topic.Name }; var configs = adminClient.DescribeConfigsAsync(new ConfigResource[] { cr }).Result; } Assert.Equal(0, Library.HandleCount); LogToFile("end CancellationDelayMax"); }
public static void AdminClient_AlterConfigs(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start AdminClient_AlterConfigs"); using (var adminClient = new AdminClient(new AdminClientConfig { BootstrapServers = bootstrapServers })) { // 1. create a new topic to play with. string topicName = Guid.NewGuid().ToString(); adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topicName, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); Thread.Sleep(TimeSpan.FromSeconds(1)); // without this, sometimes describe topic throws unknown topic/partition error. // 2. do an invalid alter configs call to change it. var configResource = new ConfigResource { Name = topicName, Type = ResourceType.Topic }; var toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> > { { configResource, new List <ConfigEntry> { new ConfigEntry { Name = "flush.ms", Value = "10001" }, new ConfigEntry { Name = "ubute.invalid.config", Value = "42" } } } }; try { adminClient.AlterConfigsAsync(toUpdate).Wait(); Assert.True(false); } catch (Exception e) { Assert.True(e.InnerException.GetType() == typeof(AlterConfigsException)); var ace = (AlterConfigsException)e.InnerException; Assert.Single(ace.Results); Assert.Contains("Unknown", ace.Results[0].Error.Reason); } // 3. test that in the failed alter configs call for the specified config resource, the // config that was specified correctly wasn't updated. List <DescribeConfigsResult> describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> { configResource }).Result; Assert.NotEqual("10001", describeConfigsResult[0].Entries["flush.ms"].Value); // 4. do a valid call, and check that the alteration did correctly happen. toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> > { { configResource, new List <ConfigEntry> { new ConfigEntry { Name = "flush.ms", Value = "10011" } } } }; adminClient.AlterConfigsAsync(toUpdate); describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> { configResource }).Result; Assert.Equal("10011", describeConfigsResult[0].Entries["flush.ms"].Value); // 4. test ValidateOnly = true does not update config entry. toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> > { { configResource, new List <ConfigEntry> { new ConfigEntry { Name = "flush.ms", Value = "20002" } } } }; adminClient.AlterConfigsAsync(toUpdate, new AlterConfigsOptions { ValidateOnly = true }).Wait(); describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> { configResource }).Result; Assert.Equal("10011", describeConfigsResult[0].Entries["flush.ms"].Value); // 5. test updating broker resource. toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> > { { new ConfigResource { Name = "0", Type = ResourceType.Broker }, new List <ConfigEntry> { new ConfigEntry { Name = "num.network.threads", Value = "2" } } } }; adminClient.AlterConfigsAsync(toUpdate).Wait(); // 6. test updating more than one resource. string topicName2 = Guid.NewGuid().ToString(); adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topicName2, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); Thread.Sleep(TimeSpan.FromSeconds(1)); // without this, sometimes describe topic throws unknown topic/partition error. var configResource2 = new ConfigResource { Name = topicName2, Type = ResourceType.Topic }; toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> > { { configResource, new List <ConfigEntry> { new ConfigEntry { Name = "flush.ms", Value = "222" } } }, { configResource2, new List <ConfigEntry> { new ConfigEntry { Name = "flush.ms", Value = "333" } } } }; adminClient.AlterConfigsAsync(toUpdate).Wait(); describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> { configResource, configResource2 }).Result; Assert.Equal(2, describeConfigsResult.Count); Assert.Equal("222", describeConfigsResult[0].Entries["flush.ms"].Value); Assert.Equal("333", describeConfigsResult[1].Entries["flush.ms"].Value); } Assert.Equal(0, Library.HandleCount); LogToFile("end AdminClient_AlterConfigs"); }