private static bool TopicExists(string topicName)
{
var config = new AdminClientConfig()
{
BootstrapServers = _server
};
using (var client = new AdminClient(config))
{
// Which call auto-creates the topic?
var metaBroker = client.GetMetadata(TimeSpan.FromSeconds(3));
var metaTopic = client.GetMetadata(_topicName, TimeSpan.FromSeconds(3)); // Auto-creates topic. Broker setting?
var resources = new List <ConfigResource>()
{
new ConfigResource()
{
Name = _topicName, Type = ResourceType.Topic
}
};
var options = new DescribeConfigsOptions()
{
RequestTimeout = TimeSpan.FromSeconds(3)
};
var configTask = client.DescribeConfigsAsync(resources, options);
var configResults = configTask.Result;
}
return(false);
}
public static void Metadata(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Metadata");
var config = new AdminClientConfig {
BootstrapServers = bootstrapServers
};
using (var adminClient = new AdminClient(config))
{
var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(10));
Assert.NotNull(metadata.Brokers);
Assert.True(metadata.Brokers.Count > 0);
var metadataAsJson = metadata.ToString();
var deserialized = (JObject)Newtonsoft.Json.JsonConvert.DeserializeObject(metadataAsJson);
Assert.Equal(deserialized.Value <int>("OriginatingBrokerId"), metadata.OriginatingBrokerId);
Assert.Equal(deserialized.Value <string>("OriginatingBrokerName"), metadata.OriginatingBrokerName);
var topics = new List <JToken>(deserialized["Topics"].Children());
Assert.Equal(metadata.Topics.Count, topics.Count);
for (int i = 0; i < metadata.Topics.Count; ++i)
{
Assert.Equal(topics[i].Value <string>("Error"), metadata.Topics[i].Error.Code.ToString());
Assert.Equal(topics[i].Value <string>("Topic"), metadata.Topics[i].Topic);
var partitions = new List <JToken>(topics[i]["Partitions"].Children());
Assert.Equal(partitions.Count, metadata.Topics[i].Partitions.Count);
for (int j = 0; j < metadata.Topics[i].Partitions.Count; ++j)
{
Assert.Equal(partitions[j].Value <string>("Error"), metadata.Topics[i].Partitions[j].Error.Code.ToString());
Assert.Equal(partitions[j].Value <int>("Leader"), metadata.Topics[i].Partitions[j].Leader);
Assert.Equal(partitions[j].Value <int>("PartitionId"), metadata.Topics[i].Partitions[j].PartitionId);
var replicas = new List <JToken>(partitions[j]["Replicas"].Children());
Assert.Equal(replicas.Count, metadata.Topics[i].Partitions[j].Replicas.Length);
for (int k = 0; k < metadata.Topics[i].Partitions[j].Replicas.Length; ++k)
{
Assert.Equal(replicas[k].Value <int>(), metadata.Topics[i].Partitions[j].Replicas[k]);
}
var inSyncReplicas = new List <JToken>(partitions[j]["InSyncReplicas"].Children());
Assert.Equal(inSyncReplicas.Count, metadata.Topics[i].Partitions[j].InSyncReplicas.Length);
for (int k = 0; k < metadata.Topics[i].Partitions[j].InSyncReplicas.Length; ++k)
{
Assert.Equal(inSyncReplicas[k].Value <int>(), metadata.Topics[i].Partitions[j].InSyncReplicas[k]);
}
}
}
var brokers = new List <JToken>(deserialized["Brokers"].Children());
Assert.Equal(metadata.Brokers.Count, brokers.Count);
for (int i = 0; i < metadata.Brokers.Count; ++i)
{
Assert.Equal(metadata.Brokers[i].BrokerId, brokers[i].Value <int>("BrokerId"));
Assert.Equal(metadata.Brokers[i].Host, brokers[i].Value <string>("Host"));
Assert.Equal(metadata.Brokers[i].Port, brokers[i].Value <int>("Port"));
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Metadata");
}
private static void CreateTopic(IConfigurationRoot config)
{
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = config["BootstrapServers"]
};
var adminClient = new AdminClient(adminClientConfig);
var topic = config["TopicName"];
var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(10));
if (metadata.Topics.Any(x => x.Topic == topic))
{
return;
}
var topicConfig = config.GetSection("TopicConfig").GetChildren().AsEnumerable()
.ToDictionary(x => x.Key, x => x.Value);
adminClient.CreateTopicsAsync(new[]
{
new TopicSpecification
{
Name = topic,
NumPartitions = int.Parse(config["NumPartitions"]),
ReplicationFactor = short.Parse(config["ReplicationFactor"]),
Configs = topicConfig
}
}).Wait();
}
public void AddBrokers(string bootstrapServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = "localhost:65533"
};
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
using (var adminClient = new AdminClient(producer.Handle))
{
try
{
var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3));
Assert.True(false, "Broker should not be reached here");
}
catch (KafkaException e)
{
Assert.Equal(ErrorCode.Local_Transport, e.Error.Code);
}
// test is > 0 note == 1 since bootstrapServers could include more than one broker.
int brokersAdded = adminClient.AddBrokers(bootstrapServers);
Assert.True(brokersAdded > 0, "Should have added one broker or more");
brokersAdded = adminClient.AddBrokers(bootstrapServers);
Assert.True(brokersAdded > 0, "Should have added one broker or more (duplicates considered added)");
var newMetadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3));
Assert.True(newMetadata.Brokers.Count == 1);
brokersAdded = adminClient.AddBrokers("");
Assert.True(brokersAdded == 0, "Should not have added brokers");
newMetadata = adminClient.GetMetadata(TimeSpan.FromSeconds(3));
Assert.True(newMetadata.Brokers.Count > 0);
}
}
private static void DeleteTopic(IConfigurationRoot config)
{
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = config["BootstrapServers"]
};
var adminClient = new AdminClient(adminClientConfig);
var topic = config["TopicName"];
var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(10));
if (metadata.Topics.Any(x => x.Topic == topic))
{
adminClient.DeleteTopicsAsync(new[] { topic }).Wait();
}
}
private static void Consume()
{
var conf = new Dictionary <string, string>
{
{ "group.id", "test-consumer-group" },
{ "bootstrap.servers", "localhost:9092" },
{ "enable.auto.commit", "false" },
//{ "auto.commit.interval.ms", 5000 },
// { "auto.offset.reset", "earliest" }
};
using (var consumer = new Consumer <string, string>(conf))
using (var admin = new AdminClient(consumer.Handle))
{
Console.WriteLine($@"Partition IDs: {
string.Join(',',
admin
.GetMetadata(TimeSpan.FromMilliseconds(1000))
.Topics
.First(t => t.Topic.Contains("my-topic"))
.Partitions
.Select(x => x.PartitionId))
}"
);
consumer.OnPartitionEOF += (_, end)
=> Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
consumer.Assign(new TopicPartitionOffset("my-topic", 1, Offset.Beginning));
//consumer.Subscribe("my-topic");
while (true)
{
var message = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (message != null)
{
Console.WriteLine(message.Value);
}
}
}
}
static void PrintMetadata(string brokerList)
{
using (var adminClient = new AdminClient(new AdminClientConfig {
BootstrapServers = brokerList
}))
{
var meta = adminClient.GetMetadata(TimeSpan.FromSeconds(20));
Console.WriteLine($"{meta.OriginatingBrokerId} {meta.OriginatingBrokerName}");
meta.Brokers.ForEach(broker =>
Console.WriteLine($"Broker: {broker.BrokerId} {broker.Host}:{broker.Port}"));
meta.Topics.ForEach(topic =>
{
Console.WriteLine($"Topic: {topic.Topic} {topic.Error}");
topic.Partitions.ForEach(partition =>
{
Console.WriteLine($" Partition: {partition.PartitionId}");
Console.WriteLine($" Replicas: {ToString(partition.Replicas)}");
Console.WriteLine($" InSyncReplicas: {ToString(partition.InSyncReplicas)}");
});
});
}
}
private IDictionary <string, QueueProperties> GetQueuesProperties()
{
var config = _options.ToAdminProperties();
try
{
using (var admin = new AdminClient(config))
{
var meta = admin.GetMetadata(_options.AdminRequestTimeout);
var props = from kafkaTopic in meta.Topics
join userTopic in _options.Topics on kafkaTopic.Topic equals userTopic
from partition in kafkaTopic.Partitions
select new QueueProperties(userTopic, (uint)partition.PartitionId);
return(props.ToDictionary(prop => prop.QueueName));
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to retrieve Kafka meta data. {@config}", config);
throw;
}
}
public static void Producer_Handles(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Producer_Handles");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer1 = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var producer2 = new DependentProducerBuilder <string, string>(producer1.Handle).Build())
using (var producer3 = new DependentProducerBuilder <byte[], byte[]>(producer1.Handle).Build())
using (var producer4 = new DependentProducerBuilder <int, string>(producer2.Handle).Build())
using (var producer5 = new DependentProducerBuilder <int, int>(producer3.Handle).Build())
using (var producer6 = new DependentProducerBuilder <string, byte[]>(producer4.Handle).Build())
using (var producer7 = new ProducerBuilder <double, double>(producerConfig).Build())
using (var adminClient = new AdminClient(producer7.Handle))
{
var r1 = producer1.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 42 }, Value = new byte[] { 33 }
}).Result;
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
var r2 = producer2.ProduceAsync(topic.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Result;
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
var r3 = producer3.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 40 }, Value = new byte[] { 31 }
}).Result;
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
var r4 = producer4.ProduceAsync(topic.Name, new Message <int, string> {
Key = 42, Value = "mellow world"
}).Result;
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
var r5 = producer5.ProduceAsync(topic.Name, new Message <int, int> {
Key = int.MaxValue, Value = int.MinValue
}).Result;
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
var r6 = producer6.ProduceAsync(topic.Name, new Message <string, byte[]> {
Key = "yellow mould", Value = new byte[] { 69 }
}).Result;
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
var r7 = producer7.ProduceAsync(topic.Name, new Message <double, double> {
Key = 44.0, Value = 234.4
}).Result;
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
var topicMetadata = adminClient.GetMetadata(singlePartitionTopic, TimeSpan.FromSeconds(10));
Assert.Single(topicMetadata.Topics);
// implicitly check this does not throw.
}
var consumerConfig = new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString()
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var r1 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
}
using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 1));
var r2 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
Assert.Equal(1, r2.Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 2));
var r3 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
Assert.Equal(2, r3.Offset);
}
using (var consumer = new ConsumerBuilder <int, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 3));
var r4 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
Assert.Equal(3, r4.Offset);
}
using (var consumer = new ConsumerBuilder <int, int>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 4));
var r5 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
Assert.Equal(4, r5.Offset);
}
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 5));
var r6 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
Assert.Equal(5, r6.Offset);
}
using (var consumer = new ConsumerBuilder <double, double>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 6));
var r7 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
Assert.Equal(6, r7.Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_Handles");
}
