private static void CreateTopic(IConfigurationRoot config)
{
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = config["BootstrapServers"]
};
var adminClient = new AdminClient(adminClientConfig);
var topic = config["TopicName"];
var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(10));
if (metadata.Topics.Any(x => x.Topic == topic))
{
return;
}
var topicConfig = config.GetSection("TopicConfig").GetChildren().AsEnumerable()
.ToDictionary(x => x.Key, x => x.Value);
adminClient.CreateTopicsAsync(new[]
{
new TopicSpecification
{
Name = topic,
NumPartitions = int.Parse(config["NumPartitions"]),
ReplicationFactor = short.Parse(config["ReplicationFactor"]),
Configs = topicConfig
}
}).Wait();
}
public TemporaryTopic(string bootstrapServers, int numPartitions)
{
Name = Guid.NewGuid().ToString();
adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build();
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = Name, NumPartitions = numPartitions, ReplicationFactor = 1
}
}).Wait();
}
private void CreatePartition()
{
if (PartitionOptions != null)
{
TopicPartitionCache = new ConcurrentDictionary <string, TopicPartition>();
if (PartitionOptions.Partition > 1)
{
using (var adminClient = new AdminClient(Producer.Handle))
{
try
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = TopicName, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
}
catch (AggregateException ex)
{
//{Confluent.Kafka.Admin.CreateTopicsException: An error occurred creating topics: [jt809]: [Topic 'jt809' already exists.].}
if (ex.InnerException is Confluent.Kafka.Admin.CreateTopicsException exception)
{
}
else
{
//记录日志
//throw ex.InnerException;
}
}
try
{
//topic IncreaseTo 只增不减
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification
{
IncreaseTo = PartitionOptions.Partition,
Topic = TopicName
}
}
).Wait();
}
catch (AggregateException ex)
{
//记录日志
// throw ex.InnerException;
}
}
}
}
}
public JT808_MsgId_Producer(
IOptions <ProducerConfig> producerConfigAccessor)
{
producer = new Producer <string, byte[]>(producerConfigAccessor.Value);
this.jT808ProducerPartitionFactory = new JT808MsgIdProducerPartitionFactoryImpl();
using (var adminClient = new AdminClient(producer.Handle))
{
try
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = TopicName, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
}
catch (AggregateException ex)
{
//{Confluent.Kafka.Admin.CreateTopicsException: An error occurred creating topics: [jt808]: [Topic 'jt808' already exists.].}
if (ex.InnerException is Confluent.Kafka.Admin.CreateTopicsException exception)
{
}
else
{
//记录日志
//throw ex.InnerException;
}
}
try
{
//topic IncreaseTo 只增不减
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification
{
IncreaseTo = 8,
Topic = TopicName
}
}
).Wait();
}
catch (AggregateException ex)
{
//记录日志
// throw ex.InnerException;
}
}
}
private static void CreateTopic()
{
using (var producer = new Producer <Null, string>(GetConfig()))
{
using (var adminClient = new AdminClient(producer.Handle))
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = TOPIC, NumPartitions = 5, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> {
new PartitionsSpecification {
Topic = TOPIC, IncreaseTo = 6
}
}).Wait();
}
}
}
private static void CreateTopic()
{
var config = new AdminClientConfig()
{
BootstrapServers = _server
};
var topic = new TopicSpecification()
{
Name = _topicName, NumPartitions = 1, ReplicationFactor = 1
};
var topics = new List <TopicSpecification>()
{
topic
};
using (var client = new AdminClient(config))
{
var topicTask = client.CreateTopicsAsync(topics) as Task <List <CreateTopicExceptionResult> >;
var topicResults = topicTask.Result;
}
}
public static void AdminClient_CreatePartitions(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start AdminClient_CreatePartitions");
var topicName1 = Guid.NewGuid().ToString();
var topicName2 = Guid.NewGuid().ToString();
var topicName3 = Guid.NewGuid().ToString();
var topicName4 = Guid.NewGuid().ToString();
var topicName5 = Guid.NewGuid().ToString();
var topicName6 = Guid.NewGuid().ToString();
// test creating a new partition works.
using (var producer = new Producer <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}))
using (var adminClient = new AdminClient(producer.Handle))
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName1, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> {
new PartitionsSpecification {
Topic = topicName1, IncreaseTo = 2
}
}).Wait();
var dr1 = producer.ProduceAsync(new TopicPartition(topicName1, 0), new Message <Null, Null> {
}).Result;
var dr2 = producer.ProduceAsync(new TopicPartition(topicName1, 1), new Message <Null, Null> {
}).Result;
try
{
producer.ProduceAsync(new TopicPartition(topicName1, 2), new Message <Null, Null> {
}).Wait();
Assert.True(false, "expecting exception");
}
catch (KafkaException ex)
{
Assert.True(ex.Error.IsError);
}
}
// check validate only works.
using (var producer = new Producer <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}))
using (var adminClient = new AdminClient(producer.Handle))
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName2, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(new List <PartitionsSpecification> {
new PartitionsSpecification {
Topic = topicName2, IncreaseTo = 10
}
}, new CreatePartitionsOptions {
ValidateOnly = true
}).Wait();
// forces a metadata request.
var dr1 = producer.ProduceAsync(new TopicPartition(topicName2, 0), new Message <Null, Null> {
}).Result;
try
{
// since we have metadata, this throws immediately (i.e. not wrapped in AggregateException)
var dr2 = producer.ProduceAsync(new TopicPartition(topicName2, 1), new Message <Null, Null> {
}).Result;
Assert.True(false, "expecting exception");
}
catch (KafkaException ex)
{
Assert.True(ex.Error.IsError);
}
}
// check valid Assignments property value works.
using (var producer = new Producer <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}))
using (var adminClient = new AdminClient(producer.Handle))
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName3, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification {
Topic = topicName2, IncreaseTo = 2, ReplicaAssignments = new List <List <int> > {
new List <int> {
0
}
}
}
},
new CreatePartitionsOptions {
ValidateOnly = true
}
).Wait();
}
// check invalid Assignments property value works.
using (var producer = new Producer <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}))
using (var adminClient = new AdminClient(producer.Handle))
{
adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification {
Name = topicName4, NumPartitions = 1, ReplicationFactor = 1
} }).Wait();
try
{
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification {
Topic = topicName2, IncreaseTo = 2, ReplicaAssignments = new List <List <int> > {
new List <int> {
42
}
}
}
},
new CreatePartitionsOptions {
ValidateOnly = true
}
).Wait();
Assert.True(false, "Expecting exception");
}
catch (AggregateException ex)
{
Assert.True(ex.InnerException.GetType() == typeof(CreatePartitionsException));
var cpe = (CreatePartitionsException)ex.InnerException;
Assert.Single(cpe.Results);
Assert.True(cpe.Results.First().Error.IsError);
}
}
// more than one.
using (var adminClient = new AdminClient(new AdminClientConfig {
BootstrapServers = bootstrapServers
}))
{
adminClient.CreateTopicsAsync(new TopicSpecification[]
{
new TopicSpecification {
Name = topicName5, NumPartitions = 1, ReplicationFactor = 1
},
new TopicSpecification {
Name = topicName6, NumPartitions = 1, ReplicationFactor = 1
}
}
).Wait();
Thread.Sleep(TimeSpan.FromSeconds(1));
// just a simple check there wasn't an exception.
adminClient.CreatePartitionsAsync(
new List <PartitionsSpecification>
{
new PartitionsSpecification {
Topic = topicName5, IncreaseTo = 2
},
new PartitionsSpecification {
Topic = topicName6, IncreaseTo = 3
}
}
).Wait();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_CreatePartitions");
}
public static void AdminClient_CreateTopics(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start AdminClient_CreateTopics");
var topicName1 = Guid.NewGuid().ToString();
var topicName2 = Guid.NewGuid().ToString();
var topicName3 = Guid.NewGuid().ToString();
var topicName4 = Guid.NewGuid().ToString();
var topicName5 = Guid.NewGuid().ToString();
// test
// - construction of admin client from configuration.
// - creation of more than one topic.
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
adminClient.CreateTopicsAsync(
new TopicSpecification[]
{
new TopicSpecification {
Name = topicName1, NumPartitions = 2, ReplicationFactor = 1
},
new TopicSpecification {
Name = topicName2, NumPartitions = 12, ReplicationFactor = 1
}
}
).Wait();
}
// test
// - construction of admin client from a producer handle
// - creation of topic
// - producing to created topics works.
using (var producer = new ProducerBuilder <Null, Null>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient2 = new AdminClient(producer.Handle))
{
adminClient2.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName3, NumPartitions = 24, ReplicationFactor = 1
}
}).Wait();
var deliveryReport1 = producer.ProduceAsync(topicName1, new Message <Null, Null>()).Result;
var deliveryReport2 = producer.ProduceAsync(topicName2, new Message <Null, Null>()).Result;
var deliveryReport3 = producer.ProduceAsync(topicName3, new Message <Null, Null>()).Result;
Assert.Equal(topicName1, deliveryReport1.Topic);
Assert.Equal(topicName2, deliveryReport2.Topic);
Assert.Equal(topicName3, deliveryReport3.Topic);
}
// test
// - create topic with same name as existing topic
// - as well as another topic that does exist (and for which create should succeed).
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
try
{
adminClient.CreateTopicsAsync(new List <TopicSpecification>
{
new TopicSpecification {
Name = topicName3, NumPartitions = 1, ReplicationFactor = 1
},
new TopicSpecification {
Name = topicName4, NumPartitions = 1, ReplicationFactor = 1
}
}
).Wait();
Assert.True(false, "Expect CreateTopics request to throw an exception.");
}
// if awaited, the CreateTopicsException is not wrapped.
// this is an annoyance if used synchronously, but not atypical.
catch (AggregateException ex)
{
Assert.True(ex.InnerException.GetType() == typeof(CreateTopicsException));
var cte = (CreateTopicsException)ex.InnerException;
Assert.Equal(2, cte.Results.Count);
Assert.Single(cte.Results.Where(r => r.Error.IsError));
Assert.Single(cte.Results.Where(r => !r.Error.IsError));
Assert.Equal(topicName3, cte.Results.Where(r => r.Error.IsError).First().Topic);
Assert.Equal(topicName4, cte.Results.Where(r => !r.Error.IsError).First().Topic);
}
}
// test
// - validate only
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName5, NumPartitions = 1, ReplicationFactor = 1
}
},
new CreateTopicsOptions {
ValidateOnly = true, RequestTimeout = TimeSpan.FromSeconds(30)
}
).Wait();
// creating for real shouldn't throw exception.
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName5, NumPartitions = 1, ReplicationFactor = 1
}
}
).Wait();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_CreateTopics");
}
public static void AdminClient_DeleteTopics(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start AdminClient_DeleteTopics");
var topicName1 = Guid.NewGuid().ToString();
var topicName2 = Guid.NewGuid().ToString();
var topicName3 = Guid.NewGuid().ToString();
// test single delete topic.
using (var adminClient = new AdminClient(new AdminClientConfig {
BootstrapServers = bootstrapServers
}))
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName1, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
Thread.Sleep(TimeSpan.FromSeconds(1));
Thread.Sleep(TimeSpan.FromSeconds(2)); // git the topic some time to be created.
adminClient.DeleteTopicsAsync(new List <string> {
topicName1
}).Wait();
}
// test
// - delete two topics, one that doesn't exist.
// - check that explicitly giving options doesn't obviously not work.
using (var adminClient = new AdminClient(new AdminClientConfig {
BootstrapServers = bootstrapServers
}))
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName2, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
Thread.Sleep(TimeSpan.FromSeconds(1));
Thread.Sleep(TimeSpan.FromSeconds(2));
try
{
adminClient.DeleteTopicsAsync(
new List <string> {
topicName2, topicName3
},
new DeleteTopicsOptions {
RequestTimeout = TimeSpan.FromSeconds(30)
}
).Wait();
}
catch (AggregateException ex)
{
var dte = (DeleteTopicsException)ex.InnerException;
Assert.Equal(2, dte.Results.Count);
Assert.Single(dte.Results.Where(r => r.Error.IsError));
Assert.Single(dte.Results.Where(r => !r.Error.IsError));
Assert.Equal(topicName2, dte.Results.Where(r => !r.Error.IsError).First().Topic);
Assert.Equal(topicName3, dte.Results.Where(r => r.Error.IsError).First().Topic);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_DeleteTopics");
}
public static void ProduceConsume(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest,
EnablePartitionEof = true
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
string topic = Guid.NewGuid().ToString();
using (var adminClient = new AdminClient(adminClientConfig))
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer = new Producer <string, User>(producerConfig,
new AvroSerializer <string>(schemaRegistry), new AvroSerializer <User>(schemaRegistry)))
{
for (int i = 0; i < 100; ++i)
{
var user = new User
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer
.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
})
.Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer = new Consumer <string, User>(consumerConfig,
new AvroDeserializer <string>(schemaRegistry), new AvroDeserializer <User>(schemaRegistry)))
{
consumer.OnError += (_, e)
=> Assert.True(false, e.Reason);
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
Assert.Equal(100, i);
consumer.Close();
}
}
public static void AdminClient_AlterConfigs(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start AdminClient_AlterConfigs");
using (var adminClient = new AdminClient(new AdminClientConfig {
BootstrapServers = bootstrapServers
}))
{
// 1. create a new topic to play with.
string topicName = Guid.NewGuid().ToString();
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
Thread.Sleep(TimeSpan.FromSeconds(1)); // without this, sometimes describe topic throws unknown topic/partition error.
// 2. do an invalid alter configs call to change it.
var configResource = new ConfigResource {
Name = topicName, Type = ResourceType.Topic
};
var toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> >
{
{
configResource,
new List <ConfigEntry> {
new ConfigEntry {
Name = "flush.ms", Value = "10001"
},
new ConfigEntry {
Name = "ubute.invalid.config", Value = "42"
}
}
}
};
try
{
adminClient.AlterConfigsAsync(toUpdate).Wait();
Assert.True(false);
}
catch (Exception e)
{
Assert.True(e.InnerException.GetType() == typeof(AlterConfigsException));
var ace = (AlterConfigsException)e.InnerException;
Assert.Single(ace.Results);
Assert.Contains("Unknown", ace.Results[0].Error.Reason);
}
// 3. test that in the failed alter configs call for the specified config resource, the
// config that was specified correctly wasn't updated.
List <DescribeConfigsResult> describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> {
configResource
}).Result;
Assert.NotEqual("10001", describeConfigsResult[0].Entries["flush.ms"].Value);
// 4. do a valid call, and check that the alteration did correctly happen.
toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> >
{
{ configResource, new List <ConfigEntry> {
new ConfigEntry {
Name = "flush.ms", Value = "10011"
}
} }
};
adminClient.AlterConfigsAsync(toUpdate);
describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> {
configResource
}).Result;
Assert.Equal("10011", describeConfigsResult[0].Entries["flush.ms"].Value);
// 4. test ValidateOnly = true does not update config entry.
toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> >
{
{ configResource, new List <ConfigEntry> {
new ConfigEntry {
Name = "flush.ms", Value = "20002"
}
} }
};
adminClient.AlterConfigsAsync(toUpdate, new AlterConfigsOptions {
ValidateOnly = true
}).Wait();
describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> {
configResource
}).Result;
Assert.Equal("10011", describeConfigsResult[0].Entries["flush.ms"].Value);
// 5. test updating broker resource.
toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> >
{
{
new ConfigResource {
Name = "0", Type = ResourceType.Broker
},
new List <ConfigEntry> {
new ConfigEntry {
Name = "num.network.threads", Value = "2"
}
}
}
};
adminClient.AlterConfigsAsync(toUpdate).Wait();
// 6. test updating more than one resource.
string topicName2 = Guid.NewGuid().ToString();
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topicName2, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
Thread.Sleep(TimeSpan.FromSeconds(1)); // without this, sometimes describe topic throws unknown topic/partition error.
var configResource2 = new ConfigResource {
Name = topicName2, Type = ResourceType.Topic
};
toUpdate = new Dictionary <ConfigResource, List <ConfigEntry> >
{
{ configResource, new List <ConfigEntry> {
new ConfigEntry {
Name = "flush.ms", Value = "222"
}
} },
{ configResource2, new List <ConfigEntry> {
new ConfigEntry {
Name = "flush.ms", Value = "333"
}
} }
};
adminClient.AlterConfigsAsync(toUpdate).Wait();
describeConfigsResult = adminClient.DescribeConfigsAsync(new List <ConfigResource> {
configResource, configResource2
}).Result;
Assert.Equal(2, describeConfigsResult.Count);
Assert.Equal("222", describeConfigsResult[0].Entries["flush.ms"].Value);
Assert.Equal("333", describeConfigsResult[1].Entries["flush.ms"].Value);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_AlterConfigs");
}