/// <summary>
/// Update partitions and others settings for a topic.
/// </summary>
/// <param name="topicConfiguration">The topic update information.</param>
/// <returns>Task holds the topic info if update complete. null is return when topic does not exist or unable to update it.</returns>
/// <remarks>this method is used to increase the number of partitions for a topic. the increasement of replica is not handled here.</remarks>
public async Task <KafkaTopicInfo> UpdateKafkaTopicAsync(KafkaTopicConfiguration topicConfiguration)
{
if (topicConfiguration == null)
{
throw new ArgumentException("topicConfiguration is null or empty.");
}
// check arguments
if (topicConfiguration == null || string.IsNullOrWhiteSpace(topicConfiguration.TopicName) ||
topicConfiguration.NumOfPartitions <= 0 || topicConfiguration.NumOfReplicas <= 0 ||
topicConfiguration.TopicName == "__consumer_offsets")
{
return(null);
}
// 1 get topic info from zookeeper
var topicInfo = await this.GetKafkaTopicAsync(topicConfiguration.TopicName).ConfigureAwait(false);
if (topicInfo == null)
{
return(null);
}
// 2 check partitions and replicas
// only support add more partitions or replicas, not support reduce them
int numOfExistingPartitions = 0;
int numOfExistingReplicas = 0;
var partitions = topicInfo.Partitions;
KafkaPartitionInfo partitionInfo = null;
if (partitions != null)
{
numOfExistingPartitions = partitions.Length;
partitionInfo = partitions.FirstOrDefault(p => p.PartitionId == "0");
if (partitionInfo != null)
{
var replicas = partitionInfo.ReplicaOwners;
if (replicas != null)
{
numOfExistingReplicas = replicas.Length;
}
}
}
if (numOfExistingPartitions == 0 || numOfExistingReplicas == 0)
{
return(null);
}
if (numOfExistingPartitions >= topicConfiguration.NumOfPartitions)
{
return(topicInfo);
}
// NOTE : if want to update replicas, need to use another call
if (numOfExistingReplicas != topicConfiguration.NumOfReplicas)
{
return(topicInfo);
}
// 3 update zookeeper node content
var partitionsToAdd = topicConfiguration.NumOfPartitions - numOfExistingPartitions;
// assign the brokers to brokers
var brokers = await this.ListKakfaBrokersAsync().ConfigureAwait(false);
var newPartitionAssignmentInfo = this.AssignPartitionsAndReplicasToBrokers(partitionsToAdd, numOfExistingReplicas, brokers, Math.Max(0, partitionInfo.ReplicaOwners[0]), numOfExistingPartitions);
// check the newPartitios to see whether some partitions get more replicats
if (newPartitionAssignmentInfo.Any(p => p.ReplicaOwners.Length != partitionInfo.ReplicaOwners.Length))
{
return(null);
}
// merge existing and new partition info
var finalAssignmentInfo = new List <KafkaPartitionInfo>();
finalAssignmentInfo.AddRange(topicInfo.Partitions);
finalAssignmentInfo.AddRange(newPartitionAssignmentInfo);
topicInfo.Partitions = finalAssignmentInfo.ToArray();
// 4 update the node in zookeeper
// update node at path 'brokers/topics/{name}', with new content
var serializer = new ZookeeperDataSerializer();
var content = serializer.SerializeKafkaTopicInfo(topicInfo);
await this.ZookeeperClient.SetDataAsync("/brokers/topics/" + topicConfiguration.TopicName, content, -1).ConfigureAwait(false);
return(topicInfo);
}
/// <summary>
/// Create a topic with creation settings.
/// </summary>
/// <param name="topicConfiguration">The topic creation configuration.</param>
/// <returns>the path to the topic. null is return when unable to create it.</returns>
public async Task <string> CreateKafkaTopicAsync(KafkaTopicConfiguration topicConfiguration)
{
if (topicConfiguration == null || !this.IsTopicNameValid(topicConfiguration.TopicName) || topicConfiguration.NumOfPartitions <= 0 || topicConfiguration.NumOfReplicas <= 0)
{
return(null);
}
//// refer to https://github.com/apache/kafka/blob/trunk/core/src/main/scala/kafka/admin/AdminUtils.scala
//// the method createTopic
// 1 figure how many brokers are available
var brokers = await this.ListKakfaBrokersAsync().ConfigureAwait(false);
if (brokers == null)
{
return(null);
}
// note:
// when actual number of active brokers is less than the expected number of brokers, technically you still can
// create topic. for example: we expected 5 brokers, but actually there are only 3, when you create below topic:
// { "name" :"test", "partitions": "3", "replicas":"3"}
// as far as the number of replicas is not bigger than the number of active brokers, you can still assign replicas
// onto different brokers.
// the only problem with this scenario is the partitions will not put on the another brokers when they are alive.
// then need manually move some partitions to the addition brokers to get better balance.
// so to make things simple, we just don't do it when this happen.
var expected = this.GetExpectedKafkaBrokers();
if (expected == null || brokers.Count() != expected.Count())
{
return(null);
}
var topics = await this.ListKafkaTopicsAsync().ConfigureAwait(false);
if (topics != null)
{
var existing = topics.FirstOrDefault(t => t.Name == topicConfiguration.TopicName);
if (existing != null)
{
return("/brokers/topics/" + topicConfiguration.TopicName);
}
}
// Note : number of replicas can not be greater than number of active brokers,
// because replicas of same topic can not be on the same broker
// 2 get partitions assignment info onto those brokers with round-robin fashion.
var assignments = this.AssignPartitionsAndReplicasToBrokers(topicConfiguration.NumOfPartitions, topicConfiguration.NumOfReplicas, brokers);
// 3 convert partition assignment into json format for zookeeper node, and create zookeepr node
// the format of json is : {"version":1,"partitions":{"0":[],...}}
var serializer = new ZookeeperDataSerializer();
var content = serializer.SerializeKafkaTopicInfo(new KafkaTopicInfo()
{
Name = topicConfiguration.TopicName, Version = 1, Partitions = assignments.ToArray()
});
// 4 create zookpper node
// create node at path 'brokers/topics/{name}', with content above
var acl = new ZookeeperZnodeAcl()
{
AclRights = ZookeeperZnodeAclRights.All,
Identity = ZookeeperZnodeAclIdentity.WorldIdentity()
};
var path = await this.ZookeeperClient.CreateAsync("/brokers/topics/" + topicConfiguration.TopicName, content, new List <ZookeeperZnodeAcl>() { acl }, ZookeeperZnodeType.Persistent).ConfigureAwait(false);
return(path);
}