/// <summary> /// Get the kafka topic info. /// </summary> /// <param name="topic">The topic name.</param> /// <returns>The topic infomation.</returns> private async Task <KafkaTopicInfo> GetTopicInfoAsync(string topic) { var path = "/brokers/topics/" + topic; var data = await this.ZookeeperClient.GetDataAsync(path, false).ConfigureAwait(false); if (data == null || data.Length == 0) { return(null); } var serializer = new ZookeeperDataSerializer(); return(serializer.DeserializeKafkaTopicInfo(topic, data)); }
/// <summary> /// Get the state of a kafka partition. /// </summary> /// <param name="topic">The name of topic.</param> /// <param name="partitionId">The id of partition.</param> /// <returns>Task holds the partition state.</returns> public async Task <KafkaPartitionState> GetKafkaPartitionStateAsync(string topic, string partitionId) { int id; if (string.IsNullOrWhiteSpace(topic) || int.TryParse(partitionId, out id) == false) { return(null); } var bytes = await this.ZookeeperClient.GetDataAsync("/brokers/topics/" + topic + "/partitions/" + partitionId + "/state", false).ConfigureAwait(false); if (bytes == null || !bytes.Any()) { return(null); } var serializer = new ZookeeperDataSerializer(); var partitionState = serializer.DeserializeKafkaPartitionState(bytes); return(partitionState); }
/// <summary> /// Update partitions and others settings for a topic. /// </summary> /// <param name="topicConfiguration">The topic update information.</param> /// <returns>Task holds the topic info if update complete. null is return when topic does not exist or unable to update it.</returns> /// <remarks>this method is used to increase the number of partitions for a topic. the increasement of replica is not handled here.</remarks> public async Task <KafkaTopicInfo> UpdateKafkaTopicAsync(KafkaTopicConfiguration topicConfiguration) { if (topicConfiguration == null) { throw new ArgumentException("topicConfiguration is null or empty."); } // check arguments if (topicConfiguration == null || string.IsNullOrWhiteSpace(topicConfiguration.TopicName) || topicConfiguration.NumOfPartitions <= 0 || topicConfiguration.NumOfReplicas <= 0 || topicConfiguration.TopicName == "__consumer_offsets") { return(null); } // 1 get topic info from zookeeper var topicInfo = await this.GetKafkaTopicAsync(topicConfiguration.TopicName).ConfigureAwait(false); if (topicInfo == null) { return(null); } // 2 check partitions and replicas // only support add more partitions or replicas, not support reduce them int numOfExistingPartitions = 0; int numOfExistingReplicas = 0; var partitions = topicInfo.Partitions; KafkaPartitionInfo partitionInfo = null; if (partitions != null) { numOfExistingPartitions = partitions.Length; partitionInfo = partitions.FirstOrDefault(p => p.PartitionId == "0"); if (partitionInfo != null) { var replicas = partitionInfo.ReplicaOwners; if (replicas != null) { numOfExistingReplicas = replicas.Length; } } } if (numOfExistingPartitions == 0 || numOfExistingReplicas == 0) { return(null); } if (numOfExistingPartitions >= topicConfiguration.NumOfPartitions) { return(topicInfo); } // NOTE : if want to update replicas, need to use another call if (numOfExistingReplicas != topicConfiguration.NumOfReplicas) { return(topicInfo); } // 3 update zookeeper node content var partitionsToAdd = topicConfiguration.NumOfPartitions - numOfExistingPartitions; // assign the brokers to brokers var brokers = await this.ListKakfaBrokersAsync().ConfigureAwait(false); var newPartitionAssignmentInfo = this.AssignPartitionsAndReplicasToBrokers(partitionsToAdd, numOfExistingReplicas, brokers, Math.Max(0, partitionInfo.ReplicaOwners[0]), numOfExistingPartitions); // check the newPartitios to see whether some partitions get more replicats if (newPartitionAssignmentInfo.Any(p => p.ReplicaOwners.Length != partitionInfo.ReplicaOwners.Length)) { return(null); } // merge existing and new partition info var finalAssignmentInfo = new List <KafkaPartitionInfo>(); finalAssignmentInfo.AddRange(topicInfo.Partitions); finalAssignmentInfo.AddRange(newPartitionAssignmentInfo); topicInfo.Partitions = finalAssignmentInfo.ToArray(); // 4 update the node in zookeeper // update node at path 'brokers/topics/{name}', with new content var serializer = new ZookeeperDataSerializer(); var content = serializer.SerializeKafkaTopicInfo(topicInfo); await this.ZookeeperClient.SetDataAsync("/brokers/topics/" + topicConfiguration.TopicName, content, -1).ConfigureAwait(false); return(topicInfo); }
/// <summary> /// Create a topic with creation settings. /// </summary> /// <param name="topicConfiguration">The topic creation configuration.</param> /// <returns>the path to the topic. null is return when unable to create it.</returns> public async Task <string> CreateKafkaTopicAsync(KafkaTopicConfiguration topicConfiguration) { if (topicConfiguration == null || !this.IsTopicNameValid(topicConfiguration.TopicName) || topicConfiguration.NumOfPartitions <= 0 || topicConfiguration.NumOfReplicas <= 0) { return(null); } //// refer to https://github.com/apache/kafka/blob/trunk/core/src/main/scala/kafka/admin/AdminUtils.scala //// the method createTopic // 1 figure how many brokers are available var brokers = await this.ListKakfaBrokersAsync().ConfigureAwait(false); if (brokers == null) { return(null); } // note: // when actual number of active brokers is less than the expected number of brokers, technically you still can // create topic. for example: we expected 5 brokers, but actually there are only 3, when you create below topic: // { "name" :"test", "partitions": "3", "replicas":"3"} // as far as the number of replicas is not bigger than the number of active brokers, you can still assign replicas // onto different brokers. // the only problem with this scenario is the partitions will not put on the another brokers when they are alive. // then need manually move some partitions to the addition brokers to get better balance. // so to make things simple, we just don't do it when this happen. var expected = this.GetExpectedKafkaBrokers(); if (expected == null || brokers.Count() != expected.Count()) { return(null); } var topics = await this.ListKafkaTopicsAsync().ConfigureAwait(false); if (topics != null) { var existing = topics.FirstOrDefault(t => t.Name == topicConfiguration.TopicName); if (existing != null) { return("/brokers/topics/" + topicConfiguration.TopicName); } } // Note : number of replicas can not be greater than number of active brokers, // because replicas of same topic can not be on the same broker // 2 get partitions assignment info onto those brokers with round-robin fashion. var assignments = this.AssignPartitionsAndReplicasToBrokers(topicConfiguration.NumOfPartitions, topicConfiguration.NumOfReplicas, brokers); // 3 convert partition assignment into json format for zookeeper node, and create zookeepr node // the format of json is : {"version":1,"partitions":{"0":[],...}} var serializer = new ZookeeperDataSerializer(); var content = serializer.SerializeKafkaTopicInfo(new KafkaTopicInfo() { Name = topicConfiguration.TopicName, Version = 1, Partitions = assignments.ToArray() }); // 4 create zookpper node // create node at path 'brokers/topics/{name}', with content above var acl = new ZookeeperZnodeAcl() { AclRights = ZookeeperZnodeAclRights.All, Identity = ZookeeperZnodeAclIdentity.WorldIdentity() }; var path = await this.ZookeeperClient.CreateAsync("/brokers/topics/" + topicConfiguration.TopicName, content, new List <ZookeeperZnodeAcl>() { acl }, ZookeeperZnodeType.Persistent).ConfigureAwait(false); return(path); }