public bool CreateTopic(TopicSpecification topicSpecification)
{
if (topicSpecification == null)
{
throw new ArgumentNullException(nameof(topicSpecification));
}
var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(30));
if (!metadata.Topics.Exists(x => x.Topic == topicSpecification.Name))
{
try
{
_adminClient.CreateTopicsAsync(new[]
{
topicSpecification
}).GetAwaiter().GetResult();
}
catch (Exception e)
{
//just in case we have a race condition
if (!e.Message.Contains("already exists"))
{
throw;
}
}
}
return(true);
}
protected static List <TopicPartition> LoadTopicPartitions(IAdminClient adminClient, string topicName)
{
try
{
var timeout = TimeSpan.FromSeconds(5);
var metadata = adminClient.GetMetadata(topicName, timeout);
if (metadata.Topics == null || metadata.Topics.Count == 0)
{
Console.WriteLine($"Could not load metadata information about topic '{topicName}'");
return(new List <TopicPartition>());
}
var topicMetadata = metadata.Topics[0];
var partitions = topicMetadata.Partitions;
if (partitions == null || partitions.Count == 0)
{
Console.WriteLine($"Could not load partition information about topic '{topicName}'");
return(new List <TopicPartition>());
}
return(partitions.Select(x => new TopicPartition(topicMetadata.Topic, new Partition(x.PartitionId))).ToList());
}
catch (Exception ex)
{
Console.WriteLine($"Failed to load partition information from topic '{topicName}' {ex.Message}: {ex.StackTrace}");
}
return(new List <TopicPartition>());
}
public static List <string> ListTopics(this IAdminClient client)
{
return(client.GetMetadata(Timeout)
.Topics
.Select(t => t.Topic)
.ToList());
}
private void PrepareTopic(IAdminClient adminClient, string topic)
{
var metadata = adminClient.GetMetadata(topic, TimeSpan.FromSeconds(20));
var partitionCount = metadata.Topics.First().Partitions.Count;
if (partitionCount == 0)
{
adminClient.CreateTopicsAsync(new[]
{
new TopicSpecification
{
NumPartitions = _options.TopicPartitionCount, Name = topic, ReplicationFactor = 1
}
}).GetAwaiter().GetResult();
}
else
{
if (partitionCount < _options.TopicPartitionCount)
{
adminClient.CreatePartitionsAsync(new[]
{
new PartitionsSpecification
{
Topic = topic, IncreaseTo = _options.TopicPartitionCount
}
}).GetAwaiter().GetResult();
}
}
}
internal static List <CreateTopicReport> GetTopicMetadataErrorReports(this IAdminClient adminClient, string topicName, int timeoutInSeconds = 15)
{
return(adminClient
.GetMetadata(topicName, TimeSpan.FromSeconds(timeoutInSeconds)).Topics
.FindAll(x => x.Error != ErrorCode.NoError)
.ConvertAll(x => new CreateTopicReport {
Topic = x.Topic, Error = x.Error
}));
}
public async Task <bool> TopicExistsAsync(string topic)
{
var timeout = TimeSpan.FromSeconds(20);
var metadata = client.GetMetadata(timeout);
bool exists = metadata.Topics.Any(y => y.Topic == topic);
return(await Task.FromResult(exists));
}
public Metadata GetMetadata(TimeSpan timeout)
{
Confluent.Kafka.Metadata metadata;
try
{
metadata = _adminClient.GetMetadata(timeout);
}
catch (KafkaException e)
{
throw new AdminClientException("Unable to get metadata.", e);
}
return(new Metadata
{
Brokers = metadata.Brokers?.Select(CommonMapper.Map).ToList(),
Topics = metadata.Topics?.Select(CommonMapper.Map).ToList(),
OriginatingBrokerId = metadata.OriginatingBrokerId,
OriginatingBrokerName = metadata.OriginatingBrokerName
});
}
private IEnumerable <TopicPartition> GetTopicPartitions(IEnumerable <string> topics)
{
var timeout = TimeSpan.FromSeconds(5);
var topicPartitions =
topics
.Select(x => _adminClient.GetMetadata(x, timeout))
.SelectMany(x => x.Topics)
.SelectMany(x => x.Partitions.Select(y => new TopicPartition(x.Topic, y.PartitionId)));
return(topicPartitions);
}
private static Func <string, FSharpOption <int> > GetPartitionCount(IAdminClient adminClient) => (string topicName) =>
{
var metadata = adminClient.GetMetadata(
topic: topicName,
timeout: TimeSpan.FromSeconds(10));
var topicMetadata = metadata.Topics.FirstOrDefault(t => t.Topic == topicName);
if (topicMetadata == null)
{
return(FSharpOption <int> .None);
}
return(FSharpOption <int> .Some(topicMetadata.Partitions.Count));
};
private IEnumerable <TopicPartition> TopicPartitionsForStore(IStateStore store)
{
var topic = topology.StoresToTopics[store.Name];
var metadata = adminClient.GetMetadata(topic, TimeSpan.FromMilliseconds(config.MetadataRequestTimeoutMs));
if (metadata == null || metadata.Topics.Count == 0)
{
throw new StreamsException($"There are no partitions available for topic {topic} when initializing global store {store.Name}");
}
var result = metadata.Topics.Single().Partitions.Select(partition => new TopicPartition(topic, partition.PartitionId));
return(result);
}
private async Task createMissingTopics(IAdminClient client)
{
var metadata = client.GetMetadata(TimeSpan.FromSeconds(30));
var missingTopics = this.Kafka.Options.Topics.Where(tOpt => metadata.Topics.TrueForAll(t => t.Topic != tOpt.Name));
var sw = Stopwatch.StartNew();
await client.CreateTopicsAsync(missingTopics.Select(t => new TopicSpecification
{
Name = t.Name,
NumPartitions = t.Partitions,
ReplicationFactor = t.ReplicationFactor
}), new CreateTopicsOptions { OperationTimeout = TimeSpan.FromSeconds(30), RequestTimeout = TimeSpan.FromSeconds(30) });
sw.Stop();
}
private static async Task EnsureTopicExists(IAdminClient admin)
{
var metadata = admin.GetMetadata(Topic, TimeSpan.FromSeconds(15));
if (!metadata.Topics.Any(m => m.Topic == Topic))
{
await admin.CreateTopicsAsync(
new[]
{
new TopicSpecification
{
Name = Topic,
},
});
}
}
private void VerifyTopicExistsBeforeSubscribing(IAdminClient adminClient, string topic)
{
var logContext = $"{nameof(VerifyTopicExistsBeforeSubscribing)} " +
$"for subscriberId='{_subscriberId}', topic='{topic}'";
try
{
_logger.LogDebug($"+{logContext}");
Metadata metadata = adminClient.GetMetadata(topic, TimeSpan.FromSeconds(AdminClientTimeoutMilliseconds));
List <PartitionMetadata> partitions = metadata.Topics[0].Partitions;
_logger.LogDebug($"-{logContext}: found partitions='{String.Join(",", partitions.Select(p => p.PartitionId))}'");
}
catch (Exception e)
{
_logger.LogError(e, $"{logContext}: Got exception: {e}");
throw;
}
}
public async Task <bool> EnsureTopicExists(string topicName)
{
_logger.LogInformation("Ensuring topic exists...");
var topicSpecification = new TopicSpecification()
{
Name = topicName,
NumPartitions = 1,
ReplicationFactor = 1
};
var topicSpecifications = new List <TopicSpecification>();
topicSpecifications.Add(topicSpecification);
// try and get metadata about topic to check that it exists
var metaData = _adminClient.GetMetadata(topicName, TimeSpan.FromSeconds(10));
if (!metaData.Topics[0].Error.IsError)
{
_logger.LogInformation(" Topic already exists.");
return(true);
}
_logger.LogInformation(" Creating topic...");
try
{
await _adminClient.CreateTopicsAsync(topicSpecifications);
_logger.LogInformation(" Topic created.");
}
catch (CreateTopicsException e)
{
// only trap fatal errors so that we let topic already exists error through
if (e.Error.IsFatal)
{
return(false);
}
}
return(true);
}
public static TopicMetadata?GetTopicMetadata(this IAdminClient client, string topic)
{
return(client.GetMetadata(Timeout)
.Topics
.FirstOrDefault(t => t.Topic == topic));
}
public IActionResult GetActionMethodName()
{
return(Ok(_client.GetMetadata(TimeSpan.FromSeconds(10))?.Brokers));
}
public Metadata GetMetadata(string topic) => _innerClient.GetMetadata(topic, TimeSpan.FromSeconds(30));
/// <inheritdoc/>
public override Task <bool> CheckHealthAsync(Dictionary <string, object> data,
CancellationToken cancellationToken = default)
{
adminClient.GetMetadata(StandardTimeout);
return(Task.FromResult(true));
}