private void PrepareTopic(IAdminClient adminClient, string topic)
{
var metadata = adminClient.GetMetadata(topic, TimeSpan.FromSeconds(20));
var partitionCount = metadata.Topics.First().Partitions.Count;
if (partitionCount == 0)
{
adminClient.CreateTopicsAsync(new[]
{
new TopicSpecification
{
NumPartitions = _options.TopicPartitionCount, Name = topic, ReplicationFactor = 1
}
}).GetAwaiter().GetResult();
}
else
{
if (partitionCount < _options.TopicPartitionCount)
{
adminClient.CreatePartitionsAsync(new[]
{
new PartitionsSpecification
{
Topic = topic, IncreaseTo = _options.TopicPartitionCount
}
}).GetAwaiter().GetResult();
}
}
}
public async Task Create(ITopicConfiguration topicConfig)
{
if (!_adminClient.TopicExists(topicConfig.Name, out _))
{
TopicSpecification topicSpecification = new TopicSpecification
{
Name = topicConfig.Name,
NumPartitions = topicConfig.Partitions,
ReplicationFactor = topicConfig.ReplicationFactor,
Configs = new Dictionary <string, string>
{
{ "retention.ms", topicConfig.RetentionMs.ToString() }
}
};
try
{
await _adminClient.CreateTopicsAsync(new[] { topicSpecification });
}
catch (CreateTopicsException ex)
{
Console.WriteLine(ex);
}
_adminClient.EnsureTopicCreation(topicConfig.Name);
}
}
public async Task CreateTopicAsync(string topic, int?numPartitions, short?replicationFactor,
Dictionary <string, string> config, TimeSpan timeout)
{
var topicSpecification = new TopicSpecification
{
Name = topic,
NumPartitions = numPartitions ?? -1,
ReplicationFactor = replicationFactor ?? -1,
Configs = config
};
var options = new CreateTopicsOptions
{
RequestTimeout = timeout
};
try
{
await _adminClient.CreateTopicsAsync(new[] { topicSpecification }, options);
}
catch (KafkaException e)
{
throw new AdminClientException("Unable to create topic.", e);
}
}
public bool CreateTopic(TopicSpecification topicSpecification)
{
if (topicSpecification == null)
{
throw new ArgumentNullException(nameof(topicSpecification));
}
var metadata = _adminClient.GetMetadata(TimeSpan.FromSeconds(30));
if (!metadata.Topics.Exists(x => x.Topic == topicSpecification.Name))
{
try
{
_adminClient.CreateTopicsAsync(new[]
{
topicSpecification
}).GetAwaiter().GetResult();
}
catch (Exception e)
{
//just in case we have a race condition
if (!e.Message.Contains("already exists"))
{
throw;
}
}
}
return(true);
}
private async Task TryCreateTopicAsync(string topicName, IAdminClient adminClient)
{
_logger.LogInformation("Setting up Kafka topic {Topic} ...", topicName);
try
{
await adminClient.CreateTopicsAsync(new[]
{
new TopicSpecification
{
Name = topicName,
ReplicationFactor = 1,
NumPartitions = 1
}
});
}
catch (CreateTopicsException ex) when(ex.Error?.Code == ErrorCode.Local_Partial)
{
_logger.LogWarning(ex, "An error occured creating topic {Topic}: {Error}", topicName, ex.Message);
}
catch (Exception ex)
{
_logger.LogError(ex, "An error occured creating topic {Topic}: {Error}", topicName, ex.Message);
}
}
private static void CreateTopicWithAdminClient(IAdminClient adminClient)
{
try
{
adminClient.CreateTopicsAsync(new TopicSpecification[]
{
new TopicSpecification
{
Name = "topic01",
ReplicationFactor = 2,
NumPartitions = 4
},
new TopicSpecification
{
Name = "topic02",
ReplicationFactor = 2,
NumPartitions = 4
}
});
Console.WriteLine("Topic(s) created Successfully");
Console.WriteLine();
}
catch (CreateTopicsException e)
{
Console.WriteLine($"An error occured creating topic {e.Results[0].Topic}: {e.Results[0].Error.Reason}");
}
}
public TemporaryTopic(string bootstrapServers, int numPartitions)
{
Name = Guid.NewGuid().ToString();
adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build();
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = Name, NumPartitions = numPartitions, ReplicationFactor = 1
}
}).Wait();
}
public static async Task CreateTopicAsync(this IAdminClient client, string topic, int partitions)
{
await client.CreateTopicsAsync(new[]
{
new TopicSpecification
{
Name = topic,
NumPartitions = partitions
}
}, new CreateTopicsOptions
{
OperationTimeout = Timeout,
RequestTimeout = Timeout
});
}
private async Task createMissingTopics(IAdminClient client)
{
var metadata = client.GetMetadata(TimeSpan.FromSeconds(30));
var missingTopics = this.Kafka.Options.Topics.Where(tOpt => metadata.Topics.TrueForAll(t => t.Topic != tOpt.Name));
var sw = Stopwatch.StartNew();
await client.CreateTopicsAsync(missingTopics.Select(t => new TopicSpecification
{
Name = t.Name,
NumPartitions = t.Partitions,
ReplicationFactor = t.ReplicationFactor
}), new CreateTopicsOptions { OperationTimeout = TimeSpan.FromSeconds(30), RequestTimeout = TimeSpan.FromSeconds(30) });
sw.Stop();
}
private static async Task EnsureTopicExists(IAdminClient admin)
{
var metadata = admin.GetMetadata(Topic, TimeSpan.FromSeconds(15));
if (!metadata.Topics.Any(m => m.Topic == Topic))
{
await admin.CreateTopicsAsync(
new[]
{
new TopicSpecification
{
Name = Topic,
},
});
}
}
public override Task StartAsync(CancellationToken cancellationToken)
{
try
{
_client.CreateTopicsAsync(new TopicSpecification[] {
new TopicSpecification {
Name = _topic, ReplicationFactor = 1, NumPartitions = 1
}
});
}
catch (CreateTopicsException e)
{
_logger.LogError($"An error occured creating topic {e.Results[0].Topic}: {e.Results[0].Error.Reason}");
}
return(Task.CompletedTask);
}
public async Task <bool> EnsureTopicExists(string topicName)
{
_logger.LogInformation("Ensuring topic exists...");
var topicSpecification = new TopicSpecification()
{
Name = topicName,
NumPartitions = 1,
ReplicationFactor = 1
};
var topicSpecifications = new List <TopicSpecification>();
topicSpecifications.Add(topicSpecification);
// try and get metadata about topic to check that it exists
var metaData = _adminClient.GetMetadata(topicName, TimeSpan.FromSeconds(10));
if (!metaData.Topics[0].Error.IsError)
{
_logger.LogInformation(" Topic already exists.");
return(true);
}
_logger.LogInformation(" Creating topic...");
try
{
await _adminClient.CreateTopicsAsync(topicSpecifications);
_logger.LogInformation(" Topic created.");
}
catch (CreateTopicsException e)
{
// only trap fatal errors so that we let topic already exists error through
if (e.Error.IsFatal)
{
return(false);
}
}
return(true);
}
public async Task AddTopicAsync(Topic topic)
{
try
{
var topicsSpecification = new TopicSpecification[]
{
new TopicSpecification
{
Name = topic.Nome,
NumPartitions = topic.Particoes.GetValueOrDefault(6),
ReplicationFactor = topic.Replicas.GetValueOrDefault(3)
}
};
await client.CreateTopicsAsync(topicsSpecification);
}
catch (CreateTopicsException exception)
{
throw exception;
}
}
public async Task CreateTopicAsync(string topic, TopicConfiguration topicConfiguration)
{
var topics = _innerClient.GetMetadata(TimeSpan.FromSeconds(30)).Topics;
if (topics.Select(t => t.Topic).Contains(topic))
{
return;
}
var initialTopics = new List <TopicSpecification>
{
new TopicSpecification
{
Name = topic,
Configs = topicConfiguration.GetConfig(),
NumPartitions = 25,
ReplicationFactor = 2
}
};
await _innerClient.CreateTopicsAsync(initialTopics);
await Task.Delay(1000);
}
public async Task <IActionResult> CreateTopicAsync([FromBody] IEnumerable <TopicSpecification> topics)
{
await _client.CreateTopicsAsync(topics);
return(Ok());
}