private void ApplySettings(ConsumerSettings <K, V> updatedSettings)
{
_settings = updatedSettings;
_pollTimeout = _settings.PollTimeout;
_positionTimeout = _settings.PositionTimeout;
_commitRefreshing = CommitRefreshing.Create <K, V>(_settings.CommitRefreshInterval);
try
{
if (_log.IsDebugEnabled)
{
_log.Debug($"Creating Kafka consumer with settings: {JsonConvert.SerializeObject(_settings)}");
}
_consumer = _settings.CreateKafkaConsumer(
consumeErrorHandler: (c, e) => ProcessError(new KafkaException(e)),
partitionAssignedHandler: (c, tp) => Self.Tell(new PartitionAssigned(tp.ToImmutableHashSet())),
partitionRevokedHandler: (c, tp) => Self.Tell(new PartitionRevoked(tp.ToImmutableHashSet())),
statisticHandler: (c, json) => _statisticsHandler.OnStatistics(c, json));
_adminClient = new DependentAdminClientBuilder(_consumer.Handle).Build();
if (_settings.ConnectionCheckerSettings.Enabled)
{
_connectionCheckerActor = Context.ActorOf(ConnectionChecker.Props(_settings.ConnectionCheckerSettings));
}
}
catch (Exception e)
{
ProcessError(e);
throw;
}
}
protected static List <TopicPartition> LoadTopicPartitions(IAdminClient adminClient, string topicName)
{
try
{
var timeout = TimeSpan.FromSeconds(5);
var metadata = adminClient.GetMetadata(topicName, timeout);
if (metadata.Topics == null || metadata.Topics.Count == 0)
{
Console.WriteLine($"Could not load metadata information about topic '{topicName}'");
return(new List <TopicPartition>());
}
var topicMetadata = metadata.Topics[0];
var partitions = topicMetadata.Partitions;
if (partitions == null || partitions.Count == 0)
{
Console.WriteLine($"Could not load partition information about topic '{topicName}'");
return(new List <TopicPartition>());
}
return(partitions.Select(x => new TopicPartition(topicMetadata.Topic, new Partition(x.PartitionId))).ToList());
}
catch (Exception ex)
{
Console.WriteLine($"Failed to load partition information from topic '{topicName}' {ex.Message}: {ex.StackTrace}");
}
return(new List <TopicPartition>());
}
public async void DeleteTopic(IAdminClient admin, string topicName)
{
string[] topics = new string[1] {
topicName
};
await admin.DeleteTopicsAsync(topics, null);
}
public KafkaAdminService(ILogger <KafkaAdminService> logger, IConfiguration cfg) : base(logger, cfg)
{
_client = new AdminClientBuilder(new AdminClientConfig
{
BootstrapServers = _bootstrapServers
}).Build();
}
private async Task TryCreateTopicAsync(string topicName, IAdminClient adminClient)
{
_logger.LogInformation("Setting up Kafka topic {Topic} ...", topicName);
try
{
await adminClient.CreateTopicsAsync(new[]
{
new TopicSpecification
{
Name = topicName,
ReplicationFactor = 1,
NumPartitions = 1
}
});
}
catch (CreateTopicsException ex) when(ex.Error?.Code == ErrorCode.Local_Partial)
{
_logger.LogWarning(ex, "An error occured creating topic {Topic}: {Error}", topicName, ex.Message);
}
catch (Exception ex)
{
_logger.LogError(ex, "An error occured creating topic {Topic}: {Error}", topicName, ex.Message);
}
}
private void PrepareTopic(IAdminClient adminClient, string topic)
{
var metadata = adminClient.GetMetadata(topic, TimeSpan.FromSeconds(20));
var partitionCount = metadata.Topics.First().Partitions.Count;
if (partitionCount == 0)
{
adminClient.CreateTopicsAsync(new[]
{
new TopicSpecification
{
NumPartitions = _options.TopicPartitionCount, Name = topic, ReplicationFactor = 1
}
}).GetAwaiter().GetResult();
}
else
{
if (partitionCount < _options.TopicPartitionCount)
{
adminClient.CreatePartitionsAsync(new[]
{
new PartitionsSpecification
{
Topic = topic, IncreaseTo = _options.TopicPartitionCount
}
}).GetAwaiter().GetResult();
}
}
}
public Worker(ILogger <Worker> logger, IOptions <ConsumerConfig> config, IOptions <AdminClientConfig> adminConfig, KafkaMetrics metrics)
{
this._logger = logger;
this.consumer = new ConsumerBuilder <Ignore, string>(config.Value).Build();
this.admin = new AdminClientBuilder(adminConfig.Value).Build();
this.metrics = metrics;
}
internal static bool TopicExists(this IAdminClient adminClient, string topicName, out List <CreateTopicReport> createTopicErrorReports)
{
createTopicErrorReports = GetTopicMetadataErrorReports(adminClient, topicName);
return
(createTopicErrorReports.Count < 1 &&
!createTopicErrorReports.Exists(x => x.Error == ErrorCode.UnknownTopicOrPart));
}
public AdminController(
IRequestFieldExtractor extractor,
IAdminClient adminClient)
{
requestFieldExtractor = extractor;
adminService = new AdminService(adminClient);
}
private static void CreateTopicWithAdminClient(IAdminClient adminClient)
{
try
{
adminClient.CreateTopicsAsync(new TopicSpecification[]
{
new TopicSpecification
{
Name = "topic01",
ReplicationFactor = 2,
NumPartitions = 4
},
new TopicSpecification
{
Name = "topic02",
ReplicationFactor = 2,
NumPartitions = 4
}
});
Console.WriteLine("Topic(s) created Successfully");
Console.WriteLine();
}
catch (CreateTopicsException e)
{
Console.WriteLine($"An error occured creating topic {e.Results[0].Topic}: {e.Results[0].Error.Reason}");
}
}
public Worker(ILogger <Worker> logger, IOptions <ProducerConfig> config, IOptions <AdminClientConfig> adminConfig, KafkaMetrics metrics)
{
this._logger = logger;
this.producer = new ProducerBuilder <Null, string>(config.Value).Build();
this.admin = new AdminClientBuilder(adminConfig.Value).Build();
this.metrics = metrics;
}
/// <summary>
///
/// </summary>
/// <param name="environment"></param>
/// <param name="serviceScopeFactory"></param>
/// <param name="busOptionsAccessor"></param>
/// <param name="transportOptionsAccessor"></param>
/// <param name="loggerFactory"></param>
public KafkaTransport(IHostEnvironment environment,
IServiceScopeFactory serviceScopeFactory,
IOptions <EventBusOptions> busOptionsAccessor,
IOptions <KafkaTransportOptions> transportOptionsAccessor,
ILoggerFactory loggerFactory)
: base(serviceScopeFactory, busOptionsAccessor, transportOptionsAccessor, loggerFactory)
{
// Should be setup the logger?
adminClient = new AdminClientBuilder(TransportOptions.AdminConfig).Build();
// create the shared producer instance
var pconfig = new ProducerConfig(TransportOptions.AdminConfig);
producer = new ProducerBuilder <string, byte[]>(pconfig)
//.SetValueSerializer((ISerializer<byte[]>)null)
.Build();
// create the shared consumer instance
var c_config = new ConsumerConfig(TransportOptions.AdminConfig)
{
GroupId = BusOptions.Naming.GetApplicationName(environment),
//EnableAutoCommit = false,
//StatisticsIntervalMs = 5000,
//SessionTimeoutMs = 6000,
//AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true,
};
consumer = new ConsumerBuilder <string, byte[]>(c_config)
//.SetValueSerializer((ISerializer<byte[]>)null)
.Build();
}
public static List <string> ListTopics(this IAdminClient client)
{
return(client.GetMetadata(Timeout)
.Topics
.Select(t => t.Topic)
.ToList());
}
public static async Task PrintMetrics(IAdminClient adminClient, IConsumer <string, string> consumer,
string topicName)
{
var topicPartitions = LoadTopicPartitions(adminClient, topicName);
var metrics = await GetMetricsAsync(topicPartitions, consumer, topicName);
Console.WriteLine(metrics.ToString());
}
internal TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient,
IChangelogReader changelogReader)
{
this.builder = builder;
this.taskCreator = taskCreator;
this.adminClient = adminClient;
this.changelogReader = changelogReader;
}
public static async Task DeleteTopicAsync(this IAdminClient client, string topic)
{
await client.DeleteTopicsAsync(new[] { topic }, new DeleteTopicsOptions
{
OperationTimeout = Timeout,
RequestTimeout = Timeout
});
}
internal static List <CreateTopicReport> GetTopicMetadataErrorReports(this IAdminClient adminClient, string topicName, int timeoutInSeconds = 15)
{
return(adminClient
.GetMetadata(topicName, TimeSpan.FromSeconds(timeoutInSeconds)).Topics
.FindAll(x => x.Error != ErrorCode.NoError)
.ConvertAll(x => new CreateTopicReport {
Topic = x.Topic, Error = x.Error
}));
}
public KafkaTopicFactory(ProducerConfig config)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
_adminClient = new AdminClientBuilder(config).Build();
}
public KafkaAdminClient(IOptions <AdminClientConfig> adminclientConfig, ILogger <KafkaAdminClient> logger)
{
_logger = logger;
_logger.LogInformation("Building admin client...");
_adminClient = new AdminClientBuilder(adminclientConfig.Value)
.SetLogHandler(OnAdminLog)
.SetErrorHandler(OnAdminError)
.Build();
}
public KafkaMessagingClient(IDistributedSearchConfiguration demoCredential, TopicAndPartition topicAndPartition)
{
var bootstrapServers = $"{demoCredential.EventHubName}.servicebus.windows.net:9093";
var saslUsername = "******";
var saslPassword = demoCredential.EventHubConnectionString;
var securityProtocol = SecurityProtocol.SaslSsl;
var saslMechanism = SaslMechanism.Plain;
var groupId = "$Default";
this.producer = new ProducerBuilder <Null, string>(new ProducerConfig
{
BootstrapServers = bootstrapServers,
SecurityProtocol = securityProtocol,
SaslMechanism = saslMechanism,
SaslUsername = saslUsername,
SaslPassword = saslPassword,
// SslCaLocation = cacertlocation,
// Debug = "security,broker,protocol",
})
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(Serializers.Utf8)
.Build();
this.consumer = new ConsumerBuilder <Ignore, string>(new ConsumerConfig
{
BootstrapServers = bootstrapServers,
SecurityProtocol = securityProtocol,
SaslMechanism = saslMechanism,
SaslUsername = saslUsername,
SaslPassword = saslPassword,
GroupId = groupId,
BrokerVersionFallback = "1.0.0",
AutoOffsetReset = AutoOffsetReset.Latest,
// SslCaLocation = cacertlocation,
// Debug = "security,broker,protocol",
})
.SetKeyDeserializer(Deserializers.Ignore)
.SetValueDeserializer(Deserializers.Utf8)
.Build();
this.adminClient = new AdminClientBuilder(new AdminClientConfig
{
BootstrapServers = bootstrapServers,
SecurityProtocol = securityProtocol,
SaslMechanism = saslMechanism,
SaslUsername = saslUsername,
SaslPassword = saslPassword,
}).Build();
this.topicPartition = new Lazy <TopicPartition>(() => new TopicPartition(
topic: topicAndPartition.TopicName,
partition: DeterminePartitionID(this.adminClient, topicAndPartition)));
}
public KafkaAdmin()
{
var config = new AdminClientConfig()
{
BootstrapServers = "localhost:9092",
};
_adminBuilder = new AdminClientBuilder(config);
_adminClient = _adminBuilder.Build();
}
public static int GetPartitionCount(this IAdminClient client, string topic)
{
var topicMetadata = GetTopicMetadata(client, topic);
if (topicMetadata == null)
{
throw new Exception("Unknown topic.");
}
return(topicMetadata.Partitions.Count);
}
private static ConfluentPartition DeterminePartitionID(IAdminClient adminClient, TopicAndPartition topicAndPartition)
{
MercuryPartition partitionId = determinePartitionID(
determinePartitionCount: GetPartitionCount(adminClient).ToFSharpFunc(),
topicAndPartition: topicAndPartition);
return(partitionId switch
{
MercuryPartition.Partition x => new ConfluentPartition(x.Item),
_ => ConfluentPartition.Any,
});
public TopicCreator(NetStreamConfiguration configuration)
{
_configuration = configuration;
var adminConfig = new AdminClientConfig
{
BootstrapServers = configuration.BootstrapServers
};
_adminClient = new AdminClientBuilder(adminConfig).Build();
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.configuration = configuration;
this.kafkaSupplier = kafkaSupplier;
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
this.processorTopology = topology.Builder.BuildTopology();
this.threads = new IThread[this.configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < this.configuration.NumStreamThreads; ++i)
{
var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}";
adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
this.threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(this.threads[i].Id, this.threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(this.threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
this.queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders);
StreamState = State.CREATED;
}
public AdminClientWrapper(string name, IDictionary <string, string> config, TimeSpan expirationTimeout) : base(
name, config, expirationTimeout)
{
try
{
_adminClient = new AdminClientBuilder(config).Build();
}
catch (Exception e)
{
throw new ClientConfigException(e);
}
}
public GlobalStreamThreadFactory(ProcessorTopology topology,
string threadClientId,
IConsumer <byte[], byte[]> globalConsumer,
IStreamConfig configuration,
IAdminClient adminClient)
{
this.adminClient = adminClient;
this.topology = topology;
this.threadClientId = threadClientId;
this.configuration = configuration;
this.globalConsumer = globalConsumer;
}
public void CantCreateLoaderWithEmptyAdminClient()
{
// Arrange
var topic = new TopicName("test");
IAdminClient client = null !;
var timeout = 1000;
// Act
var exception = Record.Exception(() => new TopicWatermarkLoader(topic, client, timeout));
// Assert
exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>();
}
public TemporaryTopic(string bootstrapServers, int numPartitions)
{
Name = Guid.NewGuid().ToString();
adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build();
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = Name, NumPartitions = numPartitions, ReplicationFactor = 1
}
}).Wait();
}
public KafkaAdminClientRepository(IOptions <Secrets> secrets)
{
var client = new AdminClientConfig
{
BootstrapServers = secrets.Value.Kafka.Host,
SaslMechanism = SaslMechanism.Plain,
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslUsername = secrets.Value.Kafka.SaslUsername,
SaslPassword = secrets.Value.Kafka.SaslPassword
};
this.client = new AdminClientBuilder(client).Build();
}
