private void ApplySettings(ConsumerSettings <K, V> updatedSettings) { _settings = updatedSettings; _pollTimeout = _settings.PollTimeout; _positionTimeout = _settings.PositionTimeout; _commitRefreshing = CommitRefreshing.Create <K, V>(_settings.CommitRefreshInterval); try { if (_log.IsDebugEnabled) { _log.Debug($"Creating Kafka consumer with settings: {JsonConvert.SerializeObject(_settings)}"); } _consumer = _settings.CreateKafkaConsumer( consumeErrorHandler: (c, e) => ProcessError(new KafkaException(e)), partitionAssignedHandler: (c, tp) => Self.Tell(new PartitionAssigned(tp.ToImmutableHashSet())), partitionRevokedHandler: (c, tp) => Self.Tell(new PartitionRevoked(tp.ToImmutableHashSet())), statisticHandler: (c, json) => _statisticsHandler.OnStatistics(c, json)); _adminClient = new DependentAdminClientBuilder(_consumer.Handle).Build(); if (_settings.ConnectionCheckerSettings.Enabled) { _connectionCheckerActor = Context.ActorOf(ConnectionChecker.Props(_settings.ConnectionCheckerSettings)); } } catch (Exception e) { ProcessError(e); throw; } }
protected static List <TopicPartition> LoadTopicPartitions(IAdminClient adminClient, string topicName) { try { var timeout = TimeSpan.FromSeconds(5); var metadata = adminClient.GetMetadata(topicName, timeout); if (metadata.Topics == null || metadata.Topics.Count == 0) { Console.WriteLine($"Could not load metadata information about topic '{topicName}'"); return(new List <TopicPartition>()); } var topicMetadata = metadata.Topics[0]; var partitions = topicMetadata.Partitions; if (partitions == null || partitions.Count == 0) { Console.WriteLine($"Could not load partition information about topic '{topicName}'"); return(new List <TopicPartition>()); } return(partitions.Select(x => new TopicPartition(topicMetadata.Topic, new Partition(x.PartitionId))).ToList()); } catch (Exception ex) { Console.WriteLine($"Failed to load partition information from topic '{topicName}' {ex.Message}: {ex.StackTrace}"); } return(new List <TopicPartition>()); }
public async void DeleteTopic(IAdminClient admin, string topicName) { string[] topics = new string[1] { topicName }; await admin.DeleteTopicsAsync(topics, null); }
public KafkaAdminService(ILogger <KafkaAdminService> logger, IConfiguration cfg) : base(logger, cfg) { _client = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = _bootstrapServers }).Build(); }
private async Task TryCreateTopicAsync(string topicName, IAdminClient adminClient) { _logger.LogInformation("Setting up Kafka topic {Topic} ...", topicName); try { await adminClient.CreateTopicsAsync(new[] { new TopicSpecification { Name = topicName, ReplicationFactor = 1, NumPartitions = 1 } }); } catch (CreateTopicsException ex) when(ex.Error?.Code == ErrorCode.Local_Partial) { _logger.LogWarning(ex, "An error occured creating topic {Topic}: {Error}", topicName, ex.Message); } catch (Exception ex) { _logger.LogError(ex, "An error occured creating topic {Topic}: {Error}", topicName, ex.Message); } }
private void PrepareTopic(IAdminClient adminClient, string topic) { var metadata = adminClient.GetMetadata(topic, TimeSpan.FromSeconds(20)); var partitionCount = metadata.Topics.First().Partitions.Count; if (partitionCount == 0) { adminClient.CreateTopicsAsync(new[] { new TopicSpecification { NumPartitions = _options.TopicPartitionCount, Name = topic, ReplicationFactor = 1 } }).GetAwaiter().GetResult(); } else { if (partitionCount < _options.TopicPartitionCount) { adminClient.CreatePartitionsAsync(new[] { new PartitionsSpecification { Topic = topic, IncreaseTo = _options.TopicPartitionCount } }).GetAwaiter().GetResult(); } } }
public Worker(ILogger <Worker> logger, IOptions <ConsumerConfig> config, IOptions <AdminClientConfig> adminConfig, KafkaMetrics metrics) { this._logger = logger; this.consumer = new ConsumerBuilder <Ignore, string>(config.Value).Build(); this.admin = new AdminClientBuilder(adminConfig.Value).Build(); this.metrics = metrics; }
internal static bool TopicExists(this IAdminClient adminClient, string topicName, out List <CreateTopicReport> createTopicErrorReports) { createTopicErrorReports = GetTopicMetadataErrorReports(adminClient, topicName); return (createTopicErrorReports.Count < 1 && !createTopicErrorReports.Exists(x => x.Error == ErrorCode.UnknownTopicOrPart)); }
public AdminController( IRequestFieldExtractor extractor, IAdminClient adminClient) { requestFieldExtractor = extractor; adminService = new AdminService(adminClient); }
private static void CreateTopicWithAdminClient(IAdminClient adminClient) { try { adminClient.CreateTopicsAsync(new TopicSpecification[] { new TopicSpecification { Name = "topic01", ReplicationFactor = 2, NumPartitions = 4 }, new TopicSpecification { Name = "topic02", ReplicationFactor = 2, NumPartitions = 4 } }); Console.WriteLine("Topic(s) created Successfully"); Console.WriteLine(); } catch (CreateTopicsException e) { Console.WriteLine($"An error occured creating topic {e.Results[0].Topic}: {e.Results[0].Error.Reason}"); } }
public Worker(ILogger <Worker> logger, IOptions <ProducerConfig> config, IOptions <AdminClientConfig> adminConfig, KafkaMetrics metrics) { this._logger = logger; this.producer = new ProducerBuilder <Null, string>(config.Value).Build(); this.admin = new AdminClientBuilder(adminConfig.Value).Build(); this.metrics = metrics; }
/// <summary> /// /// </summary> /// <param name="environment"></param> /// <param name="serviceScopeFactory"></param> /// <param name="busOptionsAccessor"></param> /// <param name="transportOptionsAccessor"></param> /// <param name="loggerFactory"></param> public KafkaTransport(IHostEnvironment environment, IServiceScopeFactory serviceScopeFactory, IOptions <EventBusOptions> busOptionsAccessor, IOptions <KafkaTransportOptions> transportOptionsAccessor, ILoggerFactory loggerFactory) : base(serviceScopeFactory, busOptionsAccessor, transportOptionsAccessor, loggerFactory) { // Should be setup the logger? adminClient = new AdminClientBuilder(TransportOptions.AdminConfig).Build(); // create the shared producer instance var pconfig = new ProducerConfig(TransportOptions.AdminConfig); producer = new ProducerBuilder <string, byte[]>(pconfig) //.SetValueSerializer((ISerializer<byte[]>)null) .Build(); // create the shared consumer instance var c_config = new ConsumerConfig(TransportOptions.AdminConfig) { GroupId = BusOptions.Naming.GetApplicationName(environment), //EnableAutoCommit = false, //StatisticsIntervalMs = 5000, //SessionTimeoutMs = 6000, //AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true, }; consumer = new ConsumerBuilder <string, byte[]>(c_config) //.SetValueSerializer((ISerializer<byte[]>)null) .Build(); }
public static List <string> ListTopics(this IAdminClient client) { return(client.GetMetadata(Timeout) .Topics .Select(t => t.Topic) .ToList()); }
public static async Task PrintMetrics(IAdminClient adminClient, IConsumer <string, string> consumer, string topicName) { var topicPartitions = LoadTopicPartitions(adminClient, topicName); var metrics = await GetMetricsAsync(topicPartitions, consumer, topicName); Console.WriteLine(metrics.ToString()); }
internal TaskManager(InternalTopologyBuilder builder, TaskCreator taskCreator, IAdminClient adminClient, IChangelogReader changelogReader) { this.builder = builder; this.taskCreator = taskCreator; this.adminClient = adminClient; this.changelogReader = changelogReader; }
public static async Task DeleteTopicAsync(this IAdminClient client, string topic) { await client.DeleteTopicsAsync(new[] { topic }, new DeleteTopicsOptions { OperationTimeout = Timeout, RequestTimeout = Timeout }); }
internal static List <CreateTopicReport> GetTopicMetadataErrorReports(this IAdminClient adminClient, string topicName, int timeoutInSeconds = 15) { return(adminClient .GetMetadata(topicName, TimeSpan.FromSeconds(timeoutInSeconds)).Topics .FindAll(x => x.Error != ErrorCode.NoError) .ConvertAll(x => new CreateTopicReport { Topic = x.Topic, Error = x.Error })); }
public KafkaTopicFactory(ProducerConfig config) { if (config == null) { throw new ArgumentNullException(nameof(config)); } _adminClient = new AdminClientBuilder(config).Build(); }
public KafkaAdminClient(IOptions <AdminClientConfig> adminclientConfig, ILogger <KafkaAdminClient> logger) { _logger = logger; _logger.LogInformation("Building admin client..."); _adminClient = new AdminClientBuilder(adminclientConfig.Value) .SetLogHandler(OnAdminLog) .SetErrorHandler(OnAdminError) .Build(); }
public KafkaMessagingClient(IDistributedSearchConfiguration demoCredential, TopicAndPartition topicAndPartition) { var bootstrapServers = $"{demoCredential.EventHubName}.servicebus.windows.net:9093"; var saslUsername = "******"; var saslPassword = demoCredential.EventHubConnectionString; var securityProtocol = SecurityProtocol.SaslSsl; var saslMechanism = SaslMechanism.Plain; var groupId = "$Default"; this.producer = new ProducerBuilder <Null, string>(new ProducerConfig { BootstrapServers = bootstrapServers, SecurityProtocol = securityProtocol, SaslMechanism = saslMechanism, SaslUsername = saslUsername, SaslPassword = saslPassword, // SslCaLocation = cacertlocation, // Debug = "security,broker,protocol", }) .SetKeySerializer(Serializers.Null) .SetValueSerializer(Serializers.Utf8) .Build(); this.consumer = new ConsumerBuilder <Ignore, string>(new ConsumerConfig { BootstrapServers = bootstrapServers, SecurityProtocol = securityProtocol, SaslMechanism = saslMechanism, SaslUsername = saslUsername, SaslPassword = saslPassword, GroupId = groupId, BrokerVersionFallback = "1.0.0", AutoOffsetReset = AutoOffsetReset.Latest, // SslCaLocation = cacertlocation, // Debug = "security,broker,protocol", }) .SetKeyDeserializer(Deserializers.Ignore) .SetValueDeserializer(Deserializers.Utf8) .Build(); this.adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers, SecurityProtocol = securityProtocol, SaslMechanism = saslMechanism, SaslUsername = saslUsername, SaslPassword = saslPassword, }).Build(); this.topicPartition = new Lazy <TopicPartition>(() => new TopicPartition( topic: topicAndPartition.TopicName, partition: DeterminePartitionID(this.adminClient, topicAndPartition))); }
public KafkaAdmin() { var config = new AdminClientConfig() { BootstrapServers = "localhost:9092", }; _adminBuilder = new AdminClientBuilder(config); _adminClient = _adminBuilder.Build(); }
public static int GetPartitionCount(this IAdminClient client, string topic) { var topicMetadata = GetTopicMetadata(client, topic); if (topicMetadata == null) { throw new Exception("Unknown topic."); } return(topicMetadata.Partitions.Count); }
private static ConfluentPartition DeterminePartitionID(IAdminClient adminClient, TopicAndPartition topicAndPartition) { MercuryPartition partitionId = determinePartitionID( determinePartitionCount: GetPartitionCount(adminClient).ToFSharpFunc(), topicAndPartition: topicAndPartition); return(partitionId switch { MercuryPartition.Partition x => new ConfluentPartition(x.Item), _ => ConfluentPartition.Any, });
public TopicCreator(NetStreamConfiguration configuration) { _configuration = configuration; var adminConfig = new AdminClientConfig { BootstrapServers = configuration.BootstrapServers }; _adminClient = new AdminClientBuilder(adminConfig).Build(); }
/// <summary> /// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" /> /// Please DO NOT FORGET to call Close to avoid resources leak ! /// </summary> /// <param name="topology">the topology specifying the computational logic</param> /// <param name="configuration">configuration about this stream</param> /// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param> public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier) { this.topology = topology; this.configuration = configuration; this.kafkaSupplier = kafkaSupplier; var processID = Guid.NewGuid(); clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId; logPrefix = $"stream-application[{configuration.ApplicationId}] "; // re-write the physical topology according to the config topology.Builder.RewriteTopology(configuration); // sanity check this.processorTopology = topology.Builder.BuildTopology(); this.threads = new IThread[this.configuration.NumStreamThreads]; var threadState = new Dictionary <long, Processors.ThreadState>(); List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>(); for (int i = 0; i < this.configuration.NumStreamThreads; ++i) { var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}"; adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId))); this.threads[i] = StreamThread.Create( threadId, clientId, this.topology.Builder, configuration, this.kafkaSupplier, adminClient, i); threadState.Add(this.threads[i].Id, this.threads[i].State); stateStoreProviders.Add(new StreamThreadStateStoreProvider(this.threads[i], this.topology.Builder)); } var manager = new StreamStateManager(this, threadState); foreach (var t in threads) { t.StateChanged += manager.OnChange; } this.queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders); StreamState = State.CREATED; }
public AdminClientWrapper(string name, IDictionary <string, string> config, TimeSpan expirationTimeout) : base( name, config, expirationTimeout) { try { _adminClient = new AdminClientBuilder(config).Build(); } catch (Exception e) { throw new ClientConfigException(e); } }
public GlobalStreamThreadFactory(ProcessorTopology topology, string threadClientId, IConsumer <byte[], byte[]> globalConsumer, IStreamConfig configuration, IAdminClient adminClient) { this.adminClient = adminClient; this.topology = topology; this.threadClientId = threadClientId; this.configuration = configuration; this.globalConsumer = globalConsumer; }
public void CantCreateLoaderWithEmptyAdminClient() { // Arrange var topic = new TopicName("test"); IAdminClient client = null !; var timeout = 1000; // Act var exception = Record.Exception(() => new TopicWatermarkLoader(topic, client, timeout)); // Assert exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>(); }
public TemporaryTopic(string bootstrapServers, int numPartitions) { Name = Guid.NewGuid().ToString(); adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build(); adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = Name, NumPartitions = numPartitions, ReplicationFactor = 1 } }).Wait(); }
public KafkaAdminClientRepository(IOptions <Secrets> secrets) { var client = new AdminClientConfig { BootstrapServers = secrets.Value.Kafka.Host, SaslMechanism = SaslMechanism.Plain, SecurityProtocol = SecurityProtocol.SaslSsl, SaslUsername = secrets.Value.Kafka.SaslUsername, SaslPassword = secrets.Value.Kafka.SaslPassword }; this.client = new AdminClientBuilder(client).Build(); }