public IProducer <Null, byte[]> CreateProducer( KafkaQueueConfiguration config, Action <IProducer <Null, byte[]>, LogMessage> logHandler = null, Action <IProducer <Null, byte[]>, Error> errorHandler = null) { config.ThrowIfNull(nameof(config)); var builder = new ProducerBuilder <Null, byte[]>(new ProducerConfig { BootstrapServers = $"{config.Server}", SaslKerberosKeytab = config.KeyTab, SaslKerberosPrincipal = config.User, SaslKerberosServiceName = config.ServiceName, SecurityProtocol = config.Protocol, SaslMechanism = config.Mechanism, Debug = config.Debug }); if (logHandler != null) { builder.SetLogHandler(logHandler); } if (errorHandler != null) { builder.SetErrorHandler(errorHandler); } return(builder.Build()); }
public IProducer <byte[], byte[]> GetProducer(ProducerConfig config) { ProducerBuilder <byte[], byte[]> builder = new ProducerBuilder <byte[], byte[]>(config); builder.SetLogHandler(loggerAdapter.LogProduce); builder.SetErrorHandler(loggerAdapter.ErrorProduce); return(builder.Build()); }
private static ProducerBuilder <T, TV> GetProducerBuilderInstance <T, TV>(ProducerConfig configuration) where T : class where TV : class { var producer = new ProducerBuilder <T, TV>(configuration); producer.SetErrorHandler((_, e) => Devon4NetLogger.Error(new ConsumerException($"Error code {e.Code} : {e.Reason}"))); producer.SetStatisticsHandler((_, json) => Devon4NetLogger.Information($"Statistics: {json}")); producer.SetLogHandler((c, partitions) => { Devon4NetLogger.Information($"Kafka log handler: [{string.Join(", ", partitions)}]"); }); return(producer); }
private async void Produce_Click(object sender, EventArgs e) { if (producer == null) { var config = new ProducerConfig { BootstrapServers = bootstrapServers.Text }; if (!string.IsNullOrWhiteSpace(username.Text) && !string.IsNullOrWhiteSpace(password.Text)) { config.SaslUsername = username.Text; config.SaslPassword = password.Text; if (securityProtocol.SelectedItem != null) { config.SecurityProtocol = ((ComboboxItem <SecurityProtocol>)securityProtocol.SelectedItem).Value; if (config.SecurityProtocol == SecurityProtocol.SaslSsl || config.SecurityProtocol == SecurityProtocol.Ssl) { config.SslCaLocation = caCertificateFileLocation.Text; } } if (saslMechanism.SelectedItem != null) { config.SaslMechanism = ((ComboboxItem <SaslMechanism>)saslMechanism.SelectedItem).Value; } } config.Debug = "all"; try { var builder = new ProducerBuilder <Null, string>(config); builder.SetLogHandler((producer, message) => { // add logging }); producer = new ProducerBuilder <Null, string>(config).Build(); } catch (ProduceException <Null, string> ex) { MessageBox.Show(this, ex.Message); return; } } try { var dr = await producer.ProduceAsync(produceToTopic.Text, new Message <Null, string> { Value = input.TextValue }).ConfigureAwait(true); status.Text = $"Delivered to '{dr.TopicPartitionOffset}'"; } catch (ProduceException <Null, string> ex) { MessageBox.Show(this, $"Delivery failed: {ex.Error.Reason}"); } }
private IProducer <byte[], byte[]> CreateBaseProducer(ProducerConfig producerConfig) { var builder = new ProducerBuilder <byte[], byte[]>(producerConfig); ILogger logger = this.loggerProvider.CreateLogger("Kafka"); builder.SetLogHandler((_, m) => { logger.Log((LogLevel)m.LevelAs(LogLevelType.MicrosoftExtensionsLogging), $"Libkafka: {m?.Message}"); }); return(builder.Build()); }
/// <summary> /// producer构造器 /// </summary> /// <returns></returns> private ProducerBuilder <string, object> CreateProducerBuilder() { if (builder == null) { lock (this) { if (builder == null) { ProducerConfig config = new ProducerConfig(); config.BootstrapServers = BootstrapServers; if (!string.IsNullOrEmpty(SaslUsername)) { config.SaslUsername = SaslUsername; config.SaslPassword = SaslPassword; config.SaslMechanism = SaslMechanism.Plain; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; } //List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>(); //config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers)); //if (!string.IsNullOrEmpty(SaslUsername)) //{ // config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT")); // config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN")); // config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername)); // config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword)); //} builder = new ProducerBuilder <string, object>(config); Action <Delegate, object> tryCatchWrap = (@delegate, arg) => { try { @delegate?.DynamicInvoke(arg); } catch { } }; builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason))); builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e)); builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e))); builder.SetValueSerializer(new KafkaConverter()); } } } return(builder); }
/// <inheritdoc cref="IConfluentProducerBuilder.Build" /> public IProducer <byte[]?, byte[]?> Build() { if (_config == null) { throw new InvalidOperationException("SetConfig must be called to provide the producer configuration."); } var builder = new ProducerBuilder <byte[]?, byte[]?>(_config); if (_statisticsHandler != null) { builder.SetStatisticsHandler(_statisticsHandler); } if (_logHandler != null) { builder.SetLogHandler(_logHandler); } return(builder.Build()); }
public IProducer <byte[], byte[]> GetProducer(ProducerConfig config) { ProducerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetProducerBuilder(config); builder.SetLogHandler(loggerAdapter.LogProduce); builder.SetErrorHandler(loggerAdapter.ErrorProduce); if (exposeLibrdKafka) { // TODO : test librdkafka statistics with IntegrationTest (WIP see #82) var producerStatisticsHandler = new ProducerStatisticsHandler( config.ClientId, streamConfig.ApplicationId, (config as StreamizProducerConfig)?.ThreadId, (config as StreamizProducerConfig)?.Id?.ToString()); producerStatisticsHandler.Register(MetricsRegistry); builder.SetStatisticsHandler((c, stat) => { var statistics = JsonConvert.DeserializeObject <Statistics>(stat); producerStatisticsHandler.Publish(statistics); }); } return(builder.Build()); }
public string ProduceMessage(int keyId) { var message = $"Producer value-{keyId}"; var eventToSend = new Event { id = Guid.NewGuid().ToString(), date = DateTime.Now.ToString("u"), message = new Message { header = new Schemas.Header { msg_id = Guid.NewGuid().ToString(), msg_date = DateTime.Now.ToString("u") }, Body = message } }; try { var kafkaConfig = new ProducerConfig { BootstrapServers = Kafka }; var prod = new ProducerBuilder <Key, Event>(kafkaConfig); prod.SetErrorHandler(ProducerErrorHandler); prod.SetLogHandler(ProducerLogHandler); using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig)) { prod.SetKeySerializer(new AvroSerializer <Key>(schemaRegistry)); prod.SetValueSerializer(new AvroSerializer <Event>(schemaRegistry)); using (var p = prod.Build()) { try { var dr = p.ProduceAsync("logs", new Message <Key, Event> { Key = new Key { id = keyId }, Value = eventToSend }).GetAwaiter().GetResult(); _logger.LogInformation($"Kafka Producer service delivered the message '{message}' to Topic: {dr.Topic}"); } catch (ProduceException <Key, Event> e) { _logger.LogInformation($"Kafka Producer service Delivery failed: {e.Error.Reason}"); } } } } catch (OperationCanceledException oce) { _logger.LogError(oce, "OperationCanceled - Error in delivered message"); } catch (Exception e) { _logger.LogError(e, "Error in delivered message"); throw; } return($"The message '{message}' has been sent to the broker."); }
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken)) { funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding); CancellationTokenSource errorCts = new CancellationTokenSource(); CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token); CancellationToken compositeCancellationToken = compositeCts.Token; bool aMessageHasBeenProcessed = false; var cConfig = new ConsumerConfig { ClientId = $"{Name}-consumer-{instanceId}", GroupId = $"{Name}-group", BootstrapServers = BootstrapServers, EnableAutoCommit = true, EnableAutoOffsetStore = false, AutoOffsetReset = AutoOffsetReset.Latest }; if (DebugContext != null) { cConfig.Debug = DebugContext; } var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig); if (InKeyDeserializer != null) { cBuilder.SetKeyDeserializer(InKeyDeserializer); } if (InValueDeserializer != null) { cBuilder.SetValueDeserializer(InValueDeserializer); } if (Logger != null) { cBuilder.SetLogHandler((_, m) => { Logger(m); }); } cBuilder.SetErrorHandler((c, e) => { if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { // Logger.Log(e); errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var pConfig = new ProducerConfig { ClientId = $"{Name}-producer-{instanceId}", BootstrapServers = BootstrapServers, EnableIdempotence = true, LingerMs = 5, DeliveryReportFields = "none" }; if (DebugContext != null) { pConfig.Debug = DebugContext; } var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig); if (OutKeySerializer != null) { pBuilder.SetKeySerializer(OutKeySerializer); } if (OutValueSerializer != null) { pBuilder.SetValueSerializer(OutValueSerializer); } if (Logger != null) { pBuilder.SetLogHandler((_, m) => { Logger(m); }); } pBuilder.SetErrorHandler((p, e) => { if (e.IsFatal) { errorCts.Cancel(); return; } if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var partitionState = new Dictionary <TopicPartition, PartitionState>(); using (var producer = pBuilder.Build()) using (var consumer = cBuilder.Build()) { consumer.Subscribe(InputTopic); try { while (true) { ConsumeResult <TInKey, TInValue> cr; try { cr = consumer.Consume(compositeCancellationToken); } catch (ConsumeException ex) { if (ex.Error.Code == ErrorCode.Local_ValueDeserialization) { // For an in-depth discussion of what to do in the event of deserialization errors, refer to: // https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues if (ConsumeErrorTolerance == ErrorTolerance.All) { continue; } errorCts.Cancel(); // no error tolerance. } Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why? continue; } if (!partitionState.ContainsKey(cr.TopicPartition)) { partitionState.Add(cr.TopicPartition, new PartitionState(this)); } partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts); aMessageHasBeenProcessed = true; } } catch (OperationCanceledException) { } } if (errorCts.IsCancellationRequested) { throw new Exception("error occured, and we're failing fast."); } }