static async Task ProducerDemoAsync() { IProducer <string, string> producer = new ProducerBuilder <string, string>(new ProducerConfig { BootstrapServers = BootstrapServers, Acks = Acks.Leader, MessageSendMaxRetries = 5, BatchSize = 20, LingerMs = 3000, }).Build(); try { while (true) { DeliveryResult <string, string> deliveryResult = await producer.ProduceAsync(Topic, new Message <string, string> { Key = Guid.NewGuid().ToString(), Value = DateTime.Now.ToString() }); Console.WriteLine($"Producer::{deliveryResult.Key}::{deliveryResult.Value}::{deliveryResult.Partition.Value}::{deliveryResult.Offset.Value}::{Thread.CurrentThread.ManagedThreadId}"); // await Task.Delay(100); } } catch (Exception ex) { Console.WriteLine(ex); } finally { producer.Flush(); producer.Dispose(); } }
public void PartitionerTransfersToProducer() { var resetEvent = new ManualResetEventSlim(initialState: false); var producer = new ProducerBuilder <Null, Null>(new ProducerConfig()) .SetKeySerializer(Serializers.Null) .SetPartitioner("test", new TestPartitioners.ResetOnDisposeTestPartitioner(resetEvent)) .Build(); Assert.False(resetEvent.IsSet); producer.Dispose(); // Partitioner, if set, will dispose when producer disposes, and Set the reset event Assert.True(resetEvent.IsSet); }
public void TypedProducer_ClosedHandle(string bootstrapServers) { LogToFile("start TypedProducer_ClosedHandle"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var producer = new ProducerBuilder <Null, Null>(producerConfig).Build(); producer.Flush(TimeSpan.FromMilliseconds(10)); producer.Dispose(); Thread.Sleep(TimeSpan.FromMilliseconds(500)); // kafka handle destroy is done on the poll thread, is not immediate. Assert.Throws <ObjectDisposedException>(() => producer.Flush(TimeSpan.FromMilliseconds(10))); Assert.Equal(0, Library.HandleCount); LogToFile("end TypedProducer_ClosedHandle"); }
public void Producer_ClosedHandle(string bootstrapServers) { LogToFile("start Producer_ClosedHandle"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers, EnableBackgroundPoll = false }; var producer = new ProducerBuilder <Null, Null>(producerConfig).Build(); producer.Poll(TimeSpan.FromMilliseconds(10)); producer.Dispose(); Assert.Throws <ObjectDisposedException>(() => producer.Poll(TimeSpan.FromMilliseconds(10))); Assert.Equal(0, Library.HandleCount); LogToFile("end Producer_ClosedHandle"); }
public static void Producer() { var props = new ProducerConfig { BootstrapServers = "<CCLOUD-ADDRESS>", SecurityProtocol = SecurityProtocol.SaslSsl, SaslMechanism = SaslMechanism.Plain, SaslUsername = "******", SaslPassword = "******", ClientId = System.Net.Dns.GetHostName(), SslCaLocation = "cacert.pem" }; Action <DeliveryReport <string, String> > onComplete = result => Console.WriteLine(!result.Error.IsError ? $"Message Successfully Delivered" : $"Delivery Error: {result.Error.Reason}"); using (var producer = new ProducerBuilder <string, string>(props).Build()) { var targetAmount = 100; try { for (int i = 0; i < targetAmount; i++) { String UUID = System.Guid.NewGuid().ToString(); producer.Produce("CCloudTopic", new Message <string, string> { Key = UUID, Value = $"This is a cool value {UUID.Substring(UUID.Length - 2)}" }, onComplete); } } catch (Exception e) { Console.WriteLine(e.ToString()); } finally { producer.Flush(TimeSpan.FromSeconds(10)); producer.Dispose(); } } }
// Produce recent-change messages from Wikipedia to a Kafka topic. // The messages are sent from the RCFeed https://www.mediawiki.org/wiki/Manual:RCFeed // to the topic with the specified name. static async Task Produce(string topicName, ClientConfig config) { Console.WriteLine($"{nameof(Produce)} starting"); // The URL of the EventStreams service. string eventStreamsUrl = "https://stream.wikimedia.org/v2/stream/recentchange"; // Declare the producer reference here to enable calling the Flush // method in the finally block, when the app shuts down. IProducer <string, string> producer = null; try { // Build a producer based on the provided configuration. // It will be disposed in the finally block. producer = new ProducerBuilder <string, string>(config).Build(); using (var httpClient = new HttpClient()) using (var stream = await httpClient.GetStreamAsync(eventStreamsUrl)) using (var reader = new StreamReader(stream)) { // Read continuously until interrupted by Ctrl+C. while (!reader.EndOfStream) { var line = reader.ReadLine(); // The Wikimedia service sends a few lines, but the lines // of interest for this demo start with the "data:" prefix. if (!line.StartsWith("data:")) { continue; } // Extract and deserialize the JSON payload. int openBraceIndex = line.IndexOf('{'); string jsonData = line.Substring(openBraceIndex); Console.WriteLine($"Data string: {jsonData}"); // Parse the JSON to extract the URI of the edited page. var jsonDoc = JsonDocument.Parse(jsonData); var metaElement = jsonDoc.RootElement.GetProperty("meta"); var uriElement = metaElement.GetProperty("uri"); var key = uriElement.GetString(); // Use the URI as the message key. // For higher throughput, use the non-blocking Produce call // and handle delivery reports out-of-band, instead of awaiting // the result of a ProduceAsync call. producer.Produce(topicName, new Message <string, string> { Key = key, Value = jsonData }, (deliveryReport) => { if (deliveryReport.Error.Code != ErrorCode.NoError) { Console.WriteLine($"Failed to deliver message: {deliveryReport.Error.Reason}"); } else { Console.WriteLine($"Produced message to: {deliveryReport.TopicPartitionOffset}"); } }); } } } catch (Exception ex) { Console.WriteLine(ex); } finally { var queueSize = producer.Flush(TimeSpan.FromSeconds(5)); if (queueSize > 0) { Console.WriteLine("WARNING: Producer event queue has " + queueSize + " pending events on exit."); } producer.Dispose(); } }
public void Run() { var pConfig = new ProducerConfig { BootstrapServers = bootstrapServers, TransactionalId = $"txn_test_{this.id}" }; int lastMessageValue = 0; var producer = new ProducerBuilder <int, int>(pConfig).Build(); producer.InitTransactions(DefaultTimeout); var currentState = ProducerState.InitState; for (int i = 0; i < conf.MessageCount;) { Console.Write($"+{i}"); Console.Out.Flush(); // finalize previous state. switch (currentState) { case ProducerState.MakingMessagesToAbort: producer.AbortTransaction(DefaultTimeout); break; case ProducerState.MakingMessagesToCommit: producer.CommitTransaction(DefaultTimeout); break; default: // no action required. break; } // transition to next state. var rnd = random.NextDouble(); if (rnd < conf.ProbabilityLevel_Abort) { currentState = ProducerState.MakingMessagesToAbort; } else { currentState = ProducerState.MakingMessagesToCommit; } producer.BeginTransaction(); int runLength = random.Next(conf.MaxRunLength); for (int j = 0; j < runLength && i < conf.MessageCount; ++j, ++i) { int val = currentState == ProducerState.MakingMessagesToCommit ? lastMessageValue++ : -1; Thread.Sleep((int)(1000 * conf.MaxPauseSeconds)); producer.Produce(conf.Topic, new Message <int, int> { Key = id, Value = val }); } } if (currentState == ProducerState.MakingMessagesToCommit) { producer.CommitTransaction(DefaultTimeout); } if (currentState == ProducerState.MakingMessagesToAbort) { producer.AbortTransaction(DefaultTimeout); } Console.WriteLine($"done: {id}"); producer.Flush(); producer.Dispose(); }
private async Task ListeningThread(string host, Func <ICommand, Task> handlerAsync, Func <ICommand, Task> handlerAwaitAsync) { canceller = new CancellationTokenSource(); retry: IConsumer <string, byte[]> consumer = null; try { await KafkaCommon.AssureTopic(host, topic); var consumerConfig = new ConsumerConfig(); consumerConfig.BootstrapServers = host; consumerConfig.GroupId = topic; consumerConfig.EnableAutoCommit = false; consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build(); consumer.Subscribe(topic); for (; ;) { Exception error = null; bool awaitResponse = false; string ackTopic = null; string ackKey = null; try { if (canceller.Token.IsCancellationRequested) { break; } var consumerResult = consumer.Consume(canceller.Token); consumer.Commit(consumerResult); awaitResponse = consumerResult.Message.Key == KafkaCommon.MessageWithAckKey; if (awaitResponse) { ackTopic = Encoding.UTF8.GetString(consumerResult.Message.Headers.GetLastBytes(KafkaCommon.AckTopicHeader)); ackKey = Encoding.UTF8.GetString(consumerResult.Message.Headers.GetLastBytes(KafkaCommon.AckKeyHeader)); } if (consumerResult.Message.Key == KafkaCommon.MessageKey || awaitResponse) { var stopwatch = new Stopwatch(); stopwatch.Start(); byte[] body = consumerResult.Message.Value; if (encryptionKey != null) { body = SymmetricEncryptor.Decrypt(encryptionAlgorithm, encryptionKey, body); } var message = KafkaCommon.Deserialize <KafkaCommandMessage>(body); if (message.Claims != null) { var claimsIdentity = new ClaimsIdentity(message.Claims.Select(x => new Claim(x[0], x[1])), "CQRS"); Thread.CurrentPrincipal = new ClaimsPrincipal(claimsIdentity); } if (awaitResponse) { await handlerAwaitAsync(message.Message); } else { await handlerAsync(message.Message); } stopwatch.Stop(); _ = Log.TraceAsync($"Received Await: {topic} {stopwatch.ElapsedMilliseconds}"); } else { _ = Log.ErrorAsync($"{nameof(KafkaServer)} unrecognized message key {consumerResult.Message.Key}"); } } catch (TaskCanceledException) { break; } catch (Exception ex) { _ = Log.TraceAsync($"Error: Received Await: {topic}"); _ = Log.ErrorAsync(ex); error = ex; } if (awaitResponse) { IProducer <string, byte[]> producer = null; try { var producerConfig = new ProducerConfig(); producerConfig.BootstrapServers = host; producerConfig.ClientId = clientID; producer = new ProducerBuilder <string, byte[]>(producerConfig).Build(); var ack = new Acknowledgement() { Success = error == null, ErrorMessage = error?.Message }; var body = KafkaCommon.Serialize(ack); if (encryptionKey != null) { body = SymmetricEncryptor.Encrypt(encryptionAlgorithm, encryptionKey, body); } await producer.ProduceAsync(ackTopic, new Message <string, byte[]>() { Key = ackKey, Value = body }); } catch (Exception ex) { _ = Log.ErrorAsync(ex); } finally { if (producer != null) { producer.Dispose(); } } } } consumer.Unsubscribe(); } catch (Exception ex) { _ = Log.ErrorAsync(ex); if (consumer != null) { consumer.Dispose(); } consumer = null; if (!canceller.IsCancellationRequested) { await Task.Delay(retryDelay); goto retry; } } canceller.Dispose(); canceller = null; if (consumer != null) { consumer.Dispose(); } IsOpen = false; }