static async Task ProducerDemoAsync()
{
IProducer <string, string> producer = new ProducerBuilder <string, string>(new ProducerConfig
{
BootstrapServers = BootstrapServers,
Acks = Acks.Leader,
MessageSendMaxRetries = 5,
BatchSize = 20,
LingerMs = 3000,
}).Build();
try
{
while (true)
{
DeliveryResult <string, string> deliveryResult = await producer.ProduceAsync(Topic, new Message <string, string> {
Key = Guid.NewGuid().ToString(), Value = DateTime.Now.ToString()
});
Console.WriteLine($"Producer::{deliveryResult.Key}::{deliveryResult.Value}::{deliveryResult.Partition.Value}::{deliveryResult.Offset.Value}::{Thread.CurrentThread.ManagedThreadId}");
// await Task.Delay(100);
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
producer.Flush();
producer.Dispose();
}
}
public void PartitionerTransfersToProducer()
{
var resetEvent = new ManualResetEventSlim(initialState: false);
var producer = new ProducerBuilder <Null, Null>(new ProducerConfig())
.SetKeySerializer(Serializers.Null)
.SetPartitioner("test", new TestPartitioners.ResetOnDisposeTestPartitioner(resetEvent))
.Build();
Assert.False(resetEvent.IsSet);
producer.Dispose(); // Partitioner, if set, will dispose when producer disposes, and Set the reset event
Assert.True(resetEvent.IsSet);
}
public void TypedProducer_ClosedHandle(string bootstrapServers)
{
LogToFile("start TypedProducer_ClosedHandle");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var producer = new ProducerBuilder <Null, Null>(producerConfig).Build();
producer.Flush(TimeSpan.FromMilliseconds(10));
producer.Dispose();
Thread.Sleep(TimeSpan.FromMilliseconds(500)); // kafka handle destroy is done on the poll thread, is not immediate.
Assert.Throws <ObjectDisposedException>(() => producer.Flush(TimeSpan.FromMilliseconds(10)));
Assert.Equal(0, Library.HandleCount);
LogToFile("end TypedProducer_ClosedHandle");
}
public void Producer_ClosedHandle(string bootstrapServers)
{
LogToFile("start Producer_ClosedHandle");
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
EnableBackgroundPoll = false
};
var producer = new ProducerBuilder <Null, Null>(producerConfig).Build();
producer.Poll(TimeSpan.FromMilliseconds(10));
producer.Dispose();
Assert.Throws <ObjectDisposedException>(() => producer.Poll(TimeSpan.FromMilliseconds(10)));
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_ClosedHandle");
}
public static void Producer()
{
var props = new ProducerConfig
{
BootstrapServers = "<CCLOUD-ADDRESS>",
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslMechanism = SaslMechanism.Plain,
SaslUsername = "******",
SaslPassword = "******",
ClientId = System.Net.Dns.GetHostName(),
SslCaLocation = "cacert.pem"
};
Action <DeliveryReport <string, String> > onComplete = result =>
Console.WriteLine(!result.Error.IsError
? $"Message Successfully Delivered" : $"Delivery Error: {result.Error.Reason}");
using (var producer = new ProducerBuilder <string, string>(props).Build())
{
var targetAmount = 100;
try
{
for (int i = 0; i < targetAmount; i++)
{
String UUID = System.Guid.NewGuid().ToString();
producer.Produce("CCloudTopic",
new Message <string, string> {
Key = UUID,
Value = $"This is a cool value {UUID.Substring(UUID.Length - 2)}"
},
onComplete);
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
finally
{
producer.Flush(TimeSpan.FromSeconds(10));
producer.Dispose();
}
}
}
// Produce recent-change messages from Wikipedia to a Kafka topic.
// The messages are sent from the RCFeed https://www.mediawiki.org/wiki/Manual:RCFeed
// to the topic with the specified name.
static async Task Produce(string topicName, ClientConfig config)
{
Console.WriteLine($"{nameof(Produce)} starting");
// The URL of the EventStreams service.
string eventStreamsUrl = "https://stream.wikimedia.org/v2/stream/recentchange";
// Declare the producer reference here to enable calling the Flush
// method in the finally block, when the app shuts down.
IProducer <string, string> producer = null;
try
{
// Build a producer based on the provided configuration.
// It will be disposed in the finally block.
producer = new ProducerBuilder <string, string>(config).Build();
using (var httpClient = new HttpClient())
using (var stream = await httpClient.GetStreamAsync(eventStreamsUrl))
using (var reader = new StreamReader(stream))
{
// Read continuously until interrupted by Ctrl+C.
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
// The Wikimedia service sends a few lines, but the lines
// of interest for this demo start with the "data:" prefix.
if (!line.StartsWith("data:"))
{
continue;
}
// Extract and deserialize the JSON payload.
int openBraceIndex = line.IndexOf('{');
string jsonData = line.Substring(openBraceIndex);
Console.WriteLine($"Data string: {jsonData}");
// Parse the JSON to extract the URI of the edited page.
var jsonDoc = JsonDocument.Parse(jsonData);
var metaElement = jsonDoc.RootElement.GetProperty("meta");
var uriElement = metaElement.GetProperty("uri");
var key = uriElement.GetString(); // Use the URI as the message key.
// For higher throughput, use the non-blocking Produce call
// and handle delivery reports out-of-band, instead of awaiting
// the result of a ProduceAsync call.
producer.Produce(topicName, new Message <string, string> {
Key = key, Value = jsonData
},
(deliveryReport) =>
{
if (deliveryReport.Error.Code != ErrorCode.NoError)
{
Console.WriteLine($"Failed to deliver message: {deliveryReport.Error.Reason}");
}
else
{
Console.WriteLine($"Produced message to: {deliveryReport.TopicPartitionOffset}");
}
});
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
var queueSize = producer.Flush(TimeSpan.FromSeconds(5));
if (queueSize > 0)
{
Console.WriteLine("WARNING: Producer event queue has " + queueSize + " pending events on exit.");
}
producer.Dispose();
}
}
public void Run()
{
var pConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
TransactionalId = $"txn_test_{this.id}"
};
int lastMessageValue = 0;
var producer = new ProducerBuilder <int, int>(pConfig).Build();
producer.InitTransactions(DefaultTimeout);
var currentState = ProducerState.InitState;
for (int i = 0; i < conf.MessageCount;)
{
Console.Write($"+{i}");
Console.Out.Flush();
// finalize previous state.
switch (currentState)
{
case ProducerState.MakingMessagesToAbort:
producer.AbortTransaction(DefaultTimeout);
break;
case ProducerState.MakingMessagesToCommit:
producer.CommitTransaction(DefaultTimeout);
break;
default:
// no action required.
break;
}
// transition to next state.
var rnd = random.NextDouble();
if (rnd < conf.ProbabilityLevel_Abort)
{
currentState = ProducerState.MakingMessagesToAbort;
}
else
{
currentState = ProducerState.MakingMessagesToCommit;
}
producer.BeginTransaction();
int runLength = random.Next(conf.MaxRunLength);
for (int j = 0; j < runLength && i < conf.MessageCount; ++j, ++i)
{
int val = currentState == ProducerState.MakingMessagesToCommit ? lastMessageValue++ : -1;
Thread.Sleep((int)(1000 * conf.MaxPauseSeconds));
producer.Produce(conf.Topic, new Message <int, int> {
Key = id, Value = val
});
}
}
if (currentState == ProducerState.MakingMessagesToCommit)
{
producer.CommitTransaction(DefaultTimeout);
}
if (currentState == ProducerState.MakingMessagesToAbort)
{
producer.AbortTransaction(DefaultTimeout);
}
Console.WriteLine($"done: {id}");
producer.Flush();
producer.Dispose();
}
private async Task ListeningThread(string host, Func <ICommand, Task> handlerAsync, Func <ICommand, Task> handlerAwaitAsync)
{
canceller = new CancellationTokenSource();
retry:
IConsumer <string, byte[]> consumer = null;
try
{
await KafkaCommon.AssureTopic(host, topic);
var consumerConfig = new ConsumerConfig();
consumerConfig.BootstrapServers = host;
consumerConfig.GroupId = topic;
consumerConfig.EnableAutoCommit = false;
consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build();
consumer.Subscribe(topic);
for (; ;)
{
Exception error = null;
bool awaitResponse = false;
string ackTopic = null;
string ackKey = null;
try
{
if (canceller.Token.IsCancellationRequested)
{
break;
}
var consumerResult = consumer.Consume(canceller.Token);
consumer.Commit(consumerResult);
awaitResponse = consumerResult.Message.Key == KafkaCommon.MessageWithAckKey;
if (awaitResponse)
{
ackTopic = Encoding.UTF8.GetString(consumerResult.Message.Headers.GetLastBytes(KafkaCommon.AckTopicHeader));
ackKey = Encoding.UTF8.GetString(consumerResult.Message.Headers.GetLastBytes(KafkaCommon.AckKeyHeader));
}
if (consumerResult.Message.Key == KafkaCommon.MessageKey || awaitResponse)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
byte[] body = consumerResult.Message.Value;
if (encryptionKey != null)
{
body = SymmetricEncryptor.Decrypt(encryptionAlgorithm, encryptionKey, body);
}
var message = KafkaCommon.Deserialize <KafkaCommandMessage>(body);
if (message.Claims != null)
{
var claimsIdentity = new ClaimsIdentity(message.Claims.Select(x => new Claim(x[0], x[1])), "CQRS");
Thread.CurrentPrincipal = new ClaimsPrincipal(claimsIdentity);
}
if (awaitResponse)
{
await handlerAwaitAsync(message.Message);
}
else
{
await handlerAsync(message.Message);
}
stopwatch.Stop();
_ = Log.TraceAsync($"Received Await: {topic} {stopwatch.ElapsedMilliseconds}");
}
else
{
_ = Log.ErrorAsync($"{nameof(KafkaServer)} unrecognized message key {consumerResult.Message.Key}");
}
}
catch (TaskCanceledException)
{
break;
}
catch (Exception ex)
{
_ = Log.TraceAsync($"Error: Received Await: {topic}");
_ = Log.ErrorAsync(ex);
error = ex;
}
if (awaitResponse)
{
IProducer <string, byte[]> producer = null;
try
{
var producerConfig = new ProducerConfig();
producerConfig.BootstrapServers = host;
producerConfig.ClientId = clientID;
producer = new ProducerBuilder <string, byte[]>(producerConfig).Build();
var ack = new Acknowledgement()
{
Success = error == null,
ErrorMessage = error?.Message
};
var body = KafkaCommon.Serialize(ack);
if (encryptionKey != null)
{
body = SymmetricEncryptor.Encrypt(encryptionAlgorithm, encryptionKey, body);
}
await producer.ProduceAsync(ackTopic, new Message <string, byte[]>() { Key = ackKey, Value = body });
}
catch (Exception ex)
{
_ = Log.ErrorAsync(ex);
}
finally
{
if (producer != null)
{
producer.Dispose();
}
}
}
}
consumer.Unsubscribe();
}
catch (Exception ex)
{
_ = Log.ErrorAsync(ex);
if (consumer != null)
{
consumer.Dispose();
}
consumer = null;
if (!canceller.IsCancellationRequested)
{
await Task.Delay(retryDelay);
goto retry;
}
}
canceller.Dispose();
canceller = null;
if (consumer != null)
{
consumer.Dispose();
}
IsOpen = false;
}
