internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config,
out LibRdKafka.PartitionerCallback partitionerDelegate)
{
// PartitionerDelegate is an out parameter as its reference must be kept outside of Topic
// it may be called after topic is GC, and it may be different for different topics
// so we can't simply make it static here
this.producer = producer;
config = config ?? new TopicConfig();
config["produce.offset.report"] = "true";
IntPtr configPtr = config.handle.Dup();
if (config.CustomPartitioner != null)
{
partitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
byte[] key = null;
if (keydata != IntPtr.Zero)
{
key = new byte[(int)keylen];
Marshal.Copy(keydata, key, 0, (int)keylen);
}
return(config.CustomPartitioner(this, key, partition_cnt));
};
LibRdKafka.topic_conf_set_partitioner_cb(configPtr, partitionerDelegate);
}
else
{
partitionerDelegate = null;
}
handle = kafkaHandle.Topic(topic, configPtr);
}
// WIP, not producing useful numbers yet. Assumes one partition.
public static async Task<long> Consume(string broker, string topic)
{
long n = 0;
var topicConfig = new TopicConfig();
topicConfig["auto.offset.reset"] = "smallest";
var config = new Config()
{
GroupId = "benchmark-consumer",
DefaultTopicConfig = topicConfig
};
using (var consumer = new EventConsumer(config, broker))
{
var signal = new SemaphoreSlim(0, 1);
consumer.OnMessage += (obj, msg) =>
{
n += 1;
};
consumer.OnEndReached += (obj, end) =>
{
Console.WriteLine($"End reached");
signal.Release();
};
consumer.Subscribe(new List<string>{topic});
consumer.Start();
await signal.WaitAsync();
Console.WriteLine($"Shutting down");
}
return n;
}
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config)
{
this.producer = producer;
config = config ?? new TopicConfig();
config["produce.offset.report"] = "true";
IntPtr configPtr = config.handle.Dup();
if (config.CustomPartitioner != null)
{
PartitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
byte[] key = null;
if (keydata != IntPtr.Zero)
{
key = new byte[(int)keylen];
Marshal.Copy(keydata, key, 0, (int)keylen);
}
return(config.CustomPartitioner(this, key, partition_cnt));
};
LibRdKafka.topic_conf_set_partitioner_cb(configPtr, PartitionerDelegate);
}
handle = kafkaHandle.Topic(topic, configPtr);
}
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config)
{
this.producer = producer;
config = config ?? new TopicConfig();
config["produce.offset.report"] = "true";
IntPtr configPtr = config.handle.Dup();
if (config.CustomPartitioner != null)
{
PartitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
byte[] key = null;
if (keydata != IntPtr.Zero)
{
key = new byte[(int) keylen];
Marshal.Copy(keydata, key, 0, (int) keylen);
}
return config.CustomPartitioner(this, key, partition_cnt);
};
LibRdKafka.topic_conf_set_partitioner_cb(configPtr, PartitionerDelegate);
}
handle = kafkaHandle.Topic(topic, configPtr);
}
private static Thread StartProducer(CancellationTokenSource tokenSource)
{
var timer = Metric.Timer("Published", Unit.Events);
var thread = new Thread(() =>
{
var topicConfig = new TopicConfig();
topicConfig["request.required.acks"] = "0"; // Don't require an ack from the broker.
var config = new Config
{
GroupId = consumerGroupName,
EnableAutoCommit = true,
StatisticsInterval = TimeSpan.FromSeconds(10),
DefaultTopicConfig = topicConfig
};
config["socket.blocking.max.ms"] = "1"; // Maximum time a broker socket operation may block.
config["queue.buffering.max.ms"] = "1"; // Maximum time to buffer data when using async mode.
using (var publisher = new Producer(config, brokers))
using (var topic = publisher.Topic(topicName))
{
while (!tokenSource.IsCancellationRequested)
{
Thread.Sleep(1000);
for (var i = 0; i < 100; i++)
{
var ticks = DateTime.UtcNow.Ticks;
topic.Produce(Encoding.UTF8.GetBytes(ticks.ToString()), partition: (int)(ticks % 2))
.ContinueWith(task =>
{
if (task.Exception != null)
{
Console.WriteLine("{0}: Error publishing message - {1}", DateTime.Now.ToLongTimeString(), task.Exception);
return;
}
timer.Record((DateTime.UtcNow.Ticks - ticks) / 10000, TimeUnit.Milliseconds);
reports.Add(task.Result);
});
}
}
Console.WriteLine("Producer cancelled.");
Thread.CurrentThread.Abort();
}
});
thread.Start();
return thread;
}
public static void Main(string[] args)
{
string brokerList = args[0];
string topicName = args[1];
var topicConfig = new TopicConfig
{
CustomPartitioner = (top, key, cnt) =>
{
var kt = (key != null) ? Encoding.UTF8.GetString(key, 0, key.Length) : "(null)";
int partition = (key?.Length ?? 0) % cnt;
bool available = top.PartitionAvailable(partition);
Console.WriteLine($"Partitioner topic: {top.Name} key: {kt} partition count: {cnt} -> {partition} {available}");
return partition;
}
};
using (Producer producer = new Producer(brokerList))
using (Topic topic = producer.Topic(topicName, topicConfig))
{
Console.WriteLine($"{producer.Name} producing on {topic.Name}. q to exit.");
string text;
while ((text = Console.ReadLine()) != "q")
{
byte[] data = Encoding.UTF8.GetBytes(text);
byte[] key = null;
// Use the first word as the key
int index = text.IndexOf(" ");
if (index != -1)
{
key = Encoding.UTF8.GetBytes(text.Substring(0, index));
}
Task<DeliveryReport> deliveryReport = topic.Produce(data, key);
var unused = deliveryReport.ContinueWith(task =>
{
Console.WriteLine($"Partition: {task.Result.Partition}, Offset: {task.Result.Offset}");
});
}
}
}
public Topic Topic(string topic, TopicConfig config = null)
{
LibRdKafka.PartitionerCallback partitionerDelegate;
var kafkaTopic = new Topic(handle, this, topic, config, out partitionerDelegate);
if (config?.CustomPartitioner != null)
{
// kafkaTopic may be GC before partitionerDelegate is called on all produced mesages
// so we need to keep a reference to it.
// We can't make it static in Topic as the partitioner will be different for each topic
// we could make it in a static collection in Topic, but we can clear it when producer is closed,
// (as it wait for all message to be produced)
// so putting it in an instance collection allows us to free it eventually
// it's not very effective for people creating a lot of topics
// we should find a way to clear the list
// when there is no more messages in queue related to the topic
topicPartitioners.Add(partitionerDelegate);
}
return(kafkaTopic);
}
public Topic Topic(string topic, TopicConfig config = null) => new Topic(handle, this, topic, config);
public Topic Topic(string topic, TopicConfig config = null)
{
return(new Topic(handle, this, topic, config));
}
public Topic Topic(string topic, TopicConfig config = null)
{
return new Topic(handle, this, topic, config);
}