/// <summary> /// Writes time-series metadata to specified Kafka <paramref name="topic"/>. /// </summary> /// <param name="metadata">Source time-series metadata object to write to Kafka.</param> /// <param name="router">Kafka router connection.</param> /// <param name="topic">Kafka topic.</param> public static void WriteToKafka(TimeSeriesMetadata metadata, BrokerRouter router, string topic) { if ((object)metadata == null) throw new ArgumentNullException(nameof(metadata)); if ((object)router == null) throw new ArgumentNullException(nameof(router)); if (string.IsNullOrWhiteSpace(topic)) throw new ArgumentNullException(nameof(topic)); using (MemoryStream stream = new MemoryStream()) { Serialize(metadata, stream, true); using (Producer producer = new Producer(router)) { Message[] messages = new Message[2]; byte[] timeKey = BitConverter.GetBytes(DateTime.UtcNow.Ticks); // First message used to serialize metadata size (since metadata messages can be large) messages[SizeMessage] = new Message { Key = timeKey, Value = BitConverter.GetBytes(stream.Length) }; // Second message used to serialize metadata value messages[ValueMessage] = new Message { Key = timeKey, Value = stream.ToArray() }; // Send meta-data to Kafka producer.SendMessageAsync(topic, messages).Wait(); } } }
static void Main(string[] args) { var options = new KafkaOptions(new Uri("http://kafka1:9092"), new Uri("http://kafka2:9092")) { Log = new ConsoleLog() }; var router = new BrokerRouter(options); var client = new Producer(router); var timing = new RollingQueue<double>(50); var rate = new RollingQueue<double>(50); var second = DateTime.Now.Second; var count = 0; Task.Run(() => { var consumer = new Consumer(new ConsumerOptions("latencies", router)); var position = consumer.GetTopicOffsetAsync("latencies"); position.Wait(); consumer.SetOffsetPosition( position.Result .Select(p => new OffsetPosition(p.PartitionId, p.Offsets.First())) .ToArray() ); foreach (var data in consumer.Consume()) { count++; var rtt = (DateTime.Now - new DateTime( long.Parse(Encoding.UTF8.GetString(data.Value)) )).TotalMilliseconds; if (rtt < 1000) timing.Enqueue(rtt); if (second != DateTime.Now.Second) { second = DateTime.Now.Second; rate.Enqueue(count); count = 0; Console.WriteLine("Rate: {0} pps.\t{1} ", rate.Average().ToString("N2") , (rtt < 1000) ? "RTT: " + timing.Average().ToString("N2") + " ms." : string.Empty ); } } }); while (true) { client.SendMessageAsync("latencies", new[] { new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()), new Message(DateTime.Now.Ticks.ToString()) }, 1); Thread.Sleep(1); } client.Dispose(); router.Dispose(); }