Example #1
2
        static void Main(string[] args)
        {
            try
            {
                _topic = ConfigurationManager.AppSettings["topic"];
                var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',')
                              select new Uri(x);
                Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic);
                var options = new KafkaOptions(brokers.ToArray());
                var router = new BrokerRouter(options);
                var coption = new ConsumerOptions(_topic, router);
                _consumer = new KafkaNet.Consumer(coption);
                var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result;
                var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max());
                _consumer.SetOffsetPosition(t.ToArray());

                foreach (var message in _consumer.Consume())
                {
                    Console.WriteLine("Response: P{0},O{1} : {2}",
                    message.Meta.PartitionId, 
                    message.Meta.Offset,
                    System.Text.Encoding.Default.GetString(message.Value));
                }
            }
            catch(Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
            Console.ReadLine();
        }
        public void Setup()
        {
            var options = new KafkaOptions(IntegrationConfig.IntegrationUri);
            var endpoint = new DefaultKafkaConnectionFactory().Resolve(options.KafkaServerUri.First(), options.Log);

            _conn = new KafkaConnection(new KafkaTcpSocket(new DefaultTraceLog(), endpoint), options.ResponseTimeoutMs, options.Log);
        }
Example #3
0
        public void Main(string[] args)
        {
            var options = GetOptions(args);
            if (options == null) return;

            StartReporting();

            var kafkaOptions = new KafkaOptions(options.KafkaNodeUri);
            using (var router = new BrokerRouter(kafkaOptions))
            using (var client = new KafkaNet.Consumer(new ConsumerOptions("TestHarness", router) { Log = new ConsoleLog(), MinimumBytes = 1 }))
            {
                Console.WriteLine("Listening for messages...");
                foreach (var message in client.Consume())
                {
                    try
                    {
                        var received = DateTime.Now;
                        var sent = new DateTime(BitConverter.ToInt64(message.Value, 0));
                        var diff = received - sent;
                        lock (sync)
                            receivedItems.Add(new ReceivedMessage { DateTime = received, TotalMilliseconds = diff.TotalMilliseconds });
                    }
                    catch (Exception ex)
                    {
                        Console.Error.WriteLine("Oops... " + ex);
                    }
                }
            }
        }
Example #4
0
 public BrokerRouter(KafkaOptions kafkaOptions)
 {
     _kafkaOptions = kafkaOptions;
     _defaultConnections
         .AddRange(kafkaOptions.KafkaServerUri.Distinct()
         .Select(uri => _kafkaOptions.KafkaConnectionFactory.Create(uri, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log)));
 }
Example #5
0
 public void Run()
 {
     var options = new KafkaOptions(new Uri("http://localhost:9092"));
     var router = new BrokerRouter(options);
     _producer = new KafkaNet.Producer(router);
     Process();
 }
Example #6
0
        static void Main(string[] args)
        {
            var options = new KafkaOptions(new Uri("http://CSDKAFKA01:9092"), new Uri("http://CSDKAFKA02:9092"))
                {
                    Log = new ConsoleLog()
                };
            var router = new BrokerRouter(options);
            var client = new Producer(router);

            Task.Factory.StartNew(() =>
                {
                    var consumer = new Consumer(new ConsumerOptions("TestHarness", router));
                    foreach (var data in consumer.Consume())
                    {
                        Console.WriteLine("Response: P{0},O{1} : {2}", data.Meta.PartitionId, data.Meta.Offset, data.Value);
                    }
                });

            Console.WriteLine("Type a message and press enter...");
            while (true)
            {
                var message = Console.ReadLine();
                if (message == "quit") break;
                client.SendMessageAsync("TestHarness", new[] {new Message {Value = message}});
            }

            using (client)
            using (router)
            {

            }
        }
 public ManualConsumerTests()
 {
     _kafkaUri = IntegrationConfig.IntegrationUri;
     _topic = IntegrationConfig.IntegrationTopic;
     _defaultConsumerGroup = IntegrationConfig.IntegrationConsumer;
     _options = new KafkaOptions(IntegrationConfig.IntegrationUri);
 }
Example #8
0
        public void Main(string[] args)
        {
            var options = GetOptions(args);
            if (options == null) return;

            var count = 0;
            var lastCount = 0;
            var reporter = new Task(() =>
            {
                while (true)
                {
                    var current = count;
                    Console.WriteLine("{0} messages in last second.", current - lastCount);
                    lastCount = current;
                    Thread.Sleep(1000);
                }
            });
            var kafkaOptions = new KafkaOptions(options.KafkaNodeUri);// { Log = new ConsoleLog() };
            using (var router = new BrokerRouter(kafkaOptions))
            using (var client = new KafkaNet.Producer(router))
            {
                reporter.Start();
                while (true)
                {
                    Thread.Sleep(100);
                    client.SendMessageAsync("TestHarness", new[] { new Message() { Value = BitConverter.GetBytes(DateTime.Now.Ticks) } });
                    count++;
                }
            }
        }
Example #9
0
 public void Run()
 {
     var options = new KafkaOptions(new Uri("http://localhost:9092"));
     var router = new BrokerRouter(options);
     var consumerOptions = new ConsumerOptions("test", router);           
     _consumer = new KafkaNet.Consumer(consumerOptions, new OffsetPosition(0, 100));
     Process();
 }
Example #10
0
        public DestinationKafka(params Uri[] servers)
        {
            var options = new KafkaOptions(servers) { Log = new ConsoleLogger() };
            _router = new BrokerRouter(options);
            _producer = new Producer(_router, maximumMessageBuffer: 5000, maximumAsyncRequests: 10) { BatchSize = 1000, BatchDelayTime = TimeSpan.FromSeconds(1) };

            StatisticsTracker.OnStatisticsHeartbeat += StatisticsTracker_OnStatisticsHeartbeat;
        }
Example #11
0
        public KafkaAgent(Uri[] kafkaServers)
        {
            var options = new KafkaOptions(kafkaServers);
            _router = new BrokerRouter(options);

            var influxDbClient = new InfluxDb("http://localhost", "reapadda", "qwerty", InfluxVersion.v09x);
            _formatter = influxDbClient.GetFormatter();
        }
Example #12
0
        static void Main(string[] args)
        {
            const string topicName = "kafka.SimpleTopic";
            TopicClient topicClient = new TopicClient();
            TopicView topicView = topicClient.getTopic(topicName);
            List<Uri> brokers = getBrokerList(topicView);
            //create an options file that sets up driver preferences
            var options = new KafkaOptions()
            {
                Log = new ConsoleLog()
            };
            options.KafkaServerUri = brokers;

            //start an out of process thread that runs a consumer that will write all received messages to the console
            Task.Run(() =>
            {
                var consumer = new Consumer(new ConsumerOptions(topicName, new BrokerRouter(options)) { Log = new ConsoleLog() });
                foreach (var data in consumer.Consume())
                {
                    Console.WriteLine("Response: P{0},O{1} : {2}", data.Meta.PartitionId, data.Meta.Offset, data.Value.ToUtf8String());
                }
            });

            //create a producer to send messages with
            var producer = new KafkaNet.Producer(new BrokerRouter(options))
            {
                BatchSize = 100,
                BatchDelayTime = TimeSpan.FromMilliseconds(2000)
            };


            //take in console read messages
            Console.WriteLine("Type a message and press enter...");
            while (true)
            {
                var message = Console.ReadLine();
                if (message == "quit") break;

                if (string.IsNullOrEmpty(message))
                {
                    //send a random batch of messages
                    SendRandomBatch(producer, topicName, 200);
                }
                else
                {
                    producer.SendMessageAsync(topicName, new[] { new Message(message) });
                }
            }

            using (producer)
            {

            }
        }
        public async Task Send(object channel, Address address, TransportMessage message, object properties)
        {
            var options = new KafkaOptions(new Uri(address.Machine));
            var router = new BrokerRouter(options);
            var topic = address.Queue;
            var messageString = System.Text.Encoding.Default.GetString(message.Body);

            using (var client = new Producer(router))
            {
                await client.SendMessageAsync(topic, new[] { new Message(messageString) });
            }
        }
        private static void Consume(string broker, string topic)
        {   
            var options = new KafkaOptions(new Uri(broker));
            var router = new BrokerRouter(options);
            var consumer = new Consumer(new ConsumerOptions(topic, router));

            //Consume returns a blocking IEnumerable (ie: never ending stream)
            foreach (var message in consumer.Consume())
            {
                Console.WriteLine("Response: Partition {0},Offset {1} : {2}",
                    message.Meta.PartitionId, message.Meta.Offset, message.Value.ToUtf8String());
            }
        }
Example #15
0
        static void Main(string[] args)
        {
            const string topicName = "TestHarness";

            //create an options file that sets up driver preferences
            var options = new KafkaOptions(new Uri("http://CSDKAFKA01:9092"), new Uri("http://CSDKAFKA02:9092"))
            {
                Log = new ConsoleLog()
            };

            //start an out of process thread that runs a consumer that will write all received messages to the console
            Task.Run(() =>
            {
                var consumer = new Consumer(new ConsumerOptions(topicName, new BrokerRouter(options)) { Log = new ConsoleLog() });
                foreach (var data in consumer.Consume())
                {
                    Console.WriteLine("Response: P{0},O{1} : {2}", data.Meta.PartitionId, data.Meta.Offset, data.Value.ToUtf8String());
                }
            });

            //create a producer to send messages with
            var producer = new Producer(new BrokerRouter(options)) 
            { 
                BatchSize = 100,
                BatchDelayTime = TimeSpan.FromMilliseconds(2000)
            };


            //take in console read messages
            Console.WriteLine("Type a message and press enter...");
            while (true)
            {
                var message = Console.ReadLine();
                if (message == "quit") break;

                if (string.IsNullOrEmpty(message))
                {
                    //send a random batch of messages
                    SendRandomBatch(producer, topicName, 200);
                }
                else
                {
                    producer.SendMessageAsync(topicName, new[] { new Message(message) });
                }
            }

            using (producer)
            {

            }
        }
        private static void Produce(string broker, string topic)
        {
            var options = new KafkaOptions(new Uri(broker));
            var router = new BrokerRouter(options);
            var client = new Producer(router);

            var current_datetime =DateTime.Now;
            var key = current_datetime.Second.ToString();
            var events = new[] { new Message("Hello World " + current_datetime.ToString(), key) };
            client.SendMessageAsync(topic, events).Wait(1500);
            Console.WriteLine("Produced: Key: {0}. Message: {1}", key, events[0].Value.ToUtf8String());

            using (client) { }
        }
Example #17
0
        public BrokerRouter(KafkaOptions kafkaOptions)
        {
            _kafkaOptions = kafkaOptions;
            _kafkaMetadataProvider = new KafkaMetadataProvider(_kafkaOptions.Log);

            foreach (var endpoint in _kafkaOptions.KafkaServerEndpoints)
            {
                var conn = _kafkaOptions.KafkaConnectionFactory.Create(endpoint, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log, _kafkaOptions.MaximumReconnectionTimeout);
                _defaultConnectionIndex.AddOrUpdate(endpoint, e => conn, (e, c) => conn);
            }

            if (_defaultConnectionIndex.Count <= 0)
                throw new ServerUnreachableException("None of the provided Kafka servers are resolvable.");
        }
        /// <summary>
        /// Sends request and response raw data to Kafka
        /// </summary>
        /// <param name="data">Request and response metrics in raw form</param>
        /// <returns>A completed task</returns>
        public override async Task Write(RequestResponseData data)
        {
            try
            {
                KafkaOptions options = new KafkaOptions(new Uri("http://SERVER1:9092"), new Uri("http://SERVER2:9092"));
                BrokerRouter router = new BrokerRouter(options);
                Producer client = new Producer(router);

                client.SendMessageAsync("API-Meter Data", new[] { new Message { Value = "NULL for now" } }).Wait();
            }
            catch (Exception ex)
            {
                Trace.Write(ex.Message);
            }
        }
        public KafkaClient(Uri[] hosts)
        {
            var options = new KafkaOptions(hosts);

            _brokerRouter = new BrokerRouter(options);
        }
        public void Setup()
        {
            var options = new KafkaOptions(new Uri(ConfigurationManager.AppSettings["IntegrationKafkaServerUrl"]));

            _conn = new KafkaConnection(options.KafkaServerUri.First(), options.ResponseTimeoutMs, options.Log);
        }
        public async void ConsumerShouldMoveToNextAvailableOffsetWhenQueryingForNextMessage()
        {
            const int expectedCount = 1000;
            var options = new KafkaOptions(IntegrationConfig.IntegrationUri) { Log = new ConsoleLog() };

            using (var producerRouter = new BrokerRouter(options))
            using (var producer = new Producer(producerRouter))
            {
                //get current offset and reset consumer to top of log
                var offsets = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).ConfigureAwait(false);

                using (var consumerRouter = new BrokerRouter(options))
                using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, consumerRouter),
                     offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray()))
                {
                    Console.WriteLine("Sending {0} test messages", expectedCount);
                    var response = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, 
                        Enumerable.Range(0, expectedCount).Select(x => new Message(x.ToString())));

                    Assert.That(response.Any(x => x.Error != (int)ErrorResponseCode.NoError), Is.False, "Error occured sending test messages to server.");

                    var stream = consumer.Consume();

                    Console.WriteLine("Reading message back out from consumer.");
                    var data = stream.Take(expectedCount).ToList();

                    var consumerOffset = consumer.GetOffsetPosition().OrderBy(x => x.Offset).ToList();
                    var serverOffset = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).ConfigureAwait(false);
                    var positionOffset = serverOffset.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max()))
                        .OrderBy(x => x.Offset)
                        .ToList();

                    Assert.That(consumerOffset, Is.EqualTo(positionOffset), "The consumerOffset position should match the server offset position.");
                    Assert.That(data.Count, Is.EqualTo(expectedCount), "We should have received 2000 messages from the server.");

                }
            }
        }
        public void Setup()
        {
            var options = new KafkaOptions(new Uri(ConfigurationManager.AppSettings["IntegrationKafkaServerUrl"]));

            _router = new BrokerRouter(options);
        }
 public IBrokerRouter GetClientRouter()
 {
     var options = new KafkaOptions(new Uri(KafkaServerUri));
     return new BrokerRouter(options);
 }
Example #24
0
        public void Setup()
        {
            var brokers = ConfigurationManager.AppSettings["IntegrationKafkaServerUrl"]
                .Split(',')
                .Select(b => new Uri(b))
                .ToArray();

            var options = new KafkaOptions(brokers);

            _router = new BrokerRouter(options);
        }