static void Main(string[] args)
{
try
{
_topic = ConfigurationManager.AppSettings["topic"];
var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',')
select new Uri(x);
Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic);
var options = new KafkaOptions(brokers.ToArray());
var router = new BrokerRouter(options);
var coption = new ConsumerOptions(_topic, router);
_consumer = new KafkaNet.Consumer(coption);
var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result;
var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max());
_consumer.SetOffsetPosition(t.ToArray());
foreach (var message in _consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}",
message.Meta.PartitionId,
message.Meta.Offset,
System.Text.Encoding.Default.GetString(message.Value));
}
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.ReadLine();
}
public void Setup()
{
var options = new KafkaOptions(IntegrationConfig.IntegrationUri);
var endpoint = new DefaultKafkaConnectionFactory().Resolve(options.KafkaServerUri.First(), options.Log);
_conn = new KafkaConnection(new KafkaTcpSocket(new DefaultTraceLog(), endpoint), options.ResponseTimeoutMs, options.Log);
}
public void Main(string[] args)
{
var options = GetOptions(args);
if (options == null) return;
StartReporting();
var kafkaOptions = new KafkaOptions(options.KafkaNodeUri);
using (var router = new BrokerRouter(kafkaOptions))
using (var client = new KafkaNet.Consumer(new ConsumerOptions("TestHarness", router) { Log = new ConsoleLog(), MinimumBytes = 1 }))
{
Console.WriteLine("Listening for messages...");
foreach (var message in client.Consume())
{
try
{
var received = DateTime.Now;
var sent = new DateTime(BitConverter.ToInt64(message.Value, 0));
var diff = received - sent;
lock (sync)
receivedItems.Add(new ReceivedMessage { DateTime = received, TotalMilliseconds = diff.TotalMilliseconds });
}
catch (Exception ex)
{
Console.Error.WriteLine("Oops... " + ex);
}
}
}
}
public BrokerRouter(KafkaOptions kafkaOptions)
{
_kafkaOptions = kafkaOptions;
_defaultConnections
.AddRange(kafkaOptions.KafkaServerUri.Distinct()
.Select(uri => _kafkaOptions.KafkaConnectionFactory.Create(uri, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log)));
}
public void Run()
{
var options = new KafkaOptions(new Uri("http://localhost:9092"));
var router = new BrokerRouter(options);
_producer = new KafkaNet.Producer(router);
Process();
}
static void Main(string[] args)
{
var options = new KafkaOptions(new Uri("http://CSDKAFKA01:9092"), new Uri("http://CSDKAFKA02:9092"))
{
Log = new ConsoleLog()
};
var router = new BrokerRouter(options);
var client = new Producer(router);
Task.Factory.StartNew(() =>
{
var consumer = new Consumer(new ConsumerOptions("TestHarness", router));
foreach (var data in consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}", data.Meta.PartitionId, data.Meta.Offset, data.Value);
}
});
Console.WriteLine("Type a message and press enter...");
while (true)
{
var message = Console.ReadLine();
if (message == "quit") break;
client.SendMessageAsync("TestHarness", new[] {new Message {Value = message}});
}
using (client)
using (router)
{
}
}
public ManualConsumerTests()
{
_kafkaUri = IntegrationConfig.IntegrationUri;
_topic = IntegrationConfig.IntegrationTopic;
_defaultConsumerGroup = IntegrationConfig.IntegrationConsumer;
_options = new KafkaOptions(IntegrationConfig.IntegrationUri);
}
public void Main(string[] args)
{
var options = GetOptions(args);
if (options == null) return;
var count = 0;
var lastCount = 0;
var reporter = new Task(() =>
{
while (true)
{
var current = count;
Console.WriteLine("{0} messages in last second.", current - lastCount);
lastCount = current;
Thread.Sleep(1000);
}
});
var kafkaOptions = new KafkaOptions(options.KafkaNodeUri);// { Log = new ConsoleLog() };
using (var router = new BrokerRouter(kafkaOptions))
using (var client = new KafkaNet.Producer(router))
{
reporter.Start();
while (true)
{
Thread.Sleep(100);
client.SendMessageAsync("TestHarness", new[] { new Message() { Value = BitConverter.GetBytes(DateTime.Now.Ticks) } });
count++;
}
}
}
public void Run()
{
var options = new KafkaOptions(new Uri("http://localhost:9092"));
var router = new BrokerRouter(options);
var consumerOptions = new ConsumerOptions("test", router);
_consumer = new KafkaNet.Consumer(consumerOptions, new OffsetPosition(0, 100));
Process();
}
public DestinationKafka(params Uri[] servers)
{
var options = new KafkaOptions(servers) { Log = new ConsoleLogger() };
_router = new BrokerRouter(options);
_producer = new Producer(_router, maximumMessageBuffer: 5000, maximumAsyncRequests: 10) { BatchSize = 1000, BatchDelayTime = TimeSpan.FromSeconds(1) };
StatisticsTracker.OnStatisticsHeartbeat += StatisticsTracker_OnStatisticsHeartbeat;
}
public KafkaAgent(Uri[] kafkaServers)
{
var options = new KafkaOptions(kafkaServers);
_router = new BrokerRouter(options);
var influxDbClient = new InfluxDb("http://localhost", "reapadda", "qwerty", InfluxVersion.v09x);
_formatter = influxDbClient.GetFormatter();
}
static void Main(string[] args)
{
const string topicName = "kafka.SimpleTopic";
TopicClient topicClient = new TopicClient();
TopicView topicView = topicClient.getTopic(topicName);
List<Uri> brokers = getBrokerList(topicView);
//create an options file that sets up driver preferences
var options = new KafkaOptions()
{
Log = new ConsoleLog()
};
options.KafkaServerUri = brokers;
//start an out of process thread that runs a consumer that will write all received messages to the console
Task.Run(() =>
{
var consumer = new Consumer(new ConsumerOptions(topicName, new BrokerRouter(options)) { Log = new ConsoleLog() });
foreach (var data in consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}", data.Meta.PartitionId, data.Meta.Offset, data.Value.ToUtf8String());
}
});
//create a producer to send messages with
var producer = new KafkaNet.Producer(new BrokerRouter(options))
{
BatchSize = 100,
BatchDelayTime = TimeSpan.FromMilliseconds(2000)
};
//take in console read messages
Console.WriteLine("Type a message and press enter...");
while (true)
{
var message = Console.ReadLine();
if (message == "quit") break;
if (string.IsNullOrEmpty(message))
{
//send a random batch of messages
SendRandomBatch(producer, topicName, 200);
}
else
{
producer.SendMessageAsync(topicName, new[] { new Message(message) });
}
}
using (producer)
{
}
}
public async Task Send(object channel, Address address, TransportMessage message, object properties)
{
var options = new KafkaOptions(new Uri(address.Machine));
var router = new BrokerRouter(options);
var topic = address.Queue;
var messageString = System.Text.Encoding.Default.GetString(message.Body);
using (var client = new Producer(router))
{
await client.SendMessageAsync(topic, new[] { new Message(messageString) });
}
}
private static void Consume(string broker, string topic)
{
var options = new KafkaOptions(new Uri(broker));
var router = new BrokerRouter(options);
var consumer = new Consumer(new ConsumerOptions(topic, router));
//Consume returns a blocking IEnumerable (ie: never ending stream)
foreach (var message in consumer.Consume())
{
Console.WriteLine("Response: Partition {0},Offset {1} : {2}",
message.Meta.PartitionId, message.Meta.Offset, message.Value.ToUtf8String());
}
}
static void Main(string[] args)
{
const string topicName = "TestHarness";
//create an options file that sets up driver preferences
var options = new KafkaOptions(new Uri("http://CSDKAFKA01:9092"), new Uri("http://CSDKAFKA02:9092"))
{
Log = new ConsoleLog()
};
//start an out of process thread that runs a consumer that will write all received messages to the console
Task.Run(() =>
{
var consumer = new Consumer(new ConsumerOptions(topicName, new BrokerRouter(options)) { Log = new ConsoleLog() });
foreach (var data in consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}", data.Meta.PartitionId, data.Meta.Offset, data.Value.ToUtf8String());
}
});
//create a producer to send messages with
var producer = new Producer(new BrokerRouter(options))
{
BatchSize = 100,
BatchDelayTime = TimeSpan.FromMilliseconds(2000)
};
//take in console read messages
Console.WriteLine("Type a message and press enter...");
while (true)
{
var message = Console.ReadLine();
if (message == "quit") break;
if (string.IsNullOrEmpty(message))
{
//send a random batch of messages
SendRandomBatch(producer, topicName, 200);
}
else
{
producer.SendMessageAsync(topicName, new[] { new Message(message) });
}
}
using (producer)
{
}
}
private static void Produce(string broker, string topic)
{
var options = new KafkaOptions(new Uri(broker));
var router = new BrokerRouter(options);
var client = new Producer(router);
var current_datetime =DateTime.Now;
var key = current_datetime.Second.ToString();
var events = new[] { new Message("Hello World " + current_datetime.ToString(), key) };
client.SendMessageAsync(topic, events).Wait(1500);
Console.WriteLine("Produced: Key: {0}. Message: {1}", key, events[0].Value.ToUtf8String());
using (client) { }
}
public BrokerRouter(KafkaOptions kafkaOptions)
{
_kafkaOptions = kafkaOptions;
_kafkaMetadataProvider = new KafkaMetadataProvider(_kafkaOptions.Log);
foreach (var endpoint in _kafkaOptions.KafkaServerEndpoints)
{
var conn = _kafkaOptions.KafkaConnectionFactory.Create(endpoint, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log, _kafkaOptions.MaximumReconnectionTimeout);
_defaultConnectionIndex.AddOrUpdate(endpoint, e => conn, (e, c) => conn);
}
if (_defaultConnectionIndex.Count <= 0)
throw new ServerUnreachableException("None of the provided Kafka servers are resolvable.");
}
/// <summary>
/// Sends request and response raw data to Kafka
/// </summary>
/// <param name="data">Request and response metrics in raw form</param>
/// <returns>A completed task</returns>
public override async Task Write(RequestResponseData data)
{
try
{
KafkaOptions options = new KafkaOptions(new Uri("http://SERVER1:9092"), new Uri("http://SERVER2:9092"));
BrokerRouter router = new BrokerRouter(options);
Producer client = new Producer(router);
client.SendMessageAsync("API-Meter Data", new[] { new Message { Value = "NULL for now" } }).Wait();
}
catch (Exception ex)
{
Trace.Write(ex.Message);
}
}
public KafkaClient(Uri[] hosts)
{
var options = new KafkaOptions(hosts);
_brokerRouter = new BrokerRouter(options);
}
public void Setup()
{
var options = new KafkaOptions(new Uri(ConfigurationManager.AppSettings["IntegrationKafkaServerUrl"]));
_conn = new KafkaConnection(options.KafkaServerUri.First(), options.ResponseTimeoutMs, options.Log);
}
public async void ConsumerShouldMoveToNextAvailableOffsetWhenQueryingForNextMessage()
{
const int expectedCount = 1000;
var options = new KafkaOptions(IntegrationConfig.IntegrationUri) { Log = new ConsoleLog() };
using (var producerRouter = new BrokerRouter(options))
using (var producer = new Producer(producerRouter))
{
//get current offset and reset consumer to top of log
var offsets = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).ConfigureAwait(false);
using (var consumerRouter = new BrokerRouter(options))
using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, consumerRouter),
offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray()))
{
Console.WriteLine("Sending {0} test messages", expectedCount);
var response = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic,
Enumerable.Range(0, expectedCount).Select(x => new Message(x.ToString())));
Assert.That(response.Any(x => x.Error != (int)ErrorResponseCode.NoError), Is.False, "Error occured sending test messages to server.");
var stream = consumer.Consume();
Console.WriteLine("Reading message back out from consumer.");
var data = stream.Take(expectedCount).ToList();
var consumerOffset = consumer.GetOffsetPosition().OrderBy(x => x.Offset).ToList();
var serverOffset = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).ConfigureAwait(false);
var positionOffset = serverOffset.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max()))
.OrderBy(x => x.Offset)
.ToList();
Assert.That(consumerOffset, Is.EqualTo(positionOffset), "The consumerOffset position should match the server offset position.");
Assert.That(data.Count, Is.EqualTo(expectedCount), "We should have received 2000 messages from the server.");
}
}
}
public void Setup()
{
var options = new KafkaOptions(new Uri(ConfigurationManager.AppSettings["IntegrationKafkaServerUrl"]));
_router = new BrokerRouter(options);
}
public IBrokerRouter GetClientRouter()
{
var options = new KafkaOptions(new Uri(KafkaServerUri));
return new BrokerRouter(options);
}
public void Setup()
{
var brokers = ConfigurationManager.AppSettings["IntegrationKafkaServerUrl"]
.Split(',')
.Select(b => new Uri(b))
.ToArray();
var options = new KafkaOptions(brokers);
_router = new BrokerRouter(options);
}