static void Main(string[] args)
{
try
{
_topic = ConfigurationManager.AppSettings["topic"];
var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',')
select new Uri(x);
Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic);
var options = new KafkaOptions(brokers.ToArray());
var router = new BrokerRouter(options);
var coption = new ConsumerOptions(_topic, router);
_consumer = new KafkaNet.Consumer(coption);
var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result;
var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max());
_consumer.SetOffsetPosition(t.ToArray());
foreach (var message in _consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}",
message.Meta.PartitionId,
message.Meta.Offset,
System.Text.Encoding.Default.GetString(message.Value));
}
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.ReadLine();
}
public void BrokerRouterConstructorThrowsServerUnreachableException()
{
var result = new BrokerRouter(new KafkaOptions
{
KafkaServerUri = new List<Uri> { new Uri("http://noaddress:1") }
});
}
public void Main(string[] args)
{
var options = GetOptions(args);
if (options == null) return;
var count = 0;
var lastCount = 0;
var reporter = new Task(() =>
{
while (true)
{
var current = count;
Console.WriteLine("{0} messages in last second.", current - lastCount);
lastCount = current;
Thread.Sleep(1000);
}
});
var kafkaOptions = new KafkaOptions(options.KafkaNodeUri);// { Log = new ConsoleLog() };
using (var router = new BrokerRouter(kafkaOptions))
using (var client = new KafkaNet.Producer(router))
{
reporter.Start();
while (true)
{
Thread.Sleep(100);
client.SendMessageAsync("TestHarness", new[] { new Message() { Value = BitConverter.GetBytes(DateTime.Now.Ticks) } });
count++;
}
}
}
public void BrokerRouterConstructorShouldIgnoreUnresolvableUriWhenAtLeastOneIsGood()
{
var result = new BrokerRouter(new KafkaOptions
{
KafkaServerUri = new List<Uri> { new Uri("http://noaddress:1"), new Uri("http://localhost:1") }
});
}
static void Main(string[] args)
{
var options = new KafkaOptions(new Uri("http://CSDKAFKA01:9092"), new Uri("http://CSDKAFKA02:9092"))
{
Log = new ConsoleLog()
};
var router = new BrokerRouter(options);
var client = new Producer(router);
Task.Factory.StartNew(() =>
{
var consumer = new Consumer(new ConsumerOptions("TestHarness", router));
foreach (var data in consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}", data.Meta.PartitionId, data.Meta.Offset, data.Value);
}
});
Console.WriteLine("Type a message and press enter...");
while (true)
{
var message = Console.ReadLine();
if (message == "quit") break;
client.SendMessageAsync("TestHarness", new[] {new Message {Value = message}});
}
using (client)
using (router)
{
}
}
public void Main(string[] args)
{
var options = GetOptions(args);
if (options == null) return;
StartReporting();
var kafkaOptions = new KafkaOptions(options.KafkaNodeUri);
using (var router = new BrokerRouter(kafkaOptions))
using (var client = new KafkaNet.Consumer(new ConsumerOptions("TestHarness", router) { Log = new ConsoleLog(), MinimumBytes = 1 }))
{
Console.WriteLine("Listening for messages...");
foreach (var message in client.Consume())
{
try
{
var received = DateTime.Now;
var sent = new DateTime(BitConverter.ToInt64(message.Value, 0));
var diff = received - sent;
lock (sync)
receivedItems.Add(new ReceivedMessage { DateTime = received, TotalMilliseconds = diff.TotalMilliseconds });
}
catch (Exception ex)
{
Console.Error.WriteLine("Oops... " + ex);
}
}
}
}
public void Run()
{
var options = new KafkaOptions(new Uri("http://localhost:9092"));
var router = new BrokerRouter(options);
_producer = new KafkaNet.Producer(router);
Process();
}
public void ConsumerShouldConsumeInSameOrderAsProduced()
{
var expected = new List<string> { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" };
var testId = Guid.NewGuid().ToString();
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var offsets = producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).Result;
using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, router),
offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray()))
{
for (int i = 0; i < 20; i++)
{
producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString(), testId) }).Wait();
}
var results = consumer.Consume().Take(20).ToList();
//ensure the produced messages arrived
Console.WriteLine("Message order: {0}", string.Join(", ", results.Select(x => x.Value.ToUtf8String()).ToList()));
Assert.That(results.Count, Is.EqualTo(20));
Assert.That(results.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected), "Expected the message list in the correct order.");
Assert.That(results.Any(x => x.Key.ToUtf8String() != testId), Is.False);
}
}
}
public void OffsetFetchRequestOfNonExistingGroupShouldReturnNoError([Values(0,1)] int version)
{
//From documentation: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetFetchRequest
//Note that if there is no offset associated with a topic-partition under that consumer group the broker does not set an error code
//(since it is not really an error), but returns empty metadata and sets the offset field to -1.
const int partitionId = 0;
using (var router = new BrokerRouter(Options))
{
var request = CreateOffsetFetchRequest(version, Guid.NewGuid().ToString(), partitionId);
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic, partitionId);
var response = conn.Connection.SendAsync(request).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
if (version == 0)
{
// Version 0 (storing in zookeeper) results in unknown topic or partition as the consumer group
// and partition are used to make up the string, and when it is missing it results in an error
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.UnknownTopicOrPartition));
}
else
{
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
}
Assert.That(response.Offset, Is.EqualTo(-1));
}
}
public async Task ProtocolGateway()
{
int partitionId = 0;
var router = new BrokerRouter(Options);
var producer = new Producer(router);
string messge1 = Guid.NewGuid().ToString();
var respose = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(messge1) }, 1, null, MessageCodec.CodecNone, partitionId);
var offset = respose.FirstOrDefault().Offset;
ProtocolGateway protocolGateway = new ProtocolGateway(IntegrationConfig.IntegrationUri);
var fetch = new Fetch
{
Topic = IntegrationConfig.IntegrationTopic,
PartitionId = partitionId,
Offset = offset,
MaxBytes = 32000,
};
var fetches = new List<Fetch> { fetch };
var fetchRequest = new FetchRequest
{
MaxWaitTime = 1000,
MinBytes = 10,
Fetches = fetches
};
var r = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
// var r1 = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
Assert.IsTrue(r.Messages.FirstOrDefault().Value.ToUtf8String() == messge1);
}
public void EnsureGzipCanDecompressMessageFromKafka()
{
var router = new BrokerRouter(_options);
var producer = new Producer(router);
var offsets = producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationCompressionTopic).Result;
var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationCompressionTopic, router) { PartitionWhitelist = new List<int>() { 0 } },
offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray());
int numberOfmessage = 3;
for (int i = 0; i < numberOfmessage; i++)
{
producer.SendMessageAsync(IntegrationConfig.IntegrationCompressionTopic, new[] { new Message(i.ToString()) }, codec: MessageCodec.CodecGzip,
partition: 0);
}
var results = consumer.Consume(new CancellationTokenSource(TimeSpan.FromMinutes(1)).Token).Take(numberOfmessage).ToList();
for (int i = 0; i < numberOfmessage; i++)
{
Assert.That(results[i].Value.ToUtf8String(), Is.EqualTo(i.ToString()));
}
using (producer)
using (consumer) { }
}
public async Task FetchMessagesCacheContainsAllRequestTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var protocolGateway = new ProtocolGateway(_kafkaUri);
Producer producer = new Producer(brokerRouter);
ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);
var offset = await consumer.FetchLastOffset();
// Creating 5 messages
List<Message> messages = CreateTestMessages(10, 1);
await producer.SendMessageAsync(_topic, messages, partition: _partitionId, timeout: TimeSpan.FromSeconds(3));
// Now let's consume
var result = (await consumer.FetchMessages(5, offset)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
// Now let's consume again
result = (await consumer.FetchMessages(5, offset + 5)).ToList();
CheckMessages(messages.Skip(5).ToList(), result);
}
public void Run()
{
var options = new KafkaOptions(new Uri("http://localhost:9092"));
var router = new BrokerRouter(options);
var consumerOptions = new ConsumerOptions("test", router);
_consumer = new KafkaNet.Consumer(consumerOptions, new OffsetPosition(0, 100));
Process();
}
public DestinationKafka(params Uri[] servers)
{
var options = new KafkaOptions(servers) { Log = new ConsoleLogger() };
_router = new BrokerRouter(options);
_producer = new Producer(_router, maximumMessageBuffer: 5000, maximumAsyncRequests: 10) { BatchSize = 1000, BatchDelayTime = TimeSpan.FromSeconds(1) };
StatisticsTracker.OnStatisticsHeartbeat += StatisticsTracker_OnStatisticsHeartbeat;
}
public KafkaAgent(Uri[] kafkaServers)
{
var options = new KafkaOptions(kafkaServers);
_router = new BrokerRouter(options);
var influxDbClient = new InfluxDb("http://localhost", "reapadda", "qwerty", InfluxVersion.v09x);
_formatter = influxDbClient.GetFormatter();
}
public async void SendAsyncShouldGetAResultForEachPartitionSentTo()
{
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var result = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message("1"), new Message("2"), new Message("3") });
Assert.That(result.Count, Is.EqualTo(2));
}
}
public async void SendAsyncShouldGetOneResultForMessage()
{
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var result = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(Guid.NewGuid().ToString()) });
Assert.That(result.Count, Is.EqualTo(1));
}
}
public void BrokerRouterCanConstruct()
{
var result = new BrokerRouter(new KafkaOptions
{
KafkaServerUri = new List<Uri> { new Uri("http://localhost:1") },
KafkaConnectionFactory = _mockKafkaConnectionFactory.Object
});
Assert.That(result, Is.Not.Null);
}
public async Task ProducerAckLevel1ResponseOffsetShouldBeEqualToLastOffset()
{
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri) { Log = IntegrationConfig.NoDebugLog }))
using (var producer = new Producer(router))
{
var responseAckLevel1 = await producer.SendMessageAsync(new Message("Ack Level 1"), IntegrationConfig.IntegrationTopic, acks: 1, partition: 0);
var offsetResponse = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic);
var maxOffset = offsetResponse.Find(x => x.PartitionId == 0);
Assert.AreEqual(responseAckLevel1.Offset, maxOffset.Offsets.Max() - 1);
}
}
public async Task ProducerAckLevel()
{
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri) { Log = IntegrationConfig.NoDebugLog }))
using (var producer = new Producer(router))
{
var responseAckLevel0 = await producer.SendMessageAsync(new Message("Ack Level 0"), IntegrationConfig.IntegrationTopic, acks: 0, partition: 0);
Assert.AreEqual(responseAckLevel0.Offset, -1);
var responseAckLevel1 = await producer.SendMessageAsync(new Message("Ack Level 1"), IntegrationConfig.IntegrationTopic, acks: 1, partition: 0);
Assert.That(responseAckLevel1.Offset, Is.GreaterThan(-1));
}
}
public async Task ManualConsumerFailure()
{
string topic = "TestTopicIssue13-3R-1P";
var manualConsumer = new ManualConsumer(0, topic, new ProtocolGateway(_options), "test client", 10000);
long offset = await manualConsumer.FetchLastOffset();
var router = new BrokerRouter(_options);
var producer = new Producer(router);
SandMessageForever(producer, topic);
await ReadMessageForever(manualConsumer, offset);
}
public static Producer GetProducer(this Kafka kafkaObj)
{
if (_producer == null)
{
var addresses = from x in kafkaObj.brokers
select new Uri(x.address);
var router = new BrokerRouter(new KafkaOptions(addresses.ToArray()));
_producer = new Producer(router);
}
return _producer;
}
public async Task Send(object channel, Address address, TransportMessage message, object properties)
{
var options = new KafkaOptions(new Uri(address.Machine));
var router = new BrokerRouter(options);
var topic = address.Queue;
var messageString = System.Text.Encoding.Default.GetString(message.Body);
using (var client = new Producer(router))
{
await client.SendMessageAsync(topic, new[] { new Message(messageString) });
}
}
public void ProducerShouldNotExpectResponseWhenAckIsZero()
{
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var sendTask = producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(Guid.NewGuid().ToString()) }, acks: 0);
sendTask.Wait(TimeSpan.FromMinutes(2));
Assert.That(sendTask.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
private static void Consume(string broker, string topic)
{
var options = new KafkaOptions(new Uri(broker));
var router = new BrokerRouter(options);
var consumer = new Consumer(new ConsumerOptions(topic, router));
//Consume returns a blocking IEnumerable (ie: never ending stream)
foreach (var message in consumer.Consume())
{
Console.WriteLine("Response: Partition {0},Offset {1} : {2}",
message.Meta.PartitionId, message.Meta.Offset, message.Value.ToUtf8String());
}
}
public void OffsetCommitShouldStoreAndReturnSuccess()
{
const int partitionId = 0;
using (var router = new BrokerRouter(Options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic, partitionId);
var commit = CreateOffsetCommitRequest(IntegrationConfig.IntegrationConsumer, partitionId, 10);
var response = conn.Connection.SendAsync(commit).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
}
}
public void BrokerRouterUsesFactoryToAddNewBrokers()
{
var router = new BrokerRouter(new KafkaNet.Model.KafkaOptions
{
KafkaServerUri = new List<Uri> { new Uri("http://localhost:1") },
KafkaConnectionFactory = _factoryMock.Object
});
_connMock1.Setup(x => x.SendAsync(It.IsAny<IKafkaRequest<MetadataResponse>>()))
.Returns(() => Task.Factory.StartNew(() => new List<MetadataResponse> { CreateMetaResponse() }));
var topics = router.GetTopicMetadata(TestTopic);
_factoryMock.Verify(x => x.Create(It.Is<Uri>(uri => uri.Port == 2), It.IsAny<int>(), It.IsAny<IKafkaLog>()), Times.Once());
}
public async Task BrokerRouterUsesFactoryToAddNewBrokers()
{
var router = new BrokerRouter(new KafkaOptions
{
KafkaServerUri = new List<Uri> { new Uri("http://localhost:1") },
KafkaConnectionFactory = _mockKafkaConnectionFactory.Object
});
_mockKafkaConnection1.Setup(x => x.SendAsync(It.IsAny<IKafkaRequest<MetadataResponse>>()))
.Returns(() => Task.Run(async () => new List<MetadataResponse> { await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers() }));
await router.RefreshMissingTopicMetadata(TestTopic);
var topics = router.GetTopicMetadataFromLocalCache(TestTopic);
_mockKafkaConnectionFactory.Verify(x => x.Create(It.Is<KafkaEndpoint>(e => e.Endpoint.Port == 2), It.IsAny<TimeSpan>(), It.IsAny<IKafkaLog>(), It.IsAny<int>(), It.IsAny<TimeSpan?>(), It.IsAny<StatisticsTrackerOptions>()), Times.Once());
}
public void BrokerRouterUsesFactoryToAddNewBrokers()
{
var router = new BrokerRouter(new KafkaOptions
{
KafkaServerUri = new List<Uri> { new Uri("http://localhost:1") },
KafkaConnectionFactory = _mockKafkaConnectionFactory.Object
});
_mockKafkaConnection1.Setup(x => x.SendAsync(It.IsAny<IKafkaRequest<MetadataResponse>>()))
.Returns(() => Task.Run(() => new List<MetadataResponse> { CreateMetaResponse() }));
var topics = router.GetTopicMetadata(TestTopic);
_mockKafkaConnectionFactory.Verify(x => x.Create(It.Is<KafkaEndpoint>(e => e.Endpoint.Port == 2), It.IsAny<TimeSpan>(), It.IsAny<IKafkaLog>(), null), Times.Once());
}
public void ConsumerFailure()
{
string topic = "TestTopicIssue13-2-3R-1P";
using (var router = new BrokerRouter(_options))
{
var producer = new Producer(router);
var offsets = producer.GetTopicOffsetAsync(topic).Result;
var maxOffsets = offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray();
var consumerOptions = new ConsumerOptions(topic, router) { PartitionWhitelist = new List<int>() { 0 }, MaxWaitTimeForMinimumBytes = TimeSpan.Zero };
SandMessageForever(producer, topic);
ReadMessageForever(consumerOptions, maxOffsets);
}
}
private async Task ProduceAndSendBatchAsync(List <TopicMessage> batchs, CancellationToken cancellationToken)
{
//we must send a different produce request for each ack level and timeout combination.
foreach (var ackLevelBatch in batchs.GroupBy(batch => new { batch.Acks, batch.Timeout }))
{
var messageByRouter = ackLevelBatch.Select(batch => new
{
TopicMessage = batch,
Route = BrokerRouter.SelectBrokerRoute(batch.Topic, batch.Message.Key),
})
.GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec });
var sendTasks = new List <BrokerRouteSendBatch>();
foreach (var group in messageByRouter)
{
var payload = new Payload
{
Codec = group.Key.Codec,
Topic = group.Key.Topic,
Partition = group.Key.Route.PartitionId,
Messages = group.Select(x => x.TopicMessage.Message).ToList()
};
var request = new ProduceRequest
{
Acks = ackLevelBatch.Key.Acks,
TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds,
Payload = new List <Payload> {
payload
}
};
await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false);
var brokerSendTask = new BrokerRouteSendBatch
{
Route = group.Key.Route,
Task = group.Key.Route.Connection.SendAsync(request),
MessagesSent = group.Select(x => x.TopicMessage).ToList()
};
//ensure the async is released as soon as each task is completed
brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken,
TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
sendTasks.Add(brokerSendTask);
}
try
{
await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false);
foreach (var task in sendTasks)
{
task.MessagesSent.ForEach(x => x.Tcs.TrySetResult(task.Task.Result.FirstOrDefault()));
}
}
catch
{
//if an error occurs here, all we know is some or all of the messages in this ackBatch failed.
var failedTask = sendTasks.FirstOrDefault(t => t.Task.IsFaulted);
if (failedTask != null)
{
foreach (var topicMessageBatch in ackLevelBatch)
{
topicMessageBatch.Tcs.TrySetException(new KafkaApplicationException("An exception occured while executing a send operation against {0}. Exception:{1}",
failedTask.Route, failedTask.Task.Exception));
}
}
}
}
}