static void Main(string[] args)
{
try
{
var options = new KafkaOptions(new Uri("http://localhost:9092"))
{
Log = new ConsoleLog()
};
var brokerRouter = new BrokerRouter(options);
_brokerRoute = brokerRouter.SelectBrokerRoute(Topic);
StartConsumer(ConsoleColor.Blue, OffsetFetchRequest().Offset + 1);
StartConsumer(ConsoleColor.Green, OffsetFetchRequest().Offset + 1);
StartConsumer(ConsoleColor.Red, OffsetFetchRequest().Offset + 1);
StartConsumer(ConsoleColor.Yellow, OffsetFetchRequest().Offset + 1);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
ConsoleLog.WaitOnKeys();
}
}
public void OffsetCommitShouldStoreMetadata()
{
const int partitionId = 0;
const long offset = 101;
const string metadata = "metadata";
using (var router = new BrokerRouter(Options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic, partitionId);
var commit = CreateOffsetCommitRequest(IntegrationConfig.IntegrationConsumer, partitionId, offset, metadata);
var commitResponse = conn.Connection.SendAsync(commit).Result.FirstOrDefault();
Assert.That(commitResponse, Is.Not.Null);
Assert.That(commitResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
var fetch = CreateOffsetFetchRequest(IntegrationConfig.IntegrationConsumer, partitionId);
var fetchResponse = conn.Connection.SendAsync(fetch).Result.FirstOrDefault();
Assert.That(fetchResponse, Is.Not.Null);
Assert.That(fetchResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
Assert.That(fetchResponse.Offset, Is.EqualTo(offset));
Assert.That(fetchResponse.MetaData, Is.EqualTo(metadata));
}
}
public void OffsetFetchRequestOfNonExistingGroupShouldReturnNoError([Values(0, 1)] int version)
{
//From documentation: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetFetchRequest
//Note that if there is no offset associated with a topic-partition under that consumer group the broker does not set an error code
//(since it is not really an error), but returns empty metadata and sets the offset field to -1.
const int partitionId = 0;
using (var router = new BrokerRouter(Options))
{
var request = CreateOffsetFetchRequest(version, Guid.NewGuid().ToString(), partitionId);
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic, partitionId);
var response = conn.Connection.SendAsync(request).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
if (version == 0)
{
// Version 0 (storing in zookeeper) results in unknown topic or partition as the consumer group
// and partition are used to make up the string, and when it is missing it results in an error
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.UnknownTopicOrPartition));
}
else
{
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
}
Assert.That(response.Offset, Is.EqualTo(-1));
}
}
static void Main()
{
try
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(Topic);
Console.ForegroundColor = ConsoleColor.White;
var options = new KafkaOptions(new Uri("http://localhost:9092"))
{
Log = new ConsoleLog()
};
var brokerRouter = new BrokerRouter(options);
_brokerRoute = brokerRouter.SelectBrokerRoute(Topic);
List <Topic> topics = brokerRouter.GetTopicMetadata(Topic);
OffsetPosition[] offsetPositions = OffsetFetchRequest(topics.First().Partitions.Select(p => new OffsetFetch
{
Topic = Topic,
PartitionId = p.PartitionId
})).Select(o => new OffsetPosition {
Offset = o.Offset + 1, PartitionId = o.PartitionId
}).ToArray();
StartConsumer(offsetPositions);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
ConsoleLog.WaitOnKeys();
}
}
/// <summary>
/// Fetches a single result for the specified <paramref name="topic"/> and <paramref name="partitionID"/>.
/// </summary>
/// <param name="router">Router used to fetch result.</param>
/// <param name="topic">Topic to fetch result from.</param>
/// <param name="partitionID">Partition ID to fetch result from.</param>
/// <param name="offset">Offset of <paramref name="topic"/> to begin fetching result.</param>
/// <param name="maxBytes">Defines maximum size of request that can be adjusted for large serializations.</param>
/// <returns>A <see cref="FetchResponse"/> instance, if available; otherwise, <c>null</c>.</returns>
public static FetchResponse Fetch(this BrokerRouter router, string topic, int partitionID = 0, long offset = 0, int maxBytes = 32768)
{
if ((object)router == null)
{
throw new ArgumentNullException(nameof(router));
}
if (string.IsNullOrWhiteSpace(topic))
{
throw new ArgumentNullException(nameof(topic));
}
// Create a fetch request with a single item to request
FetchRequest request = new FetchRequest()
{
Fetches = new List <Fetch>(new[]
{
new Fetch
{
Topic = topic,
PartitionId = partitionID,
Offset = offset,
MaxBytes = maxBytes
}
})
};
return(router.SelectBrokerRoute(topic, partitionID).Connection.SendAsync(request).Result.FirstOrDefault());
}
public void EnsureGzipCompressedMessageCanSend()
{
var conn = _router.SelectBrokerRoute(CompressTopic, 0);
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 1000,
Payload = new List <Payload>
{
new Payload
{
Codec = MessageCodec.CodecGzip,
Topic = CompressTopic,
Partition = 0,
Messages = new List <Message>
{
new Message {
Value = "0", Key = "1"
},
new Message {
Value = "1", Key = "1"
},
new Message {
Value = "2", Key = "1"
}
}
}
}
};
var response = conn.Connection.SendAsync(request).Result;
Assert.That(response.First().Error, Is.EqualTo(0));
//var offsets = producer.GetTopicOffsetAsync("NewTopic").Result;
//var consumer = new Consumer(new ConsumerOptions("NewTopic", _router),
// offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray());
//var response = producer.SendMessageAsync("NewTopic", new[]
// {
// new Message {Value = "0", Key = "1"},
// new Message {Value = "1", Key = "1"},
// new Message {Value = "2", Key = "1"}
// }, codec: MessageCodec.CodecGzip).Result;
//Assert.That(response.First().Error, Is.EqualTo(0));
//var results = consumer.Consume().Take(3).ToList();
//for (int i = 0; i < 3; i++)
//{
// Assert.That(results[i].Value, Is.EqualTo(i.ToString()));
//}
}
public async Task <OffsetCommitResponse> SetStoredOffsetAsync(string topic, long offset)
{
var request = new OffsetCommitRequest
{
ConsumerGroup = ConsumerGroupName,
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
PartitionId = 0,
Topic = topic,
Offset = offset
}
}
};
var result = await _router.SelectBrokerRoute(topic).Connection.SendAsync(request).ConfigureAwait(false);
return(result.FirstOrDefault());
}
public void OffsetCommitShouldStoreAndReturnSuccess()
{
const int partitionId = 0;
using (var router = new BrokerRouter(Options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic, partitionId);
var commit = CreateOffsetCommitRequest(IntegrationConfig.IntegrationConsumer, partitionId, 10);
var response = conn.Connection.SendAsync(commit).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
}
}
public void ConsumerMetadataRequestShouldReturnWithoutError()
{
using (var router = new BrokerRouter(Options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic);
var request = new ConsumerMetadataRequest {
ConsumerGroup = IntegrationConfig.IntegrationConsumer
};
var response = conn.Connection.SendAsync(request).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
}
}
public void OffsetFetchRequestOfNonExistingGroupShouldReturnNoError()
{
//From documentation: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetFetchRequest
//Note that if there is no offset associated with a topic-partition under that consumer group the broker does not set an error code
//(since it is not really an error), but returns empty metadata and sets the offset field to -1.
const int partitionId = 0;
using (var router = new BrokerRouter(Options))
{
var request = CreateOffsetFetchRequest(Guid.NewGuid().ToString(), partitionId);
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic, partitionId);
var response = conn.Connection.SendAsync(request).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
Assert.That(response.Offset, Is.EqualTo(-1));
}
}
public void EnsureGzipCompressedMessageCanSend()
{
//ensure topic exists
_kafkaConnection.SendAsync(new MetadataRequest {
Topics = new List <string>(new[] { IntegrationConfig.IntegrationCompressionTopic })
}).Wait();
var conn = _router.SelectBrokerRoute(IntegrationConfig.IntegrationCompressionTopic, 0);
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 1000,
Payload = new List <Payload>
{
new Payload
{
Codec = MessageCodec.CodecGzip,
Topic = IntegrationConfig.IntegrationCompressionTopic,
Partition = 0,
Messages = new List <Message>
{
new Message {
Value = "0", Key = "1"
},
new Message {
Value = "1", Key = "1"
},
new Message {
Value = "2", Key = "1"
}
}
}
}
};
var response = conn.Connection.SendAsync(request).Result;
Assert.That(response.First().Error, Is.EqualTo(0));
}
public void EnsureGzipCompressedMessageCanSend()
{
//ensure topic exists
using (var conn = GetKafkaConnection())
{
conn.SendAsync(new MetadataRequest {
Topics = new List <string>(new[] { IntegrationConfig.IntegrationCompressionTopic })
}).Wait(TimeSpan.FromSeconds(10));
}
using (var router = new BrokerRouter(_options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationCompressionTopic, 0);
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 1000,
Payload = new List <Payload>
{
new Payload
{
Codec = MessageCodec.CodecGzip,
Topic = IntegrationConfig.IntegrationCompressionTopic,
Partition = 0,
Messages = new List <Message>
{
new Message("0", "1"),
new Message("1", "1"),
new Message("2", "1")
}
}
}
};
var response = conn.Connection.SendAsync(request).Result;
Assert.That(response.First().Error, Is.EqualTo(0));
}
}
public void OffsetCommitShouldStoreOffsetValue([Values(0, 1)] int version)
{
const int partitionId = 0;
const long offset = 99;
using (var router = new BrokerRouter(Options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic, partitionId);
var commit = CreateOffsetCommitRequest(version, IntegrationConfig.IntegrationConsumer, partitionId, offset);
var commitResponse = conn.Connection.SendAsync(commit).Result.FirstOrDefault();
Assert.That(commitResponse, Is.Not.Null);
Assert.That(commitResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
var fetch = CreateOffsetFetchRequest(version, IntegrationConfig.IntegrationConsumer, partitionId);
var fetchResponse = conn.Connection.SendAsync(fetch).Result.FirstOrDefault();
Assert.That(fetchResponse, Is.Not.Null);
Assert.That(fetchResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
Assert.That(fetchResponse.Offset, Is.EqualTo(offset));
}
}