public void ConsumeTest()
{
KafkaConnection kConnection = new KafkaConnection
{
Url = "localhost",
Port = 9092,
Topic = "witsmlCreateEvent"
};
Action <string> onMessageConsumed = ConsumeMessage;
KafkaTaskManager taskManager = new KafkaTaskManager();
string group = $"test_{new Random().Next()}";
string autoOffsetReset = "earliest";
int timeOut = 10 * 1000;
// Create the token source for cancelation purpose
CancellationTokenSource cts = new CancellationTokenSource();
CancellationToken ct = cts.Token;
Assert.Throws <OperationCanceledException>(() =>
{
cts.CancelAfter(timeOut);
KafkaTaskManager.Consume(kConnection, group, autoOffsetReset, ct, onMessageConsumed);
});
cts.Dispose();
Assert.IsTrue(isMessageConsumed, "Message should be received from kafka");
}
[Test, Repeat(1)]/// [Test, Repeat(100)]
public async Task ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(_log, 8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = false;
socket.OnServerDisconnected += () => disconnected = true;
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
await TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//Wait a while for the client to notice the disconnect and log
await TaskTest.WaitFor(() => disconnected);
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny <string>(), It.IsAny <Exception>()));
await TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
/// <summary>
/// This task allows caller to produce a message
/// </summary>
/// <returns>Task<bool> true if task succeded</returns>
public static async Task <bool> Produce(KafkaConnection kConnection, int timeout, string message)
{
var config = new ProducerConfig {
BootstrapServers = $"{kConnection.Url}:{kConnection.Port}"
};
// A Producer for sending messages with null keys and UTF-8 encoded values.
using (var p = new Producer <Null, string>(config))
{
try
{
var dr = p.ProduceAsync(kConnection.Topic, new Message <Null, string> {
Value = message
});
if (await Task.WhenAny(dr, Task.Delay(timeout)) == dr)
{
var taskResult = await dr;
Console.WriteLine($"Delivered '{taskResult.Value}' to '{taskResult.TopicPartitionOffset}'");
return(true);
// task completed within timeout
}
else
{
// timeout logic
Console.WriteLine($"Request timedout");
return(false);
}
}
catch (KafkaException e)
{
Console.WriteLine($"Delivery failed: {e.Error.Reason}");
return(false);
}
}
}
public void Setup()
{
var options = new KafkaOptions(IntegrationConfig.IntegrationUri);
_kafkaConnection = new KafkaConnection(new KafkaTcpSocket(new DefaultTraceLog(), options.KafkaServerUri.First()), options.ResponseTimeoutMs, options.Log);
_router = new BrokerRouter(options);
}
public void Setup()
{
var options = new KafkaOptions(IntegrationConfig.IntegrationUri);
var endpoint = new DefaultKafkaConnectionFactory().Resolve(options.KafkaServerUri.First(), options.Log);
_conn = new KafkaConnection(new KafkaTcpSocket(new DefaultTraceLog(), endpoint), options.ResponseTimeoutMs, options.Log);
}
/// <summary>
/// Combine multiple fetch requests in one call.
/// </summary>
/// <param name="request">
/// The list of fetch requests.
/// </param>
/// <returns>
/// A list of sets of fetched messages.
/// </returns>
/// <remarks>
/// Offset is passed in on every request, allowing the user to maintain this metadata
/// however they choose.
/// </remarks>
public IList <BufferedMessageSet> MultiFetch(MultiFetchRequest request)
{
var result = new List <BufferedMessageSet>();
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
using (var conn = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SocketTimeout))
{
conn.Write(request);
int size = conn.Reader.ReadInt32();
return(BufferedMessageSet.ParseMultiFrom(conn.Reader, size, request.ConsumerRequests.Count, request.ConsumerRequests.Select(x => x.Offset).ToList()));
}
}
catch (Exception ex)
{
// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "MultiFetch reconnect due to {0}", ex);
}
}
return(result);
}
public async void SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(1000000), log: _log))
{
Console.WriteLine("SendAsync blocked by reconnection attempts...");
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Console.WriteLine("Task result should be WaitingForActivation...");
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
Console.WriteLine("Starting server to establish connection...");
using (var server = new FakeTcpServer(8999))
{
server.OnClientConnected += () => Console.WriteLine("Client connected...");
server.OnBytesReceived += (b) =>
{
server.SendDataAsync(MessageHelper.CreateMetadataResponse(1, "Test"));
};
await taskResult;
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
/// <summary>
/// Fetch a set of messages from a topic.
/// </summary>
/// <param name="request">
/// Specifies the topic name, topic partition, starting byte offset, maximum bytes to be fetched.
/// </param>
/// <returns>
/// A set of fetched messages.
/// </returns>
/// <remarks>
/// Offset is passed in on every request, allowing the user to maintain this metadata
/// however they choose.
/// </remarks>
public BufferedMessageSet Fetch(FetchRequest request)
{
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
using (var conn = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SocketTimeout))
{
conn.Write(request);
int size = conn.Reader.ReadInt32();
return(BufferedMessageSet.ParseFrom(conn.Reader, size, request.Offset));
}
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "Fetch reconnect due to {0}", ex);
}
}
return(null);
}
public async Task SendAsyncShouldTimeoutMultipleMessagesAtATime()
{
using (var server = new FakeTcpServer(_log, 8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(100), log: _log))
{
server.HasClientConnected.Wait(TimeSpan.FromSeconds(3));
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
var tasks = new[]
{
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest())
};
Task.WhenAll(tasks);
await TaskTest.WaitFor(() => tasks.All(t => t.IsFaulted));
foreach (var task in tasks)
{
Assert.That(task.IsFaulted, Is.True, "Task should have faulted.");
Assert.That(task.Exception.InnerException, Is.TypeOf <ResponseTimeoutException>(),
"Task fault should be of type ResponseTimeoutException.");
}
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover()
{
var mockLog = new Mock <IKafkaLog>();
var log = new DefaultTraceLog(LogLevel.Error);
using (var server = new FakeTcpServer(log, 8999))
using (var socket = new KafkaTcpSocket(log, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = 0;
socket.OnServerDisconnected += () => Interlocked.Increment(ref disconnected);
for (int connectionAttempt = 1; connectionAttempt < 4; connectionAttempt++)
{
await TaskTest.WaitFor(() => server.ConnectionEventcount == connectionAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
server.SendDataAsync(CreateCorrelationMessage(1)).Wait(TimeSpan.FromSeconds(5));
await TaskTest.WaitFor(() => !conn.IsOnErrorState());
Assert.IsFalse(conn.IsOnErrorState());
mockLog.Verify(x => x.InfoFormat("Polling read thread has recovered: {0}", It.IsAny <object[]>()), Times.Exactly(connectionAttempt - 1));
server.DropConnection();
await TaskTest.WaitFor(() => conn.IsOnErrorState());
Assert.AreEqual(disconnected, connectionAttempt);
Assert.IsTrue(conn.IsOnErrorState());
mockLog.Verify(x => x.ErrorFormat("Exception occured in polling read thread {0}: {1}", It.IsAny <object[]>()), Times.Exactly(connectionAttempt));
}
}
}
public void Setup()
{
var options = new KafkaOptions(IntegrationConfig.IntegrationUri);
var endpoint = new DefaultKafkaConnectionFactory().Resolve(options.KafkaServerUri.First(), options.Log);
_conn = new KafkaConnection(new KafkaTcpSocket(new DefaultTraceLog(), endpoint), options.ResponseTimeoutMs, options.Log);
}
public async Task ReadShouldIgnoreMessageWithUnknownCorrelationId()
{
const int correlationId = 99;
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(_log, 8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var receivedData = false;
socket.OnBytesReceived += i => receivedData = true;
//send correlation message
server.SendDataAsync(CreateCorrelationMessage(correlationId)).Wait(TimeSpan.FromSeconds(5));
//wait for connection
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
await TaskTest.WaitFor(() => receivedData);
//should log a warning and keep going
mockLog.Verify(x => x.WarnFormat(It.IsAny <string>(), It.Is <int>(o => o == correlationId)));
}
}
private static async Task JoinGroup(KafkaConnection kafka)
{
Console.WriteLine("----- JoinGroup");
var response = await kafka.JoinGroup(new JoinGroupRequest
{
GroupId = "test_" + Guid.NewGuid().ToString(),
SessionTimeout = 10000,
RebalanceTimeout = 300000,
ProtocolType = "consumer",
Protocols = new JoinGroupRequest.Protocol[]
{
new JoinGroupRequest.Protocol {
Name = "range",
}
}
});
Console.WriteLine($"throttle: {response.ThrottleTime}");
Console.WriteLine($"errorCode: {response.ErrorCode}");
Console.WriteLine($"genId: {response.GenerationId}");
Console.WriteLine($"ProtocolName: {response.ProtocolName}");
Console.WriteLine($"Leader: {response.Leader}");
Console.WriteLine($"memberId: {response.MemberId}");
foreach (var m in response.Members)
{
Console.WriteLine($" memberId:{m.MemberId} group:{m.GroupInstanceId} meta:{(m.Metadata?.Length.ToString() ?? "NONE")}");
}
}
private static async Task Produce(KafkaConnection kafka)
{
Console.WriteLine("----- Produce");
var msg = new Message <string, string>
{
Topic = TopicName,
Key = "fred",
Value = "flintstone " + Guid.NewGuid().ToString()
};
Console.WriteLine(msg.Value);
var response = await kafka.Produce(msg, IntrinsicSerializers.String, IntrinsicSerializers.String);
Console.WriteLine($"throttle: {response.ThrottleTime}");
foreach (var r in response.Topics)
{
Console.WriteLine($"topic: {r.Topic}");
foreach (var p in r.Partitions)
{
Console.WriteLine($" {p.Partition} (error: {p.ErrorCode.ToString()}) - {p.BaseOffset} {p.LogStartOffset} {Timestamp.UnixTimestampMsToDateTime(p.LogAppendTime)}");
}
}
}
public void SendAsyncShouldTimeoutMultipleMessagesAtATime()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(100), log: _log))
{
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
var tasks = new[]
{
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest())
};
Task.WhenAll(tasks);
TaskTest.WaitFor(() => tasks.Any(t => t.IsFaulted));
foreach (var task in tasks)
{
Assert.That(task.IsFaulted, Is.True);
Assert.That(task.Exception.InnerException, Is.TypeOf <ResponseTimeoutException>());
}
}
}
public void ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//Wait a while for the client to notice the disconnect and log
Thread.Sleep(15);
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny <string>(), It.IsAny <Exception>()));
TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
private KafkaConnection Connect()
{
TimeSpan connectBackoff = new TimeSpan(1 * TimeSpan.TicksPerMillisecond);
DateTime beginTime = DateTime.Now;
while (connection == null && !disposed)
{
try
{
connection = new KafkaConnection(Config.Host, Config.Port, Config.BufferSize, Config.SocketTimeout);
Logger.Info("Connected to " + Config.Host + ":" + Config.Port + " for producing");
}
catch (Exception e)
{
Disconnect();
DateTime endTime = DateTime.Now;
// Throw because the connection timeout has expired
if (((endTime - beginTime) + connectBackoff).TotalMilliseconds > Config.ConnectTimeout)
{
Logger.Error("Producer connection to " + Config.Host + ":" + Config.Port + " timing out after " +
Config.ConnectTimeout + " ms", e);
throw;
}
int backoff = Convert.ToInt32(connectBackoff.TotalMilliseconds);
Logger.Error("Connection attempt to " + Config.Host + ":" + Config.Port + " failed, next attempt in " + backoff + " ms", e);
System.Threading.Thread.Sleep(backoff);
backoff = Math.Min(10 * backoff, MAX_CONNECT_BACKOFF_MS);
connectBackoff = new TimeSpan(backoff * TimeSpan.TicksPerMillisecond);
}
}
return(connection);
}
private static async Task Fetch(KafkaConnection kafka, string topicName = TopicName, int partitionId = 0, int offset = 0)
{
Console.WriteLine($"----- Fetch: topic:{topicName} p:{partitionId} o:{offset}");
var req = new FetchRequest
{
ReplicaId = -1,
MaxWaitTime = 100,
MinBytes = 1,
MaxBytes = 64 * 1024,
IsolationLevel = 0,
SessionId = 0,
SessionEpoc = -1,
Topics = new FetchRequest.Topic[]
{
new FetchRequest.Topic
{
TopicName = topicName,
Partitions = new FetchRequest.Topic.Partition[]
{
new FetchRequest.Topic.Partition
{
PartitionId = partitionId,
CurrentLeaderEpoc = -1,
FetchOffset = offset,
LogStartOffset = -1,
PartitionMaxBytes = 32 * 1024
}
}
}
}
};
var response = await kafka.Fetch(req);
Console.WriteLine($"throttle: {response.ThrottleTime}");
Console.WriteLine($"errorCode: {response.ErrorCode}");
foreach (var r in response.Responses)
{
Console.WriteLine($"topic: {r.TopicName}");
foreach (var pr in r.PartitionResponses)
{
Console.WriteLine($" partition: {pr.PartitionId} error: {pr.ErrorCode.ToString()}");
foreach (var batch in pr.RecordBatches)
{
Console.WriteLine(" batch offset: " + batch.BaseOffset);
foreach (var rec in batch.Records)
{
Console.WriteLine(" offset: " + rec.Offset);
Console.WriteLine(" key : " + Encoding.UTF8.GetString(rec.Key));
Console.WriteLine(" val : " + Encoding.UTF8.GetString(rec.Value));
}
}
}
}
}
public void ShouldStartReadPollingOnConstruction()
{
using (var socket = new KafkaTcpSocket(_log, new Uri("http://localhost:8999")))
using (var conn = new KafkaConnection(socket, log: _log))
{
TaskTest.WaitFor(() => conn.ReadPolling);
Assert.That(conn.ReadPolling, Is.True);
}
}
public void ShouldStartReadPollingOnConstruction()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: _log))
{
TaskTest.WaitFor(() => conn.ReadPolling);
Assert.That(conn.ReadPolling, Is.True);
}
}
public void ShouldStartReadPollingOnConstruction()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: _log))
{
TaskTest.WaitFor(() => conn.ReadPolling);
Assert.That(conn.ReadPolling, Is.True);
}
}
public void ShouldReportServerUriOnConstruction()
{
var expectedUrl = _kafkaEndpoint;
using (var socket = new KafkaTcpSocket(_log, expectedUrl))
using (var conn = new KafkaConnection(socket, log: _log))
{
Assert.That(conn.Endpoint, Is.EqualTo(expectedUrl));
}
}
public void ShouldReportServerUriOnConstruction()
{
var expectedUrl = new Uri("http://localhost:8999");
using (var socket = new KafkaTcpSocket(_log, expectedUrl))
using (var conn = new KafkaConnection(socket, log: _log))
{
Assert.That(conn.KafkaUri, Is.EqualTo(expectedUrl));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
public SyncProducer(SyncProducerConfiguration config)
{
Guard.NotNull(config, "config");
this.Config = config;
this.connection = new KafkaConnection(
this.Config.Host,
this.Config.Port,
config.BufferSize,
config.SocketTimeout);
}
public void ShouldDisposeWithoutExceptionThrown()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
{
var conn = new KafkaConnection(socket, log: _log);
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
using (conn) { }
}
}
public void ShouldDisposeWithoutExceptionEvenWhileCallingSendAsync()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: _log))
{
var task = conn.SendAsync(new MetadataRequest());
task.Wait(TimeSpan.FromMilliseconds(1000));
Assert.That(task.IsCompleted, Is.False, "The send task should still be pending.");
}
}
public void ShouldReportServerUriOnConstruction()
{
var expectedUrl = _kafkaEndpoint;
using (var socket = new KafkaTcpSocket(_log, expectedUrl))
using (var conn = new KafkaConnection(socket, log: _log))
{
Assert.That(conn.Endpoint, Is.EqualTo(expectedUrl));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
public SyncProducer(SyncProducerConfiguration config)
{
Guard.NotNull(config, "config");
this.Config = config;
this.connection = new KafkaConnection(
this.Config.Host,
this.Config.Port,
config.BufferSize,
config.SocketTimeout);
}
public void ShouldDisposeWithoutExceptionEvenWhileCallingSendAsync()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: _log))
{
var task = conn.SendAsync(new MetadataRequest());
task.Wait(TimeSpan.FromMilliseconds(1000));
Assert.That(task.IsCompleted, Is.False, "The send task should still be pending.");
}
}
public void ShouldDisposeWithoutExceptionThrown()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
{
var conn = new KafkaConnection(socket, log: _log);
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
using (conn) { }
}
}
private static async Task GetVersion(KafkaConnection kafka)
{
Console.WriteLine("----- ApiVersion");
var apiversions = await kafka.ApiVersion();
Console.WriteLine("error: " + apiversions.ErrorCode);
foreach (var v in apiversions.ApiVersions)
{
Console.WriteLine($"{ApiKey.GetName(v.ApiKey).PadRight(20)} {v.MinVersion}->{v.MaxVersion}");
}
}
/// <summary>
/// Gets a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">
/// The offset request.
/// </param>
/// <returns>
/// The list of offsets, in descending order.
/// </returns>
public IList <long> GetOffsetsBefore(OffsetRequest request)
{
var result = new List <long>();
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
using (var conn = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SocketTimeout))
{
conn.Write(request);
int size = conn.Reader.ReadInt32();
if (size == 0)
{
return(result);
}
short errorCode = conn.Reader.ReadInt16();
if (errorCode != ErrorMapping.NoError)
{
throw new KafkaException(errorCode);
}
int count = conn.Reader.ReadInt32();
for (int i = 0; i < count; i++)
{
result.Add(conn.Reader.ReadInt64());
}
return(result);
}
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
}
}
return(result);
}
public void SendAsyncShouldTimeoutByThrowingResponseTimeoutExceptionWhenTcpConnectionIsNotAvailable()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(100), log: _log))
{
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(taskResult.IsFaulted, Is.True);
Assert.That(taskResult.Exception.InnerException, Is.TypeOf <ResponseTimeoutException>());
}
}
private static async Task FindCoordinator(KafkaConnection kafka)
{
Console.WriteLine("----- FindCoordinator");
var response = await kafka.FindCoordinator(Guid.NewGuid().ToString());
Console.WriteLine($"throttle: {response.ThrottleTime}");
Console.WriteLine($"errorCode: {response.ErrorCode}");
Console.WriteLine($"errorMessage: {response.ErrorMessage}");
Console.WriteLine($"nodeId: {response.NodeId}");
Console.WriteLine($"host: {response.Host}");
Console.WriteLine($"port: {response.Port}");
}
/// <summary>
/// Initializes a new instance of the <see cref="AsyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
/// <param name="callbackHandler">
/// The callback invoked when a request is finished being sent.
/// </param>
public AsyncProducer(
AsyncProducerConfiguration config,
ICallbackHandler callbackHandler)
{
Guard.NotNull(config, "config");
this.Config = config;
this.callbackHandler = callbackHandler;
this.connection = new KafkaConnection(
this.Config.Host,
this.Config.Port,
this.Config.BufferSize,
this.Config.SocketTimeout);
}
/// <summary>
/// Initializes a new instance of the <see cref="AsyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
/// <param name="callbackHandler">
/// The callback invoked when a request is finished being sent.
/// </param>
public AsyncProducer(
AsyncProducerConfiguration config,
ICallbackHandler callbackHandler)
{
Guard.NotNull(config, "config");
this.Config = config;
this.callbackHandler = callbackHandler;
this.connection = new KafkaConnection(
this.Config.Host,
this.Config.Port,
this.Config.BufferSize,
this.Config.SocketTimeout);
}
/// <summary>
/// Gets a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">
/// The offset request.
/// </param>
/// <returns>
/// The list of offsets, in descending order.
/// </returns>
public IList <long> GetOffsetsBefore(OffsetRequest request)
{
KafkaConnection conn = null;
int size = 0;
var result = new List <long>();
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
conn = KafkaClusterConnectionPool.GetConnection(this.host, this.port);
conn.Write(request);
size = conn.Reader.ReadInt32();
if (size > 0)
{
short errorCode = conn.Reader.ReadInt16();
if (errorCode != ErrorMapping.NoError)
{
throw new KafkaException(errorCode);
}
int count = conn.Reader.ReadInt32();
for (int i = 0; i < count; i++)
{
result.Add(conn.Reader.ReadInt64());
}
}
break;
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
}
finally
{
KafkaClusterConnectionPool.ReleaseConnection(conn);
}
}
return(result);
}
/// <summary>
/// Sends the data to a multiple topics on Kafka server synchronously
/// </summary>
/// <param name="requests">
/// The requests.
/// </param>
public void MultiSend(IEnumerable<ProducerRequest> requests)
{
Guard.Assert<ArgumentNullException>(() => requests != null);
Guard.Assert<ArgumentNullException>(
() => requests.All(
x => x != null && x.MessageSet != null && x.MessageSet.Messages != null));
Guard.Assert<ArgumentNullException>(
() => requests.All(
x => x.MessageSet.Messages.All(
y => y != null && y.PayloadSize <= this.Config.MaxMessageSize)));
var multiRequest = new MultiProducerRequest(requests);
using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
{
conn.Write(multiRequest);
}
}
/// <summary>
/// Check that brokers alive by sending topic metadata request to them
/// </summary>
/// <param name="brokers">Collection of brokers to check. If null - brokers list will be retrieved from ZooKeeper state</param>
/// <returns>
/// Dictionary where Key is Broker Id and Value indicates whether Broker responds to requests.
/// Value is true when Broker TopicMetadataRequest was successfully sent to broker and any response was recieved back.
/// Value is false when connection to Broker failed or
/// </returns>
/// <remarks>
/// If brokers not specified this method will only ping brokers that exist in ZooKeeper state
/// </remarks>
public IDictionary<int, bool> GetKafkaBrokersAliveState(ICollection<Broker> brokers = null)
{
// retrive latest brokers info
if (brokers == null)
{
this.RefreshKafkaBrokersInfo();
}
brokers = brokers ?? this.kafkaCluster.Brokers.Values;
var brokersConnectionString = string.Join(", ", this.kafkaCluster.Brokers.Values.Select(x => string.Join(":", x.Id, x.Host, x.Port)));
Logger.InfoFormat("Collecting brokers alive state for brokers {0}", brokersConnectionString);
var brokerAliveStateMap = new SortedDictionary<int, bool>();
foreach (var broker in brokers)
{
try
{
Logger.DebugFormat("Sending request to broker #{0} {1}:{2}", broker.Id, broker.Host, broker.Port);
using (var kafkaConnection = new KafkaConnection(broker.Host, broker.Port, this.config.BufferSize, this.config.SendTimeout, this.config.ReceiveTimeout, int.MaxValue))
{
// send topic offset request for random non-existing topic
var requestInfos = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
requestInfos[Guid.NewGuid().ToString("N")] = new List<PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(0, OffsetRequest.EarliestTime, 1) };
kafkaConnection.Send(new OffsetRequest(requestInfos));
}
brokerAliveStateMap[broker.Id] = true;
}
catch (Exception exc)
{
Logger.WarnFormat("Failed to send request to broker #{0} {1}:{2}. Error:", broker.Id, broker.Host, broker.Port, exc.FormatException());
brokerAliveStateMap[broker.Id] = false;
}
}
Logger.InfoFormat("Completed collecting brokers alive state for brokers {0}", brokersConnectionString);
return brokerAliveStateMap;
}
/// <summary>
/// Initializes a new instance of the <see cref="Consumer"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
/// <param name="host"></param>
/// <param name="port"></param>
public Consumer(ConsumerConfiguration config, string host, int port)
{
Guard.NotNull(config, "config");
this.config = config;
this.host = host;
this.port = port;
this.connection = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SendTimeout,
this.config.ReceiveTimeout,
this.config.ReconnectInterval);
}
public async void SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(1000000), log: _log))
{
Console.WriteLine("SendAsync blocked by reconnection attempts...");
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Console.WriteLine("Task result should be WaitingForActivation...");
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
Console.WriteLine("Starting server to establish connection...");
using (var server = new FakeTcpServer(8999))
{
server.OnClientConnected += () => Console.WriteLine("Client connected...");
server.OnBytesReceived += (b) =>
{
server.SendDataAsync(MessageHelper.CreateMetadataResponse(1, "Test"));
};
await taskResult;
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
public void SendAsyncShouldTimeoutMultipleMessagesAtATime()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(100), log: _log))
{
server.HasClientConnected.Wait(TimeSpan.FromSeconds(3));
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
var tasks = new[]
{
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest())
};
Task.WhenAll(tasks);
TaskTest.WaitFor(() => tasks.Any(t => t.IsFaulted));
foreach (var task in tasks)
{
Assert.That(task.IsFaulted, Is.True, "Task should have faulted.");
Assert.That(task.Exception.InnerException, Is.TypeOf<ResponseTimeoutException>(), "Task fault should be of type ResponseTimeoutException.");
}
}
}
public void ReadShouldIgnoreMessageWithUnknownCorrelationId()
{
const int correlationId = 99;
var mockLog = _kernel.GetMock<IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var receivedData = false;
socket.OnBytesReceived += i => receivedData = true;
//send correlation message
server.SendDataAsync(CreateCorrelationMessage(correlationId)).Wait(TimeSpan.FromSeconds(5));
//wait for connection
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
TaskTest.WaitFor(() => receivedData);
//should log a warning and keep going
mockLog.Verify(x => x.WarnFormat(It.IsAny<string>(), It.Is<int>(o => o == correlationId)));
}
}
public void SendAsyncShouldTimeoutWhenSendAsyncTakesTooLong()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(1), log: _log))
{
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromMilliseconds(100));
Assert.That(taskResult.IsFaulted, Is.True, "Task should have reported an exception.");
Assert.That(taskResult.Exception.InnerException, Is.TypeOf<ResponseTimeoutException>());
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover()
{
var mockLog =new Mock<IKafkaLog>();
var log= new DefaultTraceLog(LogLevel.Error);
using (var server = new FakeTcpServer(log, 8999))
using (var socket = new KafkaTcpSocket(log, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = 0;
socket.OnServerDisconnected += () => Interlocked.Increment(ref disconnected);
for (int connectionAttempt = 1; connectionAttempt < 4; connectionAttempt++)
{
await TaskTest.WaitFor(() => server.ConnectionEventcount == connectionAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
server.SendDataAsync(CreateCorrelationMessage(1)).Wait(TimeSpan.FromSeconds(5));
await TaskTest.WaitFor(() => !conn.IsOnErrorState());
Assert.IsFalse(conn.IsOnErrorState());
mockLog.Verify(x => x.InfoFormat("Polling read thread has recovered: {0}", It.IsAny<object[]>()), Times.Exactly(connectionAttempt-1));
server.DropConnection();
await TaskTest.WaitFor(() => conn.IsOnErrorState());
Assert.AreEqual(disconnected,connectionAttempt);
Assert.IsTrue(conn.IsOnErrorState());
mockLog.Verify(x => x.ErrorFormat("Exception occured in polling read thread {0}: {1}", It.IsAny<object[]>()), Times.Exactly(connectionAttempt ));
}
}
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (connection != null)
{
lock (this)
{
if (connection != null)
{
this.connection.Dispose();
this.connection = null;
}
}
}
}
}
/// <summary>
/// Sends request to Kafka server asynchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
/// <param name="callback">
/// The callback invoked when a request is finished being sent.
/// </param>
public void Send(ProducerRequest request, MessageSent<ProducerRequest> callback)
{
Guard.Assert<ArgumentNullException>(() => request != null);
Guard.Assert<ArgumentNullException>(() => request.MessageSet != null);
Guard.Assert<ArgumentNullException>(() => request.MessageSet.Messages != null);
Guard.Assert<ArgumentException>(
() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
{
conn.BeginWrite(request, callback);
}
}
/// <summary>
/// Sends request to Kafka server asynchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
public void Send(ProducerRequest request)
{
Guard.Assert<ArgumentNullException>(() => request != null);
Guard.Assert<ArgumentException>(() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
if (this.callbackHandler != null)
{
this.Send(request, this.callbackHandler.Handle);
}
else
{
using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
{
conn.BeginWrite(request);
}
}
}
private static void GetOffsetsBefore(KafkaConnection conn, OffsetRequest request, IList<long> offsets)
{
offsets.Clear(); // to make sure the list is clean after some previous attampts to get data
conn.Write(request);
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = conn.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets =
BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray()), 0);
for (int ix = 0; ix < numOfOffsets; ix++)
{
int position = (ix * 8) + 4;
offsets.Add(
BitConverter.ToInt64(
BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray()), 0));
}
}
}
private static void MultiFetch(KafkaConnection conn, MultiFetchRequest request, IList<BufferedMessageSet> result)
{
result.Clear();
conn.Write(request);
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0);
if (dataLength <= 0)
{
return;
}
byte[] data = conn.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code
byte[] unbufferedData = data.Skip(2).ToArray();
for (int i = 0; i < request.ConsumerRequests.Count; i++)
{
int partLength = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray()), 0);
errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(unbufferedData.Skip(4).Take(2).ToArray()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
result.Add(BufferedMessageSet.ParseFrom(unbufferedData.Skip(6).Take(partLength - 2).ToArray()));
unbufferedData = unbufferedData.Skip(partLength + 4).ToArray();
}
}
private static BufferedMessageSet Fetch(KafkaConnection conn, FetchRequest request)
{
conn.Write(request);
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = conn.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code
byte[] unbufferedData = data.Skip(2).ToArray();
return BufferedMessageSet.ParseFrom(unbufferedData);
}
return null;
}
/// <summary>
/// Combine multiple fetch requests in one call.
/// </summary>
/// <param name="request">
/// The list of fetch requests.
/// </param>
/// <returns>
/// A list of sets of fetched messages.
/// </returns>
/// <remarks>
/// Offset is passed in on every request, allowing the user to maintain this metadata
/// however they choose.
/// </remarks>
public IList<BufferedMessageSet> MultiFetch(MultiFetchRequest request)
{
var result = new List<BufferedMessageSet>();
using (var conn = new KafkaConnection(this.Host, this.Port))
{
short tryCounter = 1;
bool success = false;
while (!success && tryCounter <= this.config.NumberOfTries)
{
try
{
MultiFetch(conn, request, result);
success = true;
}
catch (Exception ex)
{
// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "MultiFetch reconnect due to {0}", ex);
}
}
}
return result;
}
/// <summary>
/// Combine multiple fetch requests in one call.
/// </summary>
/// <param name="request">
/// The list of fetch requests.
/// </param>
/// <returns>
/// A list of sets of fetched messages.
/// </returns>
/// <remarks>
/// Offset is passed in on every request, allowing the user to maintain this metadata
/// however they choose.
/// </remarks>
public IList<BufferedMessageSet> MultiFetch(MultiFetchRequest request)
{
var result = new List<BufferedMessageSet>();
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
using (var conn = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SocketTimeout))
{
conn.Write(request);
int size = conn.Reader.ReadInt32();
return BufferedMessageSet.ParseMultiFrom(conn.Reader, size, request.ConsumerRequests.Count);
}
}
catch (Exception ex)
{
// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "MultiFetch reconnect due to {0}", ex);
}
}
return result;
}
/// <summary>
/// Initializes a new instance of the <see cref="Consumer"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
public Consumer(ConsumerConfiguration config)
{
Guard.NotNull(config, "config");
this.config = config;
this.host = config.Broker.Host;
this.port = config.Broker.Port;
this.connection = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SendTimeout,
this.config.ReceiveTimeout,
this.config.ReconnectInterval);
this.CreatedTimeInUTC = DateTime.UtcNow.Ticks;
}
/// <summary>
/// Gets a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">
/// The offset request.
/// </param>
/// <returns>
/// The list of offsets, in descending order.
/// </returns>
public IList<long> GetOffsetsBefore(OffsetRequest request)
{
var result = new List<long>();
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
using (var conn = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SocketTimeout))
{
conn.Write(request);
int size = conn.Reader.ReadInt32();
if (size == 0)
{
return result;
}
short errorCode = conn.Reader.ReadInt16();
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
int count = conn.Reader.ReadInt32();
for (int i = 0; i < count; i++)
{
result.Add(conn.Reader.ReadInt64());
}
return result;
}
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
}
}
return result;
}
public void Setup()
{
var options = new KafkaOptions(new Uri(ConfigurationManager.AppSettings["IntegrationKafkaServerUrl"]));
_conn = new KafkaConnection(options.KafkaServerUri.First(), options.ResponseTimeoutMs, options.Log);
}
/// <summary>
/// Gets a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">
/// The offset request.
/// </param>
/// <returns>
/// The list of offsets, in descending order.
/// </returns>
public IList<long> GetOffsetsBefore(OffsetRequest request)
{
var offsets = new List<long>();
using (var conn = new KafkaConnection(this.Host, this.Port))
{
short tryCounter = 1;
bool success = false;
while (!success && tryCounter <= this.config.NumberOfTries)
{
try
{
GetOffsetsBefore(conn, request, offsets);
success = true;
}
catch (Exception ex)
{
// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
}
}
}
return offsets;
}
public void ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock<IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = false;
socket.OnServerDisconnected += () => disconnected = true;
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//Wait a while for the client to notice the disconnect and log
TaskTest.WaitFor(() => disconnected);
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny<string>(), It.IsAny<Exception>()));
TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
/// <summary>
/// Sends request to Kafka server synchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
public void Send(ProducerRequest request)
{
Guard.Assert<ArgumentNullException>(() => request != null);
using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
{
conn.Write(request);
}
}
