public void KafkaTcpSocketShouldConstruct()
{
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
Assert.That(test, Is.Not.Null);
Assert.That(test.Endpoint, Is.EqualTo(_fakeServerUrl));
}
}
public void ShouldStartReadPollingOnConstruction()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: _log))
{
TaskTest.WaitFor(() => conn.ReadPolling);
Assert.That(conn.ReadPolling, Is.True);
}
}
public void ShouldReportServerUriOnConstruction()
{
var expectedUrl = _kafkaEndpoint;
using (var socket = new KafkaTcpSocket(_log, expectedUrl))
using (var conn = new KafkaConnection(socket, log: _log))
{
Assert.That(conn.Endpoint, Is.EqualTo(expectedUrl));
}
}
public void KafkaTcpSocketShouldConstruct()
{
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
Assert.That(test, Is.Not.Null);
Assert.That(test.Endpoint, Is.EqualTo(_fakeServerUrl));
}
}
public void ShouldDisposeWithoutExceptionThrown()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
{
var conn = new KafkaConnection(socket, log: _log);
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
using (conn) { }
}
}
public void ShouldDisposeWithoutExceptionEvenWhileCallingSendAsync()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: _log))
{
var task = conn.SendAsync(new MetadataRequest());
task.Wait(TimeSpan.FromMilliseconds(1000));
Assert.That(task.IsCompleted, Is.False, "The send task should still be pending.");
}
}
public void ConnectionShouldStartDedicatedThreadOnConstruction()
{
var count = 0;
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
test.OnReconnectionAttempt += x => Interlocked.Increment(ref count);
TaskTest.WaitFor(() => count > 0);
Assert.That(count, Is.GreaterThan(0));
}
}
public void ConnectionShouldAttemptMultipleTimesWhenConnectionFails()
{
var count = 0;
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _badServerUrl))
{
test.WriteAsync(1.ToBytes().ToPayload()); //will force a connection
test.OnReconnectionAttempt += x => Interlocked.Increment(ref count);
TaskTest.WaitFor(() => count > 1, 10000);
Assert.That(count, Is.GreaterThan(1));
}
}
public async Task ConnectionShouldStartDedicatedThreadOnConstruction()
{
var count = 0;
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
test.OnReconnectionAttempt += x => Interlocked.Increment(ref count);
await TaskTest.WaitFor(() => count > 0);
Assert.That(count, Is.GreaterThan(0));
}
}
public void KafkaTcpSocketShouldDisposeEvenWhilePollingToReconnect()
{
var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var taskResult = test.ReadAsync(4);
using (test) { }
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(taskResult.IsFaulted, Is.True);
Assert.That(taskResult.Exception.InnerException, Is.TypeOf<ObjectDisposedException>());
}
public async Task KafkaTcpSocketShouldDisposeEvenWhilePollingToReconnect()
{
int connectionAttempt = 0;
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
test.OnReconnectionAttempt += i => connectionAttempt = i;
var taskResult = test.ReadAsync(4);
await TaskTest.WaitFor(() => connectionAttempt > 1);
test.Dispose();
await Task.WhenAny(taskResult, Task.Delay(1000)).ConfigureAwait(false);
Assert.That(taskResult.IsFaulted, Is.True);
Assert.That(taskResult.Exception.InnerException, Is.TypeOf<ObjectDisposedException>());
}
}
public void ReadShouldIgnoreMessageWithUnknownCorrelationId()
{
const int correlationId = 99;
var mockLog = _kernel.GetMock<IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var receivedData = false;
socket.OnBytesReceived += i => receivedData = true;
//send correlation message
server.SendDataAsync(CreateCorrelationMessage(correlationId)).Wait(TimeSpan.FromSeconds(5));
//wait for connection
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
TaskTest.WaitFor(() => receivedData);
//should log a warning and keep going
mockLog.Verify(x => x.WarnFormat(It.IsAny<string>(), It.Is<int>(o => o == correlationId)));
}
}
public void KafkaTcpSocketShouldDisposeEvenWhileAwaitingReadAndThrowException()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
int readSize = 0;
test.OnReadFromSocketAttempt += i => readSize = i;
var taskResult = test.ReadAsync(4);
TaskTest.WaitFor(() => readSize > 0);
using (test) { }
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(taskResult.IsFaulted, Is.True);
Assert.That(taskResult.Exception.InnerException, Is.TypeOf<ObjectDisposedException>());
}
}
public void ReadShouldNotLoseDataFromStreamOverMultipleReads()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
var payload = new KafkaMessagePacker()
.Pack(firstMessage)
.Pack(secondMessage, StringPrefixEncoding.None);
//send the combined payload
server.SendDataAsync(payload.PayloadNoLength());
var firstResponse = test.ReadAsync(4).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public IKafkaConnection Create(KafkaEndpoint endpoint, TimeSpan responseTimeoutMs, IKafkaLog log, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null)
{
KafkaTcpSocket socket = new KafkaTcpSocket(log, endpoint, maxRetry, maximumReconnectionTimeout, statisticsTrackerOptions);
return(new KafkaConnection(socket, responseTimeoutMs, log));
}
public void SendAsyncShouldTimeoutMultipleMessagesAtATime()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(100), log: _log))
{
server.HasClientConnected.Wait(TimeSpan.FromSeconds(3));
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
var tasks = new[]
{
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest()),
conn.SendAsync(new MetadataRequest())
};
Task.WhenAll(tasks);
TaskTest.WaitFor(() => tasks.Any(t => t.IsFaulted));
foreach (var task in tasks)
{
Assert.That(task.IsFaulted, Is.True, "Task should have faulted.");
Assert.That(task.Exception.InnerException, Is.TypeOf<ResponseTimeoutException>(), "Task fault should be of type ResponseTimeoutException.");
}
}
}
public void WriteAndReadShouldBeAsynchronous()
{
var write = new List<int>();
var read = new List<int>();
var expected = new List<int> { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
server.OnBytesReceived += data => write.AddRange(data.Batch(4).Select(x => x.ToArray().ToInt32()));
var tasks = Enumerable.Range(1, 10)
.SelectMany(i => new[]
{
test.WriteAsync(i.ToBytes().ToPayload()),
test.ReadAsync(4).ContinueWith(t => read.Add(t.Result.ToInt32())),
server.SendDataAsync(i.ToBytes())
}).ToArray();
Task.WaitAll(tasks);
Assert.That(write.OrderBy(x => x), Is.EqualTo(expected));
Assert.That(read.OrderBy(x => x), Is.EqualTo(expected));
}
}
public void WriteAsyncShouldAllowMoreThanOneWrite()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
const int testData = 99;
var results = new List<byte>();
server.OnBytesReceived += results.AddRange;
Task.WaitAll(test.WriteAsync(testData.ToBytes().ToPayload()), test.WriteAsync(testData.ToBytes().ToPayload()));
TaskTest.WaitFor(() => results.Count >= 8);
Assert.That(results.Count, Is.EqualTo(8));
}
}
public void WriteAsyncShouldSendData()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
const int testData = 99;
int result = 0;
server.OnBytesReceived += data => result = data.ToInt32();
test.WriteAsync(testData.ToBytes().ToPayload()).Wait(TimeSpan.FromSeconds(1));
TaskTest.WaitFor(() => result > 0);
Assert.That(result, Is.EqualTo(testData));
}
}
public void ReadShouldStackReadRequestsAndReturnOneAtATime()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var messages = new[] { "test1", "test2", "test3", "test4" };
var expectedLength = "test1".Length;
var payload = new KafkaMessagePacker().Pack(messages);
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var tasks = messages.Select(x => socket.ReadAsync(x.Length)).ToArray();
server.SendDataAsync(payload.Payload());
Task.WaitAll(tasks);
foreach (var task in tasks)
{
Assert.That(task.Result.Length, Is.EqualTo(expectedLength));
}
}
}
private async Task<byte[]> ReadFromSocketWithRetry(KafkaTcpSocket socket, int readSize)
{
byte[] buffer;
try
{
buffer = await socket.ReadAsync(readSize);
return buffer;
}
catch (Exception ex)
{
Assert.That(ex, Is.TypeOf<ServerDisconnectedException>());
}
buffer = await socket.ReadAsync(4);
return buffer;
}
public void ReadShouldReconnectAfterLosingConnection()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var disconnects = 0;
var connects = 0;
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = ReadFromSocketWithRetry(socket, 4);
//wait till connected
TaskTest.WaitFor(() => connects > 0);
//drop connection
server.DropConnection();
TaskTest.WaitFor(() => disconnects > 0);
Assert.That(disconnects, Is.EqualTo(1), "Server should have disconnected the client.");
//wait for reconnection
TaskTest.WaitFor(() => connects > 1);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
//send data and get result
server.SendDataAsync(99.ToBytes());
Assert.That(resultTask.Result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public void SendAsyncShouldTimeoutWhenSendAsyncTakesTooLong()
{
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(1), log: _log))
{
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromMilliseconds(100));
Assert.That(taskResult.IsFaulted, Is.True, "Task should have reported an exception.");
Assert.That(taskResult.Exception.InnerException, Is.TypeOf<ResponseTimeoutException>());
}
}
public async void SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(1000000), log: _log))
{
Console.WriteLine("SendAsync blocked by reconnection attempts...");
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Console.WriteLine("Task result should be WaitingForActivation...");
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
Console.WriteLine("Starting server to establish connection...");
using (var server = new FakeTcpServer(8999))
{
server.OnClientConnected += () => Console.WriteLine("Client connected...");
server.OnBytesReceived += (b) =>
{
server.SendDataAsync(MessageHelper.CreateMetadataResponse(1, "Test"));
};
await taskResult;
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
public void WriteShouldHandleLargeVolumeSendAsynchronously()
{
var write = new List<int>();
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
server.OnBytesReceived += data => write.AddRange(data.Batch(4).Select(x => x.ToArray().ToInt32()));
var tasks = Enumerable.Range(1, 10000)
.SelectMany(i => new[]
{
test.WriteAsync(i.ToBytes().ToPayload()),
}).ToArray();
Task.WaitAll(tasks);
Assert.That(write.OrderBy(x => x), Is.EqualTo(Enumerable.Range(1, 10000)));
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnceAsyncronously()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
const int firstMessage = 99;
const int secondMessage = 100;
Console.WriteLine("Sending first message to receive...");
server.SendDataAsync(firstMessage.ToBytes());
var firstResponseTask = test.ReadAsync(4);
Console.WriteLine("Sending second message to receive...");
server.SendDataAsync(secondMessage.ToBytes());
var secondResponseTask = test.ReadAsync(4);
Assert.That(firstResponseTask.Result.ToInt32(), Is.EqualTo(firstMessage));
Assert.That(secondResponseTask.Result.ToInt32(), Is.EqualTo(secondMessage));
}
}
public void WriteShouldCancelWhileSendingData()
{
var writeAttempts = 0;
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
using (var token = new CancellationTokenSource())
{
test.OnWriteToSocketAttempt += payload => Interlocked.Increment(ref writeAttempts);
test.WriteAsync(new KafkaDataPayload { Buffer = 1.ToBytes() }, token.Token);
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
TaskTest.WaitFor(() => writeAttempts > 0);
Assert.That(writeAttempts, Is.EqualTo(1), "Socket should have attempted to write.");
//create a buffer write that will take a long time
var taskResult = test.WriteAsync(
new KafkaDataPayload { Buffer = Enumerable.Range(0, 1000000).Select(b => (byte)b).ToArray() },
token.Token);
token.Cancel();
taskResult.SafeWait(TimeSpan.FromMilliseconds(5000));
Assert.That(taskResult.IsCanceled, Is.True, "Task should have cancelled.");
}
}
public void WriteShouldCancelWhileAwaitingReconnection()
{
int connectionAttempt = 0;
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
using (var token = new CancellationTokenSource())
{
test.OnReconnectionAttempt += i => connectionAttempt = i;
var taskResult = test.WriteAsync(new KafkaDataPayload { Buffer = 1.ToBytes() }, token.Token);
TaskTest.WaitFor(() => connectionAttempt > 1);
token.Cancel();
taskResult.SafeWait(TimeSpan.FromMilliseconds(1000));
Assert.That(taskResult.IsCanceled, Is.True);
}
}
public void SocketShouldReconnectEvenAfterCancelledWrite()
{
int connectionAttempt = 0;
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
using (var token = new CancellationTokenSource())
{
test.OnReconnectionAttempt += i => Interlocked.Exchange(ref connectionAttempt, i);
var taskResult = test.WriteAsync(new KafkaDataPayload { Buffer = 1.ToBytes() }, token.Token);
TaskTest.WaitFor(() => connectionAttempt > 1);
var attemptsMadeSoFar = connectionAttempt;
token.Cancel();
TaskTest.WaitFor(() => connectionAttempt > attemptsMadeSoFar);
Assert.That(connectionAttempt, Is.GreaterThan(attemptsMadeSoFar));
}
}
public void ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock<IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = false;
socket.OnServerDisconnected += () => disconnected = true;
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//Wait a while for the client to notice the disconnect and log
TaskTest.WaitFor(() => disconnected);
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny<string>(), It.IsAny<Exception>()));
TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
public void ReadShouldThrowServerDisconnectedExceptionWhenDisconnected()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = socket.ReadAsync(4);
//wait till connected
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
resultTask.ContinueWith(t => resultTask = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(resultTask.IsFaulted, Is.True);
Assert.That(resultTask.Exception.InnerException, Is.TypeOf<ServerDisconnectedException>());
}
}
