public void ReadShouldBlockUntilAllBytesRequestedAreReceived()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var count = 0;
var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = test.ReadAsync(4).ContinueWith(t =>
{
Interlocked.Increment(ref count);
return(t.Result);
});
Console.WriteLine("Sending first 3 bytes...");
var sendInitialBytes = server.SendDataAsync(new byte[] { 0, 0, 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendInitialBytes, Is.True, "First 3 bytes should have been sent.");
Console.WriteLine("Ensuring task blocks...");
var unblocked = resultTask.Wait(TimeSpan.FromMilliseconds(500));
Assert.That(unblocked, Is.False, "Wait should return false.");
Assert.That(resultTask.IsCompleted, Is.False, "Task should still be running, blocking.");
Assert.That(count, Is.EqualTo(0), "Should still block even though bytes have been received.");
Console.WriteLine("Sending last byte...");
var sendLastByte = server.SendDataAsync(new byte[] { 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendLastByte, Is.True, "Last byte should have sent.");
Console.WriteLine("Ensuring task unblocks...");
resultTask.Wait(TimeSpan.FromMilliseconds(500));
Assert.That(resultTask.IsCompleted, Is.True, "Task should have completed.");
Assert.That(count, Is.EqualTo(1), "Task ContinueWith should have executed.");
Assert.That(resultTask.Result.Length, Is.EqualTo(4), "Result of task should be 4 bytes.");
}
}
public async Task ReadShouldIgnoreMessageWithUnknownCorrelationId()
{
const int correlationId = 99;
var receivedData = false;
var mockLog = new MemoryLog();
var config = new ConnectionConfiguration(onRead: (e, buffer, elapsed) => receivedData = true);
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, config, mockLog))
using (var conn = new Connection(socket, config, log: mockLog))
{
//send correlation message
server.SendDataAsync(CreateCorrelationMessage(correlationId)).Wait(TimeSpan.FromSeconds(5));
//wait for connection
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
await TaskTest.WaitFor(() => receivedData);
// shortly after receivedData, but still after
await TaskTest.WaitFor(() => mockLog.LogEvents.Any(e => e.Item1 == LogLevel.Warn && e.Item2.Message == $"Unexpected response from {endpoint} with correlation id {correlationId} (not in request queue)."));
}
}
public async Task SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog))
using (var conn = new Connection(socket, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000)), log: TestConfig.InfoLog))
{
Console.WriteLine("SendAsync blocked by reconnection attempts...");
var taskResult = conn.SendAsync(new MetadataRequest(), CancellationToken.None);
Console.WriteLine("Task result should be WaitingForActivation...");
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
Console.WriteLine("Starting server to establish connection...");
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
{
server.OnClientConnected += () => Console.WriteLine("Client connected...");
server.OnBytesReceived += b =>
{
var request = KafkaDecoder.DecodeHeader(b);
AsyncContext.Run(async() => await server.SendDataAsync(MessageHelper.CreateMetadataResponse(request.CorrelationId, "Test")));
};
await Task.WhenAny(taskResult, Task.Delay(TimeSpan.FromSeconds(10)));
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
await taskResult;
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
public async void SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(1000000), log: _log))
{
Console.WriteLine("SendAsync blocked by reconnection attempts...");
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Console.WriteLine("Task result should be WaitingForActivation...");
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
Console.WriteLine("Starting server to establish connection...");
using (var server = new FakeTcpServer(8999))
{
server.OnClientConnected += () => Console.WriteLine("Client connected...");
server.OnBytesReceived += (b) =>
{
server.SendDataAsync(MessageHelper.CreateMetadataResponse(1, "Test"));
};
await taskResult;
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover()
{
var mockLog = new Mock <IKafkaLog>();
var log = new DefaultTraceLog(LogLevel.Error);
using (var server = new FakeTcpServer(log, 8999))
using (var socket = new KafkaTcpSocket(log, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = 0;
socket.OnServerDisconnected += () => Interlocked.Increment(ref disconnected);
for (int connectionAttempt = 1; connectionAttempt < 4; connectionAttempt++)
{
await TaskTest.WaitFor(() => server.ConnectionEventcount == connectionAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
server.SendDataAsync(CreateCorrelationMessage(1)).Wait(TimeSpan.FromSeconds(5));
await TaskTest.WaitFor(() => !conn.IsOnErrorState());
Assert.IsFalse(conn.IsOnErrorState());
mockLog.Verify(x => x.InfoFormat("Polling read thread has recovered: {0}", It.IsAny <object[]>()), Times.Exactly(connectionAttempt - 1));
server.DropConnection();
await TaskTest.WaitFor(() => conn.IsOnErrorState());
Assert.AreEqual(disconnected, connectionAttempt);
Assert.IsTrue(conn.IsOnErrorState());
mockLog.Verify(x => x.ErrorFormat("Exception occured in polling read thread {0}: {1}", It.IsAny <object[]>()), Times.Exactly(connectionAttempt));
}
}
}
public void ReadShouldReconnectAfterLosingConnection()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var disconnects = 0;
var connects = 0;
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = ReadFromSocketWithRetry(socket, 4);
//wait till connected
TaskTest.WaitFor(() => connects > 0);
//drop connection
server.DropConnection();
TaskTest.WaitFor(() => disconnects > 0);
Assert.That(disconnects, Is.EqualTo(1), "Server should have disconnected the client.");
//wait for reconnection
TaskTest.WaitFor(() => connects > 1);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
//send data and get result
server.SendDataAsync(99.ToBytes());
Assert.That(resultTask.Result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public void WriteAndReadShouldBeAsyncronous()
{
var write = new List <int>();
var read = new List <int>();
var expected = new List <int> {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10
};
using (var server = new FakeTcpServer(FakeServerPort))
{
server.OnBytesReceived += data => write.AddRange(data.Batch(4).Select(x => x.ToArray().ToInt32()));
var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var tasks = Enumerable.Range(1, 10)
.SelectMany(i => new[]
{
test.WriteAsync(i.ToBytes()),
test.ReadAsync(4).ContinueWith(t => read.Add(t.Result.ToInt32())),
server.SendDataAsync(i.ToBytes())
}).ToArray();
Task.WaitAll(tasks);
Assert.That(write.OrderBy(x => x), Is.EqualTo(expected));
Assert.That(read.OrderBy(x => x), Is.EqualTo(expected));
}
}
public void ReadShouldStackReadRequestsAndReturnOneAtATime()
{
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
{
var messages = new[] { "test1", "test2", "test3", "test4" };
var expectedLength = "test1".Length;
var payload = new KafkaWriter().Write(messages);
using (var socket = new TcpSocket(endpoint, log: TestConfig.WarnLog)) {
var tasks = messages.Select(x => socket.ReadAsync(x.Length, CancellationToken.None)).ToArray();
var send = server.SendDataAsync(payload.ToBytes());
Task.WaitAll(tasks);
foreach (var task in tasks)
{
Assert.That(task.Result.Length, Is.EqualTo(expectedLength));
}
}
}
}
public void ReadShouldNotLoseDataFromStreamOverMultipleReads()
{
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var test = new TcpSocket(endpoint, log: TestConfig.InfoLog))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
var bytes = Encoding.UTF8.GetBytes(secondMessage);
var payload = new KafkaWriter()
.Write(firstMessage)
.Write(bytes, false);
//send the combined payload
var send = server.SendDataAsync(payload.ToBytesNoLength());
var firstResponse = test.ReadAsync(4, CancellationToken.None).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length, CancellationToken.None).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public async Task ReadShouldIgnoreMessageWithUnknownCorrelationId()
{
const int correlationId = 99;
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(_log, 8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var receivedData = false;
socket.OnBytesReceived += i => receivedData = true;
//send correlation message
server.SendDataAsync(CreateCorrelationMessage(correlationId)).Wait(TimeSpan.FromSeconds(5));
//wait for connection
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
await TaskTest.WaitFor(() => receivedData);
//should log a warning and keep going
mockLog.Verify(x => x.WarnFormat(It.IsAny <string>(), It.Is <int>(o => o == correlationId)));
}
}
public async Task KafkaConnectionShouldSkipPartiallyReadMessage()
{
var mockLog = new MemoryLog();
var bytesRead = 0;
var config = new ConnectionConfiguration(onReadChunk: (e, size, remaining, read, elapsed) => Interlocked.Add(ref bytesRead, read));
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, config, TestConfig.InfoLog))
using (new Connection(socket, config, log: mockLog))
{
// send size
var size = 200;
await server.SendDataAsync(size.ToBytes());
var randomizer = Randomizer.CreateRandomizer();
var firstBytes = new byte[99];
for (var i = 0; i < firstBytes.Length; i++)
{
firstBytes[i] = randomizer.NextByte();
}
// send half of payload
await server.SendDataAsync(firstBytes);
await TaskTest.WaitFor(() => bytesRead == firstBytes.Length);
//Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Warn && e.Item2.Message.StartsWith("Skipping")), Is.EqualTo(0));
server.DropConnection();
// send half of payload should be skipped
var lastBytes = new byte[size];
for (var i = 0; i < lastBytes.Length; i++)
{
lastBytes[i] = randomizer.NextByte();
}
await server.SendDataAsync(lastBytes);
await TaskTest.WaitFor(() => bytesRead >= size);
Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Warn && e.Item2.Message.StartsWith($"Skipping {size - firstBytes.Length} bytes on")), Is.EqualTo(1));
}
}
public void ShouldDisposeEvenWhenTryingToSendWithoutExceptionThrown()
{
using (var server = new FakeTcpServer(TestConfig.WarnLog, TestConfig.ServerPort()))
{
server.SendDataAsync("test");
Thread.Sleep(500);
}
}
public void ShouldDisposeEvenWhenTryingToSendWithoutExceptionThrown()
{
using (var server = new FakeTcpServer(8999))
{
server.SendDataAsync("test");
Thread.Sleep(500);
}
}
public void ShouldDisposeEvenWhenTryingToSendWithoutExceptionThrown()
{
using (var server = new FakeTcpServer(Ilog, 8999))
{
server.SendDataAsync("test");
Thread.Sleep(500);
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnceAsyncronously()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
const int firstMessage = 99;
const int secondMessage = 100;
Console.WriteLine("Sending first message to receive...");
server.SendDataAsync(firstMessage.ToBytes());
var firstResponseTask = test.ReadAsync(4);
Console.WriteLine("Sending second message to receive...");
server.SendDataAsync(secondMessage.ToBytes());
var secondResponseTask = test.ReadAsync(4);
Assert.That(firstResponseTask.Result.ToInt32(), Is.EqualTo(firstMessage));
Assert.That(secondResponseTask.Result.ToInt32(), Is.EqualTo(secondMessage));
}
}
public async Task ReadShouldBlockUntilAllBytesRequestedAreReceived()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
var sendCompleted = 0;
var bytesReceived = 0;
test.OnBytesReceived += i => Interlocked.Add(ref bytesReceived, i);
var resultTask = test.ReadAsync(4).ContinueWith(t =>
{
Interlocked.Increment(ref sendCompleted);
return(t.Result);
});
Console.WriteLine("Sending first 3 bytes...");
server.HasClientConnected.Wait(TimeSpan.FromMilliseconds(1000));
var sendInitialBytes = server.SendDataAsync(new byte[] { 0, 0, 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendInitialBytes, Is.True, "First 3 bytes should have been sent.");
Console.WriteLine("Ensuring task blocks...");
await TaskTest.WaitFor(() => bytesReceived > 0);
Assert.That(resultTask.IsCompleted, Is.False, "Task should still be running, blocking.");
Assert.That(sendCompleted, Is.EqualTo(0), "Should still block even though bytes have been received.");
Assert.That(bytesReceived, Is.EqualTo(3), "Three bytes should have been received and we are waiting on the last byte.");
Console.WriteLine("Sending last byte...");
var sendLastByte = server.SendDataAsync(new byte[] { 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendLastByte, Is.True, "Last byte should have sent.");
Console.WriteLine("Ensuring task unblocks...");
await TaskTest.WaitFor(() => resultTask.IsCompleted);
Assert.That(bytesReceived, Is.EqualTo(4), "Should have received 4 bytes.");
Assert.That(resultTask.IsCompleted, Is.True, "Task should have completed.");
Assert.That(sendCompleted, Is.EqualTo(1), "Task ContinueWith should have executed.");
Assert.That(resultTask.Result.Length, Is.EqualTo(4), "Result of task should be 4 bytes.");
}
}
public async Task ReadShouldBlockUntilAllBytesRequestedAreReceived()
{
var sendCompleted = 0;
var bytesReceived = 0;
var config = new ConnectionConfiguration(onReadChunk: (e, size, remaining, read, elapsed) => Interlocked.Add(ref bytesReceived, read));
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var test = new TcpSocket(endpoint, config, TestConfig.InfoLog))
{
var resultTask = test.ReadAsync(4, CancellationToken.None).ContinueWith(t =>
{
Interlocked.Increment(ref sendCompleted);
return(t.Result);
});
Console.WriteLine("Sending first 3 bytes...");
server.HasClientConnected.Wait(TimeSpan.FromMilliseconds(1000));
var sendInitialBytes = server.SendDataAsync(new byte[] { 0, 0, 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendInitialBytes, Is.True, "First 3 bytes should have been sent.");
Console.WriteLine("Ensuring task blocks...");
await TaskTest.WaitFor(() => bytesReceived > 0);
Assert.That(resultTask.IsCompleted, Is.False, "Task should still be running, blocking.");
Assert.That(sendCompleted, Is.EqualTo(0), "Should still block even though bytes have been received.");
Assert.That(bytesReceived, Is.EqualTo(3), "Three bytes should have been received and we are waiting on the last byte.");
Console.WriteLine("Sending last byte...");
var sendLastByte = server.SendDataAsync(new byte[] { 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendLastByte, Is.True, "Last byte should have sent.");
Console.WriteLine("Ensuring task unblocks...");
await TaskTest.WaitFor(() => resultTask.IsCompleted);
Assert.That(bytesReceived, Is.EqualTo(4), "Should have received 4 bytes.");
Assert.That(resultTask.IsCompleted, Is.True, "Task should have completed.");
Assert.That(sendCompleted, Is.EqualTo(1), "Task ContinueWith should have executed.");
Assert.That(resultTask.Result.Length, Is.EqualTo(4), "Result of task should be 4 bytes.");
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnce()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
Console.WriteLine("Sending first message to receive...");
server.SendDataAsync(firstMessage.ToBytes());
var firstResponse = test.ReadAsync(4).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
Console.WriteLine("Sending second message to receive...");
server.SendDataAsync(secondMessage);
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnceAsyncronously()
{
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var test = new TcpSocket(endpoint, log: TestConfig.InfoLog))
{
const int firstMessage = 99;
const int secondMessage = 100;
Console.WriteLine("Sending first message to receive...");
var send1 = server.SendDataAsync(firstMessage.ToBytes());
var firstResponseTask = test.ReadAsync(4, CancellationToken.None);
Console.WriteLine("Sending second message to receive...");
var send2 = server.SendDataAsync(secondMessage.ToBytes());
var secondResponseTask = test.ReadAsync(4, CancellationToken.None);
Assert.That(firstResponseTask.Result.ToInt32(), Is.EqualTo(firstMessage));
Assert.That(secondResponseTask.Result.ToInt32(), Is.EqualTo(secondMessage));
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnce()
{
using (var server = new FakeTcpServer(8999))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
Console.WriteLine("Sending first message to receive...");
server.SendDataAsync(firstMessage.ToBytes()).Wait(TimeSpan.FromSeconds(2));
var firstResponse = test.ReadAsync(4).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
Console.WriteLine("Sending second message to receive...");
server.SendDataAsync(secondMessage).Wait(TimeSpan.FromSeconds(2));
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnce()
{
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var test = new TcpSocket(endpoint, log: TestConfig.InfoLog))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
Console.WriteLine("Sending first message to receive...");
var send = server.SendDataAsync(firstMessage.ToBytes());
var firstResponse = test.ReadAsync(4, CancellationToken.None).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
Console.WriteLine("Sending second message to receive...");
server.SendDataAsync(secondMessage);
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length, CancellationToken.None).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public async Task ShouldReconnectAfterLosingConnectionAndBeAbleToStartNewRead()
{
var log = TestConfig.InfoLog;
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
{
var disconnects = 0;
var connects = 0;
TaskCompletionSource <int> disconnectEvent = new TaskCompletionSource <int>();
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var config = new ConnectionConfiguration(onDisconnected: (e, exception) => {
log.Info(() => LogEvent.Create("inside onDisconnected"));
disconnectEvent.TrySetResult(1);
});
using (var socket = new TcpSocket(endpoint, config, log)) {
//wait till connected
await TaskTest.WaitFor(() => connects > 0);
log.Info(() => LogEvent.Create("connects > 0"));
var readTask = socket.ReadAsync(4, CancellationToken.None);
log.Info(() => LogEvent.Create("Connected: Dropping connection..."));
server.DropConnection();
//wait till Disconnected
log.Info(() => LogEvent.Create("Waiting for onDisconnected"));
await Task.WhenAny(disconnectEvent.Task, Task.Delay(100000));
Assert.IsTrue(disconnectEvent.Task.IsCompleted, "Server should have disconnected the client.");
Assert.ThrowsAsync <ConnectionException>(async() => await readTask);
await TaskTest.WaitFor(() => connects == 2, 6000);
log.Info(() => LogEvent.Create("connects == 2"));
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
var readTask2 = socket.ReadAsync(4, CancellationToken.None);
log.Info(() => LogEvent.Create("sending data (99)"));
await server.SendDataAsync(99.ToBytes());
var result = await readTask2;
Assert.That(result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
}
public void SendAsyncShouldWaitUntilClientIsConnected()
{
const int testData = 99;
using (var server = new FakeTcpServer(Ilog, 8999))
using (var client = new TcpClient())
{
server.SendDataAsync(testData.ToBytes());
Thread.Sleep(1000);
client.Connect(_fakeServerUrl.Host, _fakeServerUrl.Port);
var buffer = new byte[4];
client.GetStream().ReadAsync(buffer, 0, 4).Wait(TimeSpan.FromSeconds(5));
Assert.That(buffer.ToInt32(), Is.EqualTo(testData));
}
}
public void SendAsyncShouldWaitUntilClientIsConnected()
{
const int testData = 99;
using (var server = new FakeTcpServer(8999))
using (var client = new TcpClient())
{
server.SendDataAsync(testData.ToBytes());
Thread.Sleep(1000);
client.Connect(_fakeServerUrl.Host, _fakeServerUrl.Port);
var buffer = new byte[4];
client.GetStream().ReadAsync(buffer, 0, 4).Wait(TimeSpan.FromSeconds(5));
Assert.That(buffer.ToInt32(), Is.EqualTo(testData));
}
}
public async Task SendAsyncShouldWaitUntilClientIsConnected()
{
const int testData = 99;
var serverUri = TestConfig.ServerUri();
using (var server = new FakeTcpServer(TestConfig.WarnLog, serverUri.Port))
using (var client = new TcpClient())
{
var send = server.SendDataAsync(testData.ToBytes());
Thread.Sleep(1000);
await client.ConnectAsync(serverUri.Host, serverUri.Port);
var buffer = new byte[4];
client.GetStream().ReadAsync(buffer, 0, 4).Wait(TimeSpan.FromSeconds(5));
Assert.That(buffer.ToInt32(), Is.EqualTo(testData));
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover([Values(3, 4)] int connectionAttempts)
{
var mockLog = new MemoryLog();
var disconnected = 0;
var connected = 0;
var config = new ConnectionConfiguration(
onDisconnected: (e, exception) => {
Interlocked.Increment(ref disconnected);
},
onConnected: (e, attempt, elapsed) => {
Interlocked.Increment(ref connected);
});
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, config, TestConfig.InfoLog))
using (new Connection(socket, config, log: mockLog))
{
for (var connectionAttempt = 1; connectionAttempt <= connectionAttempts; connectionAttempt++)
{
var currentAttempt = connectionAttempt;
await TaskTest.WaitFor(() => server.ConnectionEventcount == currentAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
await server.SendDataAsync(CreateCorrelationMessage(connectionAttempt));
TestConfig.InfoLog.Write(LogLevel.Info, () => LogEvent.Create($"Sent CONNECTION attempt {connectionAttempt}"));
await TaskTest.WaitFor(() => connected == disconnected, 200);
Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Info && e.Item2.Message.StartsWith("Polling receive thread has recovered on ")), Is.EqualTo(currentAttempt - 1));
TestConfig.InfoLog.Write(LogLevel.Info, () => LogEvent.Create($"Dropping CONNECTION attempt {connectionAttempt}"));
server.DropConnection();
await TaskTest.WaitFor(() => disconnected == currentAttempt);
Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Error && e.Item2.Message.StartsWith("Polling failure on")), Is.AtLeast(currentAttempt));
}
}
}
public void ReadShouldIgnoreMessageWithUnknownCorrelationId()
{
const int correlationId = 99;
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, new Uri("http://localhost:8999")))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
//send correlation message
server.SendDataAsync(CreateCorrelationMessage(correlationId)).Wait(TimeSpan.FromSeconds(1));
//wait for connection
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
//should log an exception and keep going
mockLog.Verify(x => x.WarnFormat(It.IsAny <string>(), correlationId));
}
}
public void ReadShouldNotLoseDataFromStreamOverMultipleReads()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
var payload = new WriteByteStream();
payload.Pack(firstMessage.ToBytes(), secondMessage.ToBytes());
var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
//send the combined payload
server.SendDataAsync(payload.Payload());
var firstResponse = test.ReadAsync(4).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public async Task SendAsyncShouldUseStatictVersionInfo()
{
IRequestContext context = null;
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog))
using (var conn = new Connection(socket, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.InfoLog))
{
server.OnBytesReceived += data =>
{
context = KafkaDecoder.DecodeHeader(data);
var send = server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
};
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
await TaskTest.WaitFor(() => context != null);
Assert.That(context.ApiVersion.Value, Is.EqualTo(2));
}
}
public void ReadShouldNotLoseDataFromStreamOverMultipleReads()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
var payload = new KafkaMessagePacker()
.Pack(firstMessage)
.Pack(secondMessage, StringPrefixEncoding.None);
//send the combined payload
server.SendDataAsync(payload.PayloadNoLength());
var firstResponse = test.ReadAsync(4).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public async Task ShouldReconnectAfterLosingConnectionAndBeAbleToStartNewRead()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
{
var disconnects = 0;
var connects = 0;
TaskCompletionSource <int> disconnectEvent = new TaskCompletionSource <int>();
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry);
socket.OnServerDisconnected += () => disconnectEvent.TrySetResult(1);
//wait till connected
await TaskTest.WaitFor(() => connects > 0);
var readTask = socket.ReadAsync(4);
server.DropConnection();
//wait till Disconnected
await Task.WhenAny(disconnectEvent.Task, Task.Delay(100000));
Assert.IsTrue(disconnectEvent.Task.IsCompleted, "Server should have disconnected the client.");
Assert.Throws <ServerDisconnectedException>(async() => await readTask);
await TaskTest.WaitFor(() => connects == 2, 6000);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
var readTask2 = socket.ReadAsync(4);
await server.SendDataAsync(99.ToBytes());
var result = await readTask2;
Assert.That(result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public void ReadShouldStackReadRequestsAndReturnOneAtATime()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
{
var messages = new[] { "test1", "test2", "test3", "test4" };
var expectedLength = "test1".Length;
var payload = new KafkaMessagePacker().Pack(messages);
var socket = new KafkaTcpSocket(new DefaultTraceLog(LogLevel.Warn), _fakeServerUrl, _maxRetry);
var tasks = messages.Select(x => socket.ReadAsync(x.Length)).ToArray();
server.SendDataAsync(payload.Payload());
Task.WaitAll(tasks);
foreach (var task in tasks)
{
Assert.That(task.Result.Length, Is.EqualTo(expectedLength));
}
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnce()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
Console.WriteLine("Sending first message to receive...");
server.SendDataAsync(firstMessage.ToBytes());
var firstResponse = test.ReadAsync(4).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
Console.WriteLine("Sending second message to receive...");
server.SendDataAsync(secondMessage);
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public async Task ShouldReconnectAfterLosingConnectionAndBeAbleToStartNewRead()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
{
var disconnects = 0;
var connects = 0;
TaskCompletionSource<int> disconnectEvent = new TaskCompletionSource<int>();
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry);
socket.OnServerDisconnected += () => disconnectEvent.TrySetResult(1);
//wait till connected
await TaskTest.WaitFor(() => connects > 0);
var readTask = socket.ReadAsync(4);
server.DropConnection();
//wait till Disconnected
await Task.WhenAny(disconnectEvent.Task, Task.Delay(100000));
Assert.IsTrue(disconnectEvent.Task.IsCompleted, "Server should have disconnected the client.");
Assert.Throws<ServerDisconnectedException>(async () => await readTask);
await TaskTest.WaitFor(() => connects == 2, 6000);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
var readTask2 = socket.ReadAsync(4);
await server.SendDataAsync(99.ToBytes());
var result = await readTask2;
Assert.That(result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover()
{
var mockLog =new Mock<IKafkaLog>();
var log= new DefaultTraceLog(LogLevel.Error);
using (var server = new FakeTcpServer(log, 8999))
using (var socket = new KafkaTcpSocket(log, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = 0;
socket.OnServerDisconnected += () => Interlocked.Increment(ref disconnected);
for (int connectionAttempt = 1; connectionAttempt < 4; connectionAttempt++)
{
await TaskTest.WaitFor(() => server.ConnectionEventcount == connectionAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
server.SendDataAsync(CreateCorrelationMessage(1)).Wait(TimeSpan.FromSeconds(5));
await TaskTest.WaitFor(() => !conn.IsOnErrorState());
Assert.IsFalse(conn.IsOnErrorState());
mockLog.Verify(x => x.InfoFormat("Polling read thread has recovered: {0}", It.IsAny<object[]>()), Times.Exactly(connectionAttempt-1));
server.DropConnection();
await TaskTest.WaitFor(() => conn.IsOnErrorState());
Assert.AreEqual(disconnected,connectionAttempt);
Assert.IsTrue(conn.IsOnErrorState());
mockLog.Verify(x => x.ErrorFormat("Exception occured in polling read thread {0}: {1}", It.IsAny<object[]>()), Times.Exactly(connectionAttempt ));
}
}
}
public void WriteAndReadShouldBeAsynchronous()
{
var write = new List<int>();
var read = new List<int>();
var expected = new List<int> { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
server.OnBytesReceived += data => write.AddRange(data.Batch(4).Select(x => x.ToArray().ToInt32()));
var tasks = Enumerable.Range(1, 10)
.SelectMany(i => new[]
{
test.WriteAsync(i.ToBytes().ToPayload()),
test.ReadAsync(4).ContinueWith(t => read.Add(t.Result.ToInt32())),
server.SendDataAsync(i.ToBytes())
}).ToArray();
Task.WaitAll(tasks);
Assert.That(write.OrderBy(x => x), Is.EqualTo(expected));
Assert.That(read.OrderBy(x => x), Is.EqualTo(expected));
}
}
public void ReadShouldStackReadRequestsAndReturnOneAtATime()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var messages = new[] { "test1", "test2", "test3", "test4" };
var expectedLength = "test1".Length;
var payload = new KafkaMessagePacker().Pack(messages);
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var tasks = messages.Select(x => socket.ReadAsync(x.Length)).ToArray();
server.SendDataAsync(payload.Payload());
Task.WaitAll(tasks);
foreach (var task in tasks)
{
Assert.That(task.Result.Length, Is.EqualTo(expectedLength));
}
}
}
public void ReadShouldReconnectAfterLosingConnection()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var disconnects = 0;
var connects = 0;
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = ReadFromSocketWithRetry(socket, 4);
//wait till connected
TaskTest.WaitFor(() => connects > 0);
//drop connection
server.DropConnection();
TaskTest.WaitFor(() => disconnects > 0);
Assert.That(disconnects, Is.EqualTo(1), "Server should have disconnected the client.");
//wait for reconnection
TaskTest.WaitFor(() => connects > 1);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
//send data and get result
server.SendDataAsync(99.ToBytes());
Assert.That(resultTask.Result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public void ReadShouldNotLoseDataFromStreamOverMultipleReads()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
const int firstMessage = 99;
const string secondMessage = "testmessage";
var payload = new KafkaMessagePacker()
.Pack(firstMessage)
.Pack(secondMessage, StringPrefixEncoding.None);
//send the combined payload
server.SendDataAsync(payload.PayloadNoLength());
var firstResponse = test.ReadAsync(4).Result.ToInt32();
Assert.That(firstResponse, Is.EqualTo(firstMessage));
var secondResponse = Encoding.ASCII.GetString(test.ReadAsync(secondMessage.Length).Result);
Assert.That(secondResponse, Is.EqualTo(secondMessage));
}
}
public void WriteAndReadShouldBeAsynchronous()
{
// for (int j = 0; j < 1000; j++)
// {
var expected = new List<int> { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
var write = new ConcurrentBag<int>();
var read = new ConcurrentBag<int>();
AutoResetEvent allEventAreArrived = new AutoResetEvent(false);
AutoResetEvent allrReadEventAreArrived = new AutoResetEvent(false);
using (var server = new FakeTcpServer(_log, FakeServerPort))
using (var test = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry))
{
server.OnBytesReceived += data =>
{
var d = data.Batch(4).Select(x => x.ToArray().ToInt32());
foreach (var item in d)
{
write.Add(item);
}
if (expected.Count == write.Count)
{
allEventAreArrived.Set();
}
};
var tasks = Enumerable.Range(1, 10)
.SelectMany(i => new[]
{
test.WriteAsync(i.ToBytes().ToPayload()),
test.ReadAsync(4).ContinueWith(t =>
{
read.Add(t.Result.ToInt32());
if (read.Count == expected.Count)
{
allrReadEventAreArrived.Set();
}
}),
server.SendDataAsync(i.ToBytes())
}).ToArray();
Task.WaitAll(tasks);
Assert.IsTrue(allEventAreArrived.WaitOne(3000), "not Get all write event in time");
Assert.IsTrue(allrReadEventAreArrived.WaitOne(3000), "not Get all read event in time");
var w = write.OrderBy(x => x);
var r = read.OrderBy(x => x);
for (int i = 0; i < expected.Count; i++)
{
// _log.InfoFormat("write was {0} expected {1}", w.ElementAt(i), expected[i]);
Assert.That(w.ElementAt(i), Is.EqualTo(expected[i]));
}
for (int i = 0; i < expected.Count; i++)
{
// _log.InfoFormat("read was {0} expected {1}", r.ElementAt(i), expected[i]);
Assert.That(r.ElementAt(i), Is.EqualTo(expected[i]));
}
}
// }
}
public void ReadShouldIgnoreMessageWithUnknownCorrelationId()
{
const int correlationId = 99;
var mockLog = _kernel.GetMock<IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var receivedData = false;
socket.OnBytesReceived += i => receivedData = true;
//send correlation message
server.SendDataAsync(CreateCorrelationMessage(correlationId)).Wait(TimeSpan.FromSeconds(5));
//wait for connection
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
TaskTest.WaitFor(() => receivedData);
//should log a warning and keep going
mockLog.Verify(x => x.WarnFormat(It.IsAny<string>(), It.Is<int>(o => o == correlationId)));
}
}
public void ReadShouldBlockUntilAllBytesRequestedAreReceived()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
var sendCompleted = 0;
var bytesReceived = 0;
test.OnBytesReceived += i => Interlocked.Add(ref bytesReceived, i);
var resultTask = test.ReadAsync(4).ContinueWith(t =>
{
Interlocked.Increment(ref sendCompleted);
return t.Result;
});
Console.WriteLine("Sending first 3 bytes...");
server.HasClientConnected.Wait(TimeSpan.FromMilliseconds(1000));
var sendInitialBytes = server.SendDataAsync(new byte[] { 0, 0, 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendInitialBytes, Is.True, "First 3 bytes should have been sent.");
Console.WriteLine("Ensuring task blocks...");
TaskTest.WaitFor(() => bytesReceived > 0);
Assert.That(resultTask.IsCompleted, Is.False, "Task should still be running, blocking.");
Assert.That(sendCompleted, Is.EqualTo(0), "Should still block even though bytes have been received.");
Assert.That(bytesReceived, Is.EqualTo(3), "Three bytes should have been received and we are waiting on the last byte.");
Console.WriteLine("Sending last byte...");
var sendLastByte = server.SendDataAsync(new byte[] { 0 }).Wait(TimeSpan.FromSeconds(10));
Assert.That(sendLastByte, Is.True, "Last byte should have sent.");
Console.WriteLine("Ensuring task unblocks...");
TaskTest.WaitFor(() => resultTask.IsCompleted);
Assert.That(bytesReceived, Is.EqualTo(4), "Should have received 4 bytes.");
Assert.That(resultTask.IsCompleted, Is.True, "Task should have completed.");
Assert.That(sendCompleted, Is.EqualTo(1), "Task ContinueWith should have executed.");
Assert.That(resultTask.Result.Length, Is.EqualTo(4), "Result of task should be 4 bytes.");
}
}
public async void SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
using (var socket = new KafkaTcpSocket(_log, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, TimeSpan.FromMilliseconds(1000000), log: _log))
{
Console.WriteLine("SendAsync blocked by reconnection attempts...");
var taskResult = conn.SendAsync(new MetadataRequest());
taskResult.ContinueWith(t => taskResult = t).Wait(TimeSpan.FromSeconds(1));
Console.WriteLine("Task result should be WaitingForActivation...");
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
Console.WriteLine("Starting server to establish connection...");
using (var server = new FakeTcpServer(8999))
{
server.OnClientConnected += () => Console.WriteLine("Client connected...");
server.OnBytesReceived += (b) =>
{
server.SendDataAsync(MessageHelper.CreateMetadataResponse(1, "Test"));
};
await taskResult;
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
public void ReadShouldBeAbleToReceiveMoreThanOnceAsyncronously()
{
using (var server = new FakeTcpServer(FakeServerPort))
using (var test = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl))
{
const int firstMessage = 99;
const int secondMessage = 100;
Console.WriteLine("Sending first message to receive...");
server.SendDataAsync(firstMessage.ToBytes());
var firstResponseTask = test.ReadAsync(4);
Console.WriteLine("Sending second message to receive...");
server.SendDataAsync(secondMessage.ToBytes());
var secondResponseTask = test.ReadAsync(4);
Assert.That(firstResponseTask.Result.ToInt32(), Is.EqualTo(firstMessage));
Assert.That(secondResponseTask.Result.ToInt32(), Is.EqualTo(secondMessage));
}
}