public void ReadShouldReconnectAfterLosingConnection()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var disconnects = 0;
var connects = 0;
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = ReadFromSocketWithRetry(socket, 4);
//wait till connected
TaskTest.WaitFor(() => connects > 0);
//drop connection
server.DropConnection();
TaskTest.WaitFor(() => disconnects > 0);
Assert.That(disconnects, Is.EqualTo(1), "Server should have disconnected the client.");
//wait for reconnection
TaskTest.WaitFor(() => connects > 1);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
//send data and get result
server.SendDataAsync(99.ToBytes());
Assert.That(resultTask.Result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public void ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//Wait a while for the client to notice the disconnect and log
Thread.Sleep(15);
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny <string>(), It.IsAny <Exception>()));
TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
public async Task FakeShouldBeAbleToReconnect()
{
var serverUri = TestConfig.ServerUri();
using (var server = new FakeTcpServer(TestConfig.WarnLog, serverUri.Port))
{
byte[] received = null;
server.OnBytesReceived += data => received = data;
var t1 = new TcpClient();
await t1.ConnectAsync(serverUri.Host, serverUri.Port);
await TaskTest.WaitFor(() => server.ConnectionEventcount == 1);
server.DropConnection();
await TaskTest.WaitFor(() => server.DisconnectionEventCount == 1);
var t2 = new TcpClient();
await t2.ConnectAsync(serverUri.Host, serverUri.Port);
await TaskTest.WaitFor(() => server.ConnectionEventcount == 2);
t2.GetStream().Write(99.ToBytes(), 0, 4);
await TaskTest.WaitFor(() => received != null);
Assert.That(received.ToInt32(), Is.EqualTo(99));
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover()
{
var mockLog = new Mock <IKafkaLog>();
var log = new DefaultTraceLog(LogLevel.Error);
using (var server = new FakeTcpServer(log, 8999))
using (var socket = new KafkaTcpSocket(log, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = 0;
socket.OnServerDisconnected += () => Interlocked.Increment(ref disconnected);
for (int connectionAttempt = 1; connectionAttempt < 4; connectionAttempt++)
{
await TaskTest.WaitFor(() => server.ConnectionEventcount == connectionAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
server.SendDataAsync(CreateCorrelationMessage(1)).Wait(TimeSpan.FromSeconds(5));
await TaskTest.WaitFor(() => !conn.IsOnErrorState());
Assert.IsFalse(conn.IsOnErrorState());
mockLog.Verify(x => x.InfoFormat("Polling read thread has recovered: {0}", It.IsAny <object[]>()), Times.Exactly(connectionAttempt - 1));
server.DropConnection();
await TaskTest.WaitFor(() => conn.IsOnErrorState());
Assert.AreEqual(disconnected, connectionAttempt);
Assert.IsTrue(conn.IsOnErrorState());
mockLog.Verify(x => x.ErrorFormat("Exception occured in polling read thread {0}: {1}", It.IsAny <object[]>()), Times.Exactly(connectionAttempt));
}
}
}
[Test, Repeat(1)]/// [Test, Repeat(100)]
public async Task ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(_log, 8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = false;
socket.OnServerDisconnected += () => disconnected = true;
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
await TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//Wait a while for the client to notice the disconnect and log
await TaskTest.WaitFor(() => disconnected);
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny <string>(), It.IsAny <Exception>()));
await TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
public async Task ShouldReconnectAfterLosingConnectionAndBeAbleToStartNewRead()
{
var log = TestConfig.InfoLog;
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
{
var disconnects = 0;
var connects = 0;
TaskCompletionSource <int> disconnectEvent = new TaskCompletionSource <int>();
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var config = new ConnectionConfiguration(onDisconnected: (e, exception) => {
log.Info(() => LogEvent.Create("inside onDisconnected"));
disconnectEvent.TrySetResult(1);
});
using (var socket = new TcpSocket(endpoint, config, log)) {
//wait till connected
await TaskTest.WaitFor(() => connects > 0);
log.Info(() => LogEvent.Create("connects > 0"));
var readTask = socket.ReadAsync(4, CancellationToken.None);
log.Info(() => LogEvent.Create("Connected: Dropping connection..."));
server.DropConnection();
//wait till Disconnected
log.Info(() => LogEvent.Create("Waiting for onDisconnected"));
await Task.WhenAny(disconnectEvent.Task, Task.Delay(100000));
Assert.IsTrue(disconnectEvent.Task.IsCompleted, "Server should have disconnected the client.");
Assert.ThrowsAsync <ConnectionException>(async() => await readTask);
await TaskTest.WaitFor(() => connects == 2, 6000);
log.Info(() => LogEvent.Create("connects == 2"));
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
var readTask2 = socket.ReadAsync(4, CancellationToken.None);
log.Info(() => LogEvent.Create("sending data (99)"));
await server.SendDataAsync(99.ToBytes());
var result = await readTask2;
Assert.That(result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
}
public async Task KafkaConnectionShouldSkipPartiallyReadMessage()
{
var mockLog = new MemoryLog();
var bytesRead = 0;
var config = new ConnectionConfiguration(onReadChunk: (e, size, remaining, read, elapsed) => Interlocked.Add(ref bytesRead, read));
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, config, TestConfig.InfoLog))
using (new Connection(socket, config, log: mockLog))
{
// send size
var size = 200;
await server.SendDataAsync(size.ToBytes());
var randomizer = Randomizer.CreateRandomizer();
var firstBytes = new byte[99];
for (var i = 0; i < firstBytes.Length; i++)
{
firstBytes[i] = randomizer.NextByte();
}
// send half of payload
await server.SendDataAsync(firstBytes);
await TaskTest.WaitFor(() => bytesRead == firstBytes.Length);
//Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Warn && e.Item2.Message.StartsWith("Skipping")), Is.EqualTo(0));
server.DropConnection();
// send half of payload should be skipped
var lastBytes = new byte[size];
for (var i = 0; i < lastBytes.Length; i++)
{
lastBytes[i] = randomizer.NextByte();
}
await server.SendDataAsync(lastBytes);
await TaskTest.WaitFor(() => bytesRead >= size);
Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Warn && e.Item2.Message.StartsWith($"Skipping {size - firstBytes.Length} bytes on")), Is.EqualTo(1));
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover([Values(3, 4)] int connectionAttempts)
{
var mockLog = new MemoryLog();
var disconnected = 0;
var connected = 0;
var config = new ConnectionConfiguration(
onDisconnected: (e, exception) => {
Interlocked.Increment(ref disconnected);
},
onConnected: (e, attempt, elapsed) => {
Interlocked.Increment(ref connected);
});
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, config, TestConfig.InfoLog))
using (new Connection(socket, config, log: mockLog))
{
for (var connectionAttempt = 1; connectionAttempt <= connectionAttempts; connectionAttempt++)
{
var currentAttempt = connectionAttempt;
await TaskTest.WaitFor(() => server.ConnectionEventcount == currentAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
await server.SendDataAsync(CreateCorrelationMessage(connectionAttempt));
TestConfig.InfoLog.Write(LogLevel.Info, () => LogEvent.Create($"Sent CONNECTION attempt {connectionAttempt}"));
await TaskTest.WaitFor(() => connected == disconnected, 200);
Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Info && e.Item2.Message.StartsWith("Polling receive thread has recovered on ")), Is.EqualTo(currentAttempt - 1));
TestConfig.InfoLog.Write(LogLevel.Info, () => LogEvent.Create($"Dropping CONNECTION attempt {connectionAttempt}"));
server.DropConnection();
await TaskTest.WaitFor(() => disconnected == currentAttempt);
Assert.That(mockLog.LogEvents.Count(e => e.Item1 == LogLevel.Error && e.Item2.Message.StartsWith("Polling failure on")), Is.AtLeast(currentAttempt));
}
}
}
public void ReadShouldThrowServerDisconnectedExceptionWhenDisconnected()
{
using (var server = new FakeTcpServer(8999))
{
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = socket.ReadAsync(4);
//wait till connected
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
resultTask.ContinueWith(t => resultTask = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(resultTask.IsFaulted, Is.True);
Assert.That(resultTask.Exception.InnerException, Is.TypeOf <ServerDisconnectedException>());
}
}
public async Task ReadShouldThrowServerDisconnectedExceptionWhenDisconnected()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
{
var socket = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry);
var resultTask = socket.ReadAsync(4);
//wait till connected
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
server.DropConnection();
await TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
resultTask.ContinueWith(t => resultTask = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(resultTask.IsFaulted, Is.True);
Assert.That(resultTask.Exception.InnerException, Is.TypeOf <ServerDisconnectedException>());
}
}
public async Task ReadShouldThrowServerDisconnectedExceptionWhenDisconnected()
{
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog))
{
var resultTask = socket.ReadAsync(4, CancellationToken.None);
//wait till connected
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
server.DropConnection();
await TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
resultTask.ContinueWith(t => resultTask = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(resultTask.IsFaulted, Is.True);
Assert.That(resultTask.Exception.InnerException, Is.TypeOf <ConnectionException>());
}
}
public void ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock <IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, new Uri("http://localhost:8999")))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny <string>(), It.IsAny <Exception>()));
TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
public async Task ShouldReconnectAfterLosingConnectionAndBeAbleToStartNewRead()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
{
var disconnects = 0;
var connects = 0;
TaskCompletionSource <int> disconnectEvent = new TaskCompletionSource <int>();
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry);
socket.OnServerDisconnected += () => disconnectEvent.TrySetResult(1);
//wait till connected
await TaskTest.WaitFor(() => connects > 0);
var readTask = socket.ReadAsync(4);
server.DropConnection();
//wait till Disconnected
await Task.WhenAny(disconnectEvent.Task, Task.Delay(100000));
Assert.IsTrue(disconnectEvent.Task.IsCompleted, "Server should have disconnected the client.");
Assert.Throws <ServerDisconnectedException>(async() => await readTask);
await TaskTest.WaitFor(() => connects == 2, 6000);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
var readTask2 = socket.ReadAsync(4);
await server.SendDataAsync(99.ToBytes());
var result = await readTask2;
Assert.That(result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public async Task FakeShouldBeAbleToReconnect()
{
using (var server = new FakeTcpServer(Ilog, 8999))
{
byte[] received = null;
server.OnBytesReceived += data => received = data;
var t1 = new TcpClient();
t1.Connect(_fakeServerUrl.Host, _fakeServerUrl.Port);
await TaskTest.WaitFor(() => server.ConnectionEventcount == 1);
server.DropConnection();
await TaskTest.WaitFor(() => server.DisconnectionEventCount == 1);
var t2 = new TcpClient();
t2.Connect(_fakeServerUrl.Host, _fakeServerUrl.Port);
await TaskTest.WaitFor(() => server.ConnectionEventcount == 2);
t2.GetStream().Write(99.ToBytes(), 0, 4);
await TaskTest.WaitFor(() => received != null);
Assert.That(received.ToInt32(), Is.EqualTo(99));
}
}
public void FakeShouldBeAbleToReconnect()
{
using (var server = new FakeTcpServer(8999))
{
byte[] received = null;
server.OnBytesReceived += data => received = data;
var t1 = new TcpClient();
t1.Connect(_fakeServerUrl.Host, _fakeServerUrl.Port);
TaskTest.WaitFor(() => server.ConnectionEventcount == 1);
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount == 1);
var t2 = new TcpClient();
t2.Connect(_fakeServerUrl.Host, _fakeServerUrl.Port);
TaskTest.WaitFor(() => server.ConnectionEventcount == 2);
t2.GetStream().Write(99.ToBytes(), 0, 4);
TaskTest.WaitFor(() => received != null);
Assert.That(received.ToInt32(), Is.EqualTo(99));
}
}
public void ReadShouldLogDisconnectAndRecover()
{
var mockLog = _kernel.GetMock<IKafkaLog>();
using (var server = new FakeTcpServer(8999))
using (var socket = new KafkaTcpSocket(mockLog.Object, _kafkaEndpoint))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = false;
socket.OnServerDisconnected += () => disconnected = true;
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
Assert.That(server.ConnectionEventcount, Is.EqualTo(1));
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
Assert.That(server.DisconnectionEventCount, Is.EqualTo(1));
//Wait a while for the client to notice the disconnect and log
TaskTest.WaitFor(() => disconnected);
//should log an exception and keep going
mockLog.Verify(x => x.ErrorFormat(It.IsAny<string>(), It.IsAny<Exception>()));
TaskTest.WaitFor(() => server.ConnectionEventcount > 1);
Assert.That(server.ConnectionEventcount, Is.EqualTo(2));
}
}
public async Task ReadShouldThrowServerDisconnectedExceptionWhenDisconnected()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
{
var socket = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry);
var resultTask = socket.ReadAsync(4);
//wait till connected
await TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
server.DropConnection();
await TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
resultTask.ContinueWith(t => resultTask = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(resultTask.IsFaulted, Is.True);
Assert.That(resultTask.Exception.InnerException, Is.TypeOf<BrokerConnectionException>());
}
}
public void ReadShouldReconnectAfterLosingConnection()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var disconnects = 0;
var connects = 0;
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = ReadFromSocketWithRetry(socket, 4);
//wait till connected
TaskTest.WaitFor(() => connects > 0);
//drop connection
server.DropConnection();
TaskTest.WaitFor(() => disconnects > 0);
Assert.That(disconnects, Is.EqualTo(1), "Server should have disconnected the client.");
//wait for reconnection
TaskTest.WaitFor(() => connects > 1);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
//send data and get result
server.SendDataAsync(99.ToBytes());
Assert.That(resultTask.Result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
public void ReadShouldThrowServerDisconnectedExceptionWhenDisconnected()
{
using (var server = new FakeTcpServer(FakeServerPort))
{
var socket = new KafkaTcpSocket(new DefaultTraceLog(), _fakeServerUrl);
var resultTask = socket.ReadAsync(4);
//wait till connected
TaskTest.WaitFor(() => server.ConnectionEventcount > 0);
server.DropConnection();
TaskTest.WaitFor(() => server.DisconnectionEventCount > 0);
resultTask.ContinueWith(t => resultTask = t).Wait(TimeSpan.FromSeconds(1));
Assert.That(resultTask.IsFaulted, Is.True);
Assert.That(resultTask.Exception.InnerException, Is.TypeOf<ServerDisconnectedException>());
}
}
public async Task KafkaConnectionShouldLogDisconnectAndRecover()
{
var mockLog =new Mock<IKafkaLog>();
var log= new DefaultTraceLog(LogLevel.Error);
using (var server = new FakeTcpServer(log, 8999))
using (var socket = new KafkaTcpSocket(log, _kafkaEndpoint, _maxRetry))
using (var conn = new KafkaConnection(socket, log: mockLog.Object))
{
var disconnected = 0;
socket.OnServerDisconnected += () => Interlocked.Increment(ref disconnected);
for (int connectionAttempt = 1; connectionAttempt < 4; connectionAttempt++)
{
await TaskTest.WaitFor(() => server.ConnectionEventcount == connectionAttempt);
Assert.That(server.ConnectionEventcount, Is.EqualTo(connectionAttempt));
server.SendDataAsync(CreateCorrelationMessage(1)).Wait(TimeSpan.FromSeconds(5));
await TaskTest.WaitFor(() => !conn.IsOnErrorState());
Assert.IsFalse(conn.IsOnErrorState());
mockLog.Verify(x => x.InfoFormat("Polling read thread has recovered: {0}", It.IsAny<object[]>()), Times.Exactly(connectionAttempt-1));
server.DropConnection();
await TaskTest.WaitFor(() => conn.IsOnErrorState());
Assert.AreEqual(disconnected,connectionAttempt);
Assert.IsTrue(conn.IsOnErrorState());
mockLog.Verify(x => x.ErrorFormat("Exception occured in polling read thread {0}: {1}", It.IsAny<object[]>()), Times.Exactly(connectionAttempt ));
}
}
}
public async Task ShouldReconnectAfterLosingConnectionAndBeAbleToStartNewRead()
{
using (var server = new FakeTcpServer(_log, FakeServerPort))
{
var disconnects = 0;
var connects = 0;
TaskCompletionSource<int> disconnectEvent = new TaskCompletionSource<int>();
server.OnClientConnected += () => Interlocked.Increment(ref connects);
server.OnClientDisconnected += () => Interlocked.Increment(ref disconnects);
var socket = new KafkaTcpSocket(_log, _fakeServerUrl, _maxRetry);
socket.OnServerDisconnected += () => disconnectEvent.TrySetResult(1);
//wait till connected
await TaskTest.WaitFor(() => connects > 0);
var readTask = socket.ReadAsync(4);
server.DropConnection();
//wait till Disconnected
await Task.WhenAny(disconnectEvent.Task, Task.Delay(100000));
Assert.IsTrue(disconnectEvent.Task.IsCompleted, "Server should have disconnected the client.");
Assert.Throws<ServerDisconnectedException>(async () => await readTask);
await TaskTest.WaitFor(() => connects == 2, 6000);
Assert.That(connects, Is.EqualTo(2), "Socket should have reconnected.");
var readTask2 = socket.ReadAsync(4);
await server.SendDataAsync(99.ToBytes());
var result = await readTask2;
Assert.That(result.ToInt32(), Is.EqualTo(99), "Socket should have received the 4 bytes.");
}
}
