public void SetupData()
{
Common.Compression.ZipLevel = Level;
_request = new ProduceRequest(
Enumerable.Range(1, Partitions)
.Select(partitionId => new ProduceRequest.Topic(
"topic",
partitionId,
Enumerable.Range(1, Messages)
.Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion)),
Codec)));
var response = new ProduceResponse(new ProduceResponse.Topic("topic", 1, ErrorCode.NONE, 0));
var port = 10000;
var endpoint = new Endpoint(new IPEndPoint(IPAddress.Loopback, port), "localhost");
_server = new TcpServer(endpoint.Ip.Port)
{
OnReceivedAsync = async data => {
var header = KafkaDecoder.DecodeHeader(data.Skip(4));
var bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(header.CorrelationId), response);
await _server.SendDataAsync(bytes);
}
};
_connection = new Connection(endpoint);
}
public async Task MessagesStillLogWhenSendTimesOut()
{
var logger = new MemoryLog();
var received = 0;
var timeout = TimeSpan.FromMilliseconds(100);
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: timeout, onRead: (e, read, elapsed) => Interlocked.Increment(ref received)), logger))
{
await Task.WhenAny(server.ClientConnected, Task.Delay(TimeSpan.FromSeconds(3)));
server.OnReceivedAsync = async data => {
var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
await Task.Delay(timeout);
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new MetadataResponse()));
};
await AssertAsync.Throws <TimeoutException>(() => conn.SendAsync(new MetadataRequest(), CancellationToken.None));
await AssertAsync.ThatEventually(() => received > 0, () => $"received {received}");
await AssertAsync.ThatEventually(() => logger.LogEvents.Any(e => e.Item1 == LogLevel.Debug && e.Item2.Message.StartsWith("Timed out -----> (timed out or otherwise errored in client)")), () => logger.ToString(LogLevel.Debug));
}
}
public async Task CorrelationOverflowGuardWorks()
{
var correlationId = -1;
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromMilliseconds(5)), TestConfig.Log)) {
server.OnReceivedAsync = data => {
var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
correlationId = context.CorrelationId;
return(Task.FromResult(0));
};
try {
Connection.OverflowGuard = 10;
await AssertAsync.Throws <TimeoutException>(() => conn.SendAsync(new MetadataRequest(), CancellationToken.None));
var initialCorrelation = correlationId;
await AssertAsync.Throws <TimeoutException>(() => Task.WhenAll(Enumerable.Range(initialCorrelation, Connection.OverflowGuard - 1).Select(i => conn.SendAsync(new MetadataRequest(), CancellationToken.None))));
await AssertAsync.ThatEventually(() => correlationId > 1, () => $"correlation {correlationId}");
var currentCorrelation = correlationId;
await AssertAsync.Throws <TimeoutException>(() => Task.WhenAll(Enumerable.Range(0, Connection.OverflowGuard / 2).Select(i => conn.SendAsync(new MetadataRequest(), CancellationToken.None))));
await AssertAsync.ThatEventually(() => correlationId < currentCorrelation, () => $"correlation {correlationId}");
}
finally {
Connection.OverflowGuard = int.MaxValue >> 1;
}
}
}
public async Task SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
var endpoint = TestConfig.ServerEndpoint();
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000)), log: TestConfig.Log))
{
// SendAsync blocked by reconnection attempts
var taskResult = conn.SendAsync(new MetadataRequest(), CancellationToken.None);
// Task result should be WaitingForActivation
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
// Starting server to establish connection
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
{
server.OnConnected = () => TestConfig.Log.Info(() => LogEvent.Create("Client connected..."));
server.OnReceivedAsync = async data => {
var requestContext = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
await server.SendDataAsync(MessageHelper.CreateMetadataResponse(requestContext, "Test"));
};
await Task.WhenAny(taskResult, Task.Delay(TimeSpan.FromSeconds(5)));
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
await taskResult;
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
public async Task SendAsyncShouldNotAllowResponseToTimeoutWhileAwaitingKafkaToEstableConnection()
{
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog))
using (var conn = new Connection(socket, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000)), log: TestConfig.InfoLog))
{
Console.WriteLine("SendAsync blocked by reconnection attempts...");
var taskResult = conn.SendAsync(new MetadataRequest(), CancellationToken.None);
Console.WriteLine("Task result should be WaitingForActivation...");
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.WaitingForActivation));
Console.WriteLine("Starting server to establish connection...");
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
{
server.OnClientConnected += () => Console.WriteLine("Client connected...");
server.OnBytesReceived += b =>
{
var request = KafkaDecoder.DecodeHeader(b);
AsyncContext.Run(async() => await server.SendDataAsync(MessageHelper.CreateMetadataResponse(request.CorrelationId, "Test")));
};
await Task.WhenAny(taskResult, Task.Delay(TimeSpan.FromSeconds(10)));
Assert.That(taskResult.IsFaulted, Is.False);
Assert.That(taskResult.IsCanceled, Is.False);
await taskResult;
Assert.That(taskResult.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
}
}
public async Task SendAsyncShouldUseStatictVersionInfo()
{
IRequestContext context = null;
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.Log))
{
server.OnReceivedAsync = async data => {
context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
};
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
await AssertAsync.ThatEventually(() => context != null && context.ApiVersion.GetValueOrDefault() == 2, () => $"version {context?.ApiVersion}");
}
}
public async Task SendAsyncWithDynamicVersionInfoMakesVersionCallFirst()
{
var firstCorrelation = -1;
var correlationId = 0;
var sentVersion = (short)-1;
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(3), versionSupport: VersionSupport.Kafka8.Dynamic()), log: TestConfig.Log))
{
var apiVersion = (short)3;
server.OnReceivedAsync = async data => {
var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
if (firstCorrelation < 0)
{
firstCorrelation = context.CorrelationId;
}
correlationId = context.CorrelationId;
switch (correlationId - firstCorrelation)
{
case 0:
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new ApiVersionsResponse(ErrorCode.NONE, new[] { new ApiVersionsResponse.VersionSupport(ApiKey.Fetch, apiVersion, apiVersion) })));
break;
case 1:
sentVersion = context.ApiVersion.GetValueOrDefault();
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
break;
default:
return;
}
};
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
await AssertAsync.ThatEventually(() => correlationId - firstCorrelation >= 1, () => $"first {firstCorrelation}, current {correlationId}");
Assert.That(sentVersion, Is.EqualTo(apiVersion));
}
}
public async Task SendAsyncShouldUseStatictVersionInfo()
{
IRequestContext context = null;
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog))
using (var conn = new Connection(socket, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.InfoLog))
{
server.OnBytesReceived += data =>
{
context = KafkaDecoder.DecodeHeader(data);
var send = server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
};
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
await TaskTest.WaitFor(() => context != null);
Assert.That(context.ApiVersion.Value, Is.EqualTo(2));
}
}