public static void AssertCanEncodeDecodeResponse <T>(this T response, short version, IMembershipEncoder encoder = null, T forComparison = null) where T : class, IResponse
{
var encoders = ImmutableDictionary <string, IMembershipEncoder> .Empty;
if (encoder != null)
{
encoders = encoders.Add(encoder.ProtocolType, encoder);
}
var context = new RequestContext(16, version, "Test-Response", encoders, encoder?.ProtocolType);
var data = KafkaDecoder.EncodeResponseBytes(context, response);
var decoded = GetType <T>().ToResponse(context, data.Skip(Request.IntegerByteSize + Request.CorrelationSize));
if (forComparison == null)
{
forComparison = response;
}
Assert.That(forComparison.GetHashCode(), Is.EqualTo(decoded.GetHashCode()), "HashCode equality");
var original = forComparison.ToString();
var final = decoded.ToString();
Assert.That(original, Is.EqualTo(final), "ToString equality");
Assert.That(decoded.Equals(final), Is.False); // general test for equality
Assert.That(decoded.Equals(decoded), Is.True); // general equality test for sanity
Assert.That(forComparison.Equals(decoded), $"Original\n{original}\nFinal\n{final}");
Assert.That(forComparison.Errors.HasEqualElementsInOrder(decoded.Errors), "Errors");
}
public void SetupData()
{
Common.Compression.ZipLevel = Level;
_request = new ProduceRequest(
Enumerable.Range(1, Partitions)
.Select(partitionId => new ProduceRequest.Topic(
"topic",
partitionId,
Enumerable.Range(1, Messages)
.Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion)),
Codec)));
var response = new ProduceResponse(new ProduceResponse.Topic("topic", 1, ErrorCode.NONE, 0));
var port = 10000;
var endpoint = new Endpoint(new IPEndPoint(IPAddress.Loopback, port), "localhost");
_server = new TcpServer(endpoint.Ip.Port)
{
OnReceivedAsync = async data => {
var header = KafkaDecoder.DecodeHeader(data.Skip(4));
var bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(header.CorrelationId), response);
await _server.SendDataAsync(bytes);
}
};
_connection = new Connection(endpoint);
}
public async Task MessagesStillLogWhenSendTimesOut()
{
var logger = new MemoryLog();
var received = 0;
var timeout = TimeSpan.FromMilliseconds(100);
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: timeout, onRead: (e, read, elapsed) => Interlocked.Increment(ref received)), logger))
{
await Task.WhenAny(server.ClientConnected, Task.Delay(TimeSpan.FromSeconds(3)));
server.OnReceivedAsync = async data => {
var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
await Task.Delay(timeout);
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new MetadataResponse()));
};
await AssertAsync.Throws <TimeoutException>(() => conn.SendAsync(new MetadataRequest(), CancellationToken.None));
await AssertAsync.ThatEventually(() => received > 0, () => $"received {received}");
await AssertAsync.ThatEventually(() => logger.LogEvents.Any(e => e.Item1 == LogLevel.Debug && e.Item2.Message.StartsWith("Timed out -----> (timed out or otherwise errored in client)")), () => logger.ToString(LogLevel.Debug));
}
}
public async Task SendAsyncWithDynamicVersionInfoOnlyMakesVersionCallOnce()
{
var versionRequests = 0;
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(3), versionSupport: VersionSupport.Kafka8.Dynamic()), log: TestConfig.Log))
{
server.OnReceivedAsync = async data => {
var fullHeader = KafkaDecoder.DecodeFullHeader(data.Skip(Request.IntegerByteSize));
var context = fullHeader.Item1;
switch (fullHeader.Item2)
{
case ApiKey.ApiVersions:
Interlocked.Increment(ref versionRequests);
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new ApiVersionsResponse(ErrorCode.NONE, new[] { new ApiVersionsResponse.VersionSupport(ApiKey.Fetch, 3, 3) })));
break;
default:
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
break;
}
};
for (var i = 0; i < 3; i++)
{
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
}
Assert.That(versionRequests, Is.EqualTo(1));
}
}
public void SetupData()
{
Common.Compression.ZipLevel = Level;
var response = new FetchResponse(
Enumerable.Range(1, Partitions)
.Select(partitionId => new FetchResponse.Topic(
"topic",
partitionId,
500,
ErrorCode.NONE,
Enumerable.Range(1, Messages)
.Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion))
)));
_bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(1, Version), response);
}
public void FetchSize()
{
int partitions = 1;
short version = 0;
byte messageVersion = 0;
var results = new List <object>();
foreach (var codec in new[] { MessageCodec.None, MessageCodec.Gzip, MessageCodec.Snappy })
{
foreach (var messages in new[] { 100, 10000 })
{
foreach (var messageSize in new[] { 1, 1000 })
{
foreach (var level in new[] { CompressionLevel.Fastest })
{
Compression.ZipLevel = level;
var response = new FetchResponse(
Enumerable.Range(1, partitions)
.Select(partitionId => new FetchResponse.Topic(
"topic",
partitionId,
500,
ErrorCode.NONE,
Enumerable.Range(1, messages)
.Select(i => new Message(GenerateMessageBytes(messageSize), new ArraySegment <byte>(), (byte)codec, version: messageVersion))
)));
var bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(1, version), response);
var decoded = FetchResponse.FromBytes(new RequestContext(1, version), bytes.Skip(Request.IntegerByteSize + Request.CorrelationSize));
Assert.That(decoded.responses.Sum(t => t.Messages.Count), Is.EqualTo(response.responses.Sum(t => t.Messages.Count)));
var result = new {
Codec = codec.ToString(),
Level = codec == MessageCodec.None ? "-" : level.ToString(),
Messages = messages,
MessageSize = messageSize,
Bytes = bytes.Count
};
results.Add(result);
}
}
}
}
WriteResults(results);
}
public async Task SendAsyncShouldUseStatictVersionInfo()
{
IRequestContext context = null;
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.Log))
{
server.OnReceivedAsync = async data => {
context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
};
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
await AssertAsync.ThatEventually(() => context != null && context.ApiVersion.GetValueOrDefault() == 2, () => $"version {context?.ApiVersion}");
}
}
public async Task SendAsyncWithDynamicVersionInfoMakesVersionCallFirst()
{
var firstCorrelation = -1;
var correlationId = 0;
var sentVersion = (short)-1;
var endpoint = TestConfig.ServerEndpoint();
using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(3), versionSupport: VersionSupport.Kafka8.Dynamic()), log: TestConfig.Log))
{
var apiVersion = (short)3;
server.OnReceivedAsync = async data => {
var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
if (firstCorrelation < 0)
{
firstCorrelation = context.CorrelationId;
}
correlationId = context.CorrelationId;
switch (correlationId - firstCorrelation)
{
case 0:
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new ApiVersionsResponse(ErrorCode.NONE, new[] { new ApiVersionsResponse.VersionSupport(ApiKey.Fetch, apiVersion, apiVersion) })));
break;
case 1:
sentVersion = context.ApiVersion.GetValueOrDefault();
await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
break;
default:
return;
}
};
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
await AssertAsync.ThatEventually(() => correlationId - firstCorrelation >= 1, () => $"first {firstCorrelation}, current {correlationId}");
Assert.That(sentVersion, Is.EqualTo(apiVersion));
}
}
public async Task SendAsyncShouldUseStatictVersionInfo()
{
IRequestContext context = null;
var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);
using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog))
using (var conn = new Connection(socket, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.InfoLog))
{
server.OnBytesReceived += data =>
{
context = KafkaDecoder.DecodeHeader(data);
var send = server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
};
await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
await TaskTest.WaitFor(() => context != null);
Assert.That(context.ApiVersion.Value, Is.EqualTo(2));
}
}