public static void AssertCanEncodeDecodeResponse <T>(this T response, short version, IMembershipEncoder encoder = null, T forComparison = null) where T : class, IResponse { var encoders = ImmutableDictionary <string, IMembershipEncoder> .Empty; if (encoder != null) { encoders = encoders.Add(encoder.ProtocolType, encoder); } var context = new RequestContext(16, version, "Test-Response", encoders, encoder?.ProtocolType); var data = KafkaDecoder.EncodeResponseBytes(context, response); var decoded = GetType <T>().ToResponse(context, data.Skip(Request.IntegerByteSize + Request.CorrelationSize)); if (forComparison == null) { forComparison = response; } Assert.That(forComparison.GetHashCode(), Is.EqualTo(decoded.GetHashCode()), "HashCode equality"); var original = forComparison.ToString(); var final = decoded.ToString(); Assert.That(original, Is.EqualTo(final), "ToString equality"); Assert.That(decoded.Equals(final), Is.False); // general test for equality Assert.That(decoded.Equals(decoded), Is.True); // general equality test for sanity Assert.That(forComparison.Equals(decoded), $"Original\n{original}\nFinal\n{final}"); Assert.That(forComparison.Errors.HasEqualElementsInOrder(decoded.Errors), "Errors"); }
public void SetupData() { Common.Compression.ZipLevel = Level; _request = new ProduceRequest( Enumerable.Range(1, Partitions) .Select(partitionId => new ProduceRequest.Topic( "topic", partitionId, Enumerable.Range(1, Messages) .Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion)), Codec))); var response = new ProduceResponse(new ProduceResponse.Topic("topic", 1, ErrorCode.NONE, 0)); var port = 10000; var endpoint = new Endpoint(new IPEndPoint(IPAddress.Loopback, port), "localhost"); _server = new TcpServer(endpoint.Ip.Port) { OnReceivedAsync = async data => { var header = KafkaDecoder.DecodeHeader(data.Skip(4)); var bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(header.CorrelationId), response); await _server.SendDataAsync(bytes); } }; _connection = new Connection(endpoint); }
public async Task MessagesStillLogWhenSendTimesOut() { var logger = new MemoryLog(); var received = 0; var timeout = TimeSpan.FromMilliseconds(100); var endpoint = TestConfig.ServerEndpoint(); using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log)) using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: timeout, onRead: (e, read, elapsed) => Interlocked.Increment(ref received)), logger)) { await Task.WhenAny(server.ClientConnected, Task.Delay(TimeSpan.FromSeconds(3))); server.OnReceivedAsync = async data => { var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize)); await Task.Delay(timeout); await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new MetadataResponse())); }; await AssertAsync.Throws <TimeoutException>(() => conn.SendAsync(new MetadataRequest(), CancellationToken.None)); await AssertAsync.ThatEventually(() => received > 0, () => $"received {received}"); await AssertAsync.ThatEventually(() => logger.LogEvents.Any(e => e.Item1 == LogLevel.Debug && e.Item2.Message.StartsWith("Timed out -----> (timed out or otherwise errored in client)")), () => logger.ToString(LogLevel.Debug)); } }
public async Task SendAsyncWithDynamicVersionInfoOnlyMakesVersionCallOnce() { var versionRequests = 0; var endpoint = TestConfig.ServerEndpoint(); using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log)) using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(3), versionSupport: VersionSupport.Kafka8.Dynamic()), log: TestConfig.Log)) { server.OnReceivedAsync = async data => { var fullHeader = KafkaDecoder.DecodeFullHeader(data.Skip(Request.IntegerByteSize)); var context = fullHeader.Item1; switch (fullHeader.Item2) { case ApiKey.ApiVersions: Interlocked.Increment(ref versionRequests); await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new ApiVersionsResponse(ErrorCode.NONE, new[] { new ApiVersionsResponse.VersionSupport(ApiKey.Fetch, 3, 3) }))); break; default: await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse())); break; } }; for (var i = 0; i < 3; i++) { await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None); } Assert.That(versionRequests, Is.EqualTo(1)); } }
public void SetupData() { Common.Compression.ZipLevel = Level; var response = new FetchResponse( Enumerable.Range(1, Partitions) .Select(partitionId => new FetchResponse.Topic( "topic", partitionId, 500, ErrorCode.NONE, Enumerable.Range(1, Messages) .Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion)) ))); _bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(1, Version), response); }
public void FetchSize() { int partitions = 1; short version = 0; byte messageVersion = 0; var results = new List <object>(); foreach (var codec in new[] { MessageCodec.None, MessageCodec.Gzip, MessageCodec.Snappy }) { foreach (var messages in new[] { 100, 10000 }) { foreach (var messageSize in new[] { 1, 1000 }) { foreach (var level in new[] { CompressionLevel.Fastest }) { Compression.ZipLevel = level; var response = new FetchResponse( Enumerable.Range(1, partitions) .Select(partitionId => new FetchResponse.Topic( "topic", partitionId, 500, ErrorCode.NONE, Enumerable.Range(1, messages) .Select(i => new Message(GenerateMessageBytes(messageSize), new ArraySegment <byte>(), (byte)codec, version: messageVersion)) ))); var bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(1, version), response); var decoded = FetchResponse.FromBytes(new RequestContext(1, version), bytes.Skip(Request.IntegerByteSize + Request.CorrelationSize)); Assert.That(decoded.responses.Sum(t => t.Messages.Count), Is.EqualTo(response.responses.Sum(t => t.Messages.Count))); var result = new { Codec = codec.ToString(), Level = codec == MessageCodec.None ? "-" : level.ToString(), Messages = messages, MessageSize = messageSize, Bytes = bytes.Count }; results.Add(result); } } } } WriteResults(results); }
public async Task SendAsyncShouldUseStatictVersionInfo() { IRequestContext context = null; var endpoint = TestConfig.ServerEndpoint(); using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log)) using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.Log)) { server.OnReceivedAsync = async data => { context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize)); await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse())); }; await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None); await AssertAsync.ThatEventually(() => context != null && context.ApiVersion.GetValueOrDefault() == 2, () => $"version {context?.ApiVersion}"); } }
public async Task SendAsyncWithDynamicVersionInfoMakesVersionCallFirst() { var firstCorrelation = -1; var correlationId = 0; var sentVersion = (short)-1; var endpoint = TestConfig.ServerEndpoint(); using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log)) using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(3), versionSupport: VersionSupport.Kafka8.Dynamic()), log: TestConfig.Log)) { var apiVersion = (short)3; server.OnReceivedAsync = async data => { var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize)); if (firstCorrelation < 0) { firstCorrelation = context.CorrelationId; } correlationId = context.CorrelationId; switch (correlationId - firstCorrelation) { case 0: await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new ApiVersionsResponse(ErrorCode.NONE, new[] { new ApiVersionsResponse.VersionSupport(ApiKey.Fetch, apiVersion, apiVersion) }))); break; case 1: sentVersion = context.ApiVersion.GetValueOrDefault(); await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse())); break; default: return; } }; await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None); await AssertAsync.ThatEventually(() => correlationId - firstCorrelation >= 1, () => $"first {firstCorrelation}, current {correlationId}"); Assert.That(sentVersion, Is.EqualTo(apiVersion)); } }
public async Task SendAsyncShouldUseStatictVersionInfo() { IRequestContext context = null; var endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog); using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port)) using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog)) using (var conn = new Connection(socket, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.InfoLog)) { server.OnBytesReceived += data => { context = KafkaDecoder.DecodeHeader(data); var send = server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse())); }; await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None); await TaskTest.WaitFor(() => context != null); Assert.That(context.ApiVersion.Value, Is.EqualTo(2)); } }