Example #1
0
        public static void AssertCanEncodeDecodeResponse <T>(this T response, short version, IMembershipEncoder encoder = null, T forComparison = null) where T : class, IResponse
        {
            var encoders = ImmutableDictionary <string, IMembershipEncoder> .Empty;

            if (encoder != null)
            {
                encoders = encoders.Add(encoder.ProtocolType, encoder);
            }

            var context = new RequestContext(16, version, "Test-Response", encoders, encoder?.ProtocolType);
            var data    = KafkaDecoder.EncodeResponseBytes(context, response);
            var decoded = GetType <T>().ToResponse(context, data.Skip(Request.IntegerByteSize + Request.CorrelationSize));

            if (forComparison == null)
            {
                forComparison = response;
            }
            Assert.That(forComparison.GetHashCode(), Is.EqualTo(decoded.GetHashCode()), "HashCode equality");
            var original = forComparison.ToString();
            var final    = decoded.ToString();

            Assert.That(original, Is.EqualTo(final), "ToString equality");
            Assert.That(decoded.Equals(final), Is.False);  // general test for equality
            Assert.That(decoded.Equals(decoded), Is.True); // general equality test for sanity
            Assert.That(forComparison.Equals(decoded), $"Original\n{original}\nFinal\n{final}");
            Assert.That(forComparison.Errors.HasEqualElementsInOrder(decoded.Errors), "Errors");
        }
        public void SetupData()
        {
            Common.Compression.ZipLevel = Level;
            _request = new ProduceRequest(
                Enumerable.Range(1, Partitions)
                .Select(partitionId => new ProduceRequest.Topic(
                            "topic",
                            partitionId,
                            Enumerable.Range(1, Messages)
                            .Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion)),
                            Codec)));

            var response = new ProduceResponse(new ProduceResponse.Topic("topic", 1, ErrorCode.NONE, 0));

            var port     = 10000;
            var endpoint = new Endpoint(new IPEndPoint(IPAddress.Loopback, port), "localhost");

            _server = new TcpServer(endpoint.Ip.Port)
            {
                OnReceivedAsync = async data => {
                    var header = KafkaDecoder.DecodeHeader(data.Skip(4));
                    var bytes  = KafkaDecoder.EncodeResponseBytes(new RequestContext(header.CorrelationId), response);
                    await _server.SendDataAsync(bytes);
                }
            };
            _connection = new Connection(endpoint);
        }
        public async Task MessagesStillLogWhenSendTimesOut()
        {
            var logger   = new MemoryLog();
            var received = 0;
            var timeout  = TimeSpan.FromMilliseconds(100);

            var endpoint = TestConfig.ServerEndpoint();

            using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
                using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: timeout, onRead: (e, read, elapsed) => Interlocked.Increment(ref received)), logger))
                {
                    await Task.WhenAny(server.ClientConnected, Task.Delay(TimeSpan.FromSeconds(3)));

                    server.OnReceivedAsync = async data => {
                        var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
                        await Task.Delay(timeout);

                        await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new MetadataResponse()));
                    };

                    await AssertAsync.Throws <TimeoutException>(() => conn.SendAsync(new MetadataRequest(), CancellationToken.None));

                    await AssertAsync.ThatEventually(() => received > 0, () => $"received {received}");

                    await AssertAsync.ThatEventually(() => logger.LogEvents.Any(e => e.Item1 == LogLevel.Debug && e.Item2.Message.StartsWith("Timed out -----> (timed out or otherwise errored in client)")), () => logger.ToString(LogLevel.Debug));
                }
        }
        public async Task SendAsyncWithDynamicVersionInfoOnlyMakesVersionCallOnce()
        {
            var versionRequests = 0;

            var endpoint = TestConfig.ServerEndpoint();

            using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
                using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(3), versionSupport: VersionSupport.Kafka8.Dynamic()), log: TestConfig.Log))
                {
                    server.OnReceivedAsync = async data => {
                        var fullHeader = KafkaDecoder.DecodeFullHeader(data.Skip(Request.IntegerByteSize));
                        var context    = fullHeader.Item1;
                        switch (fullHeader.Item2)
                        {
                        case ApiKey.ApiVersions:
                            Interlocked.Increment(ref versionRequests);
                            await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new ApiVersionsResponse(ErrorCode.NONE, new[] { new ApiVersionsResponse.VersionSupport(ApiKey.Fetch, 3, 3) })));

                            break;

                        default:
                            await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));

                            break;
                        }
                    };

                    for (var i = 0; i < 3; i++)
                    {
                        await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);
                    }

                    Assert.That(versionRequests, Is.EqualTo(1));
                }
        }
        public void SetupData()
        {
            Common.Compression.ZipLevel = Level;
            var response = new FetchResponse(
                Enumerable.Range(1, Partitions)
                .Select(partitionId => new FetchResponse.Topic(
                            "topic",
                            partitionId,
                            500,
                            ErrorCode.NONE,
                            Enumerable.Range(1, Messages)
                            .Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion))
                            )));

            _bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(1, Version), response);
        }
Example #6
0
        public void FetchSize()
        {
            int   partitions     = 1;
            short version        = 0;
            byte  messageVersion = 0;

            var results = new List <object>();

            foreach (var codec in new[] { MessageCodec.None, MessageCodec.Gzip, MessageCodec.Snappy })
            {
                foreach (var messages in new[] { 100, 10000 })
                {
                    foreach (var messageSize in new[] { 1, 1000 })
                    {
                        foreach (var level in new[] { CompressionLevel.Fastest })
                        {
                            Compression.ZipLevel = level;
                            var response = new FetchResponse(
                                Enumerable.Range(1, partitions)
                                .Select(partitionId => new FetchResponse.Topic(
                                            "topic",
                                            partitionId,
                                            500,
                                            ErrorCode.NONE,
                                            Enumerable.Range(1, messages)
                                            .Select(i => new Message(GenerateMessageBytes(messageSize), new ArraySegment <byte>(), (byte)codec, version: messageVersion))
                                            )));
                            var bytes   = KafkaDecoder.EncodeResponseBytes(new RequestContext(1, version), response);
                            var decoded = FetchResponse.FromBytes(new RequestContext(1, version), bytes.Skip(Request.IntegerByteSize + Request.CorrelationSize));
                            Assert.That(decoded.responses.Sum(t => t.Messages.Count), Is.EqualTo(response.responses.Sum(t => t.Messages.Count)));
                            var result = new {
                                Codec       = codec.ToString(),
                                Level       = codec == MessageCodec.None ? "-" : level.ToString(),
                                Messages    = messages,
                                MessageSize = messageSize,
                                Bytes       = bytes.Count
                            };
                            results.Add(result);
                        }
                    }
                }
            }

            WriteResults(results);
        }
        public async Task SendAsyncShouldUseStatictVersionInfo()
        {
            IRequestContext context  = null;
            var             endpoint = TestConfig.ServerEndpoint();

            using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
                using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.Log))
                {
                    server.OnReceivedAsync = async data => {
                        context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
                        await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
                    };

                    await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);

                    await AssertAsync.ThatEventually(() => context != null && context.ApiVersion.GetValueOrDefault() == 2, () => $"version {context?.ApiVersion}");
                }
        }
        public async Task SendAsyncWithDynamicVersionInfoMakesVersionCallFirst()
        {
            var firstCorrelation = -1;
            var correlationId    = 0;
            var sentVersion      = (short)-1;

            var endpoint = TestConfig.ServerEndpoint();

            using (var server = new TcpServer(endpoint.Ip.Port, TestConfig.Log))
                using (var conn = new Connection(endpoint, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(3), versionSupport: VersionSupport.Kafka8.Dynamic()), log: TestConfig.Log))
                {
                    var apiVersion = (short)3;
                    server.OnReceivedAsync = async data => {
                        var context = KafkaDecoder.DecodeHeader(data.Skip(Request.IntegerByteSize));
                        if (firstCorrelation < 0)
                        {
                            firstCorrelation = context.CorrelationId;
                        }
                        correlationId = context.CorrelationId;
                        switch (correlationId - firstCorrelation)
                        {
                        case 0:
                            await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new ApiVersionsResponse(ErrorCode.NONE, new[] { new ApiVersionsResponse.VersionSupport(ApiKey.Fetch, apiVersion, apiVersion) })));

                            break;

                        case 1:
                            sentVersion = context.ApiVersion.GetValueOrDefault();
                            await server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));

                            break;

                        default:
                            return;
                        }
                    };

                    await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);

                    await AssertAsync.ThatEventually(() => correlationId - firstCorrelation >= 1, () => $"first {firstCorrelation}, current {correlationId}");

                    Assert.That(sentVersion, Is.EqualTo(apiVersion));
                }
        }
Example #9
0
        public async Task SendAsyncShouldUseStatictVersionInfo()
        {
            IRequestContext context  = null;
            var             endpoint = Endpoint.Resolve(TestConfig.ServerUri(), TestConfig.InfoLog);

            using (var server = new FakeTcpServer(TestConfig.InfoLog, endpoint.IP.Port))
                using (var socket = new TcpSocket(endpoint, log: TestConfig.InfoLog))
                    using (var conn = new Connection(socket, new ConnectionConfiguration(requestTimeout: TimeSpan.FromSeconds(1000), versionSupport: VersionSupport.Kafka10), log: TestConfig.InfoLog))
                    {
                        server.OnBytesReceived += data =>
                        {
                            context = KafkaDecoder.DecodeHeader(data);
                            var send = server.SendDataAsync(KafkaDecoder.EncodeResponseBytes(context, new FetchResponse()));
                        };

                        await conn.SendAsync(new FetchRequest(new FetchRequest.Topic("Foo", 0, 0)), CancellationToken.None);

                        await TaskTest.WaitFor(() => context != null);

                        Assert.That(context.ApiVersion.Value, Is.EqualTo(2));
                    }
        }