private static void EncodeFetch(KafkaEncoder encoder, Fetch fetch) { encoder .Write(fetch.PartitionId) .Write(fetch.Offset) .Write(fetch.MaxBytes); }
public void Int16Tests(Int16 number, Byte[] expectedBytes) { var buffer = new byte[2]; var encoder = new KafkaEncoder(buffer); encoder.Write(number); Assert.That(buffer, Is.EqualTo(expectedBytes)); }
internal KafkaEncoder EncodeHeader(KafkaEncoder encoder) { return(encoder .Write((Int16)apiKey) .Write(ApiVersion) .Write(CorrelationId) .Write(ClientId)); }
public void StringTests(String value, Byte[] expectedBytes) { var buffer = new byte[expectedBytes.Length]; var encoder = new KafkaEncoder(buffer); encoder.Write(value); Assert.That(encoder.Offset, Is.EqualTo(expectedBytes.Length)); Assert.That(buffer, Is.EqualTo(expectedBytes)); }
public void EnsureHeaderShouldPackCorrectByteLengths() { var result = KafkaEncoder.EncodeRequestBytes(new RequestContext(123456789, clientId: "test"), new ApiVersionsRequest()); var withoutLength = new byte[result.Length - 4]; Buffer.BlockCopy(result, 4, withoutLength, 0, result.Length - 4); Assert.That(withoutLength.Length, Is.EqualTo(14)); Assert.That(withoutLength, Is.EqualTo(new byte[] { 0, 18, 0, 0, 7, 91, 205, 21, 0, 4, 116, 101, 115, 116 })); }
/// <summary> /// Encodes a collection of messages into one byte[]. Encoded in order of list. /// </summary> /// <param name="messages">The collection of messages to encode together.</param> /// <returns>Encoded byte[] representing the collection of messages.</returns> internal static KafkaEncoder EncodeMessageSet(KafkaEncoder encoder, IEnumerable <Message> messages) { foreach (var message in messages) { encoder.Write(InitialMessageOffset); var marker = encoder.PrepareForLength(); EncodeMessage(message, encoder) .WriteLength(marker); } return(encoder); }
public void EnsureHeaderShouldPackCorrectByteLengths() { var encoder = new KafkaEncoder(new byte[14]); var request = new FetchRequest { ClientId = "test", CorrelationId = 123456789 }; request.EncodeHeader(encoder); Assert.That(encoder.Offset, Is.EqualTo(14)); Assert.That(encoder.Buffer, Is.EqualTo(new byte[] { 0, 1, 0, 0, 7, 91, 205, 21, 0, 4, 116, 101, 115, 116 })); }
/// <summary> /// Encodes a collection of messages into one byte[]. Encoded in order of list. /// </summary> /// <param name="messages">The collection of messages to encode together.</param> /// <returns>Encoded byte[] representing the collection of messages.</returns> internal static KafkaEncoder EncodeMessageSet(KafkaEncoder encoder, IEnumerable<Message> messages) { foreach (var message in messages) { encoder.Write(InitialMessageOffset); var marker = encoder.PrepareForLength(); EncodeMessage(message, encoder) .WriteLength(marker); } return encoder; }
private static void EncodeCommit(KafkaEncoder encoder, int apiVersion, OffsetCommit commit) { encoder .Write(commit.PartitionId) .Write(commit.Offset); if (apiVersion == 1) { encoder.Write(commit.TimeStamp); } encoder.Write(commit.Metadata); }
private static KafkaEncoder EncodeOffsetRequest(OffsetRequest request, KafkaEncoder encoder) { request .EncodeHeader(encoder) .Write(ReplicaId); if (request.Offsets == null) { encoder.Write(0); } else if (request.Offsets.Count == 1) { // shortcut the single request var offset = request.Offsets[0]; encoder .Write(1) .Write(offset.Topic) .Write(1) .Write(offset.PartitionId) .Write(offset.Time) .Write(offset.MaxOffsets); } else { // Full request var topicGroups = new Dictionary <string, List <Offset> >(); foreach (var offset in request.Offsets) { var offsets = topicGroups.GetOrCreate(offset.Topic, () => new List <Offset>(request.Offsets.Count)); offsets.Add(offset); } encoder.Write(topicGroups.Count); foreach (var kvp in topicGroups) { var topic = kvp.Key; var offsets = kvp.Value; encoder .Write(topic) .Write(offsets.Count); foreach (var offset in offsets) { encoder .Write(offset.PartitionId) .Write(offset.Time) .Write(offset.MaxOffsets); } } } return(encoder); }
/// <summary> /// Encodes a message object to byte[] /// </summary> /// <param name="message">Message data to encode.</param> /// <returns>Encoded byte[] representation of the message object.</returns> /// <remarks> /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// </remarks> internal static KafkaEncoder EncodeMessage(Message message, KafkaEncoder encoder) { var marker = encoder.PrepareForCrc(); encoder .Write(message.MagicNumber) .Write(message.Attribute) .Write(message.Key) .Write(message.Value) .CalculateCrc(marker); return(encoder); }
private static KafkaEncoder EncodeFetchRequest(FetchRequest request, KafkaEncoder encoder) { request .EncodeHeader(encoder) .Write(ReplicaId) .Write(request.MaxWaitTime) .Write(request.MinBytes); if (request.Fetches == null) { // no topics encoder.Write(0); } else if (request.Fetches.Count == 1) { // single topic/partition - quick mode var fetch = request.Fetches[0]; encoder .Write(1) .Write(fetch.Topic) .Write(1); EncodeFetch(encoder, fetch); } else { // Multiple topics/partitions - slower mode var topicGroups = new Dictionary <string, List <Fetch> >(); foreach (var fetch in request.Fetches) { var fetchList = topicGroups.GetOrCreate(fetch.Topic, () => new List <Fetch>(request.Fetches.Count)); fetchList.Add(fetch); } encoder.Write(topicGroups.Count); foreach (var topicGroupKvp in topicGroups) { var topicGroup = topicGroupKvp.Key; var fetches = topicGroupKvp.Value; encoder .Write(topicGroup) .Write(fetches.Count); foreach (var fetch in fetches) { EncodeFetch(encoder, fetch); } } } return(encoder); }
private static KafkaEncoder EncodeProduceRequest(ProduceRequest request, KafkaEncoder encoder) { request.EncodeHeader(encoder) .Write(request.Acks) .Write(request.TimeoutMS); if (request.Payload == null) { encoder.Write(0); } else if (request.Payload.Count == 1) { // Short cut single request var payload = request.Payload[0]; encoder .Write(1) .Write(payload.Topic) .Write(1); WritePayload(encoder, payload); } else { // More complex var topicGroups = new Dictionary <string, List <Payload> >(); foreach (var payload in request.Payload) { var payloads = topicGroups.GetOrCreate(payload.Topic, () => new List <Payload>(request.Payload.Count)); payloads.Add(payload); } encoder.Write(topicGroups.Count); foreach (var kvp in topicGroups) { var topic = kvp.Key; var payloads = kvp.Value; encoder .Write(topic) .Write(payloads.Count); foreach (var payload in payloads) { WritePayload(encoder, payload); } } } return(encoder); }
public void DecodeMessageShouldThrowWhenCrcFails() { Assert.Throws(Is.TypeOf <FailCrcCheckException>(), () => { var testMessage = new Message(value: "kafka test message.", key: "test"); var buffer = new byte[1024]; var encoder = new KafkaEncoder(buffer); Message.EncodeMessage(testMessage, encoder); buffer[0] += 1; var decoder = new KafkaDecoder(buffer, 0, encoder.Offset); var result = Message.DecodeMessage(0, 0, decoder, encoder.Offset); }); }
public void EnsureMessageEncodeAndDecodeAreCompatible(string key, string value) { var testMessage = new Message(key: key, value: value); var buffer = new byte[1024]; var encoder = new KafkaEncoder(buffer); Message.EncodeMessage(testMessage, encoder); var decoder = new KafkaDecoder(buffer); var result = Message.DecodeMessage(0, 0, decoder, encoder.Offset); Assert.That(testMessage.Key, Is.EqualTo(result.Key)); Assert.That(testMessage.Value, Is.EqualTo(result.Value)); }
/// <summary> /// Send kafka payload to server and receive a task event when response is received. /// </summary> /// <typeparam name="T">A Kafka response object return by decode function.</typeparam> /// <param name="request">The IRequest to send to the kafka servers.</param> /// <param name="context">The context for the request.</param> /// <param name="token">Cancellation token used to cancel the transfer.</param> /// <returns></returns> public async Task <T> SendAsync <T>(IRequest <T> request, CancellationToken token, IRequestContext context = null) where T : class, IResponse { var version = context?.ApiVersion; if (!version.HasValue) { version = await GetVersionAsync(request.ApiKey, token).ConfigureAwait(false); } context = new RequestContext(NextCorrelationId(), version, context?.ClientId, context?.Encoders ?? _configuration.Encoders, context?.ProtocolType ?? request.ProtocolType, context?.OnProduceRequestMessages ?? _configuration.OnProduceRequestMessages); var payload = KafkaEncoder.Encode(context, request); _log.Info(() => LogEvent.Create($"Sending {request.ApiKey} with correlation id {context.CorrelationId} (v {version.GetValueOrDefault()}, {payload.Buffer.Length} bytes) to {Endpoint}")); _log.Debug(() => LogEvent.Create($"-----> {request.ApiKey} with correlation id {context.CorrelationId} to {Endpoint}\n{request.ToFormattedString()}")); if (!request.ExpectResponse) { await _socket.WriteAsync(payload, token).ConfigureAwait(false); return(default(T)); } using (var asyncRequest = new AsyncRequestItem(context.CorrelationId, request.ApiKey, _configuration.RequestTimeout)) { try { AddAsyncRequestItemToResponseQueue(asyncRequest); ExceptionDispatchInfo exceptionDispatchInfo = null; try { await _socket.WriteAsync(payload, token).ConfigureAwait(false); } catch (Exception ex) { exceptionDispatchInfo = ExceptionDispatchInfo.Capture(ex); } asyncRequest.MarkRequestAsSent(exceptionDispatchInfo, TriggerMessageTimeout); } catch (OperationCanceledException) { TriggerMessageTimeout(asyncRequest); } var response = await asyncRequest.ReceiveTask.Task.ThrowIfCancellationRequested(token).ConfigureAwait(false); _log.Info(() => LogEvent.Create($"Receiving {request.ApiKey} with correlation id {context.CorrelationId} (v {version.GetValueOrDefault()}, {response.Length} bytes) from {Endpoint}")); var result = KafkaEncoder.Decode <T>(context, response); _log.Debug(() => LogEvent.Create($"<------- {request.ApiKey} with correlation id {context.CorrelationId} from {Endpoint}\n{result.ToFormattedString()}")); return(result); } }
private static void WritePayload(KafkaEncoder encoder, Payload payload) { encoder .Write(payload.Partition); var marker = encoder.PrepareForLength(); switch (payload.Codec) { case MessageCodec.CodecNone: Message.EncodeMessageSet(encoder, payload.Messages); break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", payload.Codec)); } encoder.WriteLength(marker); }
public void WhenMessageIsTruncatedThenBufferUnderRunExceptionIsThrown() { Assert.Throws <BufferUnderRunException>(() => { // arrange var offset = (Int64)0; var message = new Byte[] { }; var messageSize = 5; var payloadBytes = new byte[16]; var encoder = new KafkaEncoder(payloadBytes); encoder.Write(offset); encoder.Write(messageSize); encoder.Write(message); var decoder = new KafkaDecoder(payloadBytes); Message.DecodeMessageSet(0, decoder, payloadBytes.Length); }); }
private static KafkaEncoder EncodeMetadataRequest(MetadataRequest request, KafkaEncoder encoder) { request .EncodeHeader(encoder); if (request.Topics == null) { encoder.Write(0); } else { encoder.Write(request.Topics.Count); foreach (var topic in request.Topics) { encoder.Write(topic); } } return(encoder); }
public static void AssertCanEncodeDecodeResponse <T>(this T response, short version, IProtocolTypeEncoder encoder = null) where T : class, IResponse { var encoders = ImmutableDictionary <string, IProtocolTypeEncoder> .Empty; if (encoder != null) { encoders = encoders.Add(encoder.Type, encoder); } var context = new RequestContext(16, version, "Test-Response", encoders, encoder?.Type); var data = KafkaDecoder.EncodeResponseBytes(context, response); var decoded = KafkaEncoder.Decode <T>(context, data, true); if (!response.Equals(decoded)) { var original = response.ToFormattedString(); var final = decoded.ToFormattedString(); Console.WriteLine($"Original\n{original}\nFinal\n{final}"); Assert.That(final, Is.EqualTo(original)); Assert.Fail("Not equal, although strings suggest they are?"); } }
public void EncodeMessageSetEncodesMultipleMessages() { //expected generated from python library var expected = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 45, 70, 24, 62, 0, 0, 0, 0, 0, 1, 49, 0, 0, 0, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 90, 65, 40, 168, 0, 0, 0, 0, 0, 1, 49, 0, 0, 0, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 195, 72, 121, 18, 0, 0, 0, 0, 0, 1, 49, 0, 0, 0, 1, 50 }; var messages = new[] { new Message("0", "1"), new Message("1", "1"), new Message("2", "1") }; var buffer = new byte[expected.Length]; var encoder = new KafkaEncoder(buffer); Message.EncodeMessageSet(encoder, messages); Assert.That(buffer, Is.EqualTo(expected)); }
internal override KafkaEncoder Encode(KafkaEncoder encoder) { return(EncodeFetchRequest(this, encoder)); }
/// <summary> /// Encodes a message object to byte[] /// </summary> /// <param name="message">Message data to encode.</param> /// <returns>Encoded byte[] representation of the message object.</returns> /// <remarks> /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// </remarks> internal static KafkaEncoder EncodeMessage(Message message, KafkaEncoder encoder) { var marker = encoder.PrepareForCrc(); encoder .Write(message.MagicNumber) .Write(message.Attribute) .Write(message.Key) .Write(message.Value) .CalculateCrc(marker); return encoder; }
/// <summary> /// Encode this request into the Kafka wire protocol. /// </summary> /// <param name="encoder">Encoder to use</param> internal abstract KafkaEncoder Encode(KafkaEncoder encoder);
private static KafkaEncoder EncodeOffsetFetchRequest(OffsetFetchRequest request, KafkaEncoder encoder) { request .EncodeHeader(encoder) .Write(request.ConsumerGroup); if (request.Topics == null) { // nothing to fetch encoder.Write(0); } else if (request.Topics.Count == 1) { // Short cut single instance request var fetch = request.Topics[0]; encoder .Write(1) .Write(fetch.Topic) .Write(1) .Write(fetch.PartitionId); } else { // more complex var topicGroups = new Dictionary <string, List <int> >(); foreach (var fetch in request.Topics) { var partitions = topicGroups.GetOrCreate(fetch.Topic, () => new List <int>(request.Topics.Count)); partitions.Add(fetch.PartitionId); } encoder.Write(topicGroups.Count); foreach (var kvp in topicGroups) { var topic = kvp.Key; var partitions = kvp.Value; encoder .Write(topic) .Write(partitions.Count); foreach (var fetch in partitions) { encoder.Write(fetch); } } } return(encoder); }
private static KafkaEncoder EncodeConsumerMetadataRequest(ConsumerMetadataRequest request, KafkaEncoder encoder) { return (request .EncodeHeader(encoder) .Write(request.ConsumerGroup)); }
internal override KafkaEncoder Encode(KafkaEncoder encoder) { return(EncodeConsumerMetadataRequest(this, encoder)); }
internal override KafkaEncoder Encode(KafkaEncoder encoder) { return(EncodeProduceRequest(this, encoder)); }
public void MetadataResponseShouldDecode() { var response = KafkaEncoder.Decode <MetadataResponse>(new RequestContext(1), MessageHelper.CreateMetadataResponse(1, "Test").Skip(4).ToArray()); Assert.That(response.Topics[0].TopicName, Is.EqualTo("Test")); }
private static KafkaEncoder EncodeOffsetCommitRequest(OffsetCommitRequest request, KafkaEncoder encoder) { request .EncodeHeader(encoder) .Write(request.ConsumerGroup); if (request.ApiVersion == 1) { encoder .Write(request.ConsumerGroupGenerationId) .Write(request.ConsumerId); } if (request.OffsetCommits == null) { // Nothing to commit encoder.Write(0); } else if (request.OffsetCommits.Count == 1) { var commit = request.OffsetCommits[0]; // Shortcut the single version encoder .Write(1) .Write(commit.Topic) .Write(1); EncodeCommit(encoder, request.ApiVersion, commit); } else { // Complete complex request var topicGroups = new Dictionary <string, List <OffsetCommit> >(); foreach (var commit in request.OffsetCommits) { var topicGroup = topicGroups.GetOrCreate(commit.Topic, () => new List <OffsetCommit>(request.OffsetCommits.Count)); topicGroup.Add(commit); } encoder.Write(topicGroups.Count); foreach (var topicGroupKvp in topicGroups) { var topic = topicGroupKvp.Key; var commits = topicGroupKvp.Value; encoder .Write(topic) .Write(commits.Count); foreach (var commit in commits) { EncodeCommit(encoder, request.ApiVersion, commit); } } } return(encoder); }
internal override KafkaEncoder Encode(KafkaEncoder encoder) { return(EncodeOffsetCommitRequest(this, encoder)); }