public void DecodeMessageShouldThrowWhenCrcFails() { var testMessage = new Message(value: "kafka test message.", key: "test"); var encoded = Message.EncodeMessage(testMessage); encoded[0] += 1; var result = Message.DecodeMessage(0, encoded).First(); }
public void EnsureMessageEncodeAndDecodeAreCompatible(string key, string value) { var testMessage = new Message(key: key, value: value); var encoded = Message.EncodeMessage(testMessage); var result = Message.DecodeMessage(0, encoded).First(); Assert.That(testMessage.Key, Is.EqualTo(result.Key)); Assert.That(testMessage.Value, Is.EqualTo(result.Value)); }
public void DecodeMessageShouldThrowWhenCrcFails() { var testMessage = new Message { Key = "test", Value = "kafka test message." }; var encoded = Message.EncodeMessage(testMessage); encoded[0] += 1; var result = Message.DecodeMessage(0, encoded).First(); }
private void SendMessageToQueue(string message) { try { var queueMessage = new Message(message); this.GetProducer().SendMessageAsync(topic, new[] { queueMessage }, 1); } catch (Exception ex) { InternalLogger.Error("Unable to send message to kafka queue", ex); } }
private async void Process() { var counter = 1000; while (true) { Thread.Sleep(100); var msg = new Message($"message {counter++}"); msg.Key = BitConverter.GetBytes((ulong)(DateTime.UtcNow - DateTime.MinValue).TotalMilliseconds); var resp = await _producer.SendMessageAsync("test", new List<Message> {msg}); resp.ForEach(x => { Console.WriteLine($"produce - {Encoding.UTF8.GetString(msg.Value)}, partition - {x.PartitionId}, offset - {x.Offset}"); }); } }
/// <summary> /// Decode messages from a payload and assign it a given kafka offset. /// </summary> /// <param name="offset">The offset represting the log entry from kafka of this message.</param> /// <param name="payload">The byte[] encode as a message from kafka.</param> /// <returns>Enumerable representing stream of messages decoded from byte[].</returns> /// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks> public static IEnumerable<Message> DecodeMessage(long offset, byte[] payload) { var crc = payload.Take(4).ToArray(); using (var stream = new BigEndianBinaryReader(payload.Skip(4))) { if (crc.SequenceEqual(stream.CrcHash()) == false) throw new FailCrcCheckException("Buffer did not match CRC validation."); var message = new Message { Meta = new MessageMetadata { Offset = offset }, MagicNumber = stream.ReadByte(), Attribute = stream.ReadByte(), Key = stream.ReadIntPrefixedBytes() }; var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute); switch (codec) { case MessageCodec.CodecNone: message.Value = stream.ReadIntPrefixedBytes(); yield return message; break; case MessageCodec.CodecGzip: var gZipData = stream.ReadIntPrefixedBytes(); foreach (var m in DecodeMessageSet(Compression.Unzip(gZipData))) { yield return m; } break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec)); } } }
/// <summary> /// Encodes a message object to byte[] /// </summary> /// <param name="message">Message data to encode.</param> /// <returns>Encoded byte[] representation of the message object.</returns> /// <remarks> /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// </remarks> public static byte[] EncodeMessage(Message message) { using(var stream = new KafkaMessagePacker()) { return stream.Pack(message.MagicNumber) .Pack(message.Attribute) .Pack(message.Key) .Pack(message.Value) .CrcPayload(); } }
/// <summary> /// Encodes a message object to byte[] /// </summary> /// <param name="message">Message data to encode.</param> /// <returns>Encoded byte[] representation of the message object.</returns> /// <remarks> /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// </remarks> public static byte[] EncodeMessage(Message message) { var body = new WriteByteStream(); body.Pack(new[] { message.MagicNumber }, new[] { message.Attribute }, message.Key.ToIntSizedBytes(), message.BinaryValue.Length.ToBytes(), message.BinaryValue); var crc = Crc32.ComputeHash(body.Payload()); body.Prepend(crc); return body.Payload(); }
public async Task<ProduceResponse> SendMessageAsync(Message messages, string topic, int partition, Int16 acks = 1) { var result = await SendMessageAsync(topic, new Message[] { messages }, partition: partition, acks: acks); return result.FirstOrDefault(); }
public void AddDocument(string document) { Message = new Message(Key, document); }