public void DecodeMessageShouldThrowWhenCrcFails()
{
var testMessage = new Message(value: "kafka test message.", key: "test");
var encoded = Message.EncodeMessage(testMessage);
encoded[0] += 1;
var result = Message.DecodeMessage(0, encoded).First();
}
public void EnsureMessageEncodeAndDecodeAreCompatible(string key, string value)
{
var testMessage = new Message(key: key, value: value);
var encoded = Message.EncodeMessage(testMessage);
var result = Message.DecodeMessage(0, encoded).First();
Assert.That(testMessage.Key, Is.EqualTo(result.Key));
Assert.That(testMessage.Value, Is.EqualTo(result.Value));
}
public void DecodeMessageShouldThrowWhenCrcFails()
{
var testMessage = new Message
{
Key = "test",
Value = "kafka test message."
};
var encoded = Message.EncodeMessage(testMessage);
encoded[0] += 1;
var result = Message.DecodeMessage(0, encoded).First();
}
private void SendMessageToQueue(string message)
{
try
{
var queueMessage = new Message(message);
this.GetProducer().SendMessageAsync(topic, new[] { queueMessage }, 1);
}
catch (Exception ex)
{
InternalLogger.Error("Unable to send message to kafka queue", ex);
}
}
private async void Process()
{
var counter = 1000;
while (true)
{
Thread.Sleep(100);
var msg = new Message($"message {counter++}");
msg.Key = BitConverter.GetBytes((ulong)(DateTime.UtcNow - DateTime.MinValue).TotalMilliseconds);
var resp = await _producer.SendMessageAsync("test", new List<Message> {msg});
resp.ForEach(x =>
{
Console.WriteLine($"produce - {Encoding.UTF8.GetString(msg.Value)}, partition - {x.PartitionId}, offset - {x.Offset}");
});
}
}
/// <summary>
/// Decode messages from a payload and assign it a given kafka offset.
/// </summary>
/// <param name="offset">The offset represting the log entry from kafka of this message.</param>
/// <param name="payload">The byte[] encode as a message from kafka.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[].</returns>
/// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
public static IEnumerable<Message> DecodeMessage(long offset, byte[] payload)
{
var crc = payload.Take(4).ToArray();
using (var stream = new BigEndianBinaryReader(payload.Skip(4)))
{
if (crc.SequenceEqual(stream.CrcHash()) == false)
throw new FailCrcCheckException("Buffer did not match CRC validation.");
var message = new Message
{
Meta = new MessageMetadata { Offset = offset },
MagicNumber = stream.ReadByte(),
Attribute = stream.ReadByte(),
Key = stream.ReadIntPrefixedBytes()
};
var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute);
switch (codec)
{
case MessageCodec.CodecNone:
message.Value = stream.ReadIntPrefixedBytes();
yield return message;
break;
case MessageCodec.CodecGzip:
var gZipData = stream.ReadIntPrefixedBytes();
foreach (var m in DecodeMessageSet(Compression.Unzip(gZipData)))
{
yield return m;
}
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec));
}
}
}
/// <summary>
/// Encodes a message object to byte[]
/// </summary>
/// <param name="message">Message data to encode.</param>
/// <returns>Encoded byte[] representation of the message object.</returns>
/// <remarks>
/// Format:
/// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[])
/// </remarks>
public static byte[] EncodeMessage(Message message)
{
using(var stream = new KafkaMessagePacker())
{
return stream.Pack(message.MagicNumber)
.Pack(message.Attribute)
.Pack(message.Key)
.Pack(message.Value)
.CrcPayload();
}
}
/// <summary>
/// Encodes a message object to byte[]
/// </summary>
/// <param name="message">Message data to encode.</param>
/// <returns>Encoded byte[] representation of the message object.</returns>
/// <remarks>
/// Format:
/// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[])
/// </remarks>
public static byte[] EncodeMessage(Message message)
{
var body = new WriteByteStream();
body.Pack(new[] { message.MagicNumber },
new[] { message.Attribute },
message.Key.ToIntSizedBytes(),
message.BinaryValue.Length.ToBytes(),
message.BinaryValue);
var crc = Crc32.ComputeHash(body.Payload());
body.Prepend(crc);
return body.Payload();
}
public async Task<ProduceResponse> SendMessageAsync(Message messages, string topic, int partition, Int16 acks = 1)
{
var result = await SendMessageAsync(topic, new Message[] { messages }, partition: partition, acks: acks);
return result.FirstOrDefault();
}
public void AddDocument(string document)
{
Message = new Message(Key, document);
}
