/// <summary>
/// Decode a byte[] that represents a collection of messages.
/// </summary>
/// <param name="incoming">The byte[] encode as a message set from kafka.</param>
/// <param name="partition">The partition to decode the message for.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
public static IEnumerable <Message> DecodeMessages(byte[] incoming, int readOffset, int readSize, int partition)
{
var reader = new BinaryReader(incoming, readOffset, readSize);
while (reader.HasData)
{
// If the message set hits against our max bytes wall on the fetch we will have a
// 1/2 completed message downloaded. The decode should guard against this situation
if (reader.Available(MinimumMessageSize) == false)
{
yield break;
}
// We read the offset of the message and the size and wait until we actually have
// enough data to process the whole message.
var offset = reader.ReadInt64();
var messageSize = reader.ReadInt32();
if (reader.Available(messageSize) == false)
{
yield break;
}
// We now compute the hash and compare it with the checksum we received within
// the message itself.
var crc = BinaryCrc32.ComputeHash(incoming, reader.Position + 4, messageSize - 4);
if (crc[0] != reader.ReadByte() ||
crc[1] != reader.ReadByte() ||
crc[2] != reader.ReadByte() ||
crc[3] != reader.ReadByte())
{
throw new FailCrcCheckException("CRC validation mismatch during message decoding.");
}
var message = new Message
{
Offset = offset,
MagicNumber = reader.ReadByte(),
Attribute = reader.ReadByte(),
Key = reader.ReadInt32Array(),
PartitionId = partition
};
// We don't support any codecs yet
var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute);
if (codec != MessageCodec.CodecNone)
{
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec));
}
// Value is int32 prefixed byte-array
message.Value = reader.ReadInt32Array();
yield return(message);
}
}
/// <summary>
/// Writes a 4-byte placeholder and returns current position
/// </summary>
public void WriteCrcAt(int offset)
{
// Remember current position
var position = (int)this.Position;
// Calculate the length & CRC
var count = position - offset - 4;
var crc = BinaryCrc32.ComputeHash(this.GetBuffer(), offset + 4, count);
// Set the position
this.Position = offset;
// Write the value
this.Stream.Write(crc, 0, 4);
// Set the position back
this.Position = position;
}
public Partition Select(Topic topic, byte[] key)
{
if (topic == null)
{
throw new ArgumentNullException("topic");
}
if (topic.Partitions.Count <= 0)
{
throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));
}
//use round robin
var partitions = topic.Partitions;
if (key == null)
{
//use round robin
var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) =>
{
return((i + 1) % partitions.Count);
});
return(partitions[paritionIndex]);
}
//use key hash
var partitionId = BinaryCrc32.Compute(key) % partitions.Count;
var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition == null)
{
throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));
}
return(partition);
}