private static Byte[] CreatePayload(Int64 offset, Byte magicByte, Byte attributes, Byte[] key, Byte[] payload)
{
var crcContentStream = new MemoryStream();
var crcContentWriter = new BigEndianBinaryWriter(crcContentStream);
crcContentWriter.Write(magicByte);
crcContentWriter.Write(attributes);
crcContentWriter.Write((Int32)key.Length);
crcContentWriter.Write(key);
crcContentWriter.Write(payload.Length);
crcContentWriter.Write(payload);
var crcContentBytes = crcContentStream.ToArray();
var crc = Crc32Provider.Compute(crcContentBytes);
var messageStream = new MemoryStream();
var messageWriter = new BigEndianBinaryWriter(messageStream);
messageWriter.Write(crc);
messageWriter.Write(crcContentBytes);
var messageBytes = messageStream.ToArray();
var messageSetStream = new MemoryStream();
var messageSetWriter = new BigEndianBinaryWriter(messageSetStream);
var messageSize = messageBytes.Length;
messageSetWriter.Write(offset);
messageSetWriter.Write(messageSize);
messageSetWriter.Write(messageBytes);
return(messageSetStream.ToArray());
}
/// <summary>
/// Decode messages from a payload and assign it a given kafka offset.
/// </summary>
/// <param name="offset">The offset represting the log entry from kafka of this message.</param>
/// <param name="payload">The byte[] encode as a message from kafka.</param>
/// <returns>The message</returns>
/// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
internal static Message DecodeMessage(long offset, int partitionId, KafkaDecoder decoder, int messageSize)
{
var crc = decoder.ReadUInt32();
var calculatedCrc = Crc32Provider.Compute(decoder.Buffer, decoder.Offset, messageSize - 4);
if (calculatedCrc != crc)
{
throw new FailCrcCheckException("Payload did not match CRC validation.");
}
var message = new Message
{
Meta = new MessageMetadata(offset, partitionId),
MagicNumber = decoder.ReadByte(),
Attribute = decoder.ReadByte(),
Key = decoder.ReadBytes(),
};
var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute);
switch (codec)
{
case MessageCodec.CodecNone:
message.Value = decoder.ReadBytes();
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec));
}
return(message);
}
public static Notification FromFile(string filename)
{
var result = new Notification();
result.InflatedData = File.ReadAllText(filename).ToByteArray();
result.Hash = Crc32Provider.ComputeChecksum(result.InflatedData);
result.Deflate();
return(result);
}
public KafkaEncoder CalculateCrc(int crcMarker)
{
var crc = Crc32Provider.Compute(buffer, crcMarker, offset - crcMarker);
var current = offset;
offset = crcMarker - 4;
Write(crc);
offset = current;
return(this);
}
private byte[] CreateKeyForPartition(int partitionId)
{
while (true)
{
var key = Guid.NewGuid().ToString().ToIntSizedBytes();
if ((Crc32Provider.Compute(key) % 2) == partitionId)
{
return(key);
}
}
}
private static void CompileTemplate(Metafile metaFile)
{
using (var stream = new MemoryStream())
{
metaFile.Save(stream);
metaFile.InflatedData = stream.ToArray();
}
metaFile.Hash = Crc32Provider.ComputeChecksum(metaFile.InflatedData);
metaFile.Compress();
}
public static Notification FromFile(string filename)
{
var result = new Notification();
var message = File.ReadAllText(filename);
result.InflatedData = message.ToByteArray();
result.Hash = Crc32Provider.ComputeChecksum(result.InflatedData);
result.Compress();
ServerContextBase.GlobalMessage = message;
return(result);
}
/// <summary>
/// Encodes a message object to byte[]
/// </summary>
/// <param name="message">Message data to encode.</param>
/// <returns>Encoded byte[] representation of the message object.</returns>
/// <remarks>
/// Format:
/// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[])
/// </remarks>
public static byte[] EncodeMessage(Message message)
{
var body = new WriteByteStream();
body.Pack(new[] { message.MagicNumber },
new[] { message.Attribute },
message.Key.ToIntPrefixedBytes(),
message.Value.ToIntPrefixedBytes());
var crc = Crc32Provider.ComputeHash(body.Payload());
body.Prepend(crc);
return(body.Payload());
}
/// <summary>
/// Decode messages from a payload and assign it a given kafka offset.
/// </summary>
/// <param name="offset">The offset represting the log entry from kafka of this message.</param>
/// <param name="payload">The byte[] encode as a message from kafka.</param>
/// <returns>Enumerable representing stream of messages decoded from byte[].</returns>
/// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks>
public static IEnumerable <Message> DecodeMessage(long offset, byte[] payload)
{
var crc = payload.Take(4).ToArray();
var stream = new ReadByteStream(payload.Skip(4));
var hash = Crc32Provider.ComputeHash(stream.Payload);
if (crc.SequenceEqual(hash) == false)
{
throw new FailCrcCheckException("Payload did not match CRC validation.");
}
var message = new Message
{
Meta = new MessageMetadata {
Offset = offset
},
MagicNumber = stream.ReadByte(),
Attribute = stream.ReadByte(),
Key = stream.ReadIntPrefixedBytes()
};
var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute);
switch (codec)
{
case MessageCodec.CodecNone:
message.Value = stream.ReadIntPrefixedBytes();
yield return(message);
break;
case MessageCodec.CodecGzip:
var gZipData = stream.ReadIntPrefixedBytes();
foreach (var m in DecodeMessageSet(Compression.Unzip(gZipData)))
{
yield return(m);
}
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec));
}
}
public MetadataResponse.Partition Select(MetadataResponse.Topic topic, byte[] key)
{
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (topic.Partitions.Count <= 0)
{
throw new CachedMetadataException($"topic/{topic.TopicName} has no partitions.")
{
TopicName = topic.TopicName
}
}
;
long partitionId;
var partitions = topic.Partitions;
if (key == null)
{
// use round robin
var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.TopicName, p => 0, (s, i) => (i + 1) % partitions.Count);
return(partitions[paritionIndex]);
}
else
{
// use key hash
partitionId = Crc32Provider.Compute(key) % partitions.Count;
var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition != null)
{
return(partition);
}
}
throw new CachedMetadataException($"Hash function return partition/{partitionId}, but the available partitions are {string.Join(",", partitions.Select(x => x.PartitionId))}")
{
TopicName = topic.TopicName,
Partition = (int)partitionId
};
}
}
public static MServerTable FromFile(string filename)
{
MServerTable result;
using (var stream = File.OpenRead(filename))
{
result = new XmlSerializer(typeof(MServerTable)).Deserialize(stream) as MServerTable;
}
using (var stream = new MemoryStream())
{
result.Save(stream);
result.InflatedData = stream.ToArray();
}
result.Hash = Crc32Provider.ComputeChecksum(result.InflatedData);
result.Deflate();
return(result);
}
public Partition Select(Topic topic, byte[] key)
{
if (topic == null)
{
throw new ArgumentNullException("topic");
}
if (topic.Partitions.Count <= 0)
{
throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));
}
//use round robin
var partitions = topic.Partitions;
if (key == null)
{
//use round robin
var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) =>
{
return((i + 1) % partitions.Count);
});
return(partitions[paritionIndex]);
}
//use key hash
var partitionId = Crc32Provider.Compute(key) % partitions.Count;
var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition == null)
{
throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));
}
return(partition);
}
public override void Load(MemoryStream stream)
{
using (var reader = new BufferReader(stream))
{
int length = reader.ReadUInt16();
for (var i = 0; i < length; i++)
{
var node = new MetafileNode(reader.ReadStringA());
var atomSize = reader.ReadUInt16();
for (var j = 0; j < atomSize; j++)
{
node.Atoms.Add(
reader.ReadStringB());
}
Nodes.Add(node);
}
}
Hash = Crc32Provider.ComputeChecksum(InflatedData);
Name = Path.GetFileName(Filename);
}
