private static Byte[] CreatePayload(Int64 offset, Byte magicByte, Byte attributes, Byte[] key, Byte[] payload) { var crcContentStream = new MemoryStream(); var crcContentWriter = new BigEndianBinaryWriter(crcContentStream); crcContentWriter.Write(magicByte); crcContentWriter.Write(attributes); crcContentWriter.Write((Int32)key.Length); crcContentWriter.Write(key); crcContentWriter.Write(payload.Length); crcContentWriter.Write(payload); var crcContentBytes = crcContentStream.ToArray(); var crc = Crc32Provider.Compute(crcContentBytes); var messageStream = new MemoryStream(); var messageWriter = new BigEndianBinaryWriter(messageStream); messageWriter.Write(crc); messageWriter.Write(crcContentBytes); var messageBytes = messageStream.ToArray(); var messageSetStream = new MemoryStream(); var messageSetWriter = new BigEndianBinaryWriter(messageSetStream); var messageSize = messageBytes.Length; messageSetWriter.Write(offset); messageSetWriter.Write(messageSize); messageSetWriter.Write(messageBytes); return(messageSetStream.ToArray()); }
/// <summary> /// Decode messages from a payload and assign it a given kafka offset. /// </summary> /// <param name="offset">The offset represting the log entry from kafka of this message.</param> /// <param name="payload">The byte[] encode as a message from kafka.</param> /// <returns>The message</returns> /// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks> internal static Message DecodeMessage(long offset, int partitionId, KafkaDecoder decoder, int messageSize) { var crc = decoder.ReadUInt32(); var calculatedCrc = Crc32Provider.Compute(decoder.Buffer, decoder.Offset, messageSize - 4); if (calculatedCrc != crc) { throw new FailCrcCheckException("Payload did not match CRC validation."); } var message = new Message { Meta = new MessageMetadata(offset, partitionId), MagicNumber = decoder.ReadByte(), Attribute = decoder.ReadByte(), Key = decoder.ReadBytes(), }; var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute); switch (codec) { case MessageCodec.CodecNone: message.Value = decoder.ReadBytes(); break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec)); } return(message); }
public static Notification FromFile(string filename) { var result = new Notification(); result.InflatedData = File.ReadAllText(filename).ToByteArray(); result.Hash = Crc32Provider.ComputeChecksum(result.InflatedData); result.Deflate(); return(result); }
public KafkaEncoder CalculateCrc(int crcMarker) { var crc = Crc32Provider.Compute(buffer, crcMarker, offset - crcMarker); var current = offset; offset = crcMarker - 4; Write(crc); offset = current; return(this); }
private byte[] CreateKeyForPartition(int partitionId) { while (true) { var key = Guid.NewGuid().ToString().ToIntSizedBytes(); if ((Crc32Provider.Compute(key) % 2) == partitionId) { return(key); } } }
private static void CompileTemplate(Metafile metaFile) { using (var stream = new MemoryStream()) { metaFile.Save(stream); metaFile.InflatedData = stream.ToArray(); } metaFile.Hash = Crc32Provider.ComputeChecksum(metaFile.InflatedData); metaFile.Compress(); }
public static Notification FromFile(string filename) { var result = new Notification(); var message = File.ReadAllText(filename); result.InflatedData = message.ToByteArray(); result.Hash = Crc32Provider.ComputeChecksum(result.InflatedData); result.Compress(); ServerContextBase.GlobalMessage = message; return(result); }
/// <summary> /// Encodes a message object to byte[] /// </summary> /// <param name="message">Message data to encode.</param> /// <returns>Encoded byte[] representation of the message object.</returns> /// <remarks> /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// </remarks> public static byte[] EncodeMessage(Message message) { var body = new WriteByteStream(); body.Pack(new[] { message.MagicNumber }, new[] { message.Attribute }, message.Key.ToIntPrefixedBytes(), message.Value.ToIntPrefixedBytes()); var crc = Crc32Provider.ComputeHash(body.Payload()); body.Prepend(crc); return(body.Payload()); }
/// <summary> /// Decode messages from a payload and assign it a given kafka offset. /// </summary> /// <param name="offset">The offset represting the log entry from kafka of this message.</param> /// <param name="payload">The byte[] encode as a message from kafka.</param> /// <returns>Enumerable representing stream of messages decoded from byte[].</returns> /// <remarks>The return type is an Enumerable as the message could be a compressed message set.</remarks> public static IEnumerable <Message> DecodeMessage(long offset, byte[] payload) { var crc = payload.Take(4).ToArray(); var stream = new ReadByteStream(payload.Skip(4)); var hash = Crc32Provider.ComputeHash(stream.Payload); if (crc.SequenceEqual(hash) == false) { throw new FailCrcCheckException("Payload did not match CRC validation."); } var message = new Message { Meta = new MessageMetadata { Offset = offset }, MagicNumber = stream.ReadByte(), Attribute = stream.ReadByte(), Key = stream.ReadIntPrefixedBytes() }; var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute); switch (codec) { case MessageCodec.CodecNone: message.Value = stream.ReadIntPrefixedBytes(); yield return(message); break; case MessageCodec.CodecGzip: var gZipData = stream.ReadIntPrefixedBytes(); foreach (var m in DecodeMessageSet(Compression.Unzip(gZipData))) { yield return(m); } break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec)); } }
public MetadataResponse.Partition Select(MetadataResponse.Topic topic, byte[] key) { if (topic == null) { throw new ArgumentNullException(nameof(topic)); } if (topic.Partitions.Count <= 0) { throw new CachedMetadataException($"topic/{topic.TopicName} has no partitions.") { TopicName = topic.TopicName } } ; long partitionId; var partitions = topic.Partitions; if (key == null) { // use round robin var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.TopicName, p => 0, (s, i) => (i + 1) % partitions.Count); return(partitions[paritionIndex]); } else { // use key hash partitionId = Crc32Provider.Compute(key) % partitions.Count; var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId); if (partition != null) { return(partition); } } throw new CachedMetadataException($"Hash function return partition/{partitionId}, but the available partitions are {string.Join(",", partitions.Select(x => x.PartitionId))}") { TopicName = topic.TopicName, Partition = (int)partitionId }; } }
public static MServerTable FromFile(string filename) { MServerTable result; using (var stream = File.OpenRead(filename)) { result = new XmlSerializer(typeof(MServerTable)).Deserialize(stream) as MServerTable; } using (var stream = new MemoryStream()) { result.Save(stream); result.InflatedData = stream.ToArray(); } result.Hash = Crc32Provider.ComputeChecksum(result.InflatedData); result.Deflate(); return(result); }
public Partition Select(Topic topic, byte[] key) { if (topic == null) { throw new ArgumentNullException("topic"); } if (topic.Partitions.Count <= 0) { throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name)); } //use round robin var partitions = topic.Partitions; if (key == null) { //use round robin var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) => { return((i + 1) % partitions.Count); }); return(partitions[paritionIndex]); } //use key hash var partitionId = Crc32Provider.Compute(key) % partitions.Count; var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId); if (partition == null) { throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}", partitionId, string.Join(",", partitions.Select(x => x.PartitionId)))); } return(partition); }
public override void Load(MemoryStream stream) { using (var reader = new BufferReader(stream)) { int length = reader.ReadUInt16(); for (var i = 0; i < length; i++) { var node = new MetafileNode(reader.ReadStringA()); var atomSize = reader.ReadUInt16(); for (var j = 0; j < atomSize; j++) { node.Atoms.Add( reader.ReadStringB()); } Nodes.Add(node); } } Hash = Crc32Provider.ComputeChecksum(InflatedData); Name = Path.GetFileName(Filename); }