예제 #1
0
        /// <summary>
        /// Decode a byte[] that represents a collection of messages.
        /// </summary>
        /// <param name="incoming">The byte[] encode as a message set from kafka.</param>
        /// <param name="partition">The partition to decode the message for.</param>
        /// <returns>Enumerable representing stream of messages decoded from byte[]</returns>
        public static IEnumerable <Message> DecodeMessages(byte[] incoming, int readOffset, int readSize, int partition)
        {
            var reader = new BinaryReader(incoming, readOffset, readSize);

            while (reader.HasData)
            {
                // If the message set hits against our max bytes wall on the fetch we will have a
                // 1/2 completed message downloaded. The decode should guard against this situation
                if (reader.Available(MinimumMessageSize) == false)
                {
                    yield break;
                }

                // We read the offset of the message and the size and wait until we actually have
                // enough data to process the whole message.
                var offset      = reader.ReadInt64();
                var messageSize = reader.ReadInt32();
                if (reader.Available(messageSize) == false)
                {
                    yield break;
                }

                // We now compute the hash and compare it with the checksum we received within
                // the message itself.
                var crc = BinaryCrc32.ComputeHash(incoming, reader.Position + 4, messageSize - 4);
                if (crc[0] != reader.ReadByte() ||
                    crc[1] != reader.ReadByte() ||
                    crc[2] != reader.ReadByte() ||
                    crc[3] != reader.ReadByte())
                {
                    throw new FailCrcCheckException("CRC validation mismatch during message decoding.");
                }

                var message = new Message
                {
                    Offset      = offset,
                    MagicNumber = reader.ReadByte(),
                    Attribute   = reader.ReadByte(),
                    Key         = reader.ReadInt32Array(),
                    PartitionId = partition
                };

                // We don't support any codecs yet
                var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute);
                if (codec != MessageCodec.CodecNone)
                {
                    throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", codec));
                }

                // Value is int32 prefixed byte-array
                message.Value = reader.ReadInt32Array();
                yield return(message);
            }
        }
예제 #2
0
        /// <summary>
        /// Writes a 4-byte placeholder and returns current position
        /// </summary>
        public void WriteCrcAt(int offset)
        {
            // Remember current position
            var position = (int)this.Position;

            // Calculate the length & CRC
            var count = position - offset - 4;
            var crc   = BinaryCrc32.ComputeHash(this.GetBuffer(), offset + 4, count);

            // Set the position
            this.Position = offset;

            // Write the value
            this.Stream.Write(crc, 0, 4);

            // Set the position back
            this.Position = position;
        }
        public Partition Select(Topic topic, byte[] key)
        {
            if (topic == null)
            {
                throw new ArgumentNullException("topic");
            }
            if (topic.Partitions.Count <= 0)
            {
                throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));
            }

            //use round robin
            var partitions = topic.Partitions;

            if (key == null)
            {
                //use round robin
                var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) =>
                {
                    return((i + 1) % partitions.Count);
                });

                return(partitions[paritionIndex]);
            }

            //use key hash
            var partitionId = BinaryCrc32.Compute(key) % partitions.Count;
            var partition   = partitions.FirstOrDefault(x => x.PartitionId == partitionId);

            if (partition == null)
            {
                throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
                                                                  partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));
            }

            return(partition);
        }