Exemple #1
0
        public static BufferedMessageSet Decompress(Message message)
        {
            switch (message.CompressionCodec)
            {
            case CompressionCodecs.DefaultCompressionCodec:
            case CompressionCodecs.GZIPCompressionCodec:
                byte[] inputBytes = message.Payload;
                using (MemoryStream outputStream = new MemoryStream())
                {
                    using (MemoryStream inputStream = new MemoryStream(inputBytes))
                    {
                        using (GZipStream gzipInputStream = new GZipStream(inputStream, CompressionMode.Decompress))
                        {
                            try
                            {
                                gzipInputStream.CopyTo(outputStream);
                                gzipInputStream.Close();
                            }
                            catch (IOException ex)
                            {
                                Logger.Error("Error while reading from the GZIP input stream", ex);
                                throw;
                            }
                        }
                    }

                    return(BufferedMessageSet.ParseFrom(outputStream.ToArray()));
                }

            default:
                throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", message.CompressionCodec));
            }
        }
Exemple #2
0
        public static BufferedMessageSet Decompress(Message message, int partition)
        {
            switch (message.CompressionCodec)
            {
            case CompressionCodecs.DefaultCompressionCodec:
            case CompressionCodecs.GZIPCompressionCodec:
                var inputBytes = message.Payload;
                using (var outputStream = new MemoryStream())
                {
                    using (var inputStream = new MemoryStream(inputBytes))
                    {
                        using (var gzipInputStream = new GZipStream(inputStream, CompressionMode.Decompress))
                        {
                            try
                            {
                                gzipInputStream.CopyTo(outputStream);
                                gzipInputStream.Close();
                            }
                            catch (IOException ex)
                            {
                                Logger.InfoFormat("Error while reading from the GZIP input stream: {0}",
                                                  ex.FormatException());
                                throw;
                            }
                        }
                    }

                    outputStream.Position = 0;
                    using (var reader = new KafkaBinaryReader(outputStream))
                    {
                        return(BufferedMessageSet.ParseFrom(reader, (int)outputStream.Length, partition));
                    }
                }

            case CompressionCodecs.SnappyCompressionCodec:
                try
                {
                    using (var stream = new MemoryStream(SnappyHelper.Decompress(message.Payload)))
                    {
                        using (var reader = new KafkaBinaryReader(stream))
                        {
                            return(BufferedMessageSet.ParseFrom(reader, (int)stream.Length, partition));
                        }
                    }
                }
                catch (Exception ex)
                {
                    Logger.ErrorFormat("Error while reading from the Snappy input stream  {0}",
                                       ex.FormatException());
                    throw;
                }

            default:
                throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}",
                                                              message.CompressionCodec));
            }
        }
Exemple #3
0
 public ProducerRequest(string topic, int partition, BufferedMessageSet messages)
 {
     Guard.Assert<ArgumentNullException>(() => messages != null);
     int length = GetRequestLength(topic, messages.SetSize);
     this.RequestBuffer = new BoundedBuffer(length);
     this.Topic = topic;
     this.Partition = partition;
     this.MessageSet = messages;
     this.WriteTo(this.RequestBuffer);
 }
Exemple #4
0
        public ProducerRequest(string topic, int partition, BufferedMessageSet messages)
        {
            Guard.NotNull(messages, "messages");

            int length = GetRequestLength(topic, messages.SetSize);
            this.RequestBuffer = new BoundedBuffer(length);
            this.Topic = topic;
            this.Partition = partition;
            this.MessageSet = messages;
            this.WriteTo(this.RequestBuffer);
        }
Exemple #5
0
        public void BufferedMessageSetWriteToValidSequence()
        {
            byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5 };
            Message msg1 = new Message(messageBytes);
            Message msg2 = new Message(messageBytes);
            MessageSet messageSet = new BufferedMessageSet(new List<Message>() { msg1, msg2 });
            MemoryStream ms = new MemoryStream();
            messageSet.WriteTo(ms);

            ////first message

            byte[] messageLength = new byte[MessageLengthPartLength];
            Array.Copy(ms.ToArray(), MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength);
            if (BitConverter.IsLittleEndian)
            {
                Array.Reverse(messageLength);
            }

            Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0));

            Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]);    // default magic number should be 1

            byte[] checksumPart = new byte[ChecksumPartLength];
            Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength);
            Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart);

            byte[] dataPart = new byte[messageBytes.Length];
            Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length);
            Assert.AreEqual(messageBytes, dataPart);

            ////second message
            int secondMessageOffset = MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength +
                                      messageBytes.Length;

            messageLength = new byte[MessageLengthPartLength];
            Array.Copy(ms.ToArray(), secondMessageOffset + MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength);
            if (BitConverter.IsLittleEndian)
            {
                Array.Reverse(messageLength);
            }

            Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0));

            Assert.AreEqual(1, ms.ToArray()[secondMessageOffset + MagicNumberPartOffset]);    // default magic number should be 1

            checksumPart = new byte[ChecksumPartLength];
            Array.Copy(ms.ToArray(), secondMessageOffset + ChecksumPartOffset, checksumPart, 0, ChecksumPartLength);
            Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart);

            dataPart = new byte[messageBytes.Length];
            Array.Copy(ms.ToArray(), secondMessageOffset + DataPartOffset, dataPart, 0, messageBytes.Length);
            Assert.AreEqual(messageBytes, dataPart);
        }
 public void ShouldAbleToEnumerateMessages()
 {
     var msg1 = new Message(new byte[101]) {Offset = 0};
     var msg2 = new Message(new byte[102]) {Offset = 1};
     var set = new BufferedMessageSet(new List<Message>() {msg1, msg2}, 0);
     set.MoveNext().Should().BeTrue();
     set.Current.Message.Payload.Length.Should().Be(101);
     set.Current.Message.Offset.Should().Be(0);
     set.MoveNext().Should().BeTrue();
     set.Current.Message.Payload.Length.Should().Be(102);
     set.Current.Message.Offset.Should().Be(1);
     set.MoveNext().Should().BeFalse();
 }
 public void ShouldAbleToWriteMessageSetWithExtraBytes()
 {
     var stream = new MemoryStream();
     var writer = new KafkaBinaryWriter(stream);
     var msg1 = new Message(new byte[101]) {Offset = 0};
     var msg2 = new Message(new byte[102]) {Offset = 1};
     var set = new BufferedMessageSet(new List<Message>() {msg1, msg2}, 0);
     set.WriteTo(writer);
     writer.Write(new byte[10]); // less than offset and size
     var size = (int) stream.Position;
     stream.Seek(0, SeekOrigin.Begin);
     var reader = new KafkaBinaryReader(stream);
     var newSet = BufferedMessageSet.ParseFrom(reader, size, 0);
     var messages = newSet.Messages.ToList();
     messages.Count().Should().Be(2);
     messages[0].Payload.Count().Should().Be(101);
     messages[1].Payload.Count().Should().Be(102);
 }
Exemple #8
0
        public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec)
        {
            switch (compressionCodec)
            {
            case CompressionCodecs.DefaultCompressionCodec:
            case CompressionCodecs.GZIPCompressionCodec:
                using (MemoryStream outputStream = new MemoryStream())
                {
                    using (GZipStream gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
                    {
                        if (Logger.IsDebugEnabled)
                        {
                            Logger.DebugFormat(
                                CultureInfo.CurrentCulture,
                                "Allocating BufferedMessageSet of size = {0}",
                                MessageSet.GetMessageSetSize(messages));
                        }

                        var bufferedMessageSet = new BufferedMessageSet(messages);
                        using (MemoryStream inputStream = new MemoryStream(bufferedMessageSet.SetSize))
                        {
                            bufferedMessageSet.WriteTo(inputStream);
                            inputStream.Position = 0;
                            try
                            {
                                gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
                                gZipStream.Close();
                            }
                            catch (IOException ex)
                            {
                                Logger.Error("Error while writing to the GZIP stream", ex);
                                throw;
                            }
                        }

                        Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec);
                        return(oneCompressedMessage);
                    }
                }

            default:
                throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec));
            }
        }
        public static Message Compress(IEnumerable<Message> messages, CompressionCodecs compressionCodec, int partition)
        {
            switch (compressionCodec)
            {
                case CompressionCodecs.DefaultCompressionCodec:
                case CompressionCodecs.GZIPCompressionCodec:
                    using (var outputStream = new MemoryStream())
                    {
                        using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
                        {
                            if (Logger.IsDebugEnabled)
                            {
                                Logger.DebugFormat(
                                    "Allocating BufferedMessageSet of size = {0}",
                                    MessageSet.GetMessageSetSize(messages));
                            }

                            var bufferedMessageSet = new BufferedMessageSet(messages, partition);
                            using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize))
                            {
                                bufferedMessageSet.WriteTo(inputStream);
                                inputStream.Position = 0;
                                try
                                {
                                    gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
                                    gZipStream.Close();
                                }
                                catch (IOException ex)
                                {
                                    Logger.ErrorFormat("Error while writing to the GZIP stream {0}", ex.FormatException());
                                    throw;
                                }
                            }

                            Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec)
                            {
                                PartitionId = partition
                            };
                            return oneCompressedMessage;
                        }
                    }

                case CompressionCodecs.SnappyCompressionCodec:
                    Logger.DebugFormat(
                            "Allocating BufferedMessageSet of size = {0}",
                            MessageSet.GetMessageSetSize(messages));

                    var messageSet = new BufferedMessageSet(messages, partition);
                    using (var inputStream = new MemoryStream(messageSet.SetSize))
                    {
                        messageSet.WriteTo(inputStream);
                        inputStream.Position = 0;

                        try
                        {
                            return new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec)
                            {
                                PartitionId = partition
                            };
                        }
                        catch (Exception ex)
                        {
                            Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException());
                            throw;
                        }
                    }

                default:
                    throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec));
            }
        }
Exemple #10
0
        public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec, int partition)
        {
            switch (compressionCodec)
            {
            case CompressionCodecs.DefaultCompressionCodec:
            case CompressionCodecs.GZIPCompressionCodec:
                using (var outputStream = new MemoryStream())
                {
                    using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
                    {
                        //if (Logger.IsDebugEnabled)
                        {
                            Logger.DebugFormat(
                                "Allocating BufferedMessageSet of size = {0}",
                                MessageSet.GetMessageSetSize(messages));
                        }

                        var bufferedMessageSet = new BufferedMessageSet(messages, partition);
                        using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize))
                        {
                            bufferedMessageSet.WriteTo(inputStream);
                            inputStream.Position = 0;
                            try
                            {
                                gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
                                gZipStream.Close();
                            }
                            catch (IOException ex)
                            {
                                Logger.ErrorFormat("Error while writing to the GZIP stream {0}",
                                                   ex.FormatException());
                                throw;
                            }
                        }

                        var oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec)
                        {
                            PartitionId = partition
                        };
                        return(oneCompressedMessage);
                    }
                }

            case CompressionCodecs.SnappyCompressionCodec:
                Logger.DebugFormat(
                    "Allocating BufferedMessageSet of size = {0}",
                    MessageSet.GetMessageSetSize(messages));

                var messageSet = new BufferedMessageSet(messages, partition);
                using (var inputStream = new MemoryStream(messageSet.SetSize))
                {
                    messageSet.WriteTo(inputStream);
                    inputStream.Position = 0;

                    try
                    {
                        return(new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec)
                        {
                            PartitionId = partition
                        });
                    }
                    catch (Exception ex)
                    {
                        Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException());
                        throw;
                    }
                }

            default:
                throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}",
                                                              compressionCodec));
            }
        }
        /// <summary>
        /// Ads a message set to the queue
        /// </summary>
        /// <param name="messages">The message set</param>
        /// <returns>The set size</returns>
        public int Add(BufferedMessageSet messages)
        {
            int size = messages.SetSize;
            if (size > 0)
            {
                long offset = messages.Messages.Last().Offset;

                Logger.InfoFormat("{2} : Updating fetch offset = {0} with value = {1}", this.fetchedOffset, offset, this.PartitionId);
                this.chunkQueue.Add(new FetchedDataChunk(messages, this, this.fetchedOffset));
                Interlocked.Exchange(ref this.fetchedOffset, offset);
                Logger.Debug("Updated fetch offset of " + this + " to " + offset);
            }

            return size;
        }
Exemple #12
0
 public void SetSizeValid()
 {
     byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5 };
     Message msg1 = new Message(messageBytes);
     Message msg2 = new Message(messageBytes);
     MessageSet messageSet = new BufferedMessageSet(new List<Message>() { msg1, msg2 });
     Assert.AreEqual(
         2 * (MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length),
         messageSet.SetSize);
 }
Exemple #13
0
        public int Add(BufferedMessageSet messages, long fetchOffset)
        {
            int size = messages.SetSize;
            if (size > 0)
            {
                long newOffset = Interlocked.Add(ref this.fetchedOffset, size);
                Logger.Debug("Updated fetch offset of " + this + " to " + newOffset);
                this.chunkQueue.Add(new FetchedDataChunk(messages, this, fetchOffset));
            }

            return size;
        }