public List <string> SendMessagesToBrokerPartition( TempKafkaConfig config, string topic, int partition, int numMessages, CompressionCodecs compression = CompressionCodecs.NoCompressionCodec) { var header = string.Format("test-{0}-{1}", config.BrokerId, partition); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, CompressionCodec = compression, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, // custom: we need time to rebalance leader }; var producer = new Producer <int, string>(props); var ms = Enumerable.Range(0, numMessages) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(x => new KeyedMessage <int, string>(topic, partition, x)).ToArray()); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); producer.Dispose(); return(ms); }
/// <summary> /// Initializes a new instance of the <see cref="BufferedMessageSet"/> class with compression. /// </summary> /// <param name="compressionCodec"></param> /// <param name="messages">messages to add</param> public BufferedMessageSet(CompressionCodecs compressionCodec, IEnumerable <Message> messages, int partition) { this.PartitionId = partition; IEnumerable <Message> messagesToAdd; switch (compressionCodec) { case CompressionCodecs.NoCompressionCodec: messagesToAdd = messages; break; default: var message = CompressionUtils.Compress(messages, compressionCodec, partition); messagesToAdd = new List <Message>() { message }; break; } int length = GetMessageSetSize(messagesToAdd); this.Messages = messagesToAdd; this.ErrorCode = (short)ErrorMapping.NoError; this.topIterPosition = 0; }
public List <string> SendMessages( TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts) { var messages = new List <string>(); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, }; var producer = new Producer <int, string>(props); for (var partition = 0; partition < numParts; partition++) { var ms = Enumerable.Range(0, messagesPerNode) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(m => new KeyedMessage <int, string>(Topic, partition, m)).ToArray()); messages.AddRange(ms); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); } producer.Dispose(); return(messages); }
/// <summary> /// Initializes a new instance of the Message class. /// </summary> /// <param name="payload">The data for the payload.</param> /// <param name="magic">The magic identifier.</param> /// <param name="checksum">The checksum for the payload.</param> public Message(byte[] payload, byte[] checksum, CompressionCodecs compressionCodec) { Guard.NotNull(payload, "payload"); Guard.NotNull(checksum, "checksum"); Guard.Count(checksum, 4, "checksum"); int length = DefaultHeaderSize + payload.Length; this.Payload = payload; this.Magic = DefaultMagicValue; if (compressionCodec != CompressionCodecs.NoCompressionCodec) { this.Attributes |= (byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec)); } if (Magic == 1) { length++; } this.Checksum = checksum; this.Size = length; }
/// <summary> /// Initializes a new instance of the Message class. /// </summary> /// <param name="payload">The data for the payload.</param> /// <param name="magic">The magic identifier.</param> /// <param name="checksum">The checksum for the payload.</param> public Message(byte[] payload, byte[] key, CompressionCodecs compressionCodec) { Guard.NotNull(payload, "payload"); int length = DefaultHeaderSize + payload.Length; Key = key; if (key != null) { length += key.Length; } this.Payload = payload; this.Magic = DefaultMagicValue; if (compressionCodec != CompressionCodecs.NoCompressionCodec) { this.Attributes |= (byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec)); // It seems that the java producer uses magic 0 for compressed messages, so we are sticking with 0 for now // this.Magic = MagicValueWhenCompress; } this.Size = length; }
public MessageTestVal(byte[] key, byte[] payload, CompressionCodecs codec, Message message) { this.Key = key; this.Payload = payload; this.Codec = codec; this.Message = message; }
/// <summary> /// Return Producer for the targetPartition /// </summary> /// <param name="partitionParam"></param> /// <returns></returns> public Producer <int, Message> GetKafkaProducer(int partitionParam, CompressionCodecs comparessionCodec, out ProducerConfiguration producerConfiguration) { int targetPartition = this.ValidateProducerLeaderTargetPartitionInialization(partitionParam); if (this.producers.Count == 0 || !this.producers.ContainsKey(this.leaders[targetPartition].BrokerId)) { Logger.DebugFormat("GetProducerConf creatNew for leaderBrokerId={0},leaders.Count={1},targetPartition={2},leaderBrokerDetails={3}", this.leaders[targetPartition].BrokerId, this.leaders.Count(), targetPartition, this.leaders[targetPartition]); // AckTimeout deafult=1, MaxMessageSize default = 1000000; ConnectionTimout default=5000; BufferSize default=102400, DefaultSocketTimeout=30 * 1000 ProducerConfiguration producerConf = new ProducerConfiguration(new List <BrokerConfiguration>() { this.leaders[targetPartition] }) { // PartitionerClass = "Kafka.Client.Producers.Partitioning.ModPartitioner", PartitionerClass = ProducerConfiguration.DefaultPartitioner, // // https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProduceRequest // RequiredAcks: This field indicates how many acknowledgements the servers should receive before responding to the request. // If it is 0 the server will not send any response (this is the only case where the server will not reply to a request). // If it is 1, the server will wait the data is written to the local log before sending a response. // If it is -1 the server will block until the message is committed by all in sync replicas before sending a response. // For any number > 1 the server will block waiting for this number of acknowledgements to occur (but the server will never wait for // more acknowledgements than there are in-sync replicas). // Timeout: This provides a maximum time in milliseconds the server can await the receipt of the number of acknowledgements in RequiredAcks. // The timeout is not an exact limit on the request time for a few reasons: // (1) it does not include network latency, // (2) the timer begins at the beginning of the processing of this request so if many requests are queued due to server overload that wait time will not be included, // (3) we will not terminate a local write so if the local write time exceeds this timeout it will not be respected. To get a hard timeout of this type the client // should use the socket timeout. RequiredAcks = -1, CompressionCodec = comparessionCodec, SendTimeout = this.HelperConfiguration.Timeout, ReceiveTimeout = this.HelperConfiguration.Timeout, AckTimeout = this.HelperConfiguration.AckTimeout, MaxMessageSize = this.HelperConfiguration.MaxMessageSize, BufferSize = this.HelperConfiguration.MaxMessageSize, TotalNumPartitions = this.PartitionsMetadata != null?this.PartitionsMetadata.Count() : this.HelperConfiguration.TotalNumPartitions }; this.producerConfs.Add(this.leaders[targetPartition].BrokerId, producerConf); this.producers.Add(this.leaders[targetPartition].BrokerId , new Producer <int, Kafka.Client.Messages.Message>(producerConf)); } producerConfiguration = this.producerConfs[this.leaders[targetPartition].BrokerId]; return(this.producers[this.leaders[targetPartition].BrokerId]); }
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec) { var messages = new List<Message> { new Message(Encoding.UTF8.GetBytes("hi there")), new Message(Encoding.UTF8.GetBytes("I am fine")), new Message(Encoding.UTF8.GetBytes("I am not so well today")), }; var messageSet = new ByteBufferMessageSet(compressionCodec, messages); Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec); var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList(); Assert.Equal(messages, decompressed); }
public void then_codec_should_be_set(Compression codec, CompressionCodecs expected) { var message = new Message { Key = "key1", Value = "value1", Codec = codec }; var actual = message.AsProducerData("topic"); actual.Data.First().CompressionCodec.ShouldBeEquivalentTo(expected); }
public static Stream BuildReader(CompressionCodecs compressionCodec, Stream stream) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: return new GZipStream(stream, CompressionMode.Decompress); case CompressionCodecs.GZIPCompressionCodec: return new GZipStream(stream, CompressionMode.Decompress); case CompressionCodecs.SnappyCompressionCodec: return new SnappyStream(stream, CompressionMode.Decompress); default: throw new UnknownCodecException("Unknown codec " + compressionCodec); } }
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec) { var messages = new List <Message> { new Message(Encoding.UTF8.GetBytes("hi there")), new Message(Encoding.UTF8.GetBytes("I am fine")), new Message(Encoding.UTF8.GetBytes("I am not so well today")), }; var messageSet = new ByteBufferMessageSet(compressionCodec, messages); Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec); var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList(); Assert.Equal(messages, decompressed); }
/// <summary> /// A constructor to create a Message /// </summary> /// <param name="bytes">The payload of the message</param> /// <param name="key">The key of the message (null, if none)</param> /// <param name="codec">The compression codec used on the contents of the message (if any)</param> /// <param name="payloadOffset">The offset into the payload array used to extract payload</param> /// <param name="payloadSize">The size of the payload to use</param> public Message(byte[] bytes, byte[] key, CompressionCodecs codec, int payloadOffset, int payloadSize) { this.buffer = ByteBuffer.Allocate(CrcLength + MagicLength + AttributesLength + KeySizeLength + ((key == null) ? 0 : key.Length) + ValueSizeLength + ((bytes == null) ? 0 : (payloadSize >= 0 ? payloadSize : bytes.Length - payloadOffset))); this.buffer.Position = MagicOffset; this.buffer.Put(CurrentMagicValue); byte attributes = 0; if (codec != CompressionCodecs.NoCompressionCodec) { attributes = Convert.ToByte( attributes | (CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(codec))); } this.buffer.Put(attributes); if (key == null) { this.buffer.PutInt(-1); } else { this.buffer.PutInt(key.Length); this.buffer.Put(key, 0, key.Length); } var size = (bytes == null) ? -1 : (payloadSize >= 0) ? payloadSize : bytes.Length - payloadOffset; this.buffer.PutInt(size); if (bytes != null) { this.buffer.Write(bytes, payloadOffset, size); } this.buffer.Rewind(); Utils.Util.WriteUnsignedInt(this.buffer, CrcOffset, this.ComputeChecksum()); }
public static Stream BuildReader(CompressionCodecs compressionCodec, Stream stream) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: return(new GZipStream(stream, CompressionMode.Decompress)); case CompressionCodecs.GZIPCompressionCodec: return(new GZipStream(stream, CompressionMode.Decompress)); case CompressionCodecs.SnappyCompressionCodec: return(new SnappyStream(stream, CompressionMode.Decompress)); default: throw new UnknownCodecException("Unknown codec " + compressionCodec); } }
public static byte GetCompressionCodecValue(CompressionCodecs compressionCodec) { switch (compressionCodec) { case CompressionCodecs.SnappyCompressionCodec: return 2; case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: return 1; case CompressionCodecs.NoCompressionCodec: return 0; default: throw new UnknownCodecException(string.Format( CultureInfo.CurrentCulture, "{0} is an unknown compression codec", compressionCodec)); } }
public static byte GetCompressionCodecValue(CompressionCodecs compressionCodec) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: return((byte)1); case CompressionCodecs.NoCompressionCodec: return((byte)0); default: throw new UnknownCodecException(String.Format( CultureInfo.CurrentCulture, "{0} is an unknown compression codec", compressionCodec)); } }
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: using (MemoryStream outputStream = new MemoryStream()) { using (GZipStream gZipStream = new GZipStream(outputStream, CompressionMode.Compress)) { if (Logger.IsDebugEnabled) { Logger.DebugFormat( CultureInfo.CurrentCulture, "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); } var bufferedMessageSet = new BufferedMessageSet(messages); using (MemoryStream inputStream = new MemoryStream(bufferedMessageSet.SetSize)) { bufferedMessageSet.WriteTo(inputStream); inputStream.Position = 0; try { gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length); gZipStream.Close(); } catch (IOException ex) { Logger.Error("Error while writing to the GZIP stream", ex); throw; } } Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec); return(oneCompressedMessage); } } default: throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec)); } }
private static ByteBuffer Create( AtomicLong offsetCounter, CompressionCodecs compressionCodec, List <Message> messages) { if (messages == null || !messages.Any()) { return(Empty.Buffer); } else if (CompressionCodecs.NoCompressionCodec == compressionCodec) { var buffer = ByteBuffer.Allocate(MessageSetSize(messages)); foreach (var message in messages) { WriteMessage(buffer, message, offsetCounter.GetAndIncrement()); } buffer.Rewind(); return(buffer); } else { var byteArrayStream = new MemoryStream(MessageSetSize(messages)); var offset = -1L; using (var output = new KafkaBinaryWriter(CompressionFactory.BuildWriter(compressionCodec, byteArrayStream))) { foreach (var message in messages) { offset = offsetCounter.GetAndIncrement(); output.Write(offset); output.Write(message.Size); output.Write(message.Buffer.Array, message.Buffer.ArrayOffset(), message.Buffer.Limit()); } } var bytes = byteArrayStream.ToArray(); var msg = new Message(bytes, compressionCodec); var buffer = ByteBuffer.Allocate(msg.Size + LogOverhead); WriteMessage(buffer, msg, offset); buffer.Rewind(); return(buffer); } }
/// <summary> /// Initializes a new instance of the Message class. /// </summary> /// <param name="payload">The data for the payload.</param> /// <param name="magic">The magic identifier.</param> /// <param name="checksum">The checksum for the payload.</param> public Message(byte[] payload, byte[] key, CompressionCodecs compressionCodec) { Guard.NotNull(payload, "payload"); var length = DefaultHeaderSize + payload.Length; Key = key; if (key != null) { length += key.Length; } Payload = payload; Magic = DefaultMagicValue; if (compressionCodec != CompressionCodecs.NoCompressionCodec) { Attributes |= (byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec)); } Size = length; }
internal override void Parse(string[] args) { base.BuildDictionary(args); GetString("-z", "--zookeeper", ref this.Zookeeper); CheckZookeeperPort(); if (string.IsNullOrEmpty(this.Zookeeper)) { throw new ArgumentException("Must specify zookeeper with port by -z. Example: -z localhost:2181"); } GetString("-t", "--topic", ref this.Topic); if (string.IsNullOrEmpty(this.Topic)) { throw new ArgumentException("Must specify topic by -t."); } GetInt("-p", "--PartitionId", ref this.PartitionId); GetString("-l", "--PartitionerClass", ref this.PartitionerClass); GetInt("-c", "--batchcount", ref this.BatchCount); GetInt("-b", "--MessageCountPerBatch", ref this.MessageCountPerBatch); GetInt("-m", "--MessageSize", ref this.MessageSize); GetInt("-r", "--Compression", ref this.Compression); this.CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(this.Compression.ToString()); GetShort("-a", "--RequiredAcks", ref this.RequiredAcks); GetInt("-k", "--AckTimeout", ref this.AckTimeout); GetInt("-s", "--SendTimeout", ref this.SendTimeout); GetInt("-e", "--ReceiveTimeout", ref this.ReceiveTimeout); GetInt("-i", "--BufferSize", ref this.BufferSize); GetInt("-y", "--SyncProducerOfOneBroker", ref this.SyncProducerOfOneBroker); GetBool("-w", "--ConstantMessageKey", ref this.ConstantMessageKey); if (this.PartitionId == -2) { if (string.IsNullOrEmpty(this.PartitionerClass)) { throw new ArgumentException(string.Format("You specify partitionID as -2, please also specify the partitioner class full name, for example: {0}", ProducerConfiguration.DefaultPartitioner)); } } }
internal ByteBufferMessageSet AssignOffsets(AtomicLong offsetCounter, CompressionCodecs codec) { if (codec == CompressionCodecs.NoCompressionCodec) { // do as in-place conversion var position = 0; this.Buffer.Mark(); while (position < this.SizeInBytes - MessageSet.LogOverhead) { this.Buffer.Position = position; this.Buffer.PutLong(offsetCounter.GetAndIncrement()); position += MessageSet.LogOverhead + this.Buffer.GetInt(); } this.Buffer.Reset(); return(this); } else { // messages are compressed, crack open the messageset and recompress with correct offset var messages = this.InternalIterator(isShallow: false).ToEnumerable().Select(_ => _.Message); return(new ByteBufferMessageSet(codec, offsetCounter, messages.ToList())); } }
public Message(byte[] payload, CompressionCodecs compressionCodec) : this(payload, null, compressionCodec) { Guard.NotNull(payload, "payload"); }
public List<string> SendMessages( TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts) { var messages = new List<string>(); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, }; var producer = new Producer<int, string>(props); for (var partition = 0; partition < numParts; partition++) { var ms = Enumerable.Range(0, messagesPerNode) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(m => new KeyedMessage<int, string>(Topic, partition, m)).ToArray()); messages.AddRange(ms); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); } producer.Dispose(); return messages; }
public List<string> SendMessagesToBrokerPartition( TempKafkaConfig config, string topic, int partition, int numMessages, CompressionCodecs compression = CompressionCodecs.NoCompressionCodec) { var header = string.Format("test-{0}-{1}", config.BrokerId, partition); var props = new ProducerConfig { Brokers = TestUtils.GetBrokerListFromConfigs(Configs), PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName, CompressionCodec = compression, KeySerializer = typeof(IntEncoder).AssemblyQualifiedName, Serializer = typeof(StringEncoder).AssemblyQualifiedName, RetryBackoffMs = 1000, // custom: we need time to rebalance leader }; var producer = new Producer<int, string>(props); var ms = Enumerable.Range(0, numMessages) .Select(x => header + config.BrokerId + "-" + partition + "-" + x) .ToList(); producer.Send(ms.Select(x => new KeyedMessage<int, string>(topic, partition, x)).ToArray()); Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition); producer.Dispose(); return ms; }
public Message(byte[] bytes, CompressionCodecs codec) : this(bytes, null, codec) { }
public static Message Compress(IEnumerable<Message> messages, CompressionCodecs compressionCodec, int partition) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: using (var outputStream = new MemoryStream()) { using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress)) { if (Logger.IsDebugEnabled) { Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); } var bufferedMessageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize)) { bufferedMessageSet.WriteTo(inputStream); inputStream.Position = 0; try { gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length); gZipStream.Close(); } catch (IOException ex) { Logger.ErrorFormat("Error while writing to the GZIP stream {0}", ex.FormatException()); throw; } } Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec) { PartitionId = partition }; return oneCompressedMessage; } } case CompressionCodecs.SnappyCompressionCodec: Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); var messageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(messageSet.SetSize)) { messageSet.WriteTo(inputStream); inputStream.Position = 0; try { return new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec) { PartitionId = partition }; } catch (Exception ex) { Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException()); throw; } } default: throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec)); } }
public ByteBufferMessageSet( CompressionCodecs compressionCodec, AtomicLong offsetCounter, List <Message> messages) : this(Create(offsetCounter, compressionCodec, messages)) { }
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec, int partition) { switch (compressionCodec) { case CompressionCodecs.DefaultCompressionCodec: case CompressionCodecs.GZIPCompressionCodec: using (var outputStream = new MemoryStream()) { using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress)) { //if (Logger.IsDebugEnabled) { Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); } var bufferedMessageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize)) { bufferedMessageSet.WriteTo(inputStream); inputStream.Position = 0; try { gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length); gZipStream.Close(); } catch (IOException ex) { Logger.ErrorFormat("Error while writing to the GZIP stream {0}", ex.FormatException()); throw; } } var oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec) { PartitionId = partition }; return(oneCompressedMessage); } } case CompressionCodecs.SnappyCompressionCodec: Logger.DebugFormat( "Allocating BufferedMessageSet of size = {0}", MessageSet.GetMessageSetSize(messages)); var messageSet = new BufferedMessageSet(messages, partition); using (var inputStream = new MemoryStream(messageSet.SetSize)) { messageSet.WriteTo(inputStream); inputStream.Position = 0; try { return(new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec) { PartitionId = partition }); } catch (Exception ex) { Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException()); throw; } } default: throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec)); } }
internal override void Parse(string[] args) { base.BuildDictionary(args); GetString("-z", "--zookeeper", ref this.Zookeeper); CheckZookeeperPort(); if (string.IsNullOrEmpty(this.Zookeeper)) throw new ArgumentException("Must specify zookeeper with port by -z. Example: -z localhost:2181"); GetString("-t", "--topic", ref this.Topic); if (string.IsNullOrEmpty(this.Topic)) throw new ArgumentException("Must specify topic by -t."); GetInt("-p", "--PartitionId", ref this.PartitionId); GetString("-l", "--PartitionerClass", ref this.PartitionerClass); GetInt("-c", "--batchcount", ref this.BatchCount); GetInt("-b", "--MessageCountPerBatch", ref this.MessageCountPerBatch); GetInt("-m", "--MessageSize", ref this.MessageSize); GetInt("-r", "--Compression", ref this.Compression); this.CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(this.Compression.ToString()); GetShort("-a", "--RequiredAcks", ref this.RequiredAcks); GetInt("-k", "--AckTimeout", ref this.AckTimeout); GetInt("-s", "--SendTimeout", ref this.SendTimeout); GetInt("-e", "--ReceiveTimeout", ref this.ReceiveTimeout); GetInt("-i", "--BufferSize", ref this.BufferSize); GetInt("-y", "--SyncProducerOfOneBroker", ref this.SyncProducerOfOneBroker); GetBool("-w", "--ConstantMessageKey", ref this.ConstantMessageKey); if (this.PartitionId == -2) { if (string.IsNullOrEmpty(this.PartitionerClass)) throw new ArgumentException(string.Format("You specify partitionID as -2, please also specify the partitioner class full name, for example: {0}", ProducerConfiguration.DefaultPartitioner)); } }
public ByteBufferMessageSet(CompressionCodecs compressionCodec, List <Message> messages) : this(Create(new AtomicLong(0), compressionCodec, messages)) { }
public Message(byte[] bytes, byte[] key, CompressionCodecs codec) : this(bytes, key, codec, 0, -1) { }
/// <summary> /// Initializes a new instance of the Message class. /// </summary> /// <remarks> /// Initializes the checksum as null. It will be automatically computed. /// </remarks> /// <param name="payload">The data for the payload.</param> /// <param name="magic">The magic identifier.</param> public Message(byte[] payload, CompressionCodecs compressionCodec) : this(payload, Crc32Hasher.Compute(payload), compressionCodec) { Guard.NotNull(payload, "payload"); }