public List <string> SendMessagesToBrokerPartition(
TempKafkaConfig config,
string topic,
int partition,
int numMessages,
CompressionCodecs compression = CompressionCodecs.NoCompressionCodec)
{
var header = string.Format("test-{0}-{1}", config.BrokerId, partition);
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
CompressionCodec = compression,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000, // custom: we need time to rebalance leader
};
var producer = new Producer <int, string>(props);
var ms =
Enumerable.Range(0, numMessages)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(x => new KeyedMessage <int, string>(topic, partition, x)).ToArray());
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
producer.Dispose();
return(ms);
}
/// <summary>
/// Initializes a new instance of the <see cref="BufferedMessageSet"/> class with compression.
/// </summary>
/// <param name="compressionCodec"></param>
/// <param name="messages">messages to add</param>
public BufferedMessageSet(CompressionCodecs compressionCodec, IEnumerable <Message> messages, int partition)
{
this.PartitionId = partition;
IEnumerable <Message> messagesToAdd;
switch (compressionCodec)
{
case CompressionCodecs.NoCompressionCodec:
messagesToAdd = messages;
break;
default:
var message = CompressionUtils.Compress(messages, compressionCodec, partition);
messagesToAdd = new List <Message>()
{
message
};
break;
}
int length = GetMessageSetSize(messagesToAdd);
this.Messages = messagesToAdd;
this.ErrorCode = (short)ErrorMapping.NoError;
this.topIterPosition = 0;
}
public List <string> SendMessages(
TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts)
{
var messages = new List <string>();
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000,
};
var producer = new Producer <int, string>(props);
for (var partition = 0; partition < numParts; partition++)
{
var ms =
Enumerable.Range(0, messagesPerNode)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(m => new KeyedMessage <int, string>(Topic, partition, m)).ToArray());
messages.AddRange(ms);
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
}
producer.Dispose();
return(messages);
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
/// <param name="checksum">The checksum for the payload.</param>
public Message(byte[] payload, byte[] checksum, CompressionCodecs compressionCodec)
{
Guard.NotNull(payload, "payload");
Guard.NotNull(checksum, "checksum");
Guard.Count(checksum, 4, "checksum");
int length = DefaultHeaderSize + payload.Length;
this.Payload = payload;
this.Magic = DefaultMagicValue;
if (compressionCodec != CompressionCodecs.NoCompressionCodec)
{
this.Attributes |=
(byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec));
}
if (Magic == 1)
{
length++;
}
this.Checksum = checksum;
this.Size = length;
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
/// <param name="checksum">The checksum for the payload.</param>
public Message(byte[] payload, byte[] key, CompressionCodecs compressionCodec)
{
Guard.NotNull(payload, "payload");
int length = DefaultHeaderSize + payload.Length;
Key = key;
if (key != null)
{
length += key.Length;
}
this.Payload = payload;
this.Magic = DefaultMagicValue;
if (compressionCodec != CompressionCodecs.NoCompressionCodec)
{
this.Attributes |=
(byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec));
// It seems that the java producer uses magic 0 for compressed messages, so we are sticking with 0 for now
// this.Magic = MagicValueWhenCompress;
}
this.Size = length;
}
public MessageTestVal(byte[] key, byte[] payload, CompressionCodecs codec, Message message)
{
this.Key = key;
this.Payload = payload;
this.Codec = codec;
this.Message = message;
}
/// <summary>
/// Return Producer for the targetPartition
/// </summary>
/// <param name="partitionParam"></param>
/// <returns></returns>
public Producer <int, Message> GetKafkaProducer(int partitionParam,
CompressionCodecs comparessionCodec,
out ProducerConfiguration producerConfiguration)
{
int targetPartition = this.ValidateProducerLeaderTargetPartitionInialization(partitionParam);
if (this.producers.Count == 0 || !this.producers.ContainsKey(this.leaders[targetPartition].BrokerId))
{
Logger.DebugFormat("GetProducerConf creatNew for leaderBrokerId={0},leaders.Count={1},targetPartition={2},leaderBrokerDetails={3}",
this.leaders[targetPartition].BrokerId, this.leaders.Count(), targetPartition, this.leaders[targetPartition]);
// AckTimeout deafult=1, MaxMessageSize default = 1000000; ConnectionTimout default=5000; BufferSize default=102400, DefaultSocketTimeout=30 * 1000
ProducerConfiguration producerConf = new ProducerConfiguration(new List <BrokerConfiguration>()
{
this.leaders[targetPartition]
})
{
// PartitionerClass = "Kafka.Client.Producers.Partitioning.ModPartitioner",
PartitionerClass = ProducerConfiguration.DefaultPartitioner,
//
// https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProduceRequest
// RequiredAcks: This field indicates how many acknowledgements the servers should receive before responding to the request.
// If it is 0 the server will not send any response (this is the only case where the server will not reply to a request).
// If it is 1, the server will wait the data is written to the local log before sending a response.
// If it is -1 the server will block until the message is committed by all in sync replicas before sending a response.
// For any number > 1 the server will block waiting for this number of acknowledgements to occur (but the server will never wait for
// more acknowledgements than there are in-sync replicas).
// Timeout: This provides a maximum time in milliseconds the server can await the receipt of the number of acknowledgements in RequiredAcks.
// The timeout is not an exact limit on the request time for a few reasons:
// (1) it does not include network latency,
// (2) the timer begins at the beginning of the processing of this request so if many requests are queued due to server overload that wait time will not be included,
// (3) we will not terminate a local write so if the local write time exceeds this timeout it will not be respected. To get a hard timeout of this type the client
// should use the socket timeout.
RequiredAcks = -1,
CompressionCodec = comparessionCodec,
SendTimeout = this.HelperConfiguration.Timeout,
ReceiveTimeout = this.HelperConfiguration.Timeout,
AckTimeout = this.HelperConfiguration.AckTimeout,
MaxMessageSize = this.HelperConfiguration.MaxMessageSize,
BufferSize = this.HelperConfiguration.MaxMessageSize,
TotalNumPartitions = this.PartitionsMetadata != null?this.PartitionsMetadata.Count() : this.HelperConfiguration.TotalNumPartitions
};
this.producerConfs.Add(this.leaders[targetPartition].BrokerId, producerConf);
this.producers.Add(this.leaders[targetPartition].BrokerId
, new Producer <int, Kafka.Client.Messages.Message>(producerConf));
}
producerConfiguration = this.producerConfs[this.leaders[targetPartition].BrokerId];
return(this.producers[this.leaders[targetPartition].BrokerId]);
}
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec)
{
var messages = new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var messageSet = new ByteBufferMessageSet(compressionCodec, messages);
Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec);
var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressed);
}
public void then_codec_should_be_set(Compression codec, CompressionCodecs expected)
{
var message = new Message
{
Key = "key1",
Value = "value1",
Codec = codec
};
var actual = message.AsProducerData("topic");
actual.Data.First().CompressionCodec.ShouldBeEquivalentTo(expected);
}
public static Stream BuildReader(CompressionCodecs compressionCodec, Stream stream)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
return new GZipStream(stream, CompressionMode.Decompress);
case CompressionCodecs.GZIPCompressionCodec:
return new GZipStream(stream, CompressionMode.Decompress);
case CompressionCodecs.SnappyCompressionCodec:
return new SnappyStream(stream, CompressionMode.Decompress);
default:
throw new UnknownCodecException("Unknown codec " + compressionCodec);
}
}
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec)
{
var messages = new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var messageSet = new ByteBufferMessageSet(compressionCodec, messages);
Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec);
var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressed);
}
/// <summary>
/// A constructor to create a Message
/// </summary>
/// <param name="bytes">The payload of the message</param>
/// <param name="key">The key of the message (null, if none)</param>
/// <param name="codec">The compression codec used on the contents of the message (if any)</param>
/// <param name="payloadOffset">The offset into the payload array used to extract payload</param>
/// <param name="payloadSize">The size of the payload to use</param>
public Message(byte[] bytes, byte[] key, CompressionCodecs codec, int payloadOffset, int payloadSize)
{
this.buffer = ByteBuffer.Allocate(CrcLength +
MagicLength +
AttributesLength +
KeySizeLength +
((key == null) ? 0 : key.Length) +
ValueSizeLength +
((bytes == null)
? 0
: (payloadSize >= 0 ? payloadSize : bytes.Length - payloadOffset)));
this.buffer.Position = MagicOffset;
this.buffer.Put(CurrentMagicValue);
byte attributes = 0;
if (codec != CompressionCodecs.NoCompressionCodec)
{
attributes =
Convert.ToByte(
attributes | (CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(codec)));
}
this.buffer.Put(attributes);
if (key == null)
{
this.buffer.PutInt(-1);
}
else
{
this.buffer.PutInt(key.Length);
this.buffer.Put(key, 0, key.Length);
}
var size = (bytes == null)
? -1
: (payloadSize >= 0) ? payloadSize : bytes.Length - payloadOffset;
this.buffer.PutInt(size);
if (bytes != null)
{
this.buffer.Write(bytes, payloadOffset, size);
}
this.buffer.Rewind();
Utils.Util.WriteUnsignedInt(this.buffer, CrcOffset, this.ComputeChecksum());
}
public static Stream BuildReader(CompressionCodecs compressionCodec, Stream stream)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
return(new GZipStream(stream, CompressionMode.Decompress));
case CompressionCodecs.GZIPCompressionCodec:
return(new GZipStream(stream, CompressionMode.Decompress));
case CompressionCodecs.SnappyCompressionCodec:
return(new SnappyStream(stream, CompressionMode.Decompress));
default:
throw new UnknownCodecException("Unknown codec " + compressionCodec);
}
}
/// <summary>
/// A constructor to create a Message
/// </summary>
/// <param name="bytes">The payload of the message</param>
/// <param name="key">The key of the message (null, if none)</param>
/// <param name="codec">The compression codec used on the contents of the message (if any)</param>
/// <param name="payloadOffset">The offset into the payload array used to extract payload</param>
/// <param name="payloadSize">The size of the payload to use</param>
public Message(byte[] bytes, byte[] key, CompressionCodecs codec, int payloadOffset, int payloadSize)
{
this.buffer = ByteBuffer.Allocate(CrcLength +
MagicLength +
AttributesLength +
KeySizeLength +
((key == null) ? 0 : key.Length) +
ValueSizeLength +
((bytes == null)
? 0
: (payloadSize >= 0 ? payloadSize : bytes.Length - payloadOffset)));
this.buffer.Position = MagicOffset;
this.buffer.Put(CurrentMagicValue);
byte attributes = 0;
if (codec != CompressionCodecs.NoCompressionCodec)
{
attributes =
Convert.ToByte(
attributes | (CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(codec)));
}
this.buffer.Put(attributes);
if (key == null)
{
this.buffer.PutInt(-1);
}
else
{
this.buffer.PutInt(key.Length);
this.buffer.Put(key, 0, key.Length);
}
var size = (bytes == null)
? -1
: (payloadSize >= 0) ? payloadSize : bytes.Length - payloadOffset;
this.buffer.PutInt(size);
if (bytes != null)
{
this.buffer.Write(bytes, payloadOffset, size);
}
this.buffer.Rewind();
Utils.Util.WriteUnsignedInt(this.buffer, CrcOffset, this.ComputeChecksum());
}
public static byte GetCompressionCodecValue(CompressionCodecs compressionCodec)
{
switch (compressionCodec)
{
case CompressionCodecs.SnappyCompressionCodec:
return 2;
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
return 1;
case CompressionCodecs.NoCompressionCodec:
return 0;
default:
throw new UnknownCodecException(string.Format(
CultureInfo.CurrentCulture,
"{0} is an unknown compression codec",
compressionCodec));
}
}
public static byte GetCompressionCodecValue(CompressionCodecs compressionCodec)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
return((byte)1);
case CompressionCodecs.NoCompressionCodec:
return((byte)0);
default:
throw new UnknownCodecException(String.Format(
CultureInfo.CurrentCulture,
"{0} is an unknown compression codec",
compressionCodec));
}
}
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (MemoryStream outputStream = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
CultureInfo.CurrentCulture,
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages);
using (MemoryStream inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.Error("Error while writing to the GZIP stream", ex);
throw;
}
}
Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec);
return(oneCompressedMessage);
}
}
default:
throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec));
}
}
private static ByteBuffer Create(
AtomicLong offsetCounter, CompressionCodecs compressionCodec, List <Message> messages)
{
if (messages == null || !messages.Any())
{
return(Empty.Buffer);
}
else if (CompressionCodecs.NoCompressionCodec == compressionCodec)
{
var buffer = ByteBuffer.Allocate(MessageSetSize(messages));
foreach (var message in messages)
{
WriteMessage(buffer, message, offsetCounter.GetAndIncrement());
}
buffer.Rewind();
return(buffer);
}
else
{
var byteArrayStream = new MemoryStream(MessageSetSize(messages));
var offset = -1L;
using (var output = new KafkaBinaryWriter(CompressionFactory.BuildWriter(compressionCodec, byteArrayStream)))
{
foreach (var message in messages)
{
offset = offsetCounter.GetAndIncrement();
output.Write(offset);
output.Write(message.Size);
output.Write(message.Buffer.Array, message.Buffer.ArrayOffset(), message.Buffer.Limit());
}
}
var bytes = byteArrayStream.ToArray();
var msg = new Message(bytes, compressionCodec);
var buffer = ByteBuffer.Allocate(msg.Size + LogOverhead);
WriteMessage(buffer, msg, offset);
buffer.Rewind();
return(buffer);
}
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
/// <param name="checksum">The checksum for the payload.</param>
public Message(byte[] payload, byte[] key, CompressionCodecs compressionCodec)
{
Guard.NotNull(payload, "payload");
var length = DefaultHeaderSize + payload.Length;
Key = key;
if (key != null)
{
length += key.Length;
}
Payload = payload;
Magic = DefaultMagicValue;
if (compressionCodec != CompressionCodecs.NoCompressionCodec)
{
Attributes |=
(byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec));
}
Size = length;
}
internal override void Parse(string[] args)
{
base.BuildDictionary(args);
GetString("-z", "--zookeeper", ref this.Zookeeper);
CheckZookeeperPort();
if (string.IsNullOrEmpty(this.Zookeeper))
{
throw new ArgumentException("Must specify zookeeper with port by -z. Example: -z localhost:2181");
}
GetString("-t", "--topic", ref this.Topic);
if (string.IsNullOrEmpty(this.Topic))
{
throw new ArgumentException("Must specify topic by -t.");
}
GetInt("-p", "--PartitionId", ref this.PartitionId);
GetString("-l", "--PartitionerClass", ref this.PartitionerClass);
GetInt("-c", "--batchcount", ref this.BatchCount);
GetInt("-b", "--MessageCountPerBatch", ref this.MessageCountPerBatch);
GetInt("-m", "--MessageSize", ref this.MessageSize);
GetInt("-r", "--Compression", ref this.Compression);
this.CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(this.Compression.ToString());
GetShort("-a", "--RequiredAcks", ref this.RequiredAcks);
GetInt("-k", "--AckTimeout", ref this.AckTimeout);
GetInt("-s", "--SendTimeout", ref this.SendTimeout);
GetInt("-e", "--ReceiveTimeout", ref this.ReceiveTimeout);
GetInt("-i", "--BufferSize", ref this.BufferSize);
GetInt("-y", "--SyncProducerOfOneBroker", ref this.SyncProducerOfOneBroker);
GetBool("-w", "--ConstantMessageKey", ref this.ConstantMessageKey);
if (this.PartitionId == -2)
{
if (string.IsNullOrEmpty(this.PartitionerClass))
{
throw new ArgumentException(string.Format("You specify partitionID as -2, please also specify the partitioner class full name, for example: {0}", ProducerConfiguration.DefaultPartitioner));
}
}
}
internal ByteBufferMessageSet AssignOffsets(AtomicLong offsetCounter, CompressionCodecs codec)
{
if (codec == CompressionCodecs.NoCompressionCodec)
{
// do as in-place conversion
var position = 0;
this.Buffer.Mark();
while (position < this.SizeInBytes - MessageSet.LogOverhead)
{
this.Buffer.Position = position;
this.Buffer.PutLong(offsetCounter.GetAndIncrement());
position += MessageSet.LogOverhead + this.Buffer.GetInt();
}
this.Buffer.Reset();
return(this);
}
else
{
// messages are compressed, crack open the messageset and recompress with correct offset
var messages = this.InternalIterator(isShallow: false).ToEnumerable().Select(_ => _.Message);
return(new ByteBufferMessageSet(codec, offsetCounter, messages.ToList()));
}
}
public Message(byte[] payload, CompressionCodecs compressionCodec)
: this(payload, null, compressionCodec)
{
Guard.NotNull(payload, "payload");
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
/// <param name="checksum">The checksum for the payload.</param>
public Message(byte[] payload, byte[] checksum, CompressionCodecs compressionCodec)
{
Guard.NotNull(payload, "payload");
Guard.NotNull(checksum, "checksum");
Guard.Count(checksum, 4, "checksum");
int length = DefaultHeaderSize + payload.Length;
this.Payload = payload;
this.Magic = DefaultMagicValue;
if (compressionCodec != CompressionCodecs.NoCompressionCodec)
{
this.Attributes |=
(byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec));
}
if (Magic == 1)
{
length++;
}
this.Checksum = checksum;
this.Size = length;
}
public List<string> SendMessages(
TempKafkaConfig config, int messagesPerNode, string header, CompressionCodecs compressionCodec, int numParts)
{
var messages = new List<string>();
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000,
};
var producer = new Producer<int, string>(props);
for (var partition = 0; partition < numParts; partition++)
{
var ms =
Enumerable.Range(0, messagesPerNode)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(m => new KeyedMessage<int, string>(Topic, partition, m)).ToArray());
messages.AddRange(ms);
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
}
producer.Dispose();
return messages;
}
public List<string> SendMessagesToBrokerPartition(
TempKafkaConfig config,
string topic,
int partition,
int numMessages,
CompressionCodecs compression = CompressionCodecs.NoCompressionCodec)
{
var header = string.Format("test-{0}-{1}", config.BrokerId, partition);
var props = new ProducerConfig
{
Brokers = TestUtils.GetBrokerListFromConfigs(Configs),
PartitionerClass = typeof(FixedValuePartitioner).AssemblyQualifiedName,
CompressionCodec = compression,
KeySerializer = typeof(IntEncoder).AssemblyQualifiedName,
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
RetryBackoffMs = 1000, // custom: we need time to rebalance leader
};
var producer = new Producer<int, string>(props);
var ms =
Enumerable.Range(0, numMessages)
.Select(x => header + config.BrokerId + "-" + partition + "-" + x)
.ToList();
producer.Send(ms.Select(x => new KeyedMessage<int, string>(topic, partition, x)).ToArray());
Logger.DebugFormat("Sent {0} messages to broker {1} for partition [{2},{3}]", ms.Count, config.BrokerId, Topic, partition);
producer.Dispose();
return ms;
}
public Message(byte[] bytes, CompressionCodecs codec) : this(bytes, null, codec)
{
}
public Message(byte[] bytes, CompressionCodecs codec) : this(bytes, null, codec)
{
}
public Message(byte[] payload, CompressionCodecs compressionCodec)
: this(payload, null, compressionCodec)
{
Guard.NotNull(payload, "payload");
}
public static Message Compress(IEnumerable<Message> messages, CompressionCodecs compressionCodec, int partition)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (var outputStream = new MemoryStream())
{
using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.ErrorFormat("Error while writing to the GZIP stream {0}", ex.FormatException());
throw;
}
}
Message oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec)
{
PartitionId = partition
};
return oneCompressedMessage;
}
}
case CompressionCodecs.SnappyCompressionCodec:
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
var messageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(messageSet.SetSize))
{
messageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
return new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec)
{
PartitionId = partition
};
}
catch (Exception ex)
{
Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException());
throw;
}
}
default:
throw new UnknownCodecException(String.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}", compressionCodec));
}
}
public ByteBufferMessageSet(
CompressionCodecs compressionCodec, AtomicLong offsetCounter, List <Message> messages)
: this(Create(offsetCounter, compressionCodec, messages))
{
}
public static Message Compress(IEnumerable <Message> messages, CompressionCodecs compressionCodec, int partition)
{
switch (compressionCodec)
{
case CompressionCodecs.DefaultCompressionCodec:
case CompressionCodecs.GZIPCompressionCodec:
using (var outputStream = new MemoryStream())
{
using (var gZipStream = new GZipStream(outputStream, CompressionMode.Compress))
{
//if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
}
var bufferedMessageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(bufferedMessageSet.SetSize))
{
bufferedMessageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
gZipStream.Write(inputStream.ToArray(), 0, inputStream.ToArray().Length);
gZipStream.Close();
}
catch (IOException ex)
{
Logger.ErrorFormat("Error while writing to the GZIP stream {0}",
ex.FormatException());
throw;
}
}
var oneCompressedMessage = new Message(outputStream.ToArray(), compressionCodec)
{
PartitionId = partition
};
return(oneCompressedMessage);
}
}
case CompressionCodecs.SnappyCompressionCodec:
Logger.DebugFormat(
"Allocating BufferedMessageSet of size = {0}",
MessageSet.GetMessageSetSize(messages));
var messageSet = new BufferedMessageSet(messages, partition);
using (var inputStream = new MemoryStream(messageSet.SetSize))
{
messageSet.WriteTo(inputStream);
inputStream.Position = 0;
try
{
return(new Message(SnappyHelper.Compress(inputStream.GetBuffer()), compressionCodec)
{
PartitionId = partition
});
}
catch (Exception ex)
{
Logger.ErrorFormat("Error while writing to the Snappy stream {0}", ex.FormatException());
throw;
}
}
default:
throw new UnknownCodecException(string.Format(CultureInfo.CurrentCulture, "Unknown Codec: {0}",
compressionCodec));
}
}
internal override void Parse(string[] args)
{
base.BuildDictionary(args);
GetString("-z", "--zookeeper", ref this.Zookeeper);
CheckZookeeperPort();
if (string.IsNullOrEmpty(this.Zookeeper))
throw new ArgumentException("Must specify zookeeper with port by -z. Example: -z localhost:2181");
GetString("-t", "--topic", ref this.Topic);
if (string.IsNullOrEmpty(this.Topic))
throw new ArgumentException("Must specify topic by -t.");
GetInt("-p", "--PartitionId", ref this.PartitionId);
GetString("-l", "--PartitionerClass", ref this.PartitionerClass);
GetInt("-c", "--batchcount", ref this.BatchCount);
GetInt("-b", "--MessageCountPerBatch", ref this.MessageCountPerBatch);
GetInt("-m", "--MessageSize", ref this.MessageSize);
GetInt("-r", "--Compression", ref this.Compression);
this.CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(this.Compression.ToString());
GetShort("-a", "--RequiredAcks", ref this.RequiredAcks);
GetInt("-k", "--AckTimeout", ref this.AckTimeout);
GetInt("-s", "--SendTimeout", ref this.SendTimeout);
GetInt("-e", "--ReceiveTimeout", ref this.ReceiveTimeout);
GetInt("-i", "--BufferSize", ref this.BufferSize);
GetInt("-y", "--SyncProducerOfOneBroker", ref this.SyncProducerOfOneBroker);
GetBool("-w", "--ConstantMessageKey", ref this.ConstantMessageKey);
if (this.PartitionId == -2)
{
if (string.IsNullOrEmpty(this.PartitionerClass))
throw new ArgumentException(string.Format("You specify partitionID as -2, please also specify the partitioner class full name, for example: {0}", ProducerConfiguration.DefaultPartitioner));
}
}
public MessageTestVal(byte[] key, byte[] payload, CompressionCodecs codec, Message message)
{
this.Key = key;
this.Payload = payload;
this.Codec = codec;
this.Message = message;
}
public ByteBufferMessageSet(CompressionCodecs compressionCodec, List <Message> messages)
: this(Create(new AtomicLong(0), compressionCodec, messages))
{
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
/// <param name="checksum">The checksum for the payload.</param>
public Message(byte[] payload, byte[] key, CompressionCodecs compressionCodec)
{
Guard.NotNull(payload, "payload");
int length = DefaultHeaderSize + payload.Length;
Key = key;
if (key != null)
{
length += key.Length;
}
this.Payload = payload;
this.Magic = DefaultMagicValue;
if (compressionCodec != CompressionCodecs.NoCompressionCodec)
{
this.Attributes |=
(byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec));
// It seems that the java producer uses magic 0 for compressed messages, so we are sticking with 0 for now
// this.Magic = MagicValueWhenCompress;
}
this.Size = length;
}
public Message(byte[] bytes, byte[] key, CompressionCodecs codec) : this(bytes, key, codec, 0, -1)
{
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <remarks>
/// Initializes the checksum as null. It will be automatically computed.
/// </remarks>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
public Message(byte[] payload, CompressionCodecs compressionCodec)
: this(payload, Crc32Hasher.Compute(payload), compressionCodec)
{
Guard.NotNull(payload, "payload");
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <remarks>
/// Initializes the checksum as null. It will be automatically computed.
/// </remarks>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
public Message(byte[] payload, CompressionCodecs compressionCodec)
: this(payload, Crc32Hasher.Compute(payload), compressionCodec)
{
Guard.NotNull(payload, "payload");
}
public Message(byte[] bytes, byte[] key, CompressionCodecs codec) : this(bytes, key, codec, 0, -1)
{
}