public void ShouldParseResponse()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
writer.Write(1);
writer.Write(123); // correlation id
writer.Write(1); // topic count
writer.WriteShortString("topic");
writer.Write(1); // partition count
writer.Write(999); // partition id
writer.Write((short)ErrorMapping.NoError);
writer.Write(3); // number of offsets
writer.Write(111L);
writer.Write(222L);
writer.Write(333L);
stream.Seek(0, SeekOrigin.Begin);
var reader = new KafkaBinaryReader(stream);
var response = new OffsetResponse.Parser().ParseFrom(reader);
response.CorrelationId.Should().Be(123);
response.ResponseMap.Count.Should().Be(1);
var partitions = response.ResponseMap["topic"];
partitions.Count.Should().Be(1);
var info = partitions.First();
info.Error.Should().Be(ErrorMapping.NoError);
info.Offsets.Count.Should().Be(3);
info.Offsets.SequenceEqual(new List<long>() {111L, 222L, 333L}).Should().BeTrue();
info.PartitionId.Should().Be(999);
}
public void WriteTo(MemoryStream output)
{
using (var writer = new KafkaBinaryWriter(output))
{
WriteTo(writer);
}
}
public void ShouldAbleToParseFetchResponse()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
writer.Write(1);
writer.Write(123); // correlation id
writer.Write(1); // data count
writer.WriteShortString("topic1");
writer.Write(1); // partition count
writer.Write(111); //partition id
writer.Write((short)ErrorMapping.NoError);
writer.Write(1011L); // hw
var messageStream = new MemoryStream();
var messageWriter = new KafkaBinaryWriter(messageStream);
new BufferedMessageSet(new List<Message>() { new Message(new byte[100]) }, 0).WriteTo(messageWriter);
writer.Write((int)messageStream.Length);
writer.Write(messageStream.GetBuffer(), 0, (int)messageStream.Length);
stream.Seek(0, SeekOrigin.Begin);
var reader = new KafkaBinaryReader(stream);
var response = new FetchResponse.Parser().ParseFrom(reader);
var set = response.MessageSet("topic1", 111);
set.Should().NotBeNull();
var messages = set.Messages.ToList();
messages.Count().Should().Be(1);
messages.First().Payload.Length.Should().Be(100);
}
public void ShouldAbleToParseRequest()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
writer.Write(1);
writer.Write(100); // correlation id
writer.Write(2); // broker count
writer.Write(0); // broker id
writer.WriteShortString("host1");
writer.Write(9092); // port
writer.Write(1); // broker id
writer.WriteShortString("host2");
writer.Write(9093); // port
writer.Write(1); // topic count
writer.Write((short)ErrorMapping.NoError);
writer.WriteShortString("topic1");
writer.Write(1); // partitions
writer.Write((short)ErrorMapping.NoError);
writer.Write(111); // partition id
writer.Write(0); // leader broker id
writer.Write(1); // num replicas
writer.Write(1); // replica broker id
writer.Write(1); // in sync replicas
writer.Write(1); // in sync replica broker id
stream.Seek(0, SeekOrigin.Begin);
var reader = new KafkaBinaryReader(stream);
var response = new TopicMetadataRequest.Parser().ParseFrom(reader);
var enumerator = response.GetEnumerator();
enumerator.MoveNext().Should().BeTrue();
enumerator.Current.Topic.Should().Be("topic1");
enumerator.Current.Error.Should().Be(ErrorMapping.NoError);
var partitionEnumerator = enumerator.Current.PartitionsMetadata.GetEnumerator();
partitionEnumerator.MoveNext().Should().BeTrue();
partitionEnumerator.Current.PartitionId.Should().Be(111);
var leader = partitionEnumerator.Current.Leader;
leader.Id.Should().Be(0);
leader.Host.Should().Be("host1");
leader.Port.Should().Be(9092);
var replicas = partitionEnumerator.Current.Replicas.ToList();
replicas.Count.Should().Be(1);
replicas.First().Id.Should().Be(1);
replicas.First().Host.Should().Be("host2");
replicas.First().Port.Should().Be(9093);
var isrs = partitionEnumerator.Current.Isr.ToList();
isrs.Count.Should().Be(1);
isrs.First().Id.Should().Be(1);
isrs.First().Host.Should().Be("host2");
isrs.First().Port.Should().Be(9093);
}
public void ShouldAbleToWriteMessageSetWithExtraBytes()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
var msg1 = new Message(new byte[101]) {Offset = 0};
var msg2 = new Message(new byte[102]) {Offset = 1};
var set = new BufferedMessageSet(new List<Message>() {msg1, msg2}, 0);
set.WriteTo(writer);
writer.Write(new byte[10]); // less than offset and size
var size = (int) stream.Position;
stream.Seek(0, SeekOrigin.Begin);
var reader = new KafkaBinaryReader(stream);
var newSet = BufferedMessageSet.ParseFrom(reader, size, 0);
var messages = newSet.Messages.ToList();
messages.Count().Should().Be(2);
messages[0].Payload.Count().Should().Be(101);
messages[1].Payload.Count().Should().Be(102);
}
public void ShouldAbleToParseResponse()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
writer.Write(1);
writer.Write(123); // correlation id
writer.Write(1); // topic count
writer.WriteShortString("topic");
writer.Write(1); // partition count
writer.Write(999); // partition id
writer.Write((short)ErrorMapping.NoError); // error
writer.Write(111L); // offset
stream.Seek(0, SeekOrigin.Begin);
var reader = new KafkaBinaryReader(stream);
var response = new ProducerResponse.Parser().ParseFrom(reader);
response.CorrelationId.Should().Be(123);
response.Statuses.Count.Should().Be(1);
var info = response.Statuses[new TopicAndPartition("topic", 999)];
info.Error.Should().Be(ErrorMapping.NoError);
info.Offset.Should().Be(111L);
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.Write((short)this.ProducerRequests.Count());
foreach (var request in ProducerRequests)
{
request.WriteTo(writer);
}
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.Assert<ArgumentNullException>(() => writer != null);
foreach (var consumerRequest in ConsumerRequests)
{
consumerRequest.WriteTo(writer);
}
}
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.Write(this.Id);
writer.WriteShortString(this.Host, AbstractRequest.DefaultEncoding);
writer.Write(this.Port);
}
public void WriteTo(KafkaBinaryWriter writer)
{
writer.Write(PartitionId);
writer.Write(Time);
writer.Write(MaxNumOffsets);
}
public void WriteTo(KafkaBinaryWriter writer)
{
writer.Write(PartitionId);
writer.Write(Offset);
writer.Write(FetchSize);
}
/// <summary>
/// Writes content into given stream
/// </summary>
/// <param name="output">
/// The output stream.
/// </param>
public void WriteTo(System.IO.MemoryStream output)
{
Guard.Assert<ArgumentNullException>(() => output != null);
using (var writer = new KafkaBinaryWriter(output))
{
writer.Write(this.RequestBuffer.Capacity - DefaultRequestSizeSize);
writer.Write(this.RequestTypeId);
this.WriteTo(writer);
}
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.Write(this.VersionId);
writer.Write(this.CorrelationId);
writer.WriteShortString(this.ClientId);
writer.Write(this.RequiredAcks);
writer.Write(this.AckTimeout);
writer.Write(this.Data.Count());
foreach (var topicData in this.Data)
{
writer.WriteShortString(topicData.Topic);
writer.Write(topicData.PartitionData.Count());
foreach (var partitionData in topicData.PartitionData)
{
writer.Write(partitionData.Partition);
writer.Write(partitionData.MessageSet.SetSize);
partitionData.MessageSet.WriteTo(writer);
}
}
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.WriteTopic(this.Topic, DefaultEncoding);
writer.Write(this.Partition);
writer.Write(this.MessageSet.SetSize);
this.MessageSet.WriteTo(writer);
}
/// <summary>
/// Writes message data using given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
/// <param name="getBuffer"></param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.Seek(MagicOffset, SeekOrigin.Current);
var beginningPosition = writer.CurrentPos;
writer.Write(this.Magic);
writer.Write(this.Attributes);
writer.Write(this.KeyLength);
if (KeyLength != -1)
{
writer.Write(this.Key);
}
writer.Write(Payload.Length);
writer.Write(this.Payload);
var crc = ComputeChecksum(writer.Buffer, (int)beginningPosition, Size - MagicOffset);
writer.Seek(-Size, SeekOrigin.Current);
writer.Write(crc);
writer.Seek(Size - DefaultCrcLength, SeekOrigin.Current);
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.Assert<ArgumentNullException>(() => writer != null);
writer.WriteTopic(this.Topic, DefaultEncoding);
writer.Write(this.Partition);
writer.Write(this.MessageSet.SetSize);
this.MessageSet.WriteTo(writer);
}
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
// if leader exists
writer.Write(this.PartitionId);
if (this.Leader != null)
{
writer.Write((byte)1);
this.Leader.WriteTo(writer);
}
else
{
writer.Write((byte)0);
}
// number of replicas
writer.Write((short)Replicas.Count());
foreach (var replica in Replicas)
{
replica.WriteTo(writer);
}
// number of in-sync replicas
writer.Write((short)this.Isr.Count());
foreach (var isr in Isr)
{
isr.WriteTo(writer);
}
writer.Write((byte)0);
}
/// <summary>
/// Writes message data using given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.Write(this.Magic);
writer.Write(this.Attributes);
writer.Write(this.Checksum);
writer.Write(this.Payload);
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
foreach (var consumerRequest in ConsumerRequests)
{
consumerRequest.WriteTo(writer);
}
}
/// <summary>
/// Writes message data using given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.Assert<ArgumentNullException>(() => writer != null);
writer.Write(this.Magic);
writer.Write(this.Checksum);
writer.Write(this.Payload);
}
/// <summary>
/// Writes content into given stream
/// </summary>
/// <param name="output">
/// The output stream.
/// </param>
public void WriteTo(MemoryStream output)
{
Guard.NotNull(output, "output");
using (var writer = new KafkaBinaryWriter(output))
{
writer.Write(this.RequestBuffer.Capacity - DefaultRequestSizeSize);
writer.Write(this.RequestTypeId);
this.WriteTo(writer);
}
}
public void ShouldReadShortString()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
writer.WriteShortString("hello world");
stream.Seek(0, SeekOrigin.Begin);
BitWorks.ReadShortString(new KafkaBinaryReader(stream), "UTF-8").Should().Be("hello world");
}
/// <summary>
/// Writes message data into given message buffer
/// </summary>
/// <param name="output">
/// The output.
/// </param>
public void WriteTo(MemoryStream output)
{
Guard.NotNull(output, "output");
using (var writer = new KafkaBinaryWriter(output))
{
this.WriteTo(writer);
}
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.WriteTopic(this.Topic, DefaultEncoding);
writer.Write(this.Partition);
writer.Write(this.Offset);
writer.Write(this.MaxSize);
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.Assert<ArgumentNullException>(() => writer != null);
writer.Write((short)this.ProducerRequests.Count());
foreach (var request in ProducerRequests)
{
request.WriteTo(writer);
}
}
/// <summary>
/// Writes content into given stream
/// </summary>
/// <param name="output">
/// The output stream.
/// </param>
public override sealed void WriteTo(MemoryStream output)
{
Guard.Assert<ArgumentNullException>(() => output != null);
using (var writer = new KafkaBinaryWriter(output))
{
this.WriteTo(writer);
}
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.Assert<ArgumentNullException>(() => writer != null);
writer.WriteTopic(this.Topic, DefaultEncoding);
writer.Write(this.Partition);
writer.Write(this.Time);
writer.Write(this.MaxOffsets);
}
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public override sealed void WriteTo(KafkaBinaryWriter writer)
{
Guard.Assert<ArgumentNullException>(() => writer != null);
foreach (var message in this.Messages)
{
writer.Write(message.Size);
message.WriteTo(writer);
}
}
/// <summary> /// Writes content into given writer /// </summary> /// <param name="writer"> /// The writer. /// </param> public abstract void WriteTo(KafkaBinaryWriter writer);
/// <summary>
/// Writes content into given writer
/// </summary>
/// <param name="writer">
/// The writer.
/// </param>
public void WriteTo(KafkaBinaryWriter writer)
{
Guard.NotNull(writer, "writer");
writer.Write(VersionId);
writer.Write(CorrelationId);
writer.WriteShortString(ClientId);
writer.Write(ReplicaId);
writer.Write(RequestInfo.Count);
foreach (var kv in RequestInfo)
{
writer.WriteShortString(kv.Key);
writer.Write(kv.Value.Count);
foreach (var info in kv.Value)
{
info.WriteTo(writer);
}
}
}