/// <summary>
/// Gets the bytes representing the request which is used when generating a multi-request.
/// </summary>
/// <remarks>
/// The <see cref="GetBytes"/> method is used for sending a single <see cref="RequestType.Produce"/>.
/// It prefixes this byte array with the request type and the number of messages. This method
/// is used to supply the <see cref="MultiProducerRequest"/> with the contents for its message.
/// </remarks>
/// <returns>The bytes that represent this <see cref="ProducerRequest"/>.</returns>
internal byte[] GetInternalBytes()
{
List <byte> messagePack = new List <byte>();
foreach (Message message in Messages)
{
byte[] messageBytes = message.GetBytes();
messagePack.AddRange(BitWorks.GetBytesReversed(messageBytes.Length));
messagePack.AddRange(messageBytes);
}
byte[] topicLengthBytes = BitWorks.GetBytesReversed(Convert.ToInt16(Topic.Length));
byte[] topicBytes = Encoding.UTF8.GetBytes(Topic);
byte[] partitionBytes = BitWorks.GetBytesReversed(Partition);
byte[] messagePackLengthBytes = BitWorks.GetBytesReversed(messagePack.Count);
byte[] messagePackBytes = messagePack.ToArray();
List <byte> encodedMessageSet = new List <byte>();
encodedMessageSet.AddRange(topicLengthBytes);
encodedMessageSet.AddRange(topicBytes);
encodedMessageSet.AddRange(partitionBytes);
encodedMessageSet.AddRange(messagePackLengthBytes);
encodedMessageSet.AddRange(messagePackBytes);
return(encodedMessageSet.ToArray());
}
public List <byte> GetRequestBytes()
{
var request = new List <byte>();
request.AddRange(BitWorks.GetBytesReversed(partitionId));
return(request);
}
/// <summary>
/// Gets the bytes matching the expected Kafka structure.
/// </summary>
/// <returns>The byte array of the request.</returns>
public override byte[] GetBytes()
{
List <byte> messagePack = new List <byte>();
foreach (Message message in Messages)
{
byte[] messageBytes = message.GetBytes();
messagePack.AddRange(BitWorks.GetBytesReversed(messageBytes.Length));
messagePack.AddRange(messageBytes);
}
byte[] requestBytes = BitWorks.GetBytesReversed(Convert.ToInt16((int)RequestType.Produce));
byte[] topicLengthBytes = BitWorks.GetBytesReversed(Convert.ToInt16(Topic.Length));
byte[] topicBytes = Encoding.UTF8.GetBytes(Topic);
byte[] partitionBytes = BitWorks.GetBytesReversed(Partition);
byte[] messagePackLengthBytes = BitWorks.GetBytesReversed(messagePack.Count);
byte[] messagePackBytes = messagePack.ToArray();
List <byte> encodedMessageSet = new List <byte>();
encodedMessageSet.AddRange(requestBytes);
encodedMessageSet.AddRange(topicLengthBytes);
encodedMessageSet.AddRange(topicBytes);
encodedMessageSet.AddRange(partitionBytes);
encodedMessageSet.AddRange(messagePackLengthBytes);
encodedMessageSet.AddRange(messagePackBytes);
encodedMessageSet.InsertRange(0, BitWorks.GetBytesReversed(encodedMessageSet.Count));
return(encodedMessageSet.ToArray());
}
public void ReverseBytesZeroLengthArray()
{
byte[] arr = new byte[0];
byte[] reversedArr = BitWorks.ReverseBytes(arr);
Assert.IsNotNull(reversedArr);
Assert.AreEqual(0, reversedArr.Length);
}
public void GetBytesValidStructure()
{
string topicName = "topic";
ConsumerRequest request = new ConsumerRequest(topicName, 1, 10L, 100);
// REQUEST TYPE ID + TOPIC LENGTH + TOPIC + PARTITION + OFFSET + MAX SIZE
int requestSize = 2 + 2 + topicName.Length + 4 + 8 + 4;
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
// add 4 bytes for the length of the message at the beginning
Assert.AreEqual(requestSize + 4, bytes.Length);
// first 4 bytes = the message length
Assert.AreEqual(25, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the request type
Assert.AreEqual((short)RequestType.Fetch, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the topic length
Assert.AreEqual((short)topicName.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next few bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(topicName.Length).ToArray <byte>()));
// next 4 bytes = the partition
Assert.AreEqual(1, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(8 + topicName.Length).Take(4).ToArray <byte>()), 0));
// next 8 bytes = the offset
Assert.AreEqual(10, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(12 + topicName.Length).Take(8).ToArray <byte>()), 0));
// last 4 bytes = the max size
Assert.AreEqual(100, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(20 + +topicName.Length).Take(4).ToArray <byte>()), 0));
}
public void GetBytesValid()
{
string topicName = "topic";
OffsetRequest request = new OffsetRequest(topicName, 0, OffsetRequest.LatestTime, 10);
// format = len(request) + requesttype + len(topic) + topic + partition + time + max
// total byte count = 4 + (2 + 2 + 5 + 4 + 8 + 4)
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
Assert.AreEqual(29, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(25, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestType.Offsets, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the length of the topic
Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next 5 bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray <byte>()));
// next 4 bytes = the partition
Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray <byte>()), 0));
// next 8 bytes = time
Assert.AreEqual(OffsetRequest.LatestTime, BitConverter.ToInt64(BitWorks.ReverseBytes(bytes.Skip(17).Take(8).ToArray <byte>()), 0));
// next 4 bytes = max offsets
Assert.AreEqual(10, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(25).Take(4).ToArray <byte>()), 0));
}
public void GetBytesValidFormat()
{
string topicName = "topic";
ProducerRequest request = new ProducerRequest(
topicName, 0, new List <Message> {
new Message(new byte[10])
});
// format = len(request) + requesttype + len(topic) + topic + partition + len(messagepack) + message
// total byte count = 4 + (2 + 2 + 5 + 4 + 4 + 19)
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
Assert.AreEqual(40, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(36, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestType.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the length of the topic
Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next 5 bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray <byte>()));
// next 4 bytes = the partition
Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray <byte>()), 0));
// next 4 bytes = the length of the individual messages in the pack
Assert.AreEqual(19, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(17).Take(4).ToArray <byte>()), 0));
// fianl bytes = the individual messages in the pack
Assert.AreEqual(19, bytes.Skip(21).ToArray <byte>().Length);
}
public List <byte> GetRequestBytes()
{
var requestBuffer = new List <byte>();
// Get request base: ApiKey, ApiVersion, CorrelationId, ClientId
requestBuffer.AddRange(request.GetRequestBytes());
// Add Fetch request base: ReplicaId, MaxWaitTime, MinBytes
requestBuffer.AddRange(BitWorks.GetBytesReversed(Request.ReplicaId));
requestBuffer.AddRange(BitWorks.GetBytesReversed(MaxWaitTime));
requestBuffer.AddRange(BitWorks.GetBytesReversed(MinBytes));
if (Topics.Count > 0)
{
requestBuffer.AddRange(BitWorks.GetBytesReversed(Topics.Count));
// Add Topic count and all topics including partitions
foreach (var topic in Topics)
{
requestBuffer.AddRange(topic.GetRequestBytes());
}
}
requestBuffer.InsertRange(0, BitWorks.GetBytesReversed(Convert.ToInt32(requestBuffer.Count)));
return(requestBuffer);
}
public void GetBytesValidFormat()
{
List <FetchRequest> requests = new List <FetchRequest>
{
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic b", 0, 0),
new FetchRequest("topic c", 0, 0)
};
MultiFetchRequest request = new MultiFetchRequest(requests);
// format = len(request) + requesttype + requestcount + requestpackage
// total byte count = 4 + (2 + 2 + 100)
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
Assert.AreEqual(108, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(104, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestType.MultiFetch, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the number of messages
Assert.AreEqual((short)4, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
}
/// <summary>
/// Consumes messages from Kafka.
/// </summary>
/// <param name="request">The request to send to Kafka.</param>
/// <returns>A list of messages from Kafka.</returns>
public List <Message> Consume(ConsumerRequest request)
{
List <Message> messages = new List <Message>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
// TODO: need to check in on kafka error codes...assume all's good for now
byte[] unbufferedData = data.Skip(2).ToArray();
int processed = 0;
int length = unbufferedData.Length - 4;
int messageSize = 0;
while (processed <= length)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0);
messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>()));
processed += 4 + messageSize;
}
}
}
return(messages);
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public IList <long> GetOffsetsBefore(OffsetRequest request)
{
List <long> offsets = new List <long>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
// TODO: need to check in on kafka error codes...assume all's good for now
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray <byte>()), 0);
int position = 0;
for (int ix = 0; ix < numOfOffsets; ix++)
{
position = (ix * 8) + 4;
offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray <byte>()), 0));
}
}
}
return(offsets);
}
internal static Broker ParseFrom(KafkaBinaryReader reader)
{
var id = reader.ReadInt32();
var host = BitWorks.ReadShortString(reader, AbstractRequest.DefaultEncoding);
var port = reader.ReadInt32();
return(new Broker(id, host, port));
}
public void GetBytesReversedShortValid()
{
short val = (short)100;
byte[] normal = BitConverter.GetBytes(val);
byte[] reversed = BitWorks.GetBytesReversed(val);
TestReversedArray(normal, reversed);
}
public void ShouldAbleToParseBroker()
{
var broker = Broker.CreateBroker(1, @"{'host': 'host1', 'port': 1234}");
broker.Id.Should().Be(1);
broker.Host.Should().Be("host1");
broker.Port.Should().Be(1234);
broker.SizeInBytes.Should().Be(BitWorks.GetShortStringLength("host1", "UTF-8") + 8);
}
public void GetBytesReversedLongValid()
{
long val = 100L;
byte[] normal = BitConverter.GetBytes(val);
byte[] reversed = BitWorks.GetBytesReversed(val);
TestReversedArray(normal, reversed);
}
public List <byte> GetRequestBytes()
{
var request = new List <byte>();
request.AddRange(partition.GetRequestBytes());
request.AddRange(BitWorks.GetBytesReversed(FetchOffset));
request.AddRange(BitWorks.GetBytesReversed(MaxBytes));
return(request);
}
public void ShouldReadShortString()
{
var stream = new MemoryStream();
var writer = new KafkaBinaryWriter(stream);
writer.WriteShortString("hello world");
stream.Seek(0, SeekOrigin.Begin);
BitWorks.ReadShortString(new KafkaBinaryReader(stream), "UTF-8").Should().Be("hello world");
}
public void GetBytesReversedIntValid()
{
int val = 100;
byte[] normal = BitConverter.GetBytes(val);
byte[] reversed = BitWorks.GetBytesReversed(val);
TestReversedArray(normal, reversed);
}
public void ShouldAbleToCreateBroker()
{
var broker = new Broker(1, "host1", 1234);
broker.Id.Should().Be(1);
broker.Host.Should().Be("host1");
broker.Port.Should().Be(1234);
broker.SizeInBytes.Should().Be(BitWorks.GetShortStringLength("host1", "UTF-8") + 8);
}
public void ReverseBytesValid()
{
byte[] arr = BitConverter.GetBytes((short)1);
byte[] original = new byte[2];
arr.CopyTo(original, 0);
byte[] reversedArr = BitWorks.ReverseBytes(arr);
TestReversedArray(original, reversedArr);
}
public List <byte> GetRequestBytes()
{
var request = new List <byte>();
request.AddRange(partition.GetRequestBytes());
request.AddRange(BitWorks.GetBytesReversed(Time));
request.AddRange(BitWorks.GetBytesReversed(MaxNuberOfOffsets));
return(request);
}
/// <summary>
/// Converts the request to an array of bytes that is expected by Kafka.
/// </summary>
/// <returns>A list of bytes that represents the request.</returns>
public List <byte> GetBytes()
{
var request = new List <byte>();
request.AddRange(BitWorks.GetBytesReversed(ApiKey));
request.AddRange(BitWorks.GetBytesReversed(apiVersion));
request.AddRange(BitWorks.GetBytesReversed(CorrelationId));
return(request);
}
/// <summary>
/// Parses a message from a byte array given the format Kafka likes.
/// </summary>
/// <param name="data">The data for a message.</param>
/// <returns>The message.</returns>
public static Message ParseFrom(byte[] data)
{
int size = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Take(4).ToArray <byte>()), 0);
byte magic = data[4];
byte[] checksum = data.Skip(5).Take(4).ToArray <byte>();
byte[] payload = data.Skip(9).Take(size).ToArray <byte>();
return(new Message(payload, magic, checksum));
}
public List <byte> GetRequestBytes()
{
var request = new List <byte>();
request.AddRange(BitWorks.GetBytesReversed(ApiKey));
request.AddRange(BitWorks.GetBytesReversed(apiVersion));
request.AddRange(BitWorks.GetBytesReversed(CorrelationId));
request.AddRange(BitWorks.GetBytesReversed((short)ClientId.Length));
request.AddRange(Encoding.ASCII.GetBytes(ClientId));
return(request);
}
public int GetRequestLength()
{
var size = DefaultHeaderSize8 +
FetchRequest.DefaultVersionIdSize +
FetchRequest.DefaultCorrelationIdSize +
BitWorks.GetShortStringLength(clientId, DefaultEncoding) +
DefaultNumberOfTopicsSize +
Topics.Sum(x => BitWorks.GetShortStringLength(x, DefaultEncoding));
return(size);
}
public int GetRequestLength()
{
return(DefaultRequestSizeSize +
DefaultRequestIdSize +
DefaultVersionIdSize +
DefaultCorrelationIdSize +
BitWorks.GetShortStringLength(this.ClientId, DefaultEncoding) +
DefaultReplicaIdSize +
DefaultMaxWaitSize +
DefaultMinBytesSize +
DefaultOffsetInfoSizeSize + this.OffsetInfo.Keys.Sum(x => BitWorks.GetShortStringLength(x, DefaultEncoding)) + this.OffsetInfo.Values.Select(pl => 4 + pl.Sum(p => p.SizeInBytes)).Sum());
}
/// <summary>
/// Gets the bytes matching the expected Kafka structure.
/// </summary>
/// <returns>The byte array of the request.</returns>
public override byte[] GetBytes()
{
List <byte> encodedMessageSet = new List <byte>();
encodedMessageSet.AddRange(GetInternalBytes());
byte[] requestBytes = BitWorks.GetBytesReversed(Convert.ToInt16((int)RequestType.Produce));
encodedMessageSet.InsertRange(0, requestBytes);
encodedMessageSet.InsertRange(0, BitWorks.GetBytesReversed(encodedMessageSet.Count));
return(encodedMessageSet.ToArray());
}
public List <byte> GetRequestBytes()
{
var requestBytes = new List <byte>();
requestBytes.AddRange(BitWorks.GetBytesReversed(partitionId));
var messageSetBuffer = new List <byte>();
messageSetBuffer.AddRange(messageSet.GetRequestBytes());
requestBytes.AddRange(BitWorks.GetBytesReversed(messageSetBuffer.Count));
requestBytes.AddRange(messageSetBuffer);
return(requestBytes);
}
public List <byte> GetRequestBytes()
{
var requestBytes = new List <byte>();
requestBytes.AddRange(BitWorks.GetBytesReversed(messageOffset)); // Don't care
var messageBuffer = new List <byte>();
messageBuffer.AddRange(message.GetBytes());
requestBytes.AddRange(BitWorks.GetBytesReversed(messageBuffer.Count));
requestBytes.AddRange(messageBuffer);
return(requestBytes);
}
/// <summary>
/// Gets the bytes matching the expected Kafka structure.
/// </summary>
/// <returns>The byte array of the request.</returns>
public override byte[] GetBytes()
{
byte[] internalBytes = GetInternalBytes();
List <byte> request = new List <byte>();
// add the 2 for the RequestType.Fetch
request.AddRange(BitWorks.GetBytesReversed(internalBytes.Length + 2));
request.AddRange(BitWorks.GetBytesReversed((short)RequestType.Fetch));
request.AddRange(internalBytes);
return(request.ToArray <byte>());
}
