/// <summary>
/// Consumes messages from Kafka.
/// </summary>
/// <param name="request">The request to send to Kafka.</param>
/// <returns>A list of messages from Kafka.</returns>
public List <Message> Consume(ConsumerRequest request)
{
List <Message> messages = new List <Message>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
// TODO: need to check in on kafka error codes...assume all's good for now
byte[] unbufferedData = data.Skip(2).ToArray();
int processed = 0;
int length = unbufferedData.Length - 4;
int messageSize = 0;
while (processed <= length)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0);
messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>()));
processed += 4 + messageSize;
}
}
}
return(messages);
}
public void ReverseBytesZeroLengthArray()
{
byte[] arr = new byte[0];
byte[] reversedArr = BitWorks.ReverseBytes(arr);
Assert.IsNotNull(reversedArr);
Assert.AreEqual(0, reversedArr.Length);
}
public void GetBytesValidFormat()
{
List <FetchRequest> requests = new List <FetchRequest>
{
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic a", 0, 0),
new FetchRequest("topic b", 0, 0),
new FetchRequest("topic c", 0, 0)
};
MultiFetchRequest request = new MultiFetchRequest(requests);
// format = len(request) + requesttype + requestcount + requestpackage
// total byte count = 4 + (2 + 2 + 100)
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
Assert.AreEqual(108, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(104, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestType.MultiFetch, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the number of messages
Assert.AreEqual((short)4, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
}
public void GetBytesValidStructure()
{
string topicName = "topic";
ConsumerRequest request = new ConsumerRequest(topicName, 1, 10L, 100);
// REQUEST TYPE ID + TOPIC LENGTH + TOPIC + PARTITION + OFFSET + MAX SIZE
int requestSize = 2 + 2 + topicName.Length + 4 + 8 + 4;
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
// add 4 bytes for the length of the message at the beginning
Assert.AreEqual(requestSize + 4, bytes.Length);
// first 4 bytes = the message length
Assert.AreEqual(25, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the request type
Assert.AreEqual((short)RequestType.Fetch, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the topic length
Assert.AreEqual((short)topicName.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next few bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(topicName.Length).ToArray <byte>()));
// next 4 bytes = the partition
Assert.AreEqual(1, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(8 + topicName.Length).Take(4).ToArray <byte>()), 0));
// next 8 bytes = the offset
Assert.AreEqual(10, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(12 + topicName.Length).Take(8).ToArray <byte>()), 0));
// last 4 bytes = the max size
Assert.AreEqual(100, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(20 + +topicName.Length).Take(4).ToArray <byte>()), 0));
}
public void GetBytesValid()
{
string topicName = "topic";
OffsetRequest request = new OffsetRequest(topicName, 0, OffsetRequest.LatestTime, 10);
// format = len(request) + requesttype + len(topic) + topic + partition + time + max
// total byte count = 4 + (2 + 2 + 5 + 4 + 8 + 4)
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
Assert.AreEqual(29, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(25, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestType.Offsets, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the length of the topic
Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next 5 bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray <byte>()));
// next 4 bytes = the partition
Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray <byte>()), 0));
// next 8 bytes = time
Assert.AreEqual(OffsetRequest.LatestTime, BitConverter.ToInt64(BitWorks.ReverseBytes(bytes.Skip(17).Take(8).ToArray <byte>()), 0));
// next 4 bytes = max offsets
Assert.AreEqual(10, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(25).Take(4).ToArray <byte>()), 0));
}
public void GetBytesValidFormat()
{
string topicName = "topic";
ProducerRequest request = new ProducerRequest(
topicName, 0, new List <Message> {
new Message(new byte[10])
});
// format = len(request) + requesttype + len(topic) + topic + partition + len(messagepack) + message
// total byte count = 4 + (2 + 2 + 5 + 4 + 4 + 19)
byte[] bytes = request.GetBytes();
Assert.IsNotNull(bytes);
Assert.AreEqual(40, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(36, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestType.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the length of the topic
Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next 5 bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray <byte>()));
// next 4 bytes = the partition
Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray <byte>()), 0));
// next 4 bytes = the length of the individual messages in the pack
Assert.AreEqual(19, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(17).Take(4).ToArray <byte>()), 0));
// fianl bytes = the individual messages in the pack
Assert.AreEqual(19, bytes.Skip(21).ToArray <byte>().Length);
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public IList <long> GetOffsetsBefore(OffsetRequest request)
{
List <long> offsets = new List <long>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
// TODO: need to check in on kafka error codes...assume all's good for now
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray <byte>()), 0);
int position = 0;
for (int ix = 0; ix < numOfOffsets; ix++)
{
position = (ix * 8) + 4;
offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray <byte>()), 0));
}
}
}
return(offsets);
}
public void ReverseBytesValid()
{
byte[] arr = BitConverter.GetBytes((short)1);
byte[] original = new byte[2];
arr.CopyTo(original, 0);
byte[] reversedArr = BitWorks.ReverseBytes(arr);
TestReversedArray(original, reversedArr);
}
/// <summary>
/// Parses a message from a byte array given the format Kafka likes.
/// </summary>
/// <param name="data">The data for a message.</param>
/// <returns>The message.</returns>
public static Message ParseFrom(byte[] data)
{
int size = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Take(4).ToArray <byte>()), 0);
byte magic = data[4];
byte[] checksum = data.Skip(5).Take(4).ToArray <byte>();
byte[] payload = data.Skip(9).Take(size).ToArray <byte>();
return(new Message(payload, magic, checksum));
}
internal static BufferedMessageSet ParseFrom(byte[] bytes)
{
var messages = new List <Message>();
int processed = 0;
int length = bytes.Length - 4;
while (processed <= length)
{
int messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(processed).Take(4).ToArray()), 0);
messages.Add(Message.ParseFrom(bytes.Skip(processed).Take(messageSize + 4).ToArray()));
processed += 4 + messageSize;
}
return(new BufferedMessageSet(messages));
}
public FetchResponse Fetch(FetchRequest request)
{
using (KafkaConnection connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
var fetchResponse = new FetchResponse(data);
return(fetchResponse);
}
return(null);
}
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public OffsetResponse GetOffsetResponseBefore(OffsetRequest request)
{
using (var connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength == 0)
{
return(null);
}
byte[] data = connection.Read(dataLength);
var offsetResponse = new OffsetResponse(data);
return(offsetResponse);
}
}
public ProduceResponse Produce(int correlationId, string clientId, int timeOut, string topicName, int partitionId, byte[] payLoad)
{
var request = new ProduceRequest(timeOut, correlationId, clientId);
request.AddMessage(topicName, partitionId, payLoad);
using (var connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
var response = new ProduceResponse();
if (dataLength != 0)
{
byte[] data = connection.Read(dataLength);
response.Parse(data);
}
return(response);
}
}
/// <summary>
/// Get meta data for a topic
/// </summary>
/// <param name="correlationId"></param>Id used by the client to identify this transaction. Returned in the response
/// <param name="clientId"></param>Name to identify the client. Used in server logs
/// <param name="topicName"></param> Name of the requested topic. If topic name is null metadata for all topics will be returned
/// <returns></returns>
public MetadataResponse Metadata(int correlationId, string clientId, String topicName)
{
MetadataRequest request = new MetadataRequest(correlationId, clientId, topicName);
using (var connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength == 0)
{
return(null);
}
byte[] data = connection.Read(dataLength);
MetadataResponse metadataResponse = new MetadataResponse();
metadataResponse.Parse(data, 0);
return(metadataResponse);
}
}
public void WriteToValidFormat()
{
List <ProducerRequest> requests = new List <ProducerRequest>
{
new ProducerRequest("topic a", 0, new List <Message> {
new Message(new byte[10])
}),
new ProducerRequest("topic a", 0, new List <Message> {
new Message(new byte[10])
}),
new ProducerRequest("topic b", 0, new List <Message> {
new Message(new byte[10])
}),
new ProducerRequest("topic c", 0, new List <Message> {
new Message(new byte[10])
})
};
MultiProducerRequest request = new MultiProducerRequest(requests);
// format = len(request) + requesttype + requestcount + requestpackage
// total byte count = 4 + (2 + 2 + 144)
MemoryStream ms = new MemoryStream();
request.WriteTo(ms);
byte[] bytes = ms.ToArray();
Assert.IsNotNull(bytes);
Assert.AreEqual(156, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(152, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestTypes.MultiProduce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the number of messages
Assert.AreEqual((short)4, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
}
public static Message ParseFrom(byte[] data)
{
int size = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Take(4).ToArray()), 0);
byte magic = data[4];
byte[] checksum;
byte[] payload;
byte attributes;
if (magic == 1)
{
attributes = data[5];
checksum = data.Skip(6).Take(4).ToArray();
payload = data.Skip(10).Take(size).ToArray();
return(new Message(payload, checksum, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask)));
}
else
{
checksum = data.Skip(5).Take(4).ToArray();
payload = data.Skip(9).Take(size).ToArray();
return(new Message(payload, checksum));
}
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public IList <long> GetOffsetsBefore(OffsetRequest request)
{
List <long> offsets = new List <long>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray <byte>()), 0);
int position = 0;
for (int ix = 0; ix < numOfOffsets; ix++)
{
position = (ix * 8) + 4;
offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray <byte>()), 0));
}
}
}
return(offsets);
}
/// <summary>
/// Consumes messages from Kafka.
/// </summary>
/// <param name="request">The request to send to Kafka.</param>
/// <returns>A list of messages from Kafka.</returns>
public List <Message> Consume(FetchRequest request)
{
List <Message> messages = new List <Message>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
byte[] unbufferedData = data.Skip(2).ToArray();
int processed = 0;
int length = unbufferedData.Length - 4;
int messageSize = 0;
while (processed <= length)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0);
messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>()));
processed += 4 + messageSize;
}
}
}
return(messages);
}
public void GetBytesValidStructure()
{
string topicName = "topic";
int correlationId = 1;
string clientId = "TestClient";
int maxWait = 234;
int minBytes = 345;
var requestMap = new Dictionary <string, List <PartitionFetchInfo> >();
requestMap[topicName] = new List <PartitionFetchInfo>()
{
new PartitionFetchInfo(2, 4000, 777)
};
var request = new FetchRequest(correlationId, clientId, maxWait, minBytes, requestMap);
int requestSize = 4 + //request size
2 + //request type id
2 + //versionId
4 + //correlation id
BitWorks.GetShortStringLength(clientId, AbstractRequest.DefaultEncoding) + // client id length
4 + //replica id
4 + //max wait
4 + //min bytes
4 + //offset info count
//=== offset info part
request.OffsetInfo.Keys.Sum(x => BitWorks.GetShortStringLength(x, AbstractRequest.DefaultEncoding)) +
request.OffsetInfo.Values.Select(pl => 4 + pl.Sum(p => p.SizeInBytes)).Sum();
var ms = new MemoryStream();
request.WriteTo(ms);
byte[] bytes = ms.ToArray();
Assert.IsNotNull(bytes);
// add 4 bytes for the length of the message at the beginning
Assert.AreEqual(requestSize, bytes.Length);
// first 4 bytes = the message length
Assert.AreEqual(requestSize - 4, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the request type
Assert.AreEqual((short)RequestTypes.Fetch,
BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the version id
Assert.AreEqual((short)FetchRequest.CurrentVersion,
BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the correlation id
Assert.AreEqual(correlationId,
BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(8).Take(4).ToArray <byte>()), 0));
// next 2 bytes = the client id length
Assert.AreEqual((short)clientId.Length,
BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(12).Take(2).ToArray <byte>()), 0));
// next few bytes = the client id
Assert.AreEqual(clientId, Encoding.ASCII.GetString(bytes.Skip(14).Take(clientId.Length).ToArray <byte>()));
// next 4 bytes = replica id
Assert.AreEqual(-1,
BitConverter.ToInt32(
BitWorks.ReverseBytes(bytes.Skip(14 + clientId.Length).Take(4).ToArray <byte>()), 0));
// next 4 bytes = max wait
Assert.AreEqual(maxWait,
BitConverter.ToInt32(
BitWorks.ReverseBytes(bytes.Skip(18 + clientId.Length).Take(4).ToArray <byte>()), 0));
// next 4 bytes = min bytes
Assert.AreEqual(minBytes,
BitConverter.ToInt32(
BitWorks.ReverseBytes(bytes.Skip(22 + clientId.Length).Take(4).ToArray <byte>()), 0));
// next 4 bytes = offset info count
Assert.AreEqual(1,
BitConverter.ToInt32(
BitWorks.ReverseBytes(bytes.Skip(26 + clientId.Length).Take(4).ToArray <byte>()), 0));
//=== offset info part
// next 2 bytes = the topic length
Assert.AreEqual((short)topicName.Length,
BitConverter.ToInt16(
BitWorks.ReverseBytes(bytes.Skip(30 + clientId.Length).Take(2).ToArray <byte>()), 0));
// next few bytes = the topic
Assert.AreEqual(topicName,
Encoding.ASCII.GetString(
bytes.Skip(32 + clientId.Length).Take(topicName.Length).ToArray <byte>()));
// next 4 bytes = partitions count
Assert.AreEqual(1,
BitConverter.ToInt32(
BitWorks.ReverseBytes(
bytes.Skip(32 + clientId.Length + topicName.Length).Take(4).ToArray <byte>()), 0));
// next 4 bytes = partition
Assert.AreEqual(2,
BitConverter.ToInt32(
BitWorks.ReverseBytes(
bytes.Skip(36 + clientId.Length + topicName.Length).Take(4).ToArray <byte>()), 0));
// next 4 bytes = offset
Assert.AreEqual(4000,
BitConverter.ToInt64(
BitWorks.ReverseBytes(
bytes.Skip(40 + clientId.Length + topicName.Length).Take(8).ToArray <byte>()), 0));
// next 4 bytes = fetch size
Assert.AreEqual(777,
BitConverter.ToInt32(
BitWorks.ReverseBytes(
bytes.Skip(48 + clientId.Length + topicName.Length).Take(8).ToArray <byte>()), 0));
}
/// <summary>
/// Executes a multi-fetch operation.
/// </summary>
/// <param name="request">The request to push to Kafka.</param>
/// <returns>
/// A list containing sets of messages. The message sets should match the request order.
/// </returns>
public List <List <Message> > Consume(MultiFetchRequest request)
{
int fetchRequests = request.ConsumerRequests.Count;
List <List <Message> > messages = new List <List <Message> >();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int position = 0;
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
position = position + 2;
for (int ix = 0; ix < fetchRequests; ix++)
{
messages.Add(new List <Message>());
int messageSetSize = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Skip(position).Take(4).ToArray <byte>()), 0);
position = position + 4;
errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Skip(position).Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
position = position + 2;
byte[] messageSetBytes = data.Skip(position).ToArray <byte>().Take(messageSetSize).ToArray <byte>();
int processed = 0;
int messageSize = 0;
// dropped 2 bytes at the end...padding???
while (processed < messageSetBytes.Length - 2)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(messageSetBytes.Skip(processed).Take(4).ToArray <byte>()), 0);
messages[ix].Add(Message.ParseFrom(messageSetBytes.Skip(processed).Take(messageSize + 4).ToArray <byte>()));
processed += 4 + messageSize;
}
position = position + processed;
}
}
}
return(messages);
}
public void ReverseBytesNullArray()
{
byte[] arr = null;
Assert.IsNull(BitWorks.ReverseBytes(arr));
}
public void GetBytesValidStructure()
{
string topicName = "topic";
int correlationId = 1;
string clientId = "TestClient";
short requiredAcks = 5;
int ackTimeout = 345;
var partition = 2;
short error = 0;
var payload = Encoding.UTF8.GetBytes("testMessage");
BufferedMessageSet messageSet = new BufferedMessageSet(new List <Message>()
{
new Message(payload)
}, 0);
var partitionData = new PartitionData(partition, ErrorMapper.ToError(error), messageSet);
var topicData = new TopicData(topicName, new List <PartitionData>()
{
partitionData
});
var request = new ProducerRequest(correlationId, clientId, requiredAcks, ackTimeout, new List <TopicData>()
{
topicData
});
int requestSize = 2 + //request type id
2 + //versionId
4 + //correlation id
request.GetShortStringWriteLength(clientId) + // actual client id
2 + //required acks
4 + //ack timeout
4 + //data count
//=== data part
request.GetShortStringWriteLength(topicName) + //topic
4 + //partition data count
4 + //partition id
4 + //messages set size
messageSet.SetSize;
var ms = new MemoryStream();
request.WriteTo(ms);
byte[] bytes = ms.ToArray();
Assert.IsNotNull(bytes);
// add 4 bytes for the length of the message at the beginning
Assert.AreEqual(requestSize + 4, bytes.Length);
// first 4 bytes = the message length
Assert.AreEqual(requestSize, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray <byte>()), 0));
// next 2 bytes = the request type
Assert.AreEqual((short)RequestTypes.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the version id
Assert.AreEqual((short)ProducerRequest.CurrentVersion, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray <byte>()), 0));
// next 2 bytes = the correlation id
Assert.AreEqual(correlationId, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(8).Take(4).ToArray <byte>()), 0));
// next 2 bytes = the client id length
Assert.AreEqual((short)clientId.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(12).Take(2).ToArray <byte>()), 0));
// next few bytes = the client id
Assert.AreEqual(clientId, Encoding.ASCII.GetString(bytes.Skip(14).Take(clientId.Length).ToArray <byte>()));
// next 2 bytes = the required acks
Assert.AreEqual((short)requiredAcks, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(14 + clientId.Length).Take(2).ToArray <byte>()), 0));
// next 4 bytes = the ack timeout
Assert.AreEqual(ackTimeout, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(16 + clientId.Length).Take(4).ToArray <byte>()), 0));
// next 4 bytes = the data count
Assert.AreEqual(1, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(20 + clientId.Length).Take(4).ToArray <byte>()), 0));
// next 2 bytes = the tppic length
Assert.AreEqual((short)topicName.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(24 + clientId.Length).Take(2).ToArray <byte>()), 0));
// next few bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(26 + clientId.Length).Take(topicName.Length).ToArray <byte>()));
// next 4 bytes = the partition data count
Assert.AreEqual(topicData.PartitionData.Count(), BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(26 + clientId.Length + topicName.Length).Take(4).ToArray <byte>()), 0));
// next 4 bytes = the partition
Assert.AreEqual(partition, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(30 + clientId.Length + topicName.Length).Take(4).ToArray <byte>()), 0));
// skipping MessageSet check - this could be done separately in another unit test
}