|
public static ParseFrom ( byte data ) : |
||
| data | byte | The data for a message. |
| return | ||
/// <summary>
/// Consumes messages from Kafka.
/// </summary>
/// <param name="request">The request to send to Kafka.</param>
/// <returns>A list of messages from Kafka.</returns>
public List <Message> Consume(ConsumerRequest request)
{
List <Message> messages = new List <Message>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
// TODO: need to check in on kafka error codes...assume all's good for now
byte[] unbufferedData = data.Skip(2).ToArray();
int processed = 0;
int length = unbufferedData.Length - 4;
int messageSize = 0;
while (processed <= length)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0);
messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>()));
processed += 4 + messageSize;
}
}
}
return(messages);
}
/// <summary>
/// Consumes messages from Kafka.
/// </summary>
/// <param name="request">The request to send to Kafka.</param>
/// <returns>A list of messages from Kafka.</returns>
public List <Message> Consume(FetchRequest request)
{
List <Message> messages = new List <Message>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
byte[] unbufferedData = data.Skip(2).ToArray();
int processed = 0;
int length = unbufferedData.Length - 4;
int messageSize = 0;
while (processed <= length)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Skip(processed).Take(4).ToArray <byte>()), 0);
messages.Add(Message.ParseFrom(unbufferedData.Skip(processed).Take(messageSize + 4).ToArray <byte>()));
processed += 4 + messageSize;
}
}
}
return(messages);
}
/// <summary>
/// Executes a multi-fetch operation.
/// </summary>
/// <param name="request">The request to push to Kafka.</param>
/// <returns>
/// A list containing sets of messages. The message sets should match the request order.
/// </returns>
public List <List <Message> > Consume(MultiFetchRequest request)
{
int fetchRequests = request.ConsumerRequests.Count;
List <List <Message> > messages = new List <List <Message> >();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int position = 0;
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
position = position + 2;
for (int ix = 0; ix < fetchRequests; ix++)
{
messages.Add(new List <Message>());
int messageSetSize = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Skip(position).Take(4).ToArray <byte>()), 0);
position = position + 4;
errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Skip(position).Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
position = position + 2;
byte[] messageSetBytes = data.Skip(position).ToArray <byte>().Take(messageSetSize).ToArray <byte>();
int processed = 0;
int messageSize = 0;
// dropped 2 bytes at the end...padding???
while (processed < messageSetBytes.Length - 2)
{
messageSize = BitConverter.ToInt32(BitWorks.ReverseBytes(messageSetBytes.Skip(processed).Take(4).ToArray <byte>()), 0);
messages[ix].Add(Message.ParseFrom(messageSetBytes.Skip(processed).Take(messageSize + 4).ToArray <byte>()));
processed += 4 + messageSize;
}
position = position + processed;
}
}
}
return(messages);
}
