public void SetupData()
{
Common.Compression.ZipLevel = Level;
_request = new ProduceRequest(
Enumerable.Range(1, Partitions)
.Select(partitionId => new ProduceRequest.Topic(
"topic",
partitionId,
Enumerable.Range(1, Messages)
.Select(i => new Message(GenerateMessageBytes(), new ArraySegment <byte>(), (byte)Codec, version: MessageVersion)),
Codec)));
var response = new ProduceResponse(new ProduceResponse.Topic("topic", 1, ErrorCode.NONE, 0));
var port = 10000;
var endpoint = new Endpoint(new IPEndPoint(IPAddress.Loopback, port), "localhost");
_server = new TcpServer(endpoint.Ip.Port)
{
OnReceivedAsync = async data => {
var header = KafkaDecoder.DecodeHeader(data.Skip(4));
var bytes = KafkaDecoder.EncodeResponseBytes(new RequestContext(header.CorrelationId), response);
await _server.SendDataAsync(bytes);
}
};
_connection = new Connection(endpoint);
}
public async Task ConsumeByOffsetShouldGetSameMessageProducedAtSameOffset()
{
long offsetResponse;
Guid messge = Guid.NewGuid();
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)
{
Log = IntegrationConfig.NoDebugLog
}))
using (var producer = new Producer(router))
{
ProduceResponse responseAckLevel1 = await producer.SendMessageAsync(new Message(messge.ToString()), IntegrationConfig.IntegrationTopic, acks : 1, partition : 0);
offsetResponse = responseAckLevel1.Offset;
}
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)
{
Log = IntegrationConfig.NoDebugLog
}))
using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, router)
{
MaxWaitTimeForMinimumBytes = TimeSpan.Zero
}, new OffsetPosition[] { new OffsetPosition(0, offsetResponse) }))
{
var result = consumer.Consume().Take(1).ToList().FirstOrDefault();
Assert.AreEqual(messge.ToString(), result.Value.ToUtf8String());
}
}
public ReusableMemoryStream SerializeProduceBatch(int correlationId, IEnumerable <IGrouping <string, IGrouping <int, ProduceMessage> > > batch)
{
var r = new ProduceResponse {
ProducePartitionResponse = new CommonResponse <ProducePartitionResponse>
{
TopicsResponse = batch.Select(g => new TopicData <ProducePartitionResponse>
{
TopicName = g.Key,
PartitionsData = g.Select(pg => new ProducePartitionResponse
{
ErrorCode = _forceErrors && Interlocked.Increment(ref _count) % 2 == 0
? ErrorCode.LeaderNotAvailable
: _forceErrors && Interlocked.Increment(ref _count) % 3 == 0
? ErrorCode.MessageSizeTooLarge
: _metadataResponse.TopicsMeta.Where(tm => tm.TopicName == g.Key)
.Select(tm => tm.Partitions.First(p => p.Id == pg.Key).ErrorCode)
.First(),
Offset = 0,
Partition = pg.Key
}).ToArray()
}).ToArray()
}
};
_produceResponses[correlationId] = r;
return(new ReusableMemoryStream(null));
}
public void ProduceResponse()
{
var response1 = new ProduceResponse {
TopicPartitions = new[] {
new ProduceResponseTopicPartition {
TopicName = Guid.NewGuid().ToString(),
Details = new [] {
new ProduceResponseTopicPartitionDetail {
Partition = _random.Next(),
Offset = (Int64)_random.Next(),
}
}
}
}
};
Stream binary1 = new MemoryStream();
response1.Serialize(binary1);
binary1.Seek(0L, SeekOrigin.Begin);
var response2 = new ProduceResponse();
response2.Deserialize(binary1);
var compareLogic = new CompareLogic();
var result = compareLogic.Compare(response1, response2);
Assert.True(result.AreEqual);
Stream binary2 = new MemoryStream();
response2.Serialize(binary2);
Assert.Equal(binary1.Length, binary2.Length);
using (var stream1 = new MemoryStream())
using (var stream2 = new MemoryStream()) {
binary1.Seek(0L, SeekOrigin.Begin);
binary1.CopyTo(stream1);
binary2.Seek(0L, SeekOrigin.Begin);
binary2.CopyTo(stream2);
Assert.Equal(stream1.Length, stream2.Length);
stream1.Seek(0L, SeekOrigin.Begin);
var bytes1 = stream1.ToArray();
stream2.Seek(0L, SeekOrigin.Begin);
var bytes2 = stream2.ToArray();
Assert.Equal(bytes1.Length, bytes2.Length);
for (int i = 0; i < bytes1.Length; i++)
{
Assert.Equal(bytes1[i], bytes2[i]);
}
}
}
public short Produce(string topicName, int partitionId, string data)
{
try
{
if (!topicPartitionDictionary.ContainsKey(topicName))
{
// Check if topic exist and on what partition.
// This call will automatically create the topic if the brooker is set up to auto create
MetadataResponse metadataResponse = connector.Metadata(DefaultCorrelationId, clientId, topicName);
short errorCode = metadataResponse.TopicErrorCode(topicName);
if (errorCode != (short)KafkaErrorCode.NoError)
{
if (errorCode != (short)KafkaErrorCode.LeaderNotAvailable)
{
return(errorCode);
}
// Check if the topic was auto created
metadataResponse = connector.Metadata(DefaultCorrelationId, clientId, topicName);
errorCode = metadataResponse.TopicErrorCode(topicName);
if (errorCode != (short)KafkaErrorCode.NoError)
{
return(errorCode);
}
topicPartitionDictionary.Add(topicName, metadataResponse.Partitions(topicName)[0]);
}
else
{
topicPartitionDictionary.Add(topicName, metadataResponse.Partitions(topicName)[0]);
}
}
if (partitionId == -1)
{
partitionId = topicPartitionDictionary[topicName];
}
var message = Encoding.UTF8.GetBytes(data);
ProduceResponse response = connector.Produce(DefaultCorrelationId, clientId, 500, topicName, partitionId, message);
return(response.ErrorCode(topicName, 0));
}
catch (SocketException ex)
{
throw new KafkaException(ex.Message);
}
}
private async Task SetResult(List <BrokerRouteSendBatch> sendTasks)
{
foreach (var sendTask in sendTasks)
{
try
{
//all ready done don't need to await but it none blocking syntext
var batchResult = await sendTask.Task.ConfigureAwait(false);
var numberOfMessage = sendTask.MessagesSent.Count;
for (int i = 0; i < numberOfMessage; i++)
{
bool isAckLevel0 = sendTask.AckLevel == 0;
if (isAckLevel0)
{
var responce = new ProduceResponse()
{
Error = (short)ErrorResponseCode.NoError,
PartitionId = sendTask.Route.PartitionId,
Topic = sendTask.Route.Topic,
Offset = -1
};
sendTask.MessagesSent[i].Tcs.SetResult(responce);
}
else
{
var response = new ProduceResponse()
{
Error = batchResult.Error,
PartitionId = batchResult.PartitionId,
Topic = batchResult.Topic,
Offset = batchResult.Offset + i
};
sendTask.MessagesSent[i].Tcs.SetResult(response);
}
}
}
catch (Exception ex)
{
BrokerRouter.Log.ErrorFormat("failed to send batch Topic[{0}] ackLevel[{1}] partition[{2}] EndPoint[{3}] Exception[{4}] stacktrace[{5}]", sendTask.Route.Topic, sendTask.AckLevel, sendTask.Route.PartitionId, sendTask.Route.Connection.Endpoint, ex.Message, ex.StackTrace);
sendTask.MessagesSent.ForEach((x) => x.Tcs.TrySetException(ex));
}
}
}
public ProduceResponse Produce(int correlationId, string clientId, int timeOut, string topicName, int partitionId, byte[] payLoad)
{
var request = new ProduceRequest(timeOut, correlationId, clientId);
request.AddMessage(topicName, partitionId, payLoad);
using (var connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
var response = new ProduceResponse();
if (dataLength != 0)
{
byte[] data = connection.Read(dataLength);
response.Parse(data);
}
return(response);
}
}
public void ProduceResponse(
[Values(0, 1, 2)] short version,
[Values(-1, 0, 10000000)] long timestampMilliseconds,
[Values("test", "a really long name, with spaces and punctuation!")] string topicName,
[Values(1, 10)] int topicsPerRequest,
[Values(1, 5)] int totalPartitions,
[Values(
ErrorResponseCode.None,
ErrorResponseCode.CorruptMessage
)] ErrorResponseCode errorCode,
[Values(0, 100000)] int throttleTime)
{
var topics = new List <ProduceResponse.Topic>();
for (var t = 0; t < topicsPerRequest; t++)
{
topics.Add(new ProduceResponse.Topic(topicName + t, t % totalPartitions, errorCode, _randomizer.Next(), version >= 2 ? timestampMilliseconds.FromUnixEpochMilliseconds() : (DateTime?)null));
}
var response = new ProduceResponse(topics, version >= 1 ? TimeSpan.FromMilliseconds(throttleTime) : (TimeSpan?)null);
response.AssertCanEncodeDecodeResponse(version);
}
public void ProduceResponse(
[Values(0, 1, 2)] short version,
[Values(-1, 0, 10000000)] long timestampMilliseconds,
[Values("testTopic")] string topicName,
[Values(1, 10)] int topicsPerRequest,
[Values(1, 5)] int totalPartitions,
[Values(
ErrorCode.NONE,
ErrorCode.CORRUPT_MESSAGE
)] ErrorCode errorCode,
[Values(0, 100000)] int throttleTime)
{
var topics = new List <ProduceResponse.Topic>();
for (var t = 0; t < topicsPerRequest; t++)
{
topics.Add(new ProduceResponse.Topic(topicName + t, t % totalPartitions, errorCode, _randomizer.Next(), version >= 2 ? DateTimeOffset.FromUnixTimeMilliseconds(timestampMilliseconds) : (DateTimeOffset?)null));
}
var response = new ProduceResponse(topics, version >= 1 ? TimeSpan.FromMilliseconds(throttleTime) : (TimeSpan?)null);
response.AssertCanEncodeDecodeResponse(version);
}
private async Task AddFileToKafka(byte[] filename, byte[] message, string topic)
{
KafkaOptions options = new KafkaOptions(new Uri("http://sandbox.hortonworks.com:6667"));
using (BrokerRouter router = new BrokerRouter(options))
using (Producer client = new Producer(router))
{
var topicMetas = router.GetTopicMetadata(topic);
var responses = await client.SendMessageAsync(topic,
new[] {
new KafkaNet.Protocol.Message
{
Key = filename,
Value = message
}
});
ProduceResponse response = responses.FirstOrDefault();
MessageBox.Show(String.Format("File added to the queue - partition {0} offset {1}",
response.PartitionId,
response.Offset));
}
}
private async Task SetResult(List<BrokerRouteSendBatch> sendTasks)
{
foreach (var sendTask in sendTasks)
{
try
{
//all ready done don't need to await but it none blocking syntext
var batchResult = await sendTask.Task;
var numberOfMessage = sendTask.MessagesSent.Count;
for (int i = 0; i < numberOfMessage; i++)
{
bool isAckLevel0 = sendTask.AckLevel == 0;
if (isAckLevel0)
{
var responce = new ProduceResponse()
{
Error = (short)ErrorResponseCode.NoError,
PartitionId = sendTask.Route.PartitionId,
Topic = sendTask.Route.Topic,
Offset = -1
};
sendTask.MessagesSent[i].Tcs.SetResult(responce);
}
else
{
var responce = new ProduceResponse()
{
Error = batchResult.Error,
PartitionId = batchResult.PartitionId,
Topic = batchResult.Topic,
Offset = batchResult.Offset + i
};
sendTask.MessagesSent[i].Tcs.SetResult(responce);
}
}
}
catch (Exception ex)
{
BrokerRouter.Log.ErrorFormat("failed to send bach Topic[{0}] ackLevel[{1}] partition[{2}] EndPoint[{3}] Exception[{4}] stacktrace[{5}]", sendTask.Route.Topic, sendTask.AckLevel, sendTask.Route.PartitionId, sendTask.Route.Connection.Endpoint, ex.Message, ex.StackTrace);
sendTask.MessagesSent.ForEach((x) => x.Tcs.TrySetException(ex));
}
}
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ProduceResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.responses.GroupBy(t => t.topic).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
writer.Write(partition.partition_id)
.Write(partition.error_code)
.Write(partition.base_offset);
if (context.ApiVersion >= 2)
{
writer.Write(partition.timestamp?.ToUnixTimeMilliseconds() ?? -1L);
}
}
}
if (context.ApiVersion >= 1)
{
writer.Write((int?)response.throttle_time_ms?.TotalMilliseconds ?? 0);
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, ProduceResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
writer.Write(partition.PartitionId)
.Write(partition.ErrorCode)
.Write(partition.Offset);
if (context.ApiVersion >= 2)
{
writer.Write(partition.Timestamp.ToUnixEpochMilliseconds() ?? -1L);
}
}
}
if (context.ApiVersion >= 1)
{
writer.Write((int?)response.ThrottleTime?.TotalMilliseconds ?? 0);
}
return(true);
}
