public RequestResponseSerializationTest()
{
this.producerRequest = SerializationTestUtils.CreateTestProducerRequest();
this.producerResponse = SerializationTestUtils.CreateTestProducerResponse();
this.fetchRequest = SerializationTestUtils.CreateTestFetchRequest();
this.offsetRequest = SerializationTestUtils.CreateTestOffsetRequest();
this.offsetResponse = SerializationTestUtils.CreateTestOffsetResponse();
this.topicMetadataRequest = SerializationTestUtils.CreateTestTopicMetadataRequest();
this.topicMetadataResponse = SerializationTestUtils.CreateTestTopicMetadataResponse();
}
public static OffsetResponse setOffsetResponse(FSPOffset offset)
{
OffsetResponse resp = new OffsetResponse();
resp.Address = offset.Address;
resp.DataGroup = offset.Datagroup;
resp.DataType = offset.DataType;
resp.Value = offset.Value;
//resp.Version = Library.Properties.Version.getVersion();
return(resp);
}
public void TestSerializationAndDeserialization()
{
var buffer = ByteBuffer.Allocate(this.producerRequest.SizeInBytes);
this.producerRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerRequest = ProducerRequest.ReadFrom(buffer);
Assert.Equal(this.producerRequest, deserializedProducerRequest);
buffer = ByteBuffer.Allocate(this.producerResponse.SizeInBytes);
this.producerResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerResponse = ProducerResponse.ReadFrom(buffer);
Assert.Equal(this.producerResponse, deserializedProducerResponse);
buffer = ByteBuffer.Allocate(this.fetchRequest.SizeInBytes);
this.fetchRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedFetchRequest = FetchRequest.ReadFrom(buffer);
Assert.Equal(this.fetchRequest, deserializedFetchRequest);
buffer = ByteBuffer.Allocate(this.offsetRequest.SizeInBytes);
this.offsetRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetRequest = OffsetRequest.ReadFrom(buffer);
Assert.Equal(this.offsetRequest, deserializedOffsetRequest);
buffer = ByteBuffer.Allocate(this.offsetResponse.SizeInBytes);
this.offsetResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetResponse = OffsetResponse.ReadFrom(buffer);
Assert.Equal(this.offsetResponse, deserializedOffsetResponse);
buffer = ByteBuffer.Allocate(this.topicMetadataRequest.SizeInBytes);
this.topicMetadataRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataRequest = TopicMetadataRequest.ReadFrom(buffer);
Assert.Equal(this.topicMetadataRequest, deserializedTopicMetadataRequest);
buffer = ByteBuffer.Allocate(this.topicMetadataResponse.SizeInBytes);
this.topicMetadataResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataResponse = TopicMetadataResponse.ReadFrom(buffer);
Assert.Equal(this.topicMetadataResponse, deserializedTopicMetadataResponse);
}
static void Main(string[] args)
{
string topicName = "kafkatopic";
const int partitionId = 0;
const int correlationId = 0;
const int max = 4;
if (args.Length < 1)
{
Usage();
return;
}
string serverAddress = args[0].Split(':')[0];
int serverPort = Convert.ToInt32(args[0].Split(':')[1]);
if (args.Length > 1)
{
topicName = args[1];
}
var connector = new Connector(serverAddress, serverPort);
// Can't get the time to work. Always use -1 i.e. from start
const long time = -1;
const string clientId = "c# KafkaOffset util";
OffsetResponse offsetResponse = connector.GetOffsetResponse(topicName, time, max, correlationId, clientId, partitionId);
foreach (var offsetTopicName in offsetResponse.Topics())
{
Console.WriteLine("Topic:\t\t" + offsetTopicName);
foreach (var partition in offsetResponse.Partitions(offsetTopicName))
{
Console.WriteLine("PartitionId:\t" + partition);
Console.WriteLine("Error:\t\t" + offsetResponse.Errorcode(offsetTopicName, partition));
Console.WriteLine("Offset count:\t" + offsetResponse.Offsets(offsetTopicName, partition).Count);
Console.Write("Offsets:\t");
for (int index = 0; index < offsetResponse.Offsets(offsetTopicName, partition).Count; index++)
{
if (index > 0)
{
Console.Write(',');
}
var offset = offsetResponse.Offsets(offsetTopicName, partition)[index];
Console.Write(offset);
}
}
}
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public OffsetResponse GetOffsetResponseBefore(OffsetRequest request)
{
using (var connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength == 0)
{
return(null);
}
byte[] data = connection.Read(dataLength);
var offsetResponse = new OffsetResponse(data);
return(offsetResponse);
}
}
private KafkaTopic TopicFromResponse(OffsetResponse offsetResponse)
{
string topicName = offsetResponse.Topics()[0];
int partitionId = offsetResponse.Partitions(topicName)[0];
short errorCode = offsetResponse.Errorcode(topicName, partitionId);
if (errorCode != 0)
{
if (errorCode == 3)
{
throw new NoTopicException(topicName);
}
throw new KafkaException(errorCode);
}
long offset = offsetResponse.Offsets(topicName, partitionId)[0];
var kafkaTopic = new KafkaTopic(topicName, partitionId, offset);
return(kafkaTopic);
}
public HttpResponseMessage OpenPost([FromBody] FSUIPCConnect value)
{
Logger log = LogManager.GetCurrentClassLogger();
try
{
log.Info("Received OpenFSUIPC Connection call");
ConnectResponse resp = new ConnectResponse();
if (FSPFSUIPCConnection.IsOpen)
{
log.Info("Connection already is open");
resp.Message = "Connection Opened";
}
else
{
log.Info("Opening connection..");
FSPFSUIPCConnection.Open();
resp.Message = "Connection Opened";
}
FSPFSUIPCConnection.AddOffset(15616, Datatype.String, "Connect");
FSPOffset _offset = FSPFSUIPCConnection.Process("Connect").First();
OffsetResponse _resp = OffsetHelpers.setOffsetResponse(_offset);
resp.Version = FSPServerV2.Library.Properties.Version.getVersion();
resp.Simulator = FSPFSUIPCConnection.FlightSimVersionConnected.ToString();
resp.Aircraft = _resp.Value;
return(Request.CreateResponse(HttpStatusCode.OK, resp));
}
catch (Exception ee)
{
Console.WriteLine(ee.Message);
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Connection Error", ee));
//return Request.CreateResponse(HttpStatusCode.OK, "error: " + ee.Message);
}
}
public HttpResponseMessage ReadOffset(int offset, string datatype, string datagroup)
{
if (FSPFSUIPCConnection.IsOpen)
{
try
{
List <FSPOffset> offsets = FSPFSUIPCConnection.Process(datagroup);
FSPOffset _offset = (from o in offsets
where o.Address == offset
select o).First();
OffsetResponse resp = OffsetHelpers.setOffsetResponse(_offset);
return(Request.CreateResponse(HttpStatusCode.OK, resp));
}
catch (Exception ex)
{
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Exception Getting Offset", ex));
}
}
else
{
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, "FSUIPC Connection is closed!"));
}
}
static void Main(string[] args)
{
string topicName = "kafkatopic";
int partitionId = 0;
int correlationId = 0;
KafkaBusConnector busConnector = new KafkaBusConnector("192.168.0.105", 9092, "KafkaConsole");
IKafkaMessageConsumer consumer = new MessageConsumer();
consumer.Start(busConnector, topicName, partitionId, 4, DumpMessage);
while (Console.KeyAvailable == false)
{
Thread.Sleep(100);
}
consumer.Stop();
//short errorCode = busConnector.Produce(topicName, -1, "Hello");
var stream = busConnector.CreateMessageStream(topicName, partitionId, KafkaMessageStream.StreamStart.Beginning);
foreach (var kafkaMessage in stream)
{
Console.WriteLine(kafkaMessage);
if (Console.KeyAvailable == true)
{
ConsoleKeyInfo info = Console.ReadKey();
stream.Close();
}
}
stream = busConnector.CreateMessageStream(topicName, partitionId, KafkaMessageStream.StreamStart.Next);
foreach (var kafkaMessage in stream)
{
Console.WriteLine(kafkaMessage);
if (Console.KeyAvailable == true)
{
ConsoleKeyInfo info = Console.ReadKey();
stream.Close();
}
}
Connector connector = new Connector("192.168.0.105", 9092);
int max = 4;
byte[] message = new byte[] { 49 };
message = Encoding.ASCII.GetBytes("Hello from c#");
//MetadataResponse metadataResponse = connector.Metadata(correlationId, "C#", topicName);
//Console.WriteLine(metadataResponse);
//foreach (var broker in metadataResponse.Brokers)
//{
// Console.WriteLine("Brooker:" + broker);
//}
//foreach (string metadataTopicName in metadataResponse.Topics())
//{
// Console.Write("Topic:" + metadataTopicName);
// if (metadataResponse.TopicErrorCode(metadataTopicName) != 0)
// Console.WriteLine("\tTopic error:" + metadataResponse.TopicErrorCode(metadataTopicName));
// else
// Console.WriteLine();
// foreach (var partitionMetadata in metadataResponse.Partitions(topicName))
// {
// Console.WriteLine("\t" + partitionMetadata);
// }
//}
//connector.Produce(correlationId, "c#", 500, topicName, partitionId, message);
OffsetResponse offsetResponse = connector.GetOffsetResponse(topicName, OffsetRequest.LatestTime, max, correlationId, "C#", partitionId);
FetchResponse fetchResponse;
int maxOffset = 0;
foreach (var offsetTopicName in offsetResponse.Topics())
{
Console.WriteLine(offsetTopicName);
foreach (var partition in offsetResponse.Partitions(offsetTopicName))
{
Console.WriteLine("PartitionId:" + partition);
Console.WriteLine("Error:" + offsetResponse.Errorcode(offsetTopicName, partition));
Console.WriteLine("Offset count:" + offsetResponse.Offsets(offsetTopicName, partition).Count);
Console.WriteLine("Start offset:" + offsetResponse.Offsets(offsetTopicName, partition)[1]);
Console.WriteLine("End offset:" + offsetResponse.Offsets(offsetTopicName, partition)[0]);
fetchResponse = connector.Fetch(offsetTopicName, partition, correlationId, "C#", offsetResponse.Offsets(offsetTopicName, partition)[1], 5000);
foreach (var fetchTopic in fetchResponse.Topics)
{
foreach (FetchPartition fetchPartition in fetchTopic.Partitions)
{
Console.WriteLine("Error:" + fetchPartition.ErrorCode);
Console.WriteLine("Id:" + fetchPartition.PartitionId);
foreach (var messageSet in fetchPartition.MessageSets)
{
if (messageSet != null)
{
Console.Write("Offset:" + messageSet.MessageOffset + "\t");
Console.WriteLine("Message:" + messageSet.Message);
}
}
Console.WriteLine();
}
}
}
}
//fetchResponse = connector.Fetch("kafkatopic", 0, 1, "C#", 7);
//foreach (var fetchTopic in fetchResponse.Topics)
//{
// foreach (FetchPartition fetchPartition in fetchTopic.Partitions)
// {
// Console.WriteLine("Error:" + fetchPartition.ErrorCode);
// Console.WriteLine("Id:" + fetchPartition.PartitionId);
// if (fetchPartition.MessageSet != null)
// {
// Console.WriteLine(fetchPartition.MessageSet.Message);
// }
// Console.WriteLine();
// }
//}
//FetchResponse fetchResponse = connector.Fetch("kafkatopic", partitionId, correlationId, "C#", offsetResponse.Topics[0].Partitions[0].Offsets[1]);
//foreach (var topic in fetchResponse.Topics)
//{
// Console.WriteLine(topic.TopicName);
// foreach (FetchPartition partition in topic.Partitions)
// {
// Console.WriteLine("Error:" + partition.ErrorCode);
// Console.WriteLine("Id:" + partition.PartitionId);
// Console.Write("Offset:" + partition.MessageSet.MessageOffset + " ");
// Console.WriteLine(partition.MessageSet.Message);
// }
//}
//FetchResponse fetchResponse = connector.Fetch("kafkatopic", 0, 1, 1, offsetResponse.Topics[0].Partitions[0].Offsets[0]);
//long[] offsets = consumer.GetOffsetsBefore("kafkatopic", 0, OffsetRequest.LatestTime, max).ToArray();
//List<Message> messages = consumer.Consume("kafkatopic", 0, offsets[0]);
//foreach (var message in messages)
//{
// Console.WriteLine(message.ToString());
//}
}
/// <summary>
/// Get offset
/// </summary>
public void RefreshAndGetOffset(short versionId, string clientId, int correlationId, string topic, int partitionId, bool forceRefreshOffsetCache, out long earliestOffset, out long latestOffset)
{
earliestOffset = -1;
latestOffset = -1;
if (!forceRefreshOffsetCache && this.TopicOffsetEarliest.ContainsKey(topic) && this.TopicOffsetEarliest[topic].ContainsKey(partitionId))
{
earliestOffset = this.TopicOffsetEarliest[topic][partitionId];
}
if (!forceRefreshOffsetCache && this.TopicOffsetLatest.ContainsKey(topic) && this.TopicOffsetLatest[topic].ContainsKey(partitionId))
{
latestOffset = this.TopicOffsetLatest[topic][partitionId];
}
if (!forceRefreshOffsetCache && earliestOffset != -1 && latestOffset != -1)
{
return;
}
//Get
using (Consumer consumer = this.GetConsumer(topic, partitionId))
{
Dictionary <string, List <PartitionOffsetRequestInfo> > offsetRequestInfoEarliest = new Dictionary <string, List <PartitionOffsetRequestInfo> >();
List <PartitionOffsetRequestInfo> offsetRequestInfoForPartitionsEarliest = new List <PartitionOffsetRequestInfo>();
offsetRequestInfoForPartitionsEarliest.Add(new PartitionOffsetRequestInfo(partitionId, OffsetRequest.EarliestTime, 1));
offsetRequestInfoEarliest.Add(topic, offsetRequestInfoForPartitionsEarliest);
OffsetRequest offsetRequestEarliest = new OffsetRequest(offsetRequestInfoEarliest);
//Earliest
OffsetResponse offsetResponseEarliest = consumer.GetOffsetsBefore(offsetRequestEarliest);
List <PartitionOffsetsResponse> partitionOffsetEaliest = null;
if (offsetResponseEarliest.ResponseMap.TryGetValue(topic, out partitionOffsetEaliest))
{
foreach (var p in partitionOffsetEaliest)
{
if (p.Error == ErrorMapping.NoError && p.PartitionId == partitionId)
{
earliestOffset = p.Offsets[0];
//Cache
if (!this.TopicOffsetEarliest.ContainsKey(topic))
{
this.TopicOffsetEarliest.TryAdd(topic, new ConcurrentDictionary <int, long>());
}
this.TopicOffsetEarliest[topic][partitionId] = earliestOffset;
}
}
}
//Latest
Dictionary <string, List <PartitionOffsetRequestInfo> > offsetRequestInfoLatest = new Dictionary <string, List <PartitionOffsetRequestInfo> >();
List <PartitionOffsetRequestInfo> offsetRequestInfoForPartitionsLatest = new List <PartitionOffsetRequestInfo>();
offsetRequestInfoForPartitionsLatest.Add(new PartitionOffsetRequestInfo(partitionId, OffsetRequest.LatestTime, 1));
offsetRequestInfoLatest.Add(topic, offsetRequestInfoForPartitionsLatest);
OffsetRequest offsetRequestLatest = new OffsetRequest(offsetRequestInfoLatest);
OffsetResponse offsetResponseLatest = consumer.GetOffsetsBefore(offsetRequestLatest);
List <PartitionOffsetsResponse> partitionOffsetLatest = null;
if (offsetResponseLatest.ResponseMap.TryGetValue(topic, out partitionOffsetLatest))
{
foreach (var p in partitionOffsetLatest)
{
if (p.Error == ErrorMapping.NoError && p.PartitionId == partitionId)
{
latestOffset = p.Offsets[0];
//Cache
if (!this.TopicOffsetLatest.ContainsKey(topic))
{
this.TopicOffsetLatest.TryAdd(topic, new ConcurrentDictionary <int, long>());
}
this.TopicOffsetLatest[topic][partitionId] = latestOffset;
}
}
}
}
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, OffsetResponse response)
{
if (response == null)
{
return(false);
}
var groupedTopics = response.Topics.GroupBy(t => t.TopicName).ToList();
writer.Write(groupedTopics.Count);
foreach (var topic in groupedTopics)
{
var partitions = topic.ToList();
writer.Write(topic.Key)
.Write(partitions.Count);
foreach (var partition in partitions)
{
writer.Write(partition.PartitionId)
.Write(partition.ErrorCode);
if (context.ApiVersion == 0)
{
writer.Write(1)
.Write(partition.Offset);
}
else
{
writer.Write(partition.Timestamp.ToUnixEpochMilliseconds() ?? 0L)
.Write(partition.Offset);
}
}
}
return(true);
}
