public void Transactions_Commit(string bootstrapServers)
{
LogToFile("start Transactions_Commit");
var defaultTimeout = TimeSpan.FromSeconds(30);
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, wm.High));
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
producer.CommitTransaction(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 1", Value = "test val 1"
});
producer.CommitTransaction(defaultTimeout);
var cr1 = consumer.Consume();
var cr2 = consumer.Consume();
var cr3 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal(wm.High, cr1.Offset);
Assert.Equal(wm.High + 2, cr2.Offset); // there should be a skipped offset due to a commit marker in the log.
Assert.Null(cr3); // control message should not be exposed to application.
// Test that the committed offset accounts for the final ctrl message.
consumer.Commit();
}
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, AutoOffsetReset = AutoOffsetReset.Latest
}).Build())
{
consumer.Assign(new TopicPartition(topic.Name, 0));
// call InitTransactions to prevent a race conidtion between a slow txn commit and a quick offset request.
producer.InitTransactions(defaultTimeout);
var committed = consumer.Committed(defaultTimeout);
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
Assert.Equal(wm.High, committed[0].Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Commit");
}
public void Transactions_WatermarkOffsets(string bootstrapServers)
{
LogToFile("start Transactions_WatermarkOffsets");
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false
}).Build())
{
var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
Assert.Equal(Offset.Unset, wo1.Low);
Assert.Equal(Offset.Unset, wo1.High);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
producer.InitTransactions(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message1"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message2"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message3"
}).Wait();
WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr = consumer.Consume(TimeSpan.FromMilliseconds(500));
wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo2.High == 3)
{
break;
}
}
Assert.Equal(3, wo2.High);
producer.CommitTransaction(TimeSpan.FromSeconds(30));
WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr2 = consumer.Consume(TimeSpan.FromSeconds(500));
wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo3.High > 3)
{
break;
}
}
Assert.Equal(4, wo3.High);
var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
Assert.Equal(4, wo4.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_WatermarkOffsets");
}
public void WatermarkOffsets(string bootstrapServers)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
var testString = "hello world";
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
DeliveryResult <Null, string> dr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
dr = producer.ProduceAsync(topic.Name, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(0, getOffsets.Low);
// the offset of the next message to be read.
Assert.Equal(dr.Offset + 1, getOffsets.High);
var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(Offset.Unset, queryOffsets.Low);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
}
// Test empty topic case
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
var wo = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
// Refer to WatermarkOffsets class documentation for more information.
Assert.Equal(0, wo.Low);
Assert.Equal(0, wo.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
/// <summary>
/// In this example
/// - consumer group functionality (i.e. .Subscribe + offset commits) is not used.
/// - the consumer is manually assigned to a partition and always starts consumption
/// from a specific offset (0).
/// </summary>
public static void Run_ManualAssign(string brokerList, List <string> topics, CancellationToken cancellationToken)
{
var config = new ConsumerConfig
{
// the group.id property must be specified when creating a consumer, even
// if you do not intend to use any consumer group functionality.
GroupId = new Guid().ToString(),
BootstrapServers = brokerList,
// partition offsets can be committed to a group even by consumers not
// subscribed to the group. in this example, auto commit is disabled
// to prevent this from occurring.
EnableAutoCommit = true
};
using (var consumer =
new ConsumerBuilder <string, string>(config)
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
List <TopicMessages> messages = new List <TopicMessages>();
consumer.Assign(topics.Select(topic => new TopicPartitionOffset(topic, 0, Offset.Beginning)).ToList());
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cancellationToken);
// Note: End of partition notification has not been enabled, so
// it is guaranteed that the ConsumeResult instance corresponds
// to a Message, and not a PartitionEOF event.
Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: ${consumeResult.Message.Value}");
messages.Add(new TopicMessages()
{
Topic = consumeResult.Topic,
Offset = consumeResult.Offset.Value,
Key = consumeResult.Message.Key,
Value = consumeResult.Message.Value,
});
//Console.WriteLine($"Received message at topic: {consumeResult.Topic}, partition: {consumeResult.Partition}, message key: {consumeResult.Message.Key}, offset: {consumeResult.Offset}, topic partition offset: {consumeResult.TopicPartitionOffset}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
//Only save the results at the end
//We can make that it is saved when we have a certain number of elements
using (var db = new TestDbContext())
{
db.Messages.AddRange(
messages
);
var count = db.SaveChanges();
Console.WriteLine("{0} records saved to database", count);
}
}
}
public static void Timestamps(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Timestamps");
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
var drs_beginProduce = new List <DeliveryReport <Null, string> >();
var drs_task = new List <DeliveryResult <Null, string> >();
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
// --- ProduceAsync, serializer case.
drs_task.Add(producer.ProduceAsync(
singlePartitionTopic,
new Message <Null, string> {
Value = "testvalue"
}).Result);
// TimestampType: CreateTime
drs_task.Add(producer.ProduceAsync(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string>
{
Value = "test-value",
Timestamp = new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc))
}).Result);
// TimestampType: CreateTime (default)
drs_task.Add(producer.ProduceAsync(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string> {
Value = "test-value"
}).Result);
// TimestampType: LogAppendTime
Assert.Throws <ArgumentException>(() =>
producer.ProduceAsync(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string>
{
Value = "test-value",
Timestamp = new Timestamp(DateTime.Now, TimestampType.LogAppendTime)
}).Result);
// TimestampType: NotAvailable
Assert.Throws <ArgumentException>(() =>
producer.ProduceAsync(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string>
{
Value = "test-value",
Timestamp = new Timestamp(10, TimestampType.NotAvailable)
}).Result);
Action <DeliveryReport <Null, string> > dh
= (DeliveryReport <Null, string> dr) => drs_beginProduce.Add(dr);
// --- begin produce, serializer case.
producer.BeginProduce(
singlePartitionTopic,
new Message <Null, string> {
Value = "testvalue"
}, dh);
// TimestampType: CreateTime
producer.BeginProduce(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string>
{
Value = "test-value",
Timestamp = new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc))
},
dh);
// TimestampType: CreateTime (default)
producer.BeginProduce(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string> {
Value = "test-value"
},
dh);
// TimestampType: LogAppendTime
Assert.Throws <ArgumentException>(() => producer.BeginProduce(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string>
{
Value = "test-value",
Timestamp = new Timestamp(DateTime.Now, TimestampType.LogAppendTime)
},
dh));
// TimestampType: NotAvailable
Assert.Throws <ArgumentException>(() => producer.BeginProduce(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string>
{
Value = "test-value",
Timestamp = new Timestamp(10, TimestampType.NotAvailable)
},
dh));
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var drs2_beginProduce = new List <DeliveryReport <byte[], byte[]> >();
var drs2_task = new List <DeliveryResult <byte[], byte[]> >();
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
{
// --- ProduceAsync, byte[] case.
drs2_task.Add(producer.ProduceAsync(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = Timestamp.Default
}).Result);
// TimestampType: CreateTime
drs2_task.Add(producer.ProduceAsync(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc))
}).Result);
// TimestampType: CreateTime (default)
drs2_task.Add(producer.ProduceAsync(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = Timestamp.Default
}).Result);
// TimestampType: LogAppendTime
Assert.Throws <ArgumentException>(() =>
producer.ProduceAsync(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = new Timestamp(DateTime.Now, TimestampType.LogAppendTime)
}).Result);
// TimestampType: NotAvailable
Assert.Throws <ArgumentException>(() =>
producer.ProduceAsync(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = new Timestamp(10, TimestampType.NotAvailable)
}).Result);
// --- begin produce, byte[] case.
Action <DeliveryReport <byte[], byte[]> > dh = (DeliveryReport <byte[], byte[]> dr) => drs2_beginProduce.Add(dr);
producer.BeginProduce(
singlePartitionTopic, new Message <byte[], byte[]> {
Timestamp = Timestamp.Default
}, dh);
// TimestampType: CreateTime
producer.BeginProduce(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc))
},
dh);
// TimestampType: CreateTime (default)
producer.BeginProduce(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = Timestamp.Default
}, dh);
// TimestampType: LogAppendTime
Assert.Throws <ArgumentException>(() =>
producer.BeginProduce(
singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = new Timestamp(DateTime.Now, TimestampType.LogAppendTime)
}, dh));
// TimestampType: NotAvailable
Assert.Throws <ArgumentException>(() =>
producer.BeginProduce(singlePartitionTopic,
new Message <byte[], byte[]> {
Timestamp = new Timestamp(10, TimestampType.NotAvailable)
}, dh));
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig).Build())
{
// serializing async
assertCloseToNow(consumer, drs_task[0].TopicPartitionOffset);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs_task[1].TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Equal(TimestampType.CreateTime, record.Message.Timestamp.Type);
Assert.Equal(record.Message.Timestamp, new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc)));
assertCloseToNow(consumer, drs_task[2].TopicPartitionOffset);
// serializing deliveryhandler
assertCloseToNow(consumer, drs_beginProduce[0].TopicPartitionOffset);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs_beginProduce[1].TopicPartitionOffset
});
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Equal(TimestampType.CreateTime, record.Message.Timestamp.Type);
Assert.Equal(record.Message.Timestamp, new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc)));
assertCloseToNow(consumer, drs_beginProduce[2].TopicPartitionOffset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
ConsumeResult <byte[], byte[]> record;
// non-serializing async
assertCloseToNow_byte(consumer, drs2_task[0].TopicPartitionOffset);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs2_task[1].TopicPartitionOffset
});
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Equal(TimestampType.CreateTime, record.Message.Timestamp.Type);
Assert.Equal(record.Message.Timestamp, new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc)));
assertCloseToNow_byte(consumer, drs2_task[2].TopicPartitionOffset);
// non-serializing deliveryhandler
assertCloseToNow_byte(consumer, drs2_beginProduce[0].TopicPartitionOffset);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs2_beginProduce[1].TopicPartitionOffset
});
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Equal(TimestampType.CreateTime, record.Message.Timestamp.Type);
Assert.Equal(record.Message.Timestamp, new Timestamp(new DateTime(2008, 11, 12, 0, 0, 0, DateTimeKind.Utc)));
assertCloseToNow_byte(consumer, drs2_beginProduce[2].TopicPartitionOffset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Timestamps");
}
public void LogDelegate(string bootstrapServers)
{
LogToFile("start LogDelegate");
var logCount = 0;
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
Debug = "all"
};
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
Debug = "all"
};
var adminConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers,
Debug = "all"
};
DeliveryResult <byte[], byte[]> dr;
using (var producer =
new ProducerBuilder <byte[], byte[]>(producerConfig)
.SetLogHandler((_, m) => logCount += 1)
.Build())
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8("test value")
}).Result;
producer.Flush(TimeSpan.FromSeconds(10));
}
Assert.True(logCount > 0);
logCount = 0;
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetLogHandler((_, m) => logCount += 1)
.Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
consumer.Consume(TimeSpan.FromSeconds(10));
}
Assert.True(logCount > 0);
logCount = 0;
using (var adminClient =
new AdminClientBuilder(adminConfig)
.SetLogHandler((_, m) => logCount += 1)
.Build())
{
adminClient.GetMetadata(TimeSpan.FromSeconds(1));
}
Assert.True(logCount > 0);
Assert.Equal(0, Library.HandleCount);
LogToFile("end LogDelegate");
}
/**
* the primary function of the <CommandConsumer> to be run as a thread
*/
public void CommandThread()
{
Console.WriteLine("COMMANDSThread Start");
Active = true;
//objs: Kafka Consumer
//
//consumer - the Kafka consumer object
//topicp - the partition to subscribe to
var consumer = new ConsumerBuilder <string, string>(config).Build();
var topicp = new TopicPartition(topic, 0);
consumer.Assign(topicp);
while (Active)
{
try
{
//var: consumerresult
//
//the result of checking for a new message from the Kafka server
var consumeresult = consumer.Consume(canceltoken);
if (!consumeresult.IsPartitionEOF)
{
//vars: Kafka message
//
//command - the part of the message that specifys an action to take
//parameter - the part of the messag that may be passed to a function as a parameter
var input = consumeresult.Value;
string command;
string parameter;
if (input.Contains(" "))
{
command = input.Substring(0, input.IndexOf(" ")).Trim();
parameter = input.Substring(input.IndexOf(" "), input.Length - command.Length).Trim();
}
else
{
command = input;
parameter = "";
}
Console.WriteLine("COMMAND----------> " + input);
if (command.Equals("DAExit") || command.Equals("System-Shutdown"))
{
Active = false;
source.Cancel();
controller.exit();
}
}
else
{
Thread.Sleep(100);
}
}
catch (System.OperationCanceledException e)
{
}
}
}
public static void ConsumeIncompatibleTypes(string bootstrapServers, string schemaRegistryServers)
{
string topic = Guid.NewGuid().ToString();
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, User>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <User>(schemaRegistry))
.Build())
{
var user = new User
{
name = "username",
favorite_number = 107,
favorite_color = "orange"
};
producer
.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
})
.Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <User, User>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <User>(schemaRegistry))
.SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry))
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, 0)
});
bool hadError = false;
try
{
consumer.Consume(TimeSpan.FromSeconds(10));
}
catch (ConsumeException)
{
hadError = true;
}
Assert.True(hadError);
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry))
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, 0)
});
bool hadError = false;
try
{
consumer.Consume(TimeSpan.FromSeconds(10));
}
catch (ConsumeException)
{
hadError = true;
}
Assert.True(hadError);
}
}
/**
* The main loop to be run as a thread. Generates a Kafka client that checks
* for new messages COMMANDS and then calls the appropriete function from <THreadController>
*/
public void CommandThread()
{
Console.WriteLine("CommandThread Start");
Active = true;
//Objs: Kafka Client
//
//consumer - the Kafka consumer that will connect to the Kafka server
//topicp - what partition to start looking for messages in.
var consumer = new ConsumerBuilder <string, string>(config).Build();
var topicp = new TopicPartition(topic, 0);
consumer.Assign(topicp);
while (Active)
{
try
{
/**
* the result of <consumer> checking for nes messages
*/
var consumeresult = consumer.Consume(canceltoken);
if (!consumeresult.IsPartitionEOF)
{
/*
* This section breaks down the message and then exicutes the appropriete command
*/
//Vars: Commad Strings
//
//input - the raw message
//command - the part of the message that specifies what function to exicute
//parameter - the part of a message that may be passed into a function at exicution
var input = consumeresult.Value;
string command;
string parameter;
if (input.Contains(" "))
{
command = input.Substring(0, input.IndexOf(" ")).Trim();
parameter = input.Substring(input.IndexOf(" "), input.Length - command.Length).Trim();
}
else
{
command = input;
parameter = "";
}
Console.WriteLine("COMMAND----------> " + input);
if (command.Equals("Add-Channel"))
{
controller.addClient(parameter);
}
else if (command.Equals("Drop-Channel"))
{
controller.dropClient(parameter);
}
else if (command.Equals("SCPList-Channels"))
{
Console.WriteLine("The active threads are:");
controller.listThreads();
}
else if (command.Equals("SCPCount"))
{
Console.WriteLine("The message queue has " + controller.queueSize() + " messages in it right now");
}
else if (command.Equals("SCPExit") || command.Equals("System-Shutdown"))
{
controller.exit();
Active = false;
source.Cancel();
}
else if (command.Equals("SCPBlacklist"))
{
blacklist.Add(parameter);
}
else if (command.Equals("SCPUnblacklist"))
{
blacklist.Remove(parameter);
}
}
else
{
Thread.Sleep(100);
}
}
catch (System.OperationCanceledException e)
{
}
}
}
static void Main(string[] args)
{
var schema =
"{\r\n\t\"subject\":\"aa-value\",\r\n \"version\":3,\r\n \"id\":1395,\r\n \"schema\":\r\n\t{\r\n\t\t\"type\":\"record\",\r\n\t\t\"name\":\"aa\",\r\n\t\t\"namespace\":\"il.aa\",\r\n\t\t\"fields\":\r\n\t\t[\r\n\t\t\t{\"name\":\"Id\", \"type\":\"int\"},\r\n\t\t\t{\"name\":\"Name\", \"type\":\"string\"},\r\n\t\t\t{\"name\":\"BatchId\", \"type\":[\"null\", \"int\"]},\r\n\t\t\t{\"name\":\"TextData\", \"type\":[\"null\", \"string\"]},\r\n\t\t\t{\"name\":\"NumericData\", \"type\":[\"null\", \"long\"], \"default\":null}\r\n\t\t]\r\n\t}\r\n}";
const bool isFromLocalFile = true; //1
const string schemaFileName = "schema.json";
var urlSchemaPrefix = isFromLocalFile ? string.Empty : "http://localhost:9999/";
var config = new Dictionary <string, object>
{
{ KafkaPropNames.BootstrapServers, "localhost:9092" },
{ KafkaPropNames.SchemaRegistryUrl, $"{urlSchemaPrefix}{schemaFileName}" },
{ KafkaPropNames.Topic, "aa-topic" },
{ KafkaPropNames.GroupId, "aa-group" },
{ KafkaPropNames.Partition, 0 },
{ KafkaPropNames.Offset, 0 },
};
var consumer = new ConsumerBuilder <string, RecordModel>(new ConsumerConfig
{
BootstrapServers = (string)config[KafkaPropNames.BootstrapServers],
GroupId = (string)config[KafkaPropNames.GroupId],
AutoOffsetReset = AutoOffsetReset.Earliest
}).SetKeyDeserializer(Deserializers.Utf8)
.SetAvroValueDeserializer(schema)
.Build();
var topic = (string)config[KafkaPropNames.Topic];
consumer.Assign(new List <TopicPartitionOffset>
{
new TopicPartitionOffset(topic, (int)config[KafkaPropNames.Partition], (int)config[KafkaPropNames.Offset])
});
var handler = new Handler();
var kafkaConsumer = new KafkaConsumer <string, RecordModel>(
consumer,
(key, value, utcTimestamp) =>
{
Console.WriteLine($"C# {key} -> ");
Console.WriteLine($" {utcTimestamp}");
handler.Handle(value);
Console.WriteLine("");
}, CancellationToken.None)
.StartConsuming();
var producer = new ProducerBuilder <string, RecordModel>(
new ProducerConfig {
BootstrapServers = (string)config[KafkaPropNames.BootstrapServers]
})
.SetKeySerializer(Serializers.Utf8)
.SetValueSerializer(new AvroConvertSerializer <RecordModel>(schema))
.Build();
var count = 0;
var random = new Random(15);
for (var i = 0; i < 10; i++)
{
count++;
var record = new RecordModel
{
Id = count,
Name = $"{config[KafkaPropNames.GroupId]}-{count}",
BatchId = (count / 10) + 1,
TextData = "Some text data",
NumericData = (long)random.Next(0, 100_000)
};
producer.Produce(topic, new Message <string, RecordModel>
{
Key = count.ToString(),
Value = record
});
}
Console.WriteLine("Press any key to quit...");
Console.ReadKey();
kafkaConsumer.Dispose();
}
}
public static void Producer_Handles(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Producer_Handles");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer1 = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var producer2 = new DependentProducerBuilder <string, string>(producer1.Handle).Build())
using (var producer3 = new DependentProducerBuilder <byte[], byte[]>(producer1.Handle).Build())
using (var producer4 = new DependentProducerBuilder <int, string>(producer2.Handle).Build())
using (var producer5 = new DependentProducerBuilder <int, int>(producer3.Handle).Build())
using (var producer6 = new DependentProducerBuilder <string, byte[]>(producer4.Handle).Build())
using (var producer7 = new ProducerBuilder <double, double>(producerConfig).Build())
using (var adminClient = new AdminClient(producer7.Handle))
{
var r1 = producer1.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 42 }, Value = new byte[] { 33 }
}).Result;
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
var r2 = producer2.ProduceAsync(topic.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Result;
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
var r3 = producer3.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 40 }, Value = new byte[] { 31 }
}).Result;
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
var r4 = producer4.ProduceAsync(topic.Name, new Message <int, string> {
Key = 42, Value = "mellow world"
}).Result;
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
var r5 = producer5.ProduceAsync(topic.Name, new Message <int, int> {
Key = int.MaxValue, Value = int.MinValue
}).Result;
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
var r6 = producer6.ProduceAsync(topic.Name, new Message <string, byte[]> {
Key = "yellow mould", Value = new byte[] { 69 }
}).Result;
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
var r7 = producer7.ProduceAsync(topic.Name, new Message <double, double> {
Key = 44.0, Value = 234.4
}).Result;
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
var topicMetadata = adminClient.GetMetadata(singlePartitionTopic, TimeSpan.FromSeconds(10));
Assert.Single(topicMetadata.Topics);
// implicitly check this does not throw.
}
var consumerConfig = new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString()
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var r1 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
}
using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 1));
var r2 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
Assert.Equal(1, r2.Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 2));
var r3 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
Assert.Equal(2, r3.Offset);
}
using (var consumer = new ConsumerBuilder <int, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 3));
var r4 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
Assert.Equal(3, r4.Offset);
}
using (var consumer = new ConsumerBuilder <int, int>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 4));
var r5 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
Assert.Equal(4, r5.Offset);
}
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 5));
var r6 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
Assert.Equal(5, r6.Offset);
}
using (var consumer = new ConsumerBuilder <double, double>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 6));
var r7 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
Assert.Equal(6, r7.Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_Handles");
}
public void AssignOverloads(string bootstrapServers)
{
LogToFile("start AssignOverloads");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000,
EnableAutoCommit = false
};
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
var testString2 = "hello world 2";
var testString3 = "hello world 3";
var testString4 = "hello world 4";
DeliveryResult <Null, string> dr, dr3;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString2
}).Wait();
dr3 = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString3
}).Result;
producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString4
}).Wait();
producer.Flush(TimeSpan.FromSeconds(10));
}
using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig).Build())
{
// Explicitly specify partition offset.
consumer.Assign(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(dr.TopicPartition, dr.Offset)
});
var cr = consumer.Consume(TimeSpan.FromSeconds(10));
consumer.Commit();
Assert.Equal(cr.Value, testString);
// Determine offset to consume from automatically.
consumer.Assign(new List <TopicPartition>()
{
dr.TopicPartition
});
cr = consumer.Consume(TimeSpan.FromSeconds(10));
consumer.Commit();
Assert.NotNull(cr.Message);
Assert.Equal(cr.Message.Value, testString2);
// Explicitly specify partition offset.
consumer.Assign(new TopicPartitionOffset(dr.TopicPartition, dr3.Offset));
cr = consumer.Consume(TimeSpan.FromSeconds(10));
consumer.Commit();
Assert.Equal(cr.Value, testString3);
// Determine offset to consume from automatically.
consumer.Assign(dr.TopicPartition);
cr = consumer.Consume(TimeSpan.FromSeconds(10));
consumer.Commit();
Assert.NotNull(cr.Message);
Assert.Equal(cr.Message.Value, testString4);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AssignOverloads");
}
public override void Run(string brokerList, List <string> topics, CancellationToken cancellationToken)
{
var config = new ConsumerConfig
{
// the group.id property must be specified when creating a consumer, even
// if you do not intend to use any consumer group functionality.
GroupId = new Guid().ToString(),
BootstrapServers = brokerList,
// partition offsets can be committed to a group even by consumers not
// subscribed to the group. in this example, auto commit is disabled
// to prevent this from occurring.
EnableAutoCommit = true
};
Stopwatch sw = new Stopwatch();
sw.Start();
Console.WriteLine($"sw.ElapsedMilliseconds start");
using (var consumer =
new ConsumerBuilder <int, string>(config)
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Assign(topics.Select(topic => new TopicPartitionOffset(topic, 0, Offset.Beginning)).ToList());
try
{
while (true)
{
try
{
var result = consumer.ConsumeBatch <int, string>(1000, 10);
if (result.Messages.Count > 0)
{
var connString = Utility.GetConnectionString("ConnectionStrings:DefaultConnectionEF");
using var conn = new NpgsqlConnection(connString);
conn.Open();
var valuesTableSql = string.Join(",", Enumerable.Range(0, result.Messages.Count).Select(i => $"(@p1{i}, @p2{i},@p3{i},@p4{i},@p5{i})"));
using (var tx = conn.BeginTransaction())
{
using (var cmd = new NpgsqlCommand($"INSERT INTO \"Messages\" (\"Id\", \"Created\", \"Content\", \"IsReceived\", \"ReceivedTimestamp\") VALUES {valuesTableSql};", conn))
{
for (int i = 0; i < result.Messages.Count; ++i)
{
cmd.Parameters.AddWithValue($"p1{i}", result.Messages.ElementAt(i).Key);
cmd.Parameters.AddWithValue($"p2{i}", DateTime.Now);
cmd.Parameters.AddWithValue($"p3{i}", result.Messages.ElementAt(i).Value);
cmd.Parameters.AddWithValue($"p4{i}", false);
cmd.Parameters.AddWithValue($"p5{i}", DateTime.Now);
}
cmd.ExecuteNonQuery();
}
tx.Commit();
}
Console.WriteLine($"after database: {sw.ElapsedMilliseconds}, MC: {result.MessagesConsumed}, WM: {result.BatchWindowMilliseconds}");
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
sw.Stop();
Console.WriteLine($"sw.ElapsedMilliseconds: {sw.ElapsedMilliseconds}");
}
public static void AvoAndRegular(string bootstrapServers, string schemaRegistryServers)
{
using (var topic1 = new TemporaryTopic(bootstrapServers, 1))
using (var topic2 = new TemporaryTopic(bootstrapServers, 1))
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, string>(producerConfig)
.SetKeySerializer(Serializers.Utf8)
.SetValueSerializer(new AvroSerializer <string>(schemaRegistry))
.Build())
{
// implicit check that this does not fail.
producer.ProduceAsync(topic1.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Wait();
// check that the value type was registered with SR, and the key was not.
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructKeySubjectName(topic1.Name, null)).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
var s2 = schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topic1.Name, null)).Result;
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, string>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(Serializers.Utf8)
.Build())
{
// implicit check that this does not fail.
producer.ProduceAsync(topic2.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Wait();
// check that the key type was registered with SR, and the value was not.
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topic2.Name, null)).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
var s2 = schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructKeySubjectName(topic2.Name, null)).Result;
}
// check the above can be consumed (using regular / Avro serializers as appropriate)
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0));
var cr = consumer.Consume();
Assert.Equal("hello", cr.Message.Key);
Assert.Equal("world", cr.Message.Value);
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(Deserializers.Utf8).Build())
{
consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0));
var cr = consumer.Consume();
Assert.Equal("hello", cr.Message.Key);
Assert.Equal("world", cr.Message.Value);
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0));
Assert.ThrowsAny <ConsumeException>(() =>
{
try
{
consumer.Consume();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(Deserializers.Utf8)
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0));
Assert.ThrowsAny <ConsumeException>(() =>
{
try
{
consumer.Consume();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
}
}
}
public static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
DeliveryResult <Null, GenericRecord> dr;
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <Null, GenericRecord>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AsyncAvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Value = record
}).Result;
}
// produce a specific record (to later consume back as a generic record).
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <Null, User>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AsyncAvroSerializer <User>(schemaRegistry))
.Build())
{
var user = new User
{
name = "my name 3",
favorite_number = 47,
favorite_color = "orange"
};
p.ProduceAsync(topic, new Message <Null, User> {
Value = user
}).Wait();
}
Assert.Null(dr.Message.Key);
Assert.NotNull(dr.Message.Value);
dr.Message.Value.TryGetValue("name", out object name);
dr.Message.Value.TryGetValue("favorite_number", out object number);
dr.Message.Value.TryGetValue("favorite_color", out object color);
Assert.IsType <string>(name);
Assert.IsType <int>(number);
Assert.Equal("my name 2", name);
Assert.Equal(44, number);
Assert.Null(color);
var cconfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <Null, GenericRecord>(cconfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AsyncAvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync())
.Build())
{
// consume generic record produced as a generic record.
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Value.TryGetValue("name", out object msgName);
record.Message.Value.TryGetValue("favorite_number", out object msgNumber);
record.Message.Value.TryGetValue("favorite_color", out object msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.Equal("my name 2", msgName);
Assert.Equal(44, msgNumber);
Assert.Null(msgColor);
// consume generic record produced as a specific record.
record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Value.TryGetValue("name", out msgName);
record.Message.Value.TryGetValue("favorite_number", out msgNumber);
record.Message.Value.TryGetValue("favorite_color", out msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.IsType <string>(msgColor);
Assert.Equal("my name 3", msgName);
Assert.Equal(47, msgNumber);
Assert.Equal("orange", msgColor);
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <Null, User>(cconfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AsyncAvroDeserializer <User>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
Assert.Equal("my name 2", record.Message.Value.name);
Assert.Equal(44, record.Message.Value.favorite_number);
Assert.Null(record.Message.Value.favorite_color);
}
}
public void AssignPastEnd(string bootstrapServers)
{
LogToFile("start AssignPastEnd");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryResult <Null, byte[]> dr;
using (var producer = new ProducerBuilder <Null, byte[]>(producerConfig).Build())
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, byte[]> {
Value = Serializers.Utf8(testString)
}).Result;
Assert.True(dr.Offset >= 0);
producer.Flush(TimeSpan.FromSeconds(10));
}
consumerConfig.AutoOffsetReset = AutoOffsetReset.Latest;
using (var consumer = new ConsumerBuilder <Null, byte[]>(consumerConfig).Build())
{
ConsumeResult <Null, byte[]> record;
consumer.Assign(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(dr.TopicPartition, dr.Offset + 1)
});
record = consumer.Consume(TimeSpan.FromSeconds(2));
Assert.Null(record);
consumer.Assign(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(dr.TopicPartition, dr.Offset + 2)
});
consumer.Consume(TimeSpan.FromSeconds(2));
Assert.Null(record);
}
consumerConfig.AutoOffsetReset = AutoOffsetReset.Earliest;
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
ConsumeResult <byte[], byte[]> record;
consumer.Assign(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(dr.TopicPartition, dr.Offset + 1)
});
record = consumer.Consume(TimeSpan.FromSeconds(2));
Assert.Null(record);
// Note: dr.Offset+2 is an invalid (c.f. dr.Offset+1 which is valid), so auto.offset.reset will come
// into play here to determine which offset to start from (earliest). Due to the the produce call above,
// there is guarenteed to be a message on the topic, so consumer.Consume will return true.
consumer.Assign(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(dr.TopicPartition, dr.Offset + 2)
});
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record?.Message);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AssignPastEnd");
}
public static void Consumer_Commit_CommitAsync_Committed_Position(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Consumer_Commit_CommitAsync_Committed_Position");
const int N = 8;
const int Partition = 0;
var messages = ProduceMessages(bootstrapServers, singlePartitionTopic, Partition, N);
var firstMsgOffset = messages[0].Offset;
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
EnableAutoCommit = false
};
var firstMessage = messages[0];
var lastMessage = messages[N - 1];
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
// Test #0 (empty cases)
consumer.Commit(new List <TopicPartitionOffset>()); // should not throw.
var committed = consumer.Committed(new List <TopicPartition>(), TimeSpan.FromSeconds(10));
Assert.Empty(committed);
var ps = consumer.Position(new List <TopicPartition>());
Assert.Empty(ps);
// Test #0.5 (invalid cases)
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition("invalid-topic", 0)
});
Assert.Single(ps);
Assert.Equal(Offset.Invalid, ps[0].Offset);
Assert.Equal("invalid-topic", ps[0].Topic);
Assert.Equal(0, (int)ps[0].Partition);
Assert.Equal(new TopicPartition("invalid-topic", 0), ps[0].TopicPartition);
// Test #1
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var os = consumer.Commit();
Assert.Equal(firstMsgOffset + 1, os[0].Offset);
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 1, co[0].Offset);
Assert.Equal(firstMsgOffset + 1, ps[0].Offset);
// Test #2
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
os = consumer.Commit();
Assert.Equal(firstMsgOffset + 2, os[0].Offset);
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 2, ps[0].Offset);
Assert.Equal(firstMsgOffset + 2, ps[0].Offset);
}
// Test #3
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Commit(new List <TopicPartitionOffset> {
new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 5)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 5, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
Assert.Equal(firstMsgOffset + 6, ps[0].Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 5, co[0].Offset);
}
// Test #4
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
consumer.Commit(new List <TopicPartitionOffset> {
new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 3)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
Assert.Equal(firstMsgOffset + 4, ps[0].Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
// Test #5
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
consumer.Commit(record3);
var record4 = consumer.Consume(TimeSpan.FromMilliseconds(1000));
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
Assert.Equal(firstMsgOffset + 3, record.Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
// Test #6
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
consumer.Commit(record3);
var record4 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
Assert.Equal(firstMsgOffset + 3, record.Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Commit_CommitAsync_Committed_Position");
}
public void IgnoreTest(string bootstrapServers)
{
LogToFile("start IgnoreTest");
var consumerConfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
DeliveryResult <byte[], byte[]> dr;
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
{
// Assume that all these produce calls succeed.
dr = producer.ProduceAsync(new TopicPartition(singlePartitionTopic, 0), new Message <byte[], byte[]> {
Key = null, Value = null
}).Result;
producer.ProduceAsync(new TopicPartition(singlePartitionTopic, 0), new Message <byte[], byte[]> {
Key = null, Value = new byte[1] {
1
}
}).Wait();
producer.ProduceAsync(new TopicPartition(singlePartitionTopic, 0), new Message <byte[], byte[]> {
Key = new byte[1] {
0
}, Value = null
}).Wait();
producer.ProduceAsync(new TopicPartition(singlePartitionTopic, 0), new Message <byte[], byte[]> {
Key = new byte[1] {
42
}, Value = new byte[2] {
42, 240
}
}).Wait();
producer.Flush(TimeSpan.FromSeconds(10));
}
using (var consumer = new ConsumerBuilder <Ignore, Ignore>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
ConsumeResult <Ignore, Ignore> record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Null(record.Message.Key);
Assert.Null(record.Message.Value);
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Null(record.Message.Key);
Assert.Null(record.Message.Value);
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Null(record.Message.Key);
Assert.Null(record.Message.Value);
}
using (var consumer = new ConsumerBuilder <Ignore, byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(dr.TopicPartition, dr.Offset.Value + 3)
});
ConsumeResult <Ignore, byte[]> record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Null(record.Message.Key);
Assert.NotNull(record.Message.Value);
Assert.Equal(42, record.Message.Value[0]);
Assert.Equal(240, record.Message.Value[1]);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end IgnoreTest");
}
public void MessageHeaderProduceConsume(string bootstrapServers)
{
LogToFile("start MessageHeaderProduceConsume");
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
EnableIdempotence = true
};
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
var drs = new List <DeliveryReport <Null, string> >();
DeliveryResult <Null, string> dr_single, dr_empty, dr_null, dr_multiple, dr_duplicate;
DeliveryResult <Null, string> dr_ol1, dr_ol3;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
// single header value.
var headers = new Headers();
headers.Add("test-header", new byte[] { 142 });
dr_single = producer.ProduceAsync(
singlePartitionTopic,
new Message <Null, string> {
Value = "the value", Headers = headers
}).Result;
Assert.Single(dr_single.Message.Headers);
Assert.Equal("test-header", dr_single.Message.Headers[0].Key);
Assert.Equal(new byte[] { 142 }, dr_single.Message.Headers[0].GetValueBytes());
// empty header values
var headers0 = new Headers();
dr_empty = producer.ProduceAsync(
singlePartitionTopic,
new Message <Null, string> {
Value = "the value", Headers = headers0
}).Result;
Assert.Empty(dr_empty.Message.Headers);
// null header value
dr_null = producer.ProduceAsync(
singlePartitionTopic,
new Message <Null, string> {
Value = "the value"
}).Result;
Assert.Empty(dr_null.Message.Headers);
// multiple header values (also Headers no Dictionary, since order is tested).
var headers2 = new Headers();
headers2.Add("test-header-a", new byte[] { 111 });
headers2.Add("test-header-b", new byte[] { 112 });
dr_multiple = producer.ProduceAsync(
singlePartitionTopic,
new Message <Null, string> {
Value = "the value", Headers = headers2
}).Result;
Assert.Equal(2, dr_multiple.Message.Headers.Count);
Assert.Equal("test-header-a", dr_multiple.Message.Headers[0].Key);
Assert.Equal(new byte[] { 111 }, dr_multiple.Message.Headers[0].GetValueBytes());
Assert.Equal("test-header-b", dr_multiple.Message.Headers[1].Key);
Assert.Equal(new byte[] { 112 }, dr_multiple.Message.Headers[1].GetValueBytes());
// duplicate header values (also List not Dictionary)
var headers3 = new Headers();
headers3.Add(new Header("test-header-a", new byte[] { 111 }));
headers3.Add(new Header("test-header-b", new byte[] { 112 }));
headers3.Add(new Header("test-header-a", new byte[] { 113 }));
headers3.Add(new Header("test-header-b", new byte[] { 114 }));
headers3.Add(new Header("test-header-c", new byte[] { 115 }));
dr_duplicate = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = "the value", Headers = headers3
}).Result;
Assert.Equal(5, dr_duplicate.Message.Headers.Count);
Assert.Equal("test-header-a", dr_duplicate.Message.Headers[0].Key);
Assert.Equal(new byte[] { 111 }, dr_duplicate.Message.Headers[0].GetValueBytes());
Assert.Equal("test-header-a", dr_duplicate.Message.Headers[2].Key);
Assert.Equal(new byte[] { 113 }, dr_duplicate.Message.Headers[2].GetValueBytes());
// Test headers work as expected with all serializing ProduceAsync variants.
dr_ol1 = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = "the value"
}).Result;
Assert.Empty(dr_ol1.Message.Headers);
dr_ol3 = producer.ProduceAsync(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string> {
Value = "the value", Headers = headers
}
).Result;
Assert.Single(dr_ol3.Message.Headers);
Assert.Equal("test-header", dr_ol3.Message.Headers[0].Key);
Assert.Equal(new byte[] { 142 }, dr_ol3.Message.Headers[0].GetValueBytes());
Action <DeliveryReport <Null, string> > dh = (DeliveryReport <Null, string> dr) => drs.Add(dr);
// Test headers work as expected with all serializing Produce variants.
producer.Produce(singlePartitionTopic, new Message <Null, string> {
Value = "the value"
}, dh);
producer.Produce(
new TopicPartition(singlePartitionTopic, 0),
new Message <Null, string> {
Value = "the value", Headers = headers2
},
dh);
producer.Flush(TimeSpan.FromSeconds(10));
Assert.Empty(drs[0].Message.Headers); // TODO: this is intermittently not working.
Assert.Equal(2, drs[1].Message.Headers.Count);
}
List <DeliveryReport <byte[], byte[]> > drs_2 = new List <DeliveryReport <byte[], byte[]> >();
DeliveryResult <byte[], byte[]> dr_ol4, dr_ol5, dr_ol6, dr_ol7;
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
{
var headers = new Headers();
headers.Add("hkey", new byte[] { 44 });
// Test headers work as expected with all non-serializing ProduceAsync variants.
dr_ol4 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = null
}).Result;
Assert.Empty(dr_ol4.Message.Headers);
dr_ol5 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = null
}).Result;
Assert.Empty(dr_ol5.Message.Headers);
dr_ol6 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = headers
}).Result;
Assert.Single(dr_ol6.Message.Headers);
dr_ol7 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = headers
}).Result;
Assert.Single(dr_ol7.Message.Headers);
// Test headers work as expected with all non-serializing Produce variants.
Action <DeliveryReport <byte[], byte[]> > dh = (DeliveryReport <byte[], byte[]> dr) => drs_2.Add(dr);
producer.Produce(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = headers
}, dh);
producer.Produce(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = null
}, dh);
producer.Produce(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = headers
}, dh);
producer.Produce(singlePartitionTopic, new Message <byte[], byte[]> {
Headers = headers
}, dh);
producer.Flush(TimeSpan.FromSeconds(10));
Assert.Single(drs_2[0].Message.Headers);
Assert.Empty(drs_2[1].Message.Headers); // TODO: this is intermittently not working.
Assert.Single(drs_2[2].Message.Headers);
Assert.Single(drs_2[3].Message.Headers);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_single.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Single(record.Message.Headers);
Assert.Equal("test-header", record.Message.Headers[0].Key);
Assert.Equal(new byte[] { 142 }, record.Message.Headers[0].GetValueBytes());
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_empty.TopicPartitionOffset
});
var record2 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record2.Message);
// following Java, alway instantiate a new Headers instance, even in the empty case.
Assert.NotNull(record2.Message.Headers);
Assert.Empty(record2.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_null.TopicPartitionOffset
});
var record3 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record3.Message);
Assert.NotNull(record3.Message.Headers);
Assert.Empty(record3.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_multiple.TopicPartitionOffset
});
var record4 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record4.Message);
Assert.Equal(2, record4.Message.Headers.Count);
Assert.Equal("test-header-a", record4.Message.Headers[0].Key);
Assert.Equal("test-header-b", record4.Message.Headers[1].Key);
Assert.Equal(new byte[] { 111 }, record4.Message.Headers[0].GetValueBytes());
Assert.Equal(new byte[] { 112 }, record4.Message.Headers[1].GetValueBytes());
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_duplicate.TopicPartitionOffset
});
var record5 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record5.Message);
Assert.Equal(5, record5.Message.Headers.Count);
Assert.Equal("test-header-a", record5.Message.Headers[0].Key);
Assert.Equal("test-header-b", record5.Message.Headers[1].Key);
Assert.Equal("test-header-a", record5.Message.Headers[2].Key);
Assert.Equal("test-header-b", record5.Message.Headers[3].Key);
Assert.Equal("test-header-c", record5.Message.Headers[4].Key);
Assert.Equal(new byte[] { 111 }, record5.Message.Headers[0].GetValueBytes());
Assert.Equal(new byte[] { 112 }, record5.Message.Headers[1].GetValueBytes());
Assert.Equal(new byte[] { 113 }, record5.Message.Headers[2].GetValueBytes());
Assert.Equal(new byte[] { 114 }, record5.Message.Headers[3].GetValueBytes());
Assert.Equal(new byte[] { 115 }, record5.Message.Headers[4].GetValueBytes());
Assert.Equal(new byte[] { 113 }, record5.Message.Headers.GetLastBytes("test-header-a"));
Assert.Equal(new byte[] { 114 }, record5.Message.Headers.GetLastBytes("test-header-b"));
Assert.Equal(new byte[] { 115 }, record5.Message.Headers.GetLastBytes("test-header-c"));
// Test headers work with all produce method variants.
// async, serializing
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_ol1.TopicPartitionOffset
});
var record6 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record6.Message);
Assert.Empty(record6.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_ol3.TopicPartitionOffset
});
var record8 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record8.Message);
Assert.Single(record8.Message.Headers);
// delivery-handler, serializing.
consumer.Assign(new List <TopicPartitionOffset>()
{
drs[0].TopicPartitionOffset
});
var record9 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record9.Message);
Assert.Empty(record9.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs[1].TopicPartitionOffset
});
var record11 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record11.Message);
Assert.Equal(2, record11.Message.Headers.Count);
// async, non-serializing
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_ol4.TopicPartitionOffset
});
var record12 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record12.Message);
Assert.Empty(record12.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_ol5.TopicPartitionOffset
});
var record13 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record13.Message);
Assert.Empty(record13.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_ol6.TopicPartitionOffset
});
var record14 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record14.Message);
Assert.Single(record14.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
dr_ol7.TopicPartitionOffset
});
var record15 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record15.Message);
Assert.Single(record15.Message.Headers);
// delivery handler, non-serializing
consumer.Assign(new List <TopicPartitionOffset>()
{
drs_2[0].TopicPartitionOffset
});
var record16 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record16.Message);
Assert.Single(record16.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs_2[1].TopicPartitionOffset
});
var record17 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record17.Message);
Assert.Empty(record17.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs_2[2].TopicPartitionOffset
});
var record18 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record18.Message);
Assert.Single(record18.Message.Headers);
consumer.Assign(new List <TopicPartitionOffset>()
{
drs_2[3].TopicPartitionOffset
});
var record19 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record19.Message);
Assert.Single(record19.Message.Headers);
}
// null key
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
{
var headers = new Headers();
var threw = false;
try
{
headers.Add(null, new byte[] { 142 });
}
catch
{
threw = true;
}
finally
{
Assert.True(threw);
}
var headers2 = new List <Header>();
Assert.Throws <ArgumentNullException>(() => headers2.Add(new Header(null, new byte[] { 42 })));
}
// null value
DeliveryResult <Null, string> nulldr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
var headers = new Headers();
headers.Add("my-header", null);
nulldr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = "test-value", Headers = headers
}).Result;
Assert.Single(nulldr.Headers);
Assert.Null(nulldr.Headers[0].GetValueBytes());
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, nulldr.Offset));
var cr = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(cr?.Message);
Assert.Single(cr.Message.Headers);
Assert.Equal("my-header", cr.Message.Headers[0].Key);
Assert.Null(cr.Message.Headers[0].GetValueBytes());
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end MessageHeaderProduceConsume");
}
public void Transactions_Abort(string bootstrapServers)
{
LogToFile("start Transactions_Abort");
var defaultTimeout = TimeSpan.FromSeconds(30);
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
{
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
}, (dr) => {
Assert.Equal(0, dr.Offset);
});
Thread.Sleep(4000); // ensure the abort ctrl message makes it into the log.
producer.AbortTransaction(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 1", Value = "test val 1"
}, (dr) => {
// abort marker will be at offset 1.
Assert.Equal(2, dr.Offset);
});
producer.CommitTransaction(defaultTimeout);
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var cr1 = consumer.Consume();
var cr2 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal("test val 1", cr1.Message.Value);
Assert.Equal(2, cr1.Offset); // there should be skipped offsets due to the aborted txn and commit marker in the log.
Assert.Null(cr2); // control message should not be exposed to application.
}
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadUncommitted, BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var cr1 = consumer.Consume();
var cr2 = consumer.Consume();
var cr3 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal("test val 0", cr1.Message.Value);
Assert.Equal(0, cr1.Offset); // the aborted message should not be skipped.
Assert.Equal("test val 1", cr2.Message.Value);
Assert.Equal(2, cr2.Offset); // there should be a skipped offset due to a commit marker in the log.
Assert.Null(cr3); // control message should not be exposed to application.
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Abort");
}
public static void WatermarkOffsets(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryResult <Null, string> dr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
using (var adminClient = new AdminClient(producer.Handle))
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
var queryOffsets = adminClient.QueryWatermarkOffsets(new TopicPartition(singlePartitionTopic, 0), TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Invalid);
Assert.NotEqual(queryOffsets.High, Offset.Invalid);
// TODO: can anything be said about the high watermark offset c.f. dr.Offset?
// I have seen queryOffsets.High < dr.Offset and also queryOffsets.High = dr.Offset + 1.
// The former only once (or was I in error?). request.required.acks has a default value
// of 1, so with only one broker, I assume the former should never happen.
// Console.WriteLine($"Query Offsets: [{queryOffsets.Low} {queryOffsets.High}]. DR Offset: {dr.Offset}");
Assert.True(queryOffsets.Low < queryOffsets.High);
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
using (var adminClient = new AdminClient(consumer.Handle))
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = adminClient.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(getOffsets.Low, Offset.Invalid);
// the offset of the next message to be read.
Assert.Equal(getOffsets.High, dr.Offset + 1);
var queryOffsets = adminClient.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Invalid);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
public static void PrimitiveTypes(string bootstrapServers, string schemaRegistryServers)
{
var schemaRegistryConfig = new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest
};
var stringTopic = Guid.NewGuid().ToString();
var bytesTopic = Guid.NewGuid().ToString();
var intTopic = Guid.NewGuid().ToString();
var longTopic = Guid.NewGuid().ToString();
var boolTopic = Guid.NewGuid().ToString();
var floatTopic = Guid.NewGuid().ToString();
var doubleTopic = Guid.NewGuid().ToString();
var nullTopic = Guid.NewGuid().ToString();
using (var producer =
new ProducerBuilder <string, string>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <string>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(stringTopic, new Message <string, string> {
Key = "hello", Value = "world"
})
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var producer =
new ProducerBuilder <byte[], byte[]>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <byte[]>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <byte[]>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(bytesTopic, new Message <byte[], byte[]> {
Key = new byte[] { 1, 4, 11 }, Value = new byte[] {}
})
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var producer =
new ProducerBuilder <int, int>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <int>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(intTopic, new Message <int, int> {
Key = 42, Value = 43
})
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var producer =
new ProducerBuilder <long, long>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <long>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <long>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(longTopic, new Message <long, long> {
Key = -32, Value = -33
})
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var producer =
new ProducerBuilder <bool, bool>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <bool>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <bool>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(boolTopic, new Message <bool, bool> {
Key = true, Value = false
})
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var producer =
new ProducerBuilder <float, float>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <float>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <float>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(floatTopic, new Message <float, float> {
Key = 44.0f, Value = 45.0f
})
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var producer =
new ProducerBuilder <double, double>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <double>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <double>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(doubleTopic, new Message <double, double> {
Key = 46.0, Value = 47.0
})
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var producer =
new ProducerBuilder <Null, Null>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <Null>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <Null>(schemaRegistry))
.Build())
{
producer
.ProduceAsync(nullTopic, new Message <Null, Null>())
.Wait();
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(stringTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", result.Message.Key);
Assert.Equal("world", result.Message.Value);
}
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <byte[]>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <byte[]>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(bytesTopic, 0, 0)
});
var result2 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 1, 4, 11 }, result2.Message.Key);
Assert.Equal(new byte[] { }, result2.Message.Value);
}
using (var consumer =
new ConsumerBuilder <int, int>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <int>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <int>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(intTopic, 0, 0)
});
var result3 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, result3.Message.Key);
Assert.Equal(43, result3.Message.Value);
}
using (var consumer =
new ConsumerBuilder <long, long>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <long>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <long>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(longTopic, 0, 0)
});
var result4 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(-32, result4.Message.Key);
Assert.Equal(-33, result4.Message.Value);
}
using (var consumer =
new ConsumerBuilder <bool, bool>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <bool>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <bool>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(boolTopic, 0, 0)
});
var result5 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.True(result5.Message.Key);
Assert.False(result5.Message.Value);
}
using (var consumer =
new ConsumerBuilder <float, float>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <float>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <float>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(floatTopic, 0, 0)
});
var result6 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0f, result6.Message.Key);
Assert.Equal(45.0f, result6.Message.Value);
}
using (var consumer =
new ConsumerBuilder <double, double>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <double>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <double>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(doubleTopic, 0, 0)
});
var result7 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(46.0, result7.Message.Key);
Assert.Equal(47.0, result7.Message.Value);
}
using (var consumer =
new ConsumerBuilder <Null, Null>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <Null>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <Null>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(nullTopic, 0, 0)
});
var result8 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Null(result8.Key);
Assert.Null(result8.Value);
}
}
}
/**
* This is the primary function of <TwitchCommandConsumer> and is to be run as a thread
*/
public void TCCThread()
{
Console.WriteLine("TwitchCommandThread Start");
Active = true;
//objs: Kafka Consumer client
//
//consumer - the Kafka consumer client
//topicp - the topic partition from <consumer> to consume from
var consumer = new ConsumerBuilder <string, string>(config).Build();
var topicp = new TopicPartition(topic, 0);
consumer.Assign(topicp);
while (Active)
{
try
{
//var: consumerresult
//The result of checking for the next new message on the Kafka server
var consumerresult = consumer.Consume(canceltoken);
if (!consumerresult.IsPartitionEOF)
{
//strings: Kafka Message
//
//input - the raw message from Kafka
//channel - the channel that the message was received from
//uname - the username of the user that sent the message
//message - the actual message portion
//parameter - a part of the message intended to be passed as a paremeter
var input = consumerresult.Value;
Console.WriteLine(input);
var channel = input.Substring(0, input.IndexOf(" "));
var uname = input.Substring(input.IndexOf(" ")).Trim();
uname = uname.TrimStart(new char[] { '\0', ':' });
var message = uname.Substring(uname.IndexOf(" ")).Trim();
uname = uname.Substring(0, uname.IndexOf(" ")).Trim();
string parameter = "";
if (message.Contains(" "))
{
var message2 = message.Substring(0, message.IndexOf(" ")).Trim();
parameter = message.Substring(message.IndexOf(" "), message.Length - message2.Length).Trim();
message = message2;
}
Console.WriteLine("TwitchCommand----------> " + input);
if (message.ToLower().Equals("!djoin"))
{
bool joined = false;
do
{
dbAdapter.SelectCommand = new OdbcCommand("SELECT * FROM users WHERE twitch_name='" + uname + "';", connDB);
dbAdapter.Fill(data);
if (data.Tables[0].Rows.Count == 0 || data.Tables[0].Rows[0]["discord_name"].ToString().Equals(""))
{
if (data.Tables[0].Rows.Count == 0)
{
commandMessageQueue.Enqueue("SCPUnblacklist " + uname);
if (parameter.Equals(""))
{
OdbcCommand comm = new OdbcCommand("INSERT INTO users (twitch_name) VALUES ('" + uname + "' )", connDB);
comm.ExecuteNonQueryAsync().Wait();
}
else
{
OdbcCommand comm = new OdbcCommand("INSERT INTO users (twitch_name, discord_name) VALUES ( '" + uname + "','" + parameter + "')", connDB);
comm.ExecuteNonQueryAsync().Wait();
}
}
else if (!parameter.Equals(""))
{
data.Tables[0].Rows[0]["discord_name"] = parameter;
dbAdapter.Update(data);
}
}
try
{
commandMessageQueue.Enqueue("SCPUnblacklist " + uname);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
}
data.Reset();
dbAdapter.Fill(data);
if (data.Tables[0].Rows.Count > 0)
{
joined = true;
data.Reset();
}
} while (joined == false);
}
}
else
{
Thread.Sleep(100);
}
}
catch (System.OperationCanceledException e)
{
}
}
}
public void Subscribe(Action <T> onchange, CacheNotifyAction cacheNotifyAction)
{
if (ClientConfig == null)
{
MemoryCacheNotify.Subscribe(onchange, cacheNotifyAction);
return;
}
var channelName = GetChannelName(cacheNotifyAction);
Cts[channelName] = new CancellationTokenSource();
Actions[channelName] = onchange;
void action()
{
var conf = new ConsumerConfig(ClientConfig)
{
GroupId = Guid.NewGuid().ToString()
};
using var c = new ConsumerBuilder <AscCacheItem, T>(conf)
.SetErrorHandler((_, e) => Log.Error(e))
.SetKeyDeserializer(KeyDeserializer)
.SetValueDeserializer(ValueDeserializer)
.Build();
c.Assign(new TopicPartition(channelName, new Partition()));
try
{
while (true)
{
try
{
var cr = c.Consume(Cts[channelName].Token);
if (cr != null && cr.Value != null && !(new Guid(cr.Key.Id.ToByteArray())).Equals(Key) && Actions.TryGetValue(channelName, out var act))
{
try
{
act(cr.Value);
}
catch (Exception e)
{
Log.Error("Kafka onmessage", e);
}
}
}
catch (ConsumeException e)
{
Log.Error(e);
}
}
}
catch (OperationCanceledException)
{
c.Close();
}
}
var task = new Task(action, TaskCreationOptions.LongRunning);
task.Start();
}
public async Task <JsonElement?> postTransactionAsync([FromBody] JsonElement body)
{
//The reply topic name
string topicName = "reply-some1";
//Topic creation with AdminClient
Console.WriteLine("Create Topic Before:" + Timestamp.DateTimeToUnixTimestampMs(DateTime.UtcNow));
await createTopic(topicName);
Console.WriteLine("Create Topic After:" + Timestamp.DateTimeToUnixTimestampMs(DateTime.UtcNow));
//Publish to the topic
await publish(body.ToString());
//Consumer config to read from Reply topic
var conf = new ConsumerConfig
{
GroupId = "some",
BootstrapServers = "localhost:9092",
AutoOffsetReset = AutoOffsetReset.Earliest,
//EnableAutoCommit = false,
SessionTimeoutMs = 6000,
QueuedMinMessages = 1000000
};
//Read from the reply topic
var consumer = new ConsumerBuilder <Ignore, string>(conf).Build();
Console.WriteLine("kafka subscribe Topic: {0}, at: {1}", "transactions", DateTime.Now);
Console.WriteLine("Before Subscribe:" + Timestamp.DateTimeToUnixTimestampMs(DateTime.UtcNow));
consumer.Assign(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(topicName, 0, 0)
});
CancellationTokenSource cts = new CancellationTokenSource();
cts.CancelAfter(10000);
try
{
bool flag = true;
while (flag)
{
Console.WriteLine("Cosume message at: {0}", DateTime.Now);
var consumedMessage = consumer.Consume(cts.Token);
consumer.Commit(consumedMessage);
flag = false;
Console.WriteLine("After Consume:" + Timestamp.DateTimeToUnixTimestampMs(DateTime.UtcNow));
}
}
catch (Exception e)
{
Console.WriteLine("Failed While Consuming topic transaction-posted Aggregator Request and Exception: {0} at {1}", e, DateTime.Now);
Console.WriteLine("Failed While Consumiong Aggregator Request: " + e.Message);
//consumer.Close();
}
//Once the message is read, use Admin client to delete the randomly generate topic
Console.WriteLine("Delete Topic Before:" + Timestamp.DateTimeToUnixTimestampMs(DateTime.UtcNow));
await deleteTopic(topicName);
Console.WriteLine("Delete Topic After:" + Timestamp.DateTimeToUnixTimestampMs(DateTime.UtcNow));
return(body);
}