public void Transactions_SendOffsets(string bootstrapServers)
{
LogToFile("start Transactions_SendOffsets");
var defaultTimeout = TimeSpan.FromSeconds(30);
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false, Debug = "all"
}).Build())
{
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
producer.SendOffsetsToTransaction(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic.Name, 0, 7324)
}, consumer.ConsumerGroupMetadata, TimeSpan.FromSeconds(30));
producer.CommitTransaction(defaultTimeout);
var committed = consumer.Committed(new List <TopicPartition> {
new TopicPartition(topic.Name, 0)
}, TimeSpan.FromSeconds(30));
Assert.Single(committed);
Assert.Equal(7324, committed[0].Offset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_SendOffsets");
}
public void Transactions_Abort(string bootstrapServers)
{
LogToFile("start Transactions_Abort");
var defaultTimeout = TimeSpan.FromSeconds(30);
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
{
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
Thread.Sleep(1000); // ensure the abort ctrl message makes it into the log.
producer.AbortTransaction(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 1", Value = "test val 1"
});
producer.CommitTransaction(defaultTimeout);
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var cr1 = consumer.Consume();
var cr2 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal(2, cr1.Offset); // there should be skipped offsets due to the aborted txn and commit marker in the log.
Assert.Null(cr2); // control message should not be exposed to application.
}
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadUncommitted, BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var cr1 = consumer.Consume();
var cr2 = consumer.Consume();
var cr3 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal(0, cr1.Offset); // the aborted message should not be skipped.
Assert.Equal(2, cr2.Offset); // there should be a skipped offset due to a commit marker in the log.
Assert.Null(cr3); // control message should not be exposed to application.
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Abort");
}
//It is a dead simple example of Transacions on Apache Kafka. This is not a production ready code.
//To go further, you need to deal with rebalance scenarios and handle SetPartitionsRevokedHandler and SetPartitionsAssignedHandler
//This is a 1:1 (1 read for 1 write) example. Read 1 message, process and commit the message
public void SimpleReadWriteTransaction(string inputTopicName, string outputTopicName, CancellationToken ct)
{
using var consumer = new ConsumerBuilder <int, string>(_consumerConfig).Build();
using var producer = new ProducerBuilder <int, string>(_producerConfig).Build();
producer.InitTransactions(_timeout);
consumer.Subscribe(inputTopicName);
while (!ct.IsCancellationRequested)
{
producer.BeginTransaction();
string currentProcessedMessage = "";
//Read phase
try
{
var consumerResult = consumer.Consume(ct);
//Process phase
currentProcessedMessage = ProcessMessage(consumerResult.Message);
//Write phase - Produce new message on output topic
producer.Produce(outputTopicName, new Message <int, string>()
{
Key = consumerResult.Message.Key, Value = currentProcessedMessage
});
//Commit
producer.SendOffsetsToTransaction(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(consumerResult.TopicPartition, consumerResult.Offset + 1)
},
consumer.ConsumerGroupMetadata,
_timeout);
producer.CommitTransaction(_timeout);
}
catch (Exception e)
{
_logger.LogError("Transaction Fault", e.Message, e.StackTrace);
producer.AbortTransaction(_timeout);
}
_logger.LogInformation($"Successful transaction: {currentProcessedMessage}");
}
}
public void Transactions_Error(string bootstrapServers)
{
LogToFile("start Transactions_Error");
var defaultTimeout = TimeSpan.FromSeconds(30);
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
{
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
Assert.Throws <KafkaException>(() => { producer.BeginTransaction(); });
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Error");
}
public virtual IProducer <string, byte[]> Get(string connectionName = null)
{
connectionName ??= KafkaConnections.DefaultConnectionName;
return(Producers.GetOrAdd(
connectionName, connection => new Lazy <IProducer <string, byte[]> >(() =>
{
var producerConfig = new ProducerConfig(Options.Connections.GetOrDefault(connection));
Options.ConfigureProducer?.Invoke(producerConfig);
if (producerConfig.TransactionalId.IsNullOrWhiteSpace())
{
producerConfig.TransactionalId = Guid.NewGuid().ToString();
}
var producer = new ProducerBuilder <string, byte[]>(producerConfig).Build();
producer.InitTransactions(DefaultTransactionsWaitDuration);
return producer;
})).Value);
}
/// <summary>
/// A transactional (exactly once) processing loop that reads individual words and updates
/// the corresponding total count state.
///
/// When a rebalance occurs (including on startup), the total count state for all assigned
/// partitions is reloaded before the loop commences to update it.
/// </summary>
/// <remarks>
/// Refer to Processor_MapWords for more detailed comments.
/// </remarks>
public static void Processor_AggregateWords(string brokerList, string clientId, RocksDb db, CancellationToken ct)
{
var TxnCommitPeriod = TimeSpan.FromSeconds(10);
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = ConsumerGroup_Aggregate,
AutoOffsetReset = AutoOffsetReset.Earliest,
// This should be greater than the maximum amount of time required to read in
// existing count state.
MaxPollIntervalMs = 86400000,
EnableAutoCommit = false
};
ColumnFamilyHandle columnFamily = null;
var lastTxnCommit = DateTime.Now;
var producerState = new Dictionary <TopicPartition, ProducerState <string, int> >();
using (var consumer = new ConsumerBuilder <string, Null>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
// clear rocksdb state.
db.DropColumnFamily("counts");
db.CreateColumnFamily(new ColumnFamilyOptions(), "counts");
var tasks = new List <Task>();
foreach (var p in producerState.Values)
{
tasks.Add(Task.Run(() => {
p.Producer.AbortTransaction(DefaultTimeout); // Note: Not cancellable yet.
p.Producer.Dispose();
}, ct));
}
if (tasks.Count > 0)
{
Console.WriteLine("Aborting current AggregateWords transactions.");
}
Task.WaitAll(tasks.ToArray());
producerState.Clear();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group rebalanced. Partition assignment: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
Trace.Assert(producerState.Count == 0, "Unexpected producer state");
var tasks = new List <Task>();
foreach (var tp in partitions)
{
tasks.Add(Task.Run(() => {
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
TransactionalId = TransactionalId_Aggregate + "-" + clientId + "-" + tp.Partition
};
var p = new ProducerBuilder <string, int>(pConfig).Build();
p.InitTransactions(DefaultTimeout); // Note: Not cancellable yet.
p.BeginTransaction();
lock (producerState)
{
producerState.Add(tp, new ProducerState <string, int> {
Producer = p, Offset = Offset.Unset
});
}
}, ct));
}
Task.WaitAll(tasks.ToArray());
columnFamily = db.GetColumnFamily("counts");
LoadCountState(db, brokerList, partitions.Select(p => p.Partition), columnFamily, ct);
})
.Build())
{
consumer.Subscribe(Topic_Words);
var wCount = 0;
while (true)
{
try
{
var cr = consumer.Consume(ct);
producerState[cr.TopicPartition].Offset = cr.Offset;
var kBytes = Encoding.UTF8.GetBytes(cr.Message.Key);
var vBytes = db.Get(kBytes, columnFamily);
var v = vBytes == null ? 0 : BitConverter.ToInt32(vBytes);
var updatedV = v + 1;
db.Put(kBytes, BitConverter.GetBytes(updatedV), columnFamily);
while (true)
{
try
{
producerState[cr.TopicPartition].Producer.Produce(
Topic_Counts, new Message <string, int> {
Key = cr.Message.Key, Value = updatedV
});
}
catch (KafkaException e)
{
if (e.Error.Code == ErrorCode.Local_QueueFull)
{
Thread.Sleep(TimeSpan.FromSeconds(1000));
continue;
}
throw;
}
break;
}
wCount += 1;
if (DateTime.Now > lastTxnCommit + TxnCommitPeriod)
{
// Execute the transaction commits for each producer in parallel.
var tasks = new List <Task>();
foreach (var state in producerState)
{
if (state.Value.Offset == Offset.Unset)
{
continue;
}
tasks.Add(Task.Run(() =>
{
state.Value.Producer.SendOffsetsToTransaction(
new List <TopicPartitionOffset> {
new TopicPartitionOffset(state.Key, state.Value.Offset + 1)
},
consumer.ConsumerGroupMetadata, DefaultTimeout);
state.Value.Producer.CommitTransaction(DefaultTimeout);
state.Value.Offset = Offset.Unset;
state.Value.Producer.BeginTransaction();
}));
}
Task.WaitAll(tasks.ToArray(), ct);
Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words.");
lastTxnCommit = DateTime.Now;
wCount = 0;
}
}
catch (Exception)
{
consumer.Close();
break;
}
}
}
}
/// <summary>
/// A transactional (exactly once) processing loop that reads lines of text from
/// Topic_InputLines, splits them into words, and outputs the result to Topic_Words.
/// </summary>
static void Processor_MapWords(string brokerList, string clientId, CancellationToken ct)
{
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = ConsumerGroup_MapWords,
AutoOffsetReset = AutoOffsetReset.Earliest,
// Offsets are committed using the producer as part of the
// transaction - not the consumer. When using transactions,
// you must turn off auto commit on the consumer, which is
// enabled by default!
EnableAutoCommit = false,
};
var TxnCommitPeriod = TimeSpan.FromSeconds(10);
var lastTxnCommit = DateTime.Now;
// Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is
// currently necessary to use a separate producer per input partition. The
// producerState dictionary is used to keep track of these, and the current
// consumed offset.
// https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics
var producerState = new Dictionary <TopicPartition, ProducerState <string, Null> >();
using (var consumer = new ConsumerBuilder <Null, string>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
// Note: All handlers (except the log handler) are executed
// as a side-effect of, and on the same thread as the Consume
// or Close methods. Any exception thrown in a handler (with
// the exception of the log and error handlers) will
// be propagated to the application via the initiating
// call. i.e. in this example, any exceptions thrown in this
// handler will be exposed via the Consume method in the main
// consume loop and handler and handled by the try/catch block
// there.
// Abort any outstanding transactions & dispose producers
// corresponding to the previous generation of the consumer group.
var tasks = new List <Task>();
foreach (var p in producerState.Values)
{
tasks.Add(Task.Run(() => {
p.Producer.AbortTransaction(DefaultTimeout); // Note: Not cancellable yet.
p.Producer.Dispose();
}, ct));
}
if (tasks.Count > 0)
{
Console.WriteLine("Aborting current MapWords transactions.");
Task.WaitAll(tasks.ToArray());
}
producerState.Clear();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** MapWords consumer group rebalanced. Partition assignment: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
Trace.Assert(producerState.Count == 0, "Unexpected producer state.");
// Then create a new set of producers for then new partition assignment
// and initialize.
var tasks = new List <Task>();
foreach (var tp in partitions)
{
tasks.Add(Task.Run(() => {
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
TransactionalId = TransactionalId_MapWords + "-" + clientId + "-" + tp.Partition
};
var p = new ProducerBuilder <string, Null>(pConfig).Build();
p.InitTransactions(DefaultTimeout); // Note: Not cancellable yet.
p.BeginTransaction();
lock (producerState)
{
producerState.Add(tp, new ProducerState <string, Null> {
Producer = p, Offset = Offset.Unset
});
}
}, ct));
}
Task.WaitAll(tasks.ToArray());
// The PartitionsAssigned handler is called immediately after a
// new assignment set is received from the group coordinator and
// before that set is assigned to be read from. Since we have
// called init transactions already for the partitions relevant
// to this consumer, we can be sure that consumption will resume
// from the correct offsets (determined after this handler
// completed).
})
.Build())
{
consumer.Subscribe(Topic_InputLines); // Note: Subscribe is not blocking.
var wCount = 0;
var lCount = 0;
while (true)
{
try
{
ConsumeResult <Null, string> cr = consumer.Consume(ct);
lCount += 1;
producerState[cr.TopicPartition].Offset = cr.Offset;
var words = Regex.Split(cr.Message.Value.ToLower(), @"[^a-zA-Z_]").Where(s => s != String.Empty);
foreach (var w in words)
{
while (true)
{
try
{
producerState[cr.TopicPartition].Producer.Produce(Topic_Words, new Message <string, Null> {
Key = w
});
// Note: when using transactions, there is no need to check for errors of individual
// produce call delivery reports because if the transaction commits successfully, you
// can be sure that all the constituent messages were delivered successfully and in order.
wCount += 1;
}
catch (KafkaException e)
{
// An immediate failure of the produce call is most often caused by the
// local message queue being full, and appropriate response to that is
// to retry.
if (e.Error.Code == ErrorCode.Local_QueueFull)
{
Thread.Sleep(TimeSpan.FromSeconds(1000));
continue;
}
throw;
}
break;
}
}
// Commit transactions every TxnCommitPeriod
if (DateTime.Now > lastTxnCommit + TxnCommitPeriod)
{
// Execute the transaction commits for each producer in parallel.
var tasks = new List <Task>();
foreach (var state in producerState)
{
if (state.Value.Offset == Offset.Unset)
{
continue;
}
tasks.Add(Task.Run(() =>
{
state.Value.Producer.SendOffsetsToTransaction(
// Note: committed offsets reflect the next message to consume, not last
// message consumed, so we need to add one to the last consumed offset
// values here.
new List <TopicPartitionOffset> {
new TopicPartitionOffset(state.Key, state.Value.Offset + 1)
},
consumer.ConsumerGroupMetadata, DefaultTimeout); // Note: Not cancellable yet.
state.Value.Producer.CommitTransaction(DefaultTimeout); // Note: Not cancellable yet.
// Note: Exceptions thrown by SendOffsetsToTransaction and
// CommitTransaction that are not marked as fatal can be
// recovered from. However, in order to keep this example
// short(er), the additional bookkeeping/logic required to
// achieve this has been omitted. Since this should happen
// only rarely, requiring a process restart in this case
// isn't a huge compromise.
state.Value.Offset = Offset.Unset;
state.Value.Producer.BeginTransaction();
}));
}
Task.WaitAll(tasks.ToArray(), ct);
Console.WriteLine($"Committed MapWords transaction(s) comprising {wCount} words from {lCount} lines.");
lastTxnCommit = DateTime.Now;
wCount = 0;
lCount = 0;
}
}
catch (Exception)
{
// Note: transactions are aborted in the partitions revoked handler during close.
consumer.Close();
break;
}
// To simplify error handling, we assume the presence of a supervisor
// process that monitors whether worker processes have died, and restarts
// new instances as required. This is typical.
}
}
}
public void Transactions_Statistics(string bootstrapServers)
{
LogToFile("start Transactions_Statistics");
var groupName = Guid.NewGuid().ToString();
var cConfig = new ConsumerConfig
{
IsolationLevel = IsolationLevel.ReadCommitted,
BootstrapServers = bootstrapServers,
GroupId = groupName,
EnableAutoCommit = false,
StatisticsIntervalMs = 1000
};
var cts = new CancellationTokenSource();
int ls_offset = -1;
int hi_offset = -1;
bool done = false;
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(cConfig)
.SetStatisticsHandler((_, json) => {
var stats = JObject.Parse(json);
ls_offset = (int)stats["topics"][topic.Name]["partitions"]["0"]["ls_offset"];
hi_offset = (int)stats["topics"][topic.Name]["partitions"]["0"]["hi_offset"];
if (hi_offset > 4)
{
done = true;
}
})
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
producer.InitTransactions(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message_a"
}).Wait();
producer.CommitTransaction(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message_b"
}).Wait();
producer.CommitTransaction(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message1"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message2"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message3"
}).Wait();
for (int i = 0; i < 10; ++i)
{
consumer.Consume(TimeSpan.FromMilliseconds(500));
if (done)
{
break;
}
}
Assert.Equal(4, ls_offset);
Assert.Equal(7, hi_offset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Statistics");
}
/// <summary>
/// A transactional (exactly once) processing loop that reads individual words and updates
/// the corresponding total count state.
///
/// When a rebalance occurs (including on startup), the count state for the incrementally
/// assigned partitions is reloaded before the loop commences to update it. For this use-
/// case, the CooperativeSticky assignor is much more efficient than the Range or RoundRobin
/// assignors since it keeps to a minimum the count state that needs to be materialized.
/// </summary>
/// <remarks>
/// Refer to Processor_MapWords for more detailed comments.
/// </remarks>
public static void Processor_AggregateWords(string brokerList, string clientId, CancellationToken ct)
{
if (clientId == null)
{
throw new Exception("Aggregate words processor requires that a client id is specified.");
}
var txnCommitPeriod = TimeSpan.FromSeconds(10);
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_producer",
TransactionalId = TransactionalIdPrefix_Aggregate + "-" + clientId
};
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_consumer",
GroupId = ConsumerGroup_Aggregate,
AutoOffsetReset = AutoOffsetReset.Earliest,
// This should be greater than the maximum amount of time required to read in
// existing count state. It should not be too large, since a rebalance may be
// blocked for this long.
MaxPollIntervalMs = 600000, // 10 minutes.
EnableAutoCommit = false,
// Enable incremental rebalancing by using the CooperativeSticky
// assignor (avoid stop-the-world rebalances). This is particularly important,
// in the AggregateWords case, since the entire count state for newly assigned
// partitions is loaded in the partitions assigned handler.
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky,
};
var lastTxnCommit = DateTime.Now;
using (var producer = new ProducerBuilder <string, int>(pConfig).Build())
using (var consumer = new ConsumerBuilder <string, Null>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0);
Console.WriteLine(
"** AggregateWords consumer group partitions revoked: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], remaining: [" +
string.Join(',', remaining.Select(p => p.Partition.Value)) +
"]");
// Remove materialized word count state for the partitions that have been revoked.
foreach (var tp in partitions)
{
WordCountState[tp.Partition].Dispose();
WordCountState.Remove(tp.Partition);
}
producer.SendOffsetsToTransaction(
c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))),
c.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
})
.SetPartitionsLostHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group partitions lost: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
// clear materialized word count state for all partitions.
foreach (var tp in partitions)
{
WordCountState[tp.Partition].Dispose();
WordCountState.Remove(tp.Partition);
}
producer.AbortTransaction();
producer.BeginTransaction();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group partition assigned: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], all: [" +
string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) +
"]");
if (partitions.Count > 0)
{
// Initialize FASTER KV stores for each new partition.
foreach (var tp in partitions)
{
var partitionState = new FasterState(tp.Partition);
WordCountState.Add(tp.Partition, partitionState);
}
// Materialize count state for partitions into the FASTER KV stores.
// Note: the partiioning of Topic_Counts matches Topic_Words.
LoadCountState(brokerList, partitions.Select(tp => tp.Partition), ct);
}
})
.Build())
{
consumer.Subscribe(Topic_Words);
producer.InitTransactions(DefaultTimeout);
producer.BeginTransaction();
var wCount = 0;
while (true)
{
try
{
ct.ThrowIfCancellationRequested();
var cr = consumer.Consume(TimeSpan.FromSeconds(1));
if (cr != null)
{
string key = cr.Message.Key;
var(_, count) = WordCountState[cr.Partition].Session.Read(cr.Message.Key);
count += 1;
WordCountState[cr.Partition].Session.Upsert(key, count);
producer.Produce(Topic_Counts, new Message <string, int> {
Key = cr.Message.Key, Value = count
});
wCount += 1;
}
if (DateTime.Now > lastTxnCommit + txnCommitPeriod)
{
producer.SendOffsetsToTransaction(
// Note: committed offsets reflect the next message to consume, not last
// message consumed. consumer.Position returns the last consumed offset
// values + 1, as required.
consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))),
consumer.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words.");
lastTxnCommit = DateTime.Now;
wCount = 0;
}
}
catch (Exception e)
{
producer.AbortTransaction();
Console.WriteLine("Exiting AggregateWords consume loop due to an exception: " + e);
consumer.Close();
Console.WriteLine("AggregateWords consumer closed");
break;
}
}
}
}
/// <summary>
/// A transactional (exactly once) processing loop that reads lines of text from
/// Topic_InputLines, splits them into words, and outputs the result to Topic_Words.
/// </summary>
static void Processor_MapWords(string brokerList, string clientId, CancellationToken ct)
{
if (clientId == null)
{
throw new Exception("Map processor requires that a client id is specified.");
}
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_producer",
// The TransactionalId identifies this instance of the map words processor.
// If you start another instance with the same transactional id, the existing
// instance will be fenced.
TransactionalId = TransactionalIdPrefix_MapWords + "-" + clientId
};
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_consumer",
GroupId = ConsumerGroup_MapWords,
// AutoOffsetReset specifies the action to take when there
// are no committed offsets for a partition, or an error
// occurs retrieving offsets. If there are committed offsets,
// it has no effect.
AutoOffsetReset = AutoOffsetReset.Earliest,
// Offsets are committed using the producer as part of the
// transaction - not the consumer. When using transactions,
// you must turn off auto commit on the consumer, which is
// enabled by default!
EnableAutoCommit = false,
// Enable incremental rebalancing by using the CooperativeSticky
// assignor (avoid stop-the-world rebalances).
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky
};
var txnCommitPeriod = TimeSpan.FromSeconds(10);
var lastTxnCommit = DateTime.Now;
using (var producer = new ProducerBuilder <string, Null>(pConfig).Build())
using (var consumer = new ConsumerBuilder <Null, string>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0);
Console.WriteLine(
"** MapWords consumer group partitions revoked: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], remaining: [" +
string.Join(',', remaining.Select(p => p.Partition.Value)) +
"]");
// All handlers (except the log handler) are executed as a
// side-effect of, and on the same thread as the Consume or
// Close methods. Any exception thrown in a handler (with
// the exception of the log and error handlers) will
// be propagated to the application via the initiating
// call. i.e. in this example, any exceptions thrown in this
// handler will be exposed via the Consume method in the main
// consume loop and handled by the try/catch block there.
producer.SendOffsetsToTransaction(
c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))),
c.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
})
.SetPartitionsLostHandler((c, partitions) => {
// Ownership of the partitions has been involuntarily lost and
// are now likely already owned by another consumer.
Console.WriteLine(
"** MapWords consumer group partitions lost: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
producer.AbortTransaction();
producer.BeginTransaction();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** MapWords consumer group additional partitions assigned: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], all: [" +
string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) +
"]");
// No action is required here related to transactions - offsets
// for the newly assigned partitions will be committed in the
// main consume loop along with those for already assigned
// partitions as per usual.
})
.Build())
{
consumer.Subscribe(Topic_InputLines);
producer.InitTransactions(DefaultTimeout);
producer.BeginTransaction();
var wCount = 0;
var lCount = 0;
while (true)
{
try
{
ct.ThrowIfCancellationRequested();
// Do not block on Consume indefinitely to avoid the possibility of a transaction timeout.
var cr = consumer.Consume(TimeSpan.FromSeconds(1));
if (cr != null)
{
lCount += 1;
var words = Regex.Split(cr.Message.Value.ToLower(), @"[^a-zA-Z_]").Where(s => s != String.Empty);
foreach (var w in words)
{
while (true)
{
try
{
producer.Produce(Topic_Words, new Message <string, Null> {
Key = w
});
// Note: when using transactions, there is no need to check for errors of individual
// produce call delivery reports because if the transaction commits successfully, you
// can be sure that all the constituent messages were delivered successfully and in order.
wCount += 1;
}
catch (KafkaException e)
{
// An immediate failure of the produce call is most often caused by the
// local message queue being full, and appropriate response to that is
// to wait a bit and retry.
if (e.Error.Code == ErrorCode.Local_QueueFull)
{
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
continue;
}
throw;
}
break;
}
}
}
// Commit transactions every TxnCommitPeriod
if (DateTime.Now > lastTxnCommit + txnCommitPeriod)
{
// Note: Exceptions thrown by SendOffsetsToTransaction and
// CommitTransaction that are not marked as fatal can be
// recovered from. However, in order to keep this example
// short(er), the additional logic required to achieve this
// has been omitted. This should happen only rarely, so
// requiring a process restart in this case is not necessarily
// a bad compromise, even in production scenarios.
producer.SendOffsetsToTransaction(
// Note: committed offsets reflect the next message to consume, not last
// message consumed. consumer.Position returns the last consumed offset
// values + 1, as required.
consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))),
consumer.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
Console.WriteLine($"Committed MapWords transaction(s) comprising {wCount} words from {lCount} lines.");
lastTxnCommit = DateTime.Now;
wCount = 0;
lCount = 0;
}
}
catch (Exception e)
{
// Attempt to abort the transaction (but ignore any errors) as a measure
// against stalling consumption of Topic_Words.
producer.AbortTransaction();
Console.WriteLine("Exiting MapWords consume loop due to an exception: " + e);
// Note: transactions may be committed / aborted in the partitions
// revoked / lost handler as a side effect of the call to close.
consumer.Close();
Console.WriteLine("MapWords consumer closed");
break;
}
// Assume the presence of an external system that monitors whether worker
// processes have died, and restarts new instances as required. This
// setup is typical, and avoids complex error handling logic in the
// client code.
}
}
}
public void Transactions_WatermarkOffsets(string bootstrapServers)
{
LogToFile("start Transactions_WatermarkOffsets");
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false
}).Build())
{
var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
Assert.Equal(Offset.Unset, wo1.Low);
Assert.Equal(Offset.Unset, wo1.High);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
producer.InitTransactions(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message1"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message2"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message3"
}).Wait();
WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr = consumer.Consume(TimeSpan.FromMilliseconds(500));
wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo2.High == 3)
{
break;
}
}
Assert.Equal(3, wo2.High);
producer.CommitTransaction(TimeSpan.FromSeconds(30));
WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr2 = consumer.Consume(TimeSpan.FromSeconds(500));
wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo3.High > 3)
{
break;
}
}
Assert.Equal(4, wo3.High);
var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
Assert.Equal(4, wo4.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_WatermarkOffsets");
}
public void Transactions_Commit(string bootstrapServers)
{
LogToFile("start Transactions_Commit");
var defaultTimeout = TimeSpan.FromSeconds(30);
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, wm.High));
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
producer.CommitTransaction(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 1", Value = "test val 1"
});
producer.CommitTransaction(defaultTimeout);
var cr1 = consumer.Consume();
var cr2 = consumer.Consume();
var cr3 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal(wm.High, cr1.Offset);
Assert.Equal(wm.High + 2, cr2.Offset); // there should be a skipped offset due to a commit marker in the log.
Assert.Null(cr3); // control message should not be exposed to application.
// Test that the committed offset accounts for the final ctrl message.
consumer.Commit();
}
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, AutoOffsetReset = AutoOffsetReset.Latest
}).Build())
{
consumer.Assign(new TopicPartition(topic.Name, 0));
// call InitTransactions to prevent a race conidtion between a slow txn commit and a quick offset request.
producer.InitTransactions(defaultTimeout);
var committed = consumer.Committed(defaultTimeout);
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
Assert.Equal(wm.High, committed[0].Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Commit");
}
public void Run()
{
var pConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
TransactionalId = $"txn_test_{this.id}"
};
int lastMessageValue = 0;
var producer = new ProducerBuilder <int, int>(pConfig).Build();
producer.InitTransactions(DefaultTimeout);
var currentState = ProducerState.InitState;
for (int i = 0; i < conf.MessageCount;)
{
Console.Write($"+{i}");
Console.Out.Flush();
// finalize previous state.
switch (currentState)
{
case ProducerState.MakingMessagesToAbort:
producer.AbortTransaction(DefaultTimeout);
break;
case ProducerState.MakingMessagesToCommit:
producer.CommitTransaction(DefaultTimeout);
break;
default:
// no action required.
break;
}
// transition to next state.
var rnd = random.NextDouble();
if (rnd < conf.ProbabilityLevel_Abort)
{
currentState = ProducerState.MakingMessagesToAbort;
}
else
{
currentState = ProducerState.MakingMessagesToCommit;
}
producer.BeginTransaction();
int runLength = random.Next(conf.MaxRunLength);
for (int j = 0; j < runLength && i < conf.MessageCount; ++j, ++i)
{
int val = currentState == ProducerState.MakingMessagesToCommit ? lastMessageValue++ : -1;
Thread.Sleep((int)(1000 * conf.MaxPauseSeconds));
producer.Produce(conf.Topic, new Message <int, int> {
Key = id, Value = val
});
}
}
if (currentState == ProducerState.MakingMessagesToCommit)
{
producer.CommitTransaction(DefaultTimeout);
}
if (currentState == ProducerState.MakingMessagesToAbort)
{
producer.AbortTransaction(DefaultTimeout);
}
Console.WriteLine($"done: {id}");
producer.Flush();
producer.Dispose();
}
