public void Transactions_SendOffsets(string bootstrapServers)
{
LogToFile("start Transactions_SendOffsets");
var defaultTimeout = TimeSpan.FromSeconds(30);
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false, Debug = "all"
}).Build())
{
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
producer.SendOffsetsToTransaction(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic.Name, 0, 7324)
}, consumer.ConsumerGroupMetadata, TimeSpan.FromSeconds(30));
producer.CommitTransaction(defaultTimeout);
var committed = consumer.Committed(new List <TopicPartition> {
new TopicPartition(topic.Name, 0)
}, TimeSpan.FromSeconds(30));
Assert.Single(committed);
Assert.Equal(7324, committed[0].Offset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_SendOffsets");
}
//It is a dead simple example of Transacions on Apache Kafka. This is not a production ready code.
//To go further, you need to deal with rebalance scenarios and handle SetPartitionsRevokedHandler and SetPartitionsAssignedHandler
//This is a 1:1 (1 read for 1 write) example. Read 1 message, process and commit the message
public void SimpleReadWriteTransaction(string inputTopicName, string outputTopicName, CancellationToken ct)
{
using var consumer = new ConsumerBuilder <int, string>(_consumerConfig).Build();
using var producer = new ProducerBuilder <int, string>(_producerConfig).Build();
producer.InitTransactions(_timeout);
consumer.Subscribe(inputTopicName);
while (!ct.IsCancellationRequested)
{
producer.BeginTransaction();
string currentProcessedMessage = "";
//Read phase
try
{
var consumerResult = consumer.Consume(ct);
//Process phase
currentProcessedMessage = ProcessMessage(consumerResult.Message);
//Write phase - Produce new message on output topic
producer.Produce(outputTopicName, new Message <int, string>()
{
Key = consumerResult.Message.Key, Value = currentProcessedMessage
});
//Commit
producer.SendOffsetsToTransaction(new List <TopicPartitionOffset>()
{
new TopicPartitionOffset(consumerResult.TopicPartition, consumerResult.Offset + 1)
},
consumer.ConsumerGroupMetadata,
_timeout);
producer.CommitTransaction(_timeout);
}
catch (Exception e)
{
_logger.LogError("Transaction Fault", e.Message, e.StackTrace);
producer.AbortTransaction(_timeout);
}
_logger.LogInformation($"Successful transaction: {currentProcessedMessage}");
}
}
/// <summary>
/// A transactional (exactly once) processing loop that reads individual words and updates
/// the corresponding total count state.
///
/// When a rebalance occurs (including on startup), the count state for the incrementally
/// assigned partitions is reloaded before the loop commences to update it. For this use-
/// case, the CooperativeSticky assignor is much more efficient than the Range or RoundRobin
/// assignors since it keeps to a minimum the count state that needs to be materialized.
/// </summary>
/// <remarks>
/// Refer to Processor_MapWords for more detailed comments.
/// </remarks>
public static void Processor_AggregateWords(string brokerList, string clientId, CancellationToken ct)
{
if (clientId == null)
{
throw new Exception("Aggregate words processor requires that a client id is specified.");
}
var txnCommitPeriod = TimeSpan.FromSeconds(10);
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_producer",
TransactionalId = TransactionalIdPrefix_Aggregate + "-" + clientId
};
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_consumer",
GroupId = ConsumerGroup_Aggregate,
AutoOffsetReset = AutoOffsetReset.Earliest,
// This should be greater than the maximum amount of time required to read in
// existing count state. It should not be too large, since a rebalance may be
// blocked for this long.
MaxPollIntervalMs = 600000, // 10 minutes.
EnableAutoCommit = false,
// Enable incremental rebalancing by using the CooperativeSticky
// assignor (avoid stop-the-world rebalances). This is particularly important,
// in the AggregateWords case, since the entire count state for newly assigned
// partitions is loaded in the partitions assigned handler.
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky,
};
var lastTxnCommit = DateTime.Now;
using (var producer = new ProducerBuilder <string, int>(pConfig).Build())
using (var consumer = new ConsumerBuilder <string, Null>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0);
Console.WriteLine(
"** AggregateWords consumer group partitions revoked: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], remaining: [" +
string.Join(',', remaining.Select(p => p.Partition.Value)) +
"]");
// Remove materialized word count state for the partitions that have been revoked.
foreach (var tp in partitions)
{
WordCountState[tp.Partition].Dispose();
WordCountState.Remove(tp.Partition);
}
producer.SendOffsetsToTransaction(
c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))),
c.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
})
.SetPartitionsLostHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group partitions lost: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
// clear materialized word count state for all partitions.
foreach (var tp in partitions)
{
WordCountState[tp.Partition].Dispose();
WordCountState.Remove(tp.Partition);
}
producer.AbortTransaction();
producer.BeginTransaction();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group partition assigned: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], all: [" +
string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) +
"]");
if (partitions.Count > 0)
{
// Initialize FASTER KV stores for each new partition.
foreach (var tp in partitions)
{
var partitionState = new FasterState(tp.Partition);
WordCountState.Add(tp.Partition, partitionState);
}
// Materialize count state for partitions into the FASTER KV stores.
// Note: the partiioning of Topic_Counts matches Topic_Words.
LoadCountState(brokerList, partitions.Select(tp => tp.Partition), ct);
}
})
.Build())
{
consumer.Subscribe(Topic_Words);
producer.InitTransactions(DefaultTimeout);
producer.BeginTransaction();
var wCount = 0;
while (true)
{
try
{
ct.ThrowIfCancellationRequested();
var cr = consumer.Consume(TimeSpan.FromSeconds(1));
if (cr != null)
{
string key = cr.Message.Key;
var(_, count) = WordCountState[cr.Partition].Session.Read(cr.Message.Key);
count += 1;
WordCountState[cr.Partition].Session.Upsert(key, count);
producer.Produce(Topic_Counts, new Message <string, int> {
Key = cr.Message.Key, Value = count
});
wCount += 1;
}
if (DateTime.Now > lastTxnCommit + txnCommitPeriod)
{
producer.SendOffsetsToTransaction(
// Note: committed offsets reflect the next message to consume, not last
// message consumed. consumer.Position returns the last consumed offset
// values + 1, as required.
consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))),
consumer.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words.");
lastTxnCommit = DateTime.Now;
wCount = 0;
}
}
catch (Exception e)
{
producer.AbortTransaction();
Console.WriteLine("Exiting AggregateWords consume loop due to an exception: " + e);
consumer.Close();
Console.WriteLine("AggregateWords consumer closed");
break;
}
}
}
}
/// <summary>
/// A transactional (exactly once) processing loop that reads lines of text from
/// Topic_InputLines, splits them into words, and outputs the result to Topic_Words.
/// </summary>
static void Processor_MapWords(string brokerList, string clientId, CancellationToken ct)
{
if (clientId == null)
{
throw new Exception("Map processor requires that a client id is specified.");
}
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_producer",
// The TransactionalId identifies this instance of the map words processor.
// If you start another instance with the same transactional id, the existing
// instance will be fenced.
TransactionalId = TransactionalIdPrefix_MapWords + "-" + clientId
};
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_consumer",
GroupId = ConsumerGroup_MapWords,
// AutoOffsetReset specifies the action to take when there
// are no committed offsets for a partition, or an error
// occurs retrieving offsets. If there are committed offsets,
// it has no effect.
AutoOffsetReset = AutoOffsetReset.Earliest,
// Offsets are committed using the producer as part of the
// transaction - not the consumer. When using transactions,
// you must turn off auto commit on the consumer, which is
// enabled by default!
EnableAutoCommit = false,
// Enable incremental rebalancing by using the CooperativeSticky
// assignor (avoid stop-the-world rebalances).
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky
};
var txnCommitPeriod = TimeSpan.FromSeconds(10);
var lastTxnCommit = DateTime.Now;
using (var producer = new ProducerBuilder <string, Null>(pConfig).Build())
using (var consumer = new ConsumerBuilder <Null, string>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0);
Console.WriteLine(
"** MapWords consumer group partitions revoked: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], remaining: [" +
string.Join(',', remaining.Select(p => p.Partition.Value)) +
"]");
// All handlers (except the log handler) are executed as a
// side-effect of, and on the same thread as the Consume or
// Close methods. Any exception thrown in a handler (with
// the exception of the log and error handlers) will
// be propagated to the application via the initiating
// call. i.e. in this example, any exceptions thrown in this
// handler will be exposed via the Consume method in the main
// consume loop and handled by the try/catch block there.
producer.SendOffsetsToTransaction(
c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))),
c.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
})
.SetPartitionsLostHandler((c, partitions) => {
// Ownership of the partitions has been involuntarily lost and
// are now likely already owned by another consumer.
Console.WriteLine(
"** MapWords consumer group partitions lost: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
producer.AbortTransaction();
producer.BeginTransaction();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** MapWords consumer group additional partitions assigned: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], all: [" +
string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) +
"]");
// No action is required here related to transactions - offsets
// for the newly assigned partitions will be committed in the
// main consume loop along with those for already assigned
// partitions as per usual.
})
.Build())
{
consumer.Subscribe(Topic_InputLines);
producer.InitTransactions(DefaultTimeout);
producer.BeginTransaction();
var wCount = 0;
var lCount = 0;
while (true)
{
try
{
ct.ThrowIfCancellationRequested();
// Do not block on Consume indefinitely to avoid the possibility of a transaction timeout.
var cr = consumer.Consume(TimeSpan.FromSeconds(1));
if (cr != null)
{
lCount += 1;
var words = Regex.Split(cr.Message.Value.ToLower(), @"[^a-zA-Z_]").Where(s => s != String.Empty);
foreach (var w in words)
{
while (true)
{
try
{
producer.Produce(Topic_Words, new Message <string, Null> {
Key = w
});
// Note: when using transactions, there is no need to check for errors of individual
// produce call delivery reports because if the transaction commits successfully, you
// can be sure that all the constituent messages were delivered successfully and in order.
wCount += 1;
}
catch (KafkaException e)
{
// An immediate failure of the produce call is most often caused by the
// local message queue being full, and appropriate response to that is
// to wait a bit and retry.
if (e.Error.Code == ErrorCode.Local_QueueFull)
{
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
continue;
}
throw;
}
break;
}
}
}
// Commit transactions every TxnCommitPeriod
if (DateTime.Now > lastTxnCommit + txnCommitPeriod)
{
// Note: Exceptions thrown by SendOffsetsToTransaction and
// CommitTransaction that are not marked as fatal can be
// recovered from. However, in order to keep this example
// short(er), the additional logic required to achieve this
// has been omitted. This should happen only rarely, so
// requiring a process restart in this case is not necessarily
// a bad compromise, even in production scenarios.
producer.SendOffsetsToTransaction(
// Note: committed offsets reflect the next message to consume, not last
// message consumed. consumer.Position returns the last consumed offset
// values + 1, as required.
consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))),
consumer.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
Console.WriteLine($"Committed MapWords transaction(s) comprising {wCount} words from {lCount} lines.");
lastTxnCommit = DateTime.Now;
wCount = 0;
lCount = 0;
}
}
catch (Exception e)
{
// Attempt to abort the transaction (but ignore any errors) as a measure
// against stalling consumption of Topic_Words.
producer.AbortTransaction();
Console.WriteLine("Exiting MapWords consume loop due to an exception: " + e);
// Note: transactions may be committed / aborted in the partitions
// revoked / lost handler as a side effect of the call to close.
consumer.Close();
Console.WriteLine("MapWords consumer closed");
break;
}
// Assume the presence of an external system that monitors whether worker
// processes have died, and restarts new instances as required. This
// setup is typical, and avoids complex error handling logic in the
// client code.
}
}
}