public static void Consumer_Seek(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Consumer_Seek");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers
};
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var consumer =
new ConsumerBuilder <Null, string>(consumerConfig)
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
const string checkValue = "check value";
var dr = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize(checkValue, true, null, null)
}).Result;
var dr2 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("second value", true, null, null)
}).Result;
var dr3 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("third value", true, null, null)
}).Result;
consumer.Assign(new TopicPartitionOffset[] { new TopicPartitionOffset(singlePartitionTopic, 0, dr.Offset) });
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
consumer.Seek(dr.TopicPartitionOffset);
// position is that of the last consumed offset. it shouldn't be equal to the seek position.
var pos = consumer.Position(new List <TopicPartition> {
dr.TopicPartition
}).First();
Assert.NotEqual(dr.Offset, pos.Offset);
record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
Assert.Equal(checkValue, record.Message.Value);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Seek");
}
public static void Consume()
{
var conf = new ConsumerConfig
{
GroupId = "g3",
BootstrapServers = "localhost:9092",
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var c = new ConsumerBuilder <Ignore, string>(conf).Build())
{
c.Subscribe("Topic2");
var listTopicPartition = c.Assignment;
foreach (var topicPartition in c.Assignment)
{
var offset = c.Position(topicPartition);
}
System.Threading.CancellationTokenSource cts = new System.Threading.CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
var cr = c.Consume(cts.Token);
Console.WriteLine($"Consumed message '{cr.Message?.Value}' at: '{cr.TopicPartitionOffset}'.");
UpdateValue();
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
c.Close();
}
}
}
static internal async Task Start(string[] args)
{
// var config = new ProducerConfig
// {
// BootstrapServers = "kafka:9092",
// MessageTimeoutMs = 30000.
// ApiVersionRequest = false,
// };
// // If serializers are not specified, default serializers from
// // `Confluent.Kafka.Serializers` will be automatically used where
// // available. Note: by default strings are encoded as UTF8.
// using (var p = new ProducerBuilder<Null, string>(config).Build())
// {
// try
// {
// var dr = await p.ProduceAsync("test-topic", new Message<Null, string> { Value = "tes2t" });
// Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'");
// }
// catch (ProduceException<Null, string> e)
// {
// Console.WriteLine($"Delivery failed: {e.Error.Reason}");
// }
// }
var config = new ConsumerConfig
{
BootstrapServers = "kafka:9092",
AutoOffsetReset = AutoOffsetReset.Earliest,
GroupId = "hz",
EnableAutoCommit = false,
ApiVersionRequest = false,
};
// If serializers are not specified, default serializers from
// `Confluent.Kafka.Serializers` will be automatically used where
// available. Note: by default strings are encoded as UTF8.
using (var c = new ConsumerBuilder <Null, string>(config).Build())
{
try
{
c.Subscribe("test-topic");
var o = c.Position(new TopicPartition("test-topic", new Partition(0)));
var a = c.GetWatermarkOffsets(new TopicPartition("test-topic", new Partition(0)));
var cr = c.Consume();
}
catch (ConsumeException e)
{
}
}
}
public void Consumer_Commit_Committed_Position(string bootstrapServers)
{
LogToFile("start Consumer_Commit_Committed_Position");
const int N = 8;
const int Partition = 0;
var messages = ProduceMessages(bootstrapServers, singlePartitionTopic, Partition, N);
var firstMsgOffset = messages[0].Offset;
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
EnableAutoCommit = false
};
var firstMessage = messages[0];
var lastMessage = messages[N - 1];
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
// Test #0 (empty cases)
consumer.Commit(new List <TopicPartitionOffset>()); // should not throw.
var committed = consumer.Committed(new List <TopicPartition>(), TimeSpan.FromSeconds(10));
Assert.Empty(committed);
var ps = consumer.Position(new List <TopicPartition>());
Assert.Empty(ps);
// Test #0.5 (invalid cases)
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition("invalid-topic", 0)
});
Assert.Single(ps);
Assert.Equal(Offset.Invalid, ps[0].Offset);
Assert.Equal("invalid-topic", ps[0].Topic);
Assert.Equal(0, (int)ps[0].Partition);
Assert.Equal(new TopicPartition("invalid-topic", 0), ps[0].TopicPartition);
// Test #1
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var os = consumer.Commit();
Assert.Equal(firstMsgOffset + 1, os[0].Offset);
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 1, co[0].Offset);
Assert.Equal(firstMsgOffset + 1, ps[0].Offset);
// Test #2
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
os = consumer.Commit();
Assert.Equal(firstMsgOffset + 2, os[0].Offset);
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 2, ps[0].Offset);
Assert.Equal(firstMsgOffset + 2, ps[0].Offset);
}
// Test #3
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Commit(new List <TopicPartitionOffset> {
new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 5)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 5, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
Assert.Equal(firstMsgOffset + 6, ps[0].Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 5, co[0].Offset);
}
// Test #4
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
consumer.Commit(new List <TopicPartitionOffset> {
new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 3)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
Assert.Equal(firstMsgOffset + 4, ps[0].Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
// Test #5
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
consumer.Commit(record3);
var record4 = consumer.Consume(TimeSpan.FromMilliseconds(1000));
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
Assert.Equal(firstMsgOffset + 3, record.Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
// Test #6
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
consumer.Commit(record3);
var record4 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
Assert.Equal(firstMsgOffset + 3, record.Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Commit_Committed_Position");
}
/// <summary>
/// A transactional (exactly once) processing loop that reads individual words and updates
/// the corresponding total count state.
///
/// When a rebalance occurs (including on startup), the count state for the incrementally
/// assigned partitions is reloaded before the loop commences to update it. For this use-
/// case, the CooperativeSticky assignor is much more efficient than the Range or RoundRobin
/// assignors since it keeps to a minimum the count state that needs to be materialized.
/// </summary>
/// <remarks>
/// Refer to Processor_MapWords for more detailed comments.
/// </remarks>
public static void Processor_AggregateWords(string brokerList, string clientId, CancellationToken ct)
{
if (clientId == null)
{
throw new Exception("Aggregate words processor requires that a client id is specified.");
}
var txnCommitPeriod = TimeSpan.FromSeconds(10);
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_producer",
TransactionalId = TransactionalIdPrefix_Aggregate + "-" + clientId
};
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_consumer",
GroupId = ConsumerGroup_Aggregate,
AutoOffsetReset = AutoOffsetReset.Earliest,
// This should be greater than the maximum amount of time required to read in
// existing count state. It should not be too large, since a rebalance may be
// blocked for this long.
MaxPollIntervalMs = 600000, // 10 minutes.
EnableAutoCommit = false,
// Enable incremental rebalancing by using the CooperativeSticky
// assignor (avoid stop-the-world rebalances). This is particularly important,
// in the AggregateWords case, since the entire count state for newly assigned
// partitions is loaded in the partitions assigned handler.
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky,
};
var lastTxnCommit = DateTime.Now;
using (var producer = new ProducerBuilder <string, int>(pConfig).Build())
using (var consumer = new ConsumerBuilder <string, Null>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0);
Console.WriteLine(
"** AggregateWords consumer group partitions revoked: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], remaining: [" +
string.Join(',', remaining.Select(p => p.Partition.Value)) +
"]");
// Remove materialized word count state for the partitions that have been revoked.
foreach (var tp in partitions)
{
WordCountState[tp.Partition].Dispose();
WordCountState.Remove(tp.Partition);
}
producer.SendOffsetsToTransaction(
c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))),
c.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
})
.SetPartitionsLostHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group partitions lost: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
// clear materialized word count state for all partitions.
foreach (var tp in partitions)
{
WordCountState[tp.Partition].Dispose();
WordCountState.Remove(tp.Partition);
}
producer.AbortTransaction();
producer.BeginTransaction();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group partition assigned: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], all: [" +
string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) +
"]");
if (partitions.Count > 0)
{
// Initialize FASTER KV stores for each new partition.
foreach (var tp in partitions)
{
var partitionState = new FasterState(tp.Partition);
WordCountState.Add(tp.Partition, partitionState);
}
// Materialize count state for partitions into the FASTER KV stores.
// Note: the partiioning of Topic_Counts matches Topic_Words.
LoadCountState(brokerList, partitions.Select(tp => tp.Partition), ct);
}
})
.Build())
{
consumer.Subscribe(Topic_Words);
producer.InitTransactions(DefaultTimeout);
producer.BeginTransaction();
var wCount = 0;
while (true)
{
try
{
ct.ThrowIfCancellationRequested();
var cr = consumer.Consume(TimeSpan.FromSeconds(1));
if (cr != null)
{
string key = cr.Message.Key;
var(_, count) = WordCountState[cr.Partition].Session.Read(cr.Message.Key);
count += 1;
WordCountState[cr.Partition].Session.Upsert(key, count);
producer.Produce(Topic_Counts, new Message <string, int> {
Key = cr.Message.Key, Value = count
});
wCount += 1;
}
if (DateTime.Now > lastTxnCommit + txnCommitPeriod)
{
producer.SendOffsetsToTransaction(
// Note: committed offsets reflect the next message to consume, not last
// message consumed. consumer.Position returns the last consumed offset
// values + 1, as required.
consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))),
consumer.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words.");
lastTxnCommit = DateTime.Now;
wCount = 0;
}
}
catch (Exception e)
{
producer.AbortTransaction();
Console.WriteLine("Exiting AggregateWords consume loop due to an exception: " + e);
consumer.Close();
Console.WriteLine("AggregateWords consumer closed");
break;
}
}
}
}
/// <summary>
/// A transactional (exactly once) processing loop that reads lines of text from
/// Topic_InputLines, splits them into words, and outputs the result to Topic_Words.
/// </summary>
static void Processor_MapWords(string brokerList, string clientId, CancellationToken ct)
{
if (clientId == null)
{
throw new Exception("Map processor requires that a client id is specified.");
}
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_producer",
// The TransactionalId identifies this instance of the map words processor.
// If you start another instance with the same transactional id, the existing
// instance will be fenced.
TransactionalId = TransactionalIdPrefix_MapWords + "-" + clientId
};
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
ClientId = clientId + "_consumer",
GroupId = ConsumerGroup_MapWords,
// AutoOffsetReset specifies the action to take when there
// are no committed offsets for a partition, or an error
// occurs retrieving offsets. If there are committed offsets,
// it has no effect.
AutoOffsetReset = AutoOffsetReset.Earliest,
// Offsets are committed using the producer as part of the
// transaction - not the consumer. When using transactions,
// you must turn off auto commit on the consumer, which is
// enabled by default!
EnableAutoCommit = false,
// Enable incremental rebalancing by using the CooperativeSticky
// assignor (avoid stop-the-world rebalances).
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky
};
var txnCommitPeriod = TimeSpan.FromSeconds(10);
var lastTxnCommit = DateTime.Now;
using (var producer = new ProducerBuilder <string, Null>(pConfig).Build())
using (var consumer = new ConsumerBuilder <Null, string>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0);
Console.WriteLine(
"** MapWords consumer group partitions revoked: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], remaining: [" +
string.Join(',', remaining.Select(p => p.Partition.Value)) +
"]");
// All handlers (except the log handler) are executed as a
// side-effect of, and on the same thread as the Consume or
// Close methods. Any exception thrown in a handler (with
// the exception of the log and error handlers) will
// be propagated to the application via the initiating
// call. i.e. in this example, any exceptions thrown in this
// handler will be exposed via the Consume method in the main
// consume loop and handled by the try/catch block there.
producer.SendOffsetsToTransaction(
c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))),
c.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
})
.SetPartitionsLostHandler((c, partitions) => {
// Ownership of the partitions has been involuntarily lost and
// are now likely already owned by another consumer.
Console.WriteLine(
"** MapWords consumer group partitions lost: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
producer.AbortTransaction();
producer.BeginTransaction();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** MapWords consumer group additional partitions assigned: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"], all: [" +
string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) +
"]");
// No action is required here related to transactions - offsets
// for the newly assigned partitions will be committed in the
// main consume loop along with those for already assigned
// partitions as per usual.
})
.Build())
{
consumer.Subscribe(Topic_InputLines);
producer.InitTransactions(DefaultTimeout);
producer.BeginTransaction();
var wCount = 0;
var lCount = 0;
while (true)
{
try
{
ct.ThrowIfCancellationRequested();
// Do not block on Consume indefinitely to avoid the possibility of a transaction timeout.
var cr = consumer.Consume(TimeSpan.FromSeconds(1));
if (cr != null)
{
lCount += 1;
var words = Regex.Split(cr.Message.Value.ToLower(), @"[^a-zA-Z_]").Where(s => s != String.Empty);
foreach (var w in words)
{
while (true)
{
try
{
producer.Produce(Topic_Words, new Message <string, Null> {
Key = w
});
// Note: when using transactions, there is no need to check for errors of individual
// produce call delivery reports because if the transaction commits successfully, you
// can be sure that all the constituent messages were delivered successfully and in order.
wCount += 1;
}
catch (KafkaException e)
{
// An immediate failure of the produce call is most often caused by the
// local message queue being full, and appropriate response to that is
// to wait a bit and retry.
if (e.Error.Code == ErrorCode.Local_QueueFull)
{
Thread.Sleep(TimeSpan.FromMilliseconds(1000));
continue;
}
throw;
}
break;
}
}
}
// Commit transactions every TxnCommitPeriod
if (DateTime.Now > lastTxnCommit + txnCommitPeriod)
{
// Note: Exceptions thrown by SendOffsetsToTransaction and
// CommitTransaction that are not marked as fatal can be
// recovered from. However, in order to keep this example
// short(er), the additional logic required to achieve this
// has been omitted. This should happen only rarely, so
// requiring a process restart in this case is not necessarily
// a bad compromise, even in production scenarios.
producer.SendOffsetsToTransaction(
// Note: committed offsets reflect the next message to consume, not last
// message consumed. consumer.Position returns the last consumed offset
// values + 1, as required.
consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))),
consumer.ConsumerGroupMetadata,
DefaultTimeout);
producer.CommitTransaction();
producer.BeginTransaction();
Console.WriteLine($"Committed MapWords transaction(s) comprising {wCount} words from {lCount} lines.");
lastTxnCommit = DateTime.Now;
wCount = 0;
lCount = 0;
}
}
catch (Exception e)
{
// Attempt to abort the transaction (but ignore any errors) as a measure
// against stalling consumption of Topic_Words.
producer.AbortTransaction();
Console.WriteLine("Exiting MapWords consume loop due to an exception: " + e);
// Note: transactions may be committed / aborted in the partitions
// revoked / lost handler as a side effect of the call to close.
consumer.Close();
Console.WriteLine("MapWords consumer closed");
break;
}
// Assume the presence of an external system that monitors whether worker
// processes have died, and restarts new instances as required. This
// setup is typical, and avoids complex error handling logic in the
// client code.
}
}
}