public static ColumnFamilyHandle GetOrCreateColumnFamily(this RocksDb db, string familyName, ColumnFamilyOptions?options = null)
{
if (!db.TryGetColumnFamily(familyName, out var familyHandle))
{
familyHandle = db.CreateColumnFamily(options ?? defaultColumnFamilyOptions, familyName);
}
return(familyHandle);
}
private ColumnFamilyHandle CreateColumnFamilyIfMissing(RocksDb db, string columnFamily)
{
try
{
return(db.GetColumnFamily(columnFamily));
}
catch
{
var options = new ColumnFamilyOptions();
return(db.CreateColumnFamily(options, columnFamily));
}
}
private ColumnFamilyHandle GetColFamily(string name)
{
if (_columnFamiliesDictionary.ContainsKey(name))
{
return(_columnFamiliesDictionary[name]);
}
var cf = _db.CreateColumnFamily(new ColumnFamilyOptions(), name);
_columnFamiliesDictionary.Add(name, cf);
return(cf);
}
public IStore GetStore(string?path)
{
if (path == null)
{
return(new RocksDbStore(db, db.GetDefaultColumnFamily(), readOnly, shared: true));
}
if (db.TryGetColumnFamily(path, out var columnFamily))
{
return(new RocksDbStore(db, columnFamily, readOnly, shared: true));
}
if (!readOnly)
{
columnFamily = db.CreateColumnFamily(defaultColumnFamilyOptions, path);
return(new RocksDbStore(db, columnFamily, readOnly, shared: true));
}
throw new InvalidOperationException("invalid store path");
}
private ColumnFamilyHandle GetColumnFamily(RocksDb db, Guid?chainId = null)
{
if (chainId is null)
{
return(null);
}
var cfName = chainId.ToString();
ColumnFamilyHandle cf;
try
{
cf = db.GetColumnFamily(cfName);
}
catch (KeyNotFoundException)
{
cf = db.CreateColumnFamily(_options, cfName);
}
return(cf);
}
public ColumnFamilyHandle GetFamily(byte table)
{
if (!_families.TryGetValue(table, out var family))
{
try
{
// Try to find the family
family = db.GetColumnFamily(table.ToString());
_families.Add(table, family);
}
catch (KeyNotFoundException)
{
// Try to create the family
family = db.CreateColumnFamily(new ColumnFamilyOptions(), table.ToString());
_families.Add(table, family);
}
}
return(family);
}
/// <summary>
/// A transactional (exactly once) processing loop that reads individual words and updates
/// the corresponding total count state.
///
/// When a rebalance occurs (including on startup), the total count state for all assigned
/// partitions is reloaded before the loop commences to update it.
/// </summary>
/// <remarks>
/// Refer to Processor_MapWords for more detailed comments.
/// </remarks>
public static void Processor_AggregateWords(string brokerList, string clientId, RocksDb db, CancellationToken ct)
{
var TxnCommitPeriod = TimeSpan.FromSeconds(10);
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = ConsumerGroup_Aggregate,
AutoOffsetReset = AutoOffsetReset.Earliest,
// This should be greater than the maximum amount of time required to read in
// existing count state.
MaxPollIntervalMs = 86400000,
EnableAutoCommit = false
};
ColumnFamilyHandle columnFamily = null;
var lastTxnCommit = DateTime.Now;
var producerState = new Dictionary <TopicPartition, ProducerState <string, int> >();
using (var consumer = new ConsumerBuilder <string, Null>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
// clear rocksdb state.
db.DropColumnFamily("counts");
db.CreateColumnFamily(new ColumnFamilyOptions(), "counts");
var tasks = new List <Task>();
foreach (var p in producerState.Values)
{
tasks.Add(Task.Run(() => {
p.Producer.AbortTransaction(DefaultTimeout); // Note: Not cancellable yet.
p.Producer.Dispose();
}, ct));
}
if (tasks.Count > 0)
{
Console.WriteLine("Aborting current AggregateWords transactions.");
}
Task.WaitAll(tasks.ToArray());
producerState.Clear();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group rebalanced. Partition assignment: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
Trace.Assert(producerState.Count == 0, "Unexpected producer state");
var tasks = new List <Task>();
foreach (var tp in partitions)
{
tasks.Add(Task.Run(() => {
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
TransactionalId = TransactionalId_Aggregate + "-" + clientId + "-" + tp.Partition
};
var p = new ProducerBuilder <string, int>(pConfig).Build();
p.InitTransactions(DefaultTimeout); // Note: Not cancellable yet.
p.BeginTransaction();
lock (producerState)
{
producerState.Add(tp, new ProducerState <string, int> {
Producer = p, Offset = Offset.Unset
});
}
}, ct));
}
Task.WaitAll(tasks.ToArray());
columnFamily = db.GetColumnFamily("counts");
LoadCountState(db, brokerList, partitions.Select(p => p.Partition), columnFamily, ct);
})
.Build())
{
consumer.Subscribe(Topic_Words);
var wCount = 0;
while (true)
{
try
{
var cr = consumer.Consume(ct);
producerState[cr.TopicPartition].Offset = cr.Offset;
var kBytes = Encoding.UTF8.GetBytes(cr.Message.Key);
var vBytes = db.Get(kBytes, columnFamily);
var v = vBytes == null ? 0 : BitConverter.ToInt32(vBytes);
var updatedV = v + 1;
db.Put(kBytes, BitConverter.GetBytes(updatedV), columnFamily);
while (true)
{
try
{
producerState[cr.TopicPartition].Producer.Produce(
Topic_Counts, new Message <string, int> {
Key = cr.Message.Key, Value = updatedV
});
}
catch (KafkaException e)
{
if (e.Error.Code == ErrorCode.Local_QueueFull)
{
Thread.Sleep(TimeSpan.FromSeconds(1000));
continue;
}
throw;
}
break;
}
wCount += 1;
if (DateTime.Now > lastTxnCommit + TxnCommitPeriod)
{
// Execute the transaction commits for each producer in parallel.
var tasks = new List <Task>();
foreach (var state in producerState)
{
if (state.Value.Offset == Offset.Unset)
{
continue;
}
tasks.Add(Task.Run(() =>
{
state.Value.Producer.SendOffsetsToTransaction(
new List <TopicPartitionOffset> {
new TopicPartitionOffset(state.Key, state.Value.Offset + 1)
},
consumer.ConsumerGroupMetadata, DefaultTimeout);
state.Value.Producer.CommitTransaction(DefaultTimeout);
state.Value.Offset = Offset.Unset;
state.Value.Producer.BeginTransaction();
}));
}
Task.WaitAll(tasks.ToArray(), ct);
Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words.");
lastTxnCommit = DateTime.Now;
wCount = 0;
}
}
catch (Exception)
{
consumer.Close();
break;
}
}
}
}
public void CreateCF(string name)
{
var cf = new ColumnFamilyOptions();
_db.CreateColumnFamily(cf, name);
}
