public async static Task PeriodicallyDisplayTopCountsState(string brokerList, RocksDb db, CancellationToken ct)
{
while (true)
{
await Task.Delay(10000, ct);
var it = db.NewIterator(db.GetColumnFamily("counts")).SeekToFirst();
var N = 5;
var maxWords = new List<(int, string)>();
while (it.Valid())
{
var wc = (BitConverter.ToInt32(it.Value()), Encoding.UTF8.GetString(it.Key()));
if (maxWords.Count < N) { maxWords.Add(wc); }
else { if (wc.Item1 > maxWords[N-1].Item1) { maxWords[N-1] = wc; } }
maxWords.Sort((x, y) => y.Item1.CompareTo(x.Item1));
it.Next();
}
if (maxWords.Count > 0) { Console.WriteLine("Most frequently occuring words known to this instance:"); }
foreach (var wc in maxWords)
{
Console.WriteLine(" " + wc.Item2 + " " + wc.Item1);
}
}
}
public ColumnDb(RocksDb rocksDb, DbOnTheRocks mainDb, string name)
{
_rocksDb = rocksDb;
_mainDb = mainDb;
_columnFamily = _rocksDb.GetColumnFamily(name);
Name = name;
}
public Database(string path)
{
var columnFamilies = new ColumnFamilies {
{ BLOCKS_FAMILY, new ColumnFamilyOptions() },
{ BLOCK_INDEX_FAMILY, new ColumnFamilyOptions() },
{ TRANSACTIONS_FAMILY, new ColumnFamilyOptions() }
};
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
db = RocksDb.Open(options, path, columnFamilies);
blocksFamily = db.GetColumnFamily(BLOCKS_FAMILY);
blockIndexFamily = db.GetColumnFamily(BLOCK_INDEX_FAMILY);
transactionsFamily = db.GetColumnFamily(TRANSACTIONS_FAMILY);
}
private ColumnFamilyHandle CreateColumnFamilyIfMissing(RocksDb db, string columnFamily)
{
try
{
return(db.GetColumnFamily(columnFamily));
}
catch
{
var options = new ColumnFamilyOptions();
return(db.CreateColumnFamily(options, columnFamily));
}
}
public override byte[] Get(byte prefix, byte[] key)
{
var keyArray = RocksDbStore.GetKey(prefix, key);
if (generalStorage.TryGetValue(keyArray, out var value))
{
return(value);
}
var columnFamily = db.GetColumnFamily(RocksDbStore.GENERAL_STORAGE_FAMILY);
return(db.Get(keyArray, columnFamily));
}
public DataContext(string path = "ether.db")
{
_lastTxIdDictionary = new Dictionary <string, long>();
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
// Create column families
IEnumerable <string> cols = null;
ColumnFamilies columnFamilies = new ColumnFamilies();
try
{
cols = RocksDb.ListColumnFamilies(options, path);
foreach (var col in cols)
{
columnFamilies.Add(col, new ColumnFamilyOptions());
}
}
catch (Exception e)
{
// Database not exist nothing todo
}
finally
{
_db = RocksDb.Open(options, path, columnFamilies);
// Load column families to the dictionary
_columnFamiliesDictionary = new Dictionary <string, ColumnFamilyHandle>();
if (cols != null)
{
foreach (var col in cols)
{
_columnFamiliesDictionary.Add(col, _db.GetColumnFamily(col));
}
foreach (var col in cols)
{
// Load latest transaction Ids
if (!col.Contains(':') && col != "default")
{
var lastTxIdstr = _db.Get("lastTxId", GetColFamily(col + ":lastid"));
_lastTxIdDictionary.Add(col,
string.IsNullOrEmpty(lastTxIdstr) ? 0 : long.Parse(lastTxIdstr));
}
}
}
}
}
public CheckpointStore(string path)
{
db = RocksDb.OpenReadOnly(new DbOptions(), path, RocksDbStore.ColumnFamilies, false);
var metadataColumnHandle = db.GetColumnFamily(RocksDbStore.METADATA_FAMILY);
blocks = new DataTracker <UInt256, BlockState>(db, RocksDbStore.BLOCK_FAMILY);
transactions = new DataTracker <UInt256, TransactionState>(db, RocksDbStore.TX_FAMILY);
accounts = new DataTracker <UInt160, AccountState>(db, RocksDbStore.ACCOUNT_FAMILY);
_unspentCoins = new DataTracker <UInt256, UnspentCoinState>(db, RocksDbStore.UNSPENT_COIN_FAMILY);
spentCoins = new DataTracker <UInt256, SpentCoinState>(db, RocksDbStore.SPENT_COIN_FAMILY);
validators = new DataTracker <ECPoint, ValidatorState>(db, RocksDbStore.VALIDATOR_FAMILY);
assets = new DataTracker <UInt256, AssetState>(db, RocksDbStore.ASSET_FAMILY);
contracts = new DataTracker <UInt160, ContractState>(db, RocksDbStore.CONTRACT_FAMILY);
storages = new DataTracker <StorageKey, StorageItem>(db, RocksDbStore.STORAGE_FAMILY);
headerHashList = new DataTracker <UInt32Wrapper, HeaderHashList>(db, RocksDbStore.HEADER_HASH_LIST_FAMILY);
validatorsCount = new MetadataTracker <ValidatorsCountState>(db, RocksDbStore.VALIDATORS_COUNT_KEY, metadataColumnHandle);
blockHashIndex = new MetadataTracker <HashIndexState>(db, RocksDbStore.CURRENT_BLOCK_KEY, metadataColumnHandle);
headerHashIndex = new MetadataTracker <HashIndexState>(db, RocksDbStore.CURRENT_HEADER_KEY, metadataColumnHandle);
}
public ColumnFamilyHandle GetFamily(byte table)
{
if (!_families.TryGetValue(table, out var family))
{
try
{
// Try to find the family
family = db.GetColumnFamily(table.ToString());
_families.Add(table, family);
}
catch (KeyNotFoundException)
{
// Try to create the family
family = db.CreateColumnFamily(new ColumnFamilyOptions(), table.ToString());
_families.Add(table, family);
}
}
return(family);
}
private ColumnFamilyHandle GetColumnFamily(RocksDb db, Guid?chainId = null)
{
if (chainId is null)
{
return(null);
}
var cfName = chainId.ToString();
ColumnFamilyHandle cf;
try
{
cf = db.GetColumnFamily(cfName);
}
catch (KeyNotFoundException)
{
cf = db.CreateColumnFamily(_options, cfName);
}
return(cf);
}
/// <summary>
/// Provides access to and/or creates a RocksDb persistent key-value store.
/// </summary>
/// <param name="storeDirectory">
/// The directory containing the key-value store.
/// </param>
/// <param name="defaultColumnKeyTracked">
/// Whether the default column should be key-tracked.
/// This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// </param>
/// <param name="additionalColumns">
/// The names of any additional column families in the key-value store.
/// If no additional column families are provided, all entries will be stored
/// in the default column.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="additionalKeyTrackedColumns">
/// The names of any additional column families in the key-value store that
/// should also be key-tracked. This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="readOnly">
/// Whether the database should be opened read-only. This prevents modifications and
/// creating unnecessary metadata files related to write sessions.
/// </param>
/// <param name="dropMismatchingColumns">
/// If a store already exists at the given directory, whether any columns that mismatch the the columns that were passed into the constructor
/// should be dropped. This will cause data loss and can only be applied in read-write mode.
/// </param>
/// <param name="rotateLogs">
/// Have RocksDb rotate logs, useful for debugging performance issues. It will rotate logs every 12 hours,
/// up to a maximum of 60 logs (i.e. 30 days). When the maximum amount of logs is reached, the oldest logs
/// are overwritten in a circular fashion.
///
/// Every time the RocksDb instance is open, the current log file is truncated, which means that if you
/// open the DB more than once in a 12 hour period, you will only have partial information.
/// </param>
public RocksDbStore(
string storeDirectory,
bool defaultColumnKeyTracked = false,
IEnumerable <string> additionalColumns = null,
IEnumerable <string> additionalKeyTrackedColumns = null,
bool readOnly = false,
bool dropMismatchingColumns = false,
bool rotateLogs = false)
{
m_storeDirectory = storeDirectory;
m_defaults.DbOptions = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
// The background compaction threads run in low priority, so they should not hamper the rest of
// the system. The number of cores in the system is what we want here according to official docs,
// and we are setting this to the number of logical processors, which may be higher.
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(1)
.IncreaseParallelism(Environment.ProcessorCount / 2)
// Ensure we have performance statistics for profiling
.EnableStatistics();
// A small comment on things tested that did not work:
// * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop
// * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups,
// it produces a dramatic performance drop otherwise.
m_defaults.WriteOptions = new WriteOptions()
// Disable the write ahead log to reduce disk IO. The write ahead log
// is used to recover the store on crashes, so a crash will lose some writes.
// Writes will be made in-memory only until the write buffer size
// is reached and then they will be flushed to storage files.
.DisableWal(1)
// This option is off by default, but just making sure that the C# wrapper
// doesn't change anything. The idea is that the DB won't wait for fsync to
// return before acknowledging the write as successful. This affects
// correctness, because a write may be ACKd before it is actually on disk,
// but it is much faster.
.SetSync(false);
var blockBasedTableOptions = new BlockBasedTableOptions()
// Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a
// ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which
// means its optimized for not having a key.
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
// Use a hash index in SST files to speed up point lookup.
.SetIndexType(BlockBasedTableIndexType.HashSearch)
// Whether to use the whole key or a prefix of it (obtained through the prefix extractor below).
// Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e.
// setting it to true).
.SetWholeKeyFiltering(true);
m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions()
.SetBlockBasedTableFactory(blockBasedTableOptions)
.SetPrefixExtractor(SliceTransform.CreateNoOp());
if (rotateLogs)
{
// Maximum number of information log files
m_defaults.DbOptions.SetKeepLogFileNum(60);
// Do not rotate information logs based on file size
m_defaults.DbOptions.SetMaxLogFileSize(0);
// How long before we rotate the current information log file
m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)TimeSpan.FromHours(12).Seconds);
}
m_columns = new Dictionary <string, ColumnFamilyInfo>();
additionalColumns = additionalColumns ?? CollectionUtilities.EmptyArray <string>();
additionalKeyTrackedColumns = additionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>();
// The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor
HashSet <string> existingColumns;
try
{
existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory));
}
catch (RocksDbException)
{
// If there is no existing store, an exception will be thrown, ignore it
existingColumns = new HashSet <string>();
}
// In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families
if (readOnly)
{
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false);
}
else
{
// For read-write mode, column families may be added, so set up column families schema
var columnsSchema = new HashSet <string>(additionalColumns);
// Default column
columnsSchema.Add(ColumnFamilies.DefaultName);
// For key-tracked column familiies, create two columns:
// 1: Normal column of { key : value }
// 2: Key-tracking column of { key : empty-value }
if (defaultColumnKeyTracked)
{
// To be robust to the RocksDB-selected default column name changing,
// just name the default column's key-tracking column KeyColumnSuffix
columnsSchema.Add(KeyColumnSuffix);
}
foreach (var name in additionalKeyTrackedColumns)
{
columnsSchema.Add(name);
columnsSchema.Add(name + KeyColumnSuffix);
}
// Figure out which columns are not part of the schema
var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema));
// RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns
// with the columns schema that was passed into the constructor
existingColumns.UnionWith(columnsSchema);
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies);
// Provide an opportunity to update the store to the new column family schema
if (dropMismatchingColumns)
{
foreach (var name in outsideSchemaColumns)
{
m_store.DropColumnFamily(name);
existingColumns.Remove(name);
}
}
}
var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix));
foreach (var name in userFacingColumns)
{
var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix);
m_columns.Add(name, new ColumnFamilyInfo()
{
Handle = m_store.GetColumnFamily(name),
UseKeyTracking = isKeyTracked,
KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null,
});
}
m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo);
}
static ColumnFamilyHandle GetColumnFamilyHandle(RocksDb db, string?columnFamilyName)
{
return(string.IsNullOrEmpty(columnFamilyName)
? db.GetDefaultColumnFamily() : db.GetColumnFamily(columnFamilyName));
}
/// <summary>
/// Provides access to and/or creates a RocksDb persistent key-value store.
/// </summary>
public RocksDbStore(RocksDbStoreArguments arguments)
{
m_storeDirectory = arguments.StoreDirectory;
m_openBulkLoad = arguments.OpenBulkLoad;
m_defaults.DbOptions = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
// The background compaction threads run in low priority, so they should not hamper the rest of
// the system. The number of cores in the system is what we want here according to official docs,
// and we are setting this to the number of logical processors, which may be higher.
// See: https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide#parallelism-options
#if !PLATFORM_OSX
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(1)
#else
// The memtable uses significant chunks of available system memory on macOS, we increase the number
// of background flushing threads (low priority) and set the DB write buffer size. This allows for
// up to 128 MB in memtables across all column families before we flush to disk.
.SetMaxBackgroundCompactions(Environment.ProcessorCount / 4)
.SetMaxBackgroundFlushes(Environment.ProcessorCount / 4)
.SetDbWriteBufferSize(128 << 20)
#endif
.IncreaseParallelism(Environment.ProcessorCount / 2);
if (arguments.EnableStatistics)
{
m_defaults.DbOptions.EnableStatistics();
}
if (arguments.OpenBulkLoad)
{
m_defaults.DbOptions.PrepareForBulkLoad();
}
// Maximum number of information log files
if (arguments.RotateLogsNumFiles != null)
{
m_defaults.DbOptions.SetKeepLogFileNum(arguments.RotateLogsNumFiles.Value);
}
// Do not rotate information logs based on file size
if (arguments.RotateLogsMaxFileSizeBytes != null)
{
m_defaults.DbOptions.SetMaxLogFileSize(arguments.RotateLogsMaxFileSizeBytes.Value);
}
// How long before we rotate the current information log file
if (arguments.RotateLogsMaxAge != null)
{
m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)arguments.RotateLogsMaxAge.Value.Seconds);
}
if (arguments.FastOpen)
{
// max_file_opening_threads is defaulted to 16, so no need to update here.
RocksDbSharp.Native.Instance.rocksdb_options_set_skip_stats_update_on_db_open(m_defaults.DbOptions.Handle, true);
}
if (arguments.DisableAutomaticCompactions)
{
m_defaults.DbOptions.SetDisableAutoCompactions(1);
}
// A small comment on things tested that did not work:
// * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop
// * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups,
// it produces a dramatic performance drop otherwise.
m_defaults.WriteOptions = new WriteOptions()
// Disable the write ahead log to reduce disk IO. The write ahead log
// is used to recover the store on crashes, so a crash will lose some writes.
// Writes will be made in-memory only until the write buffer size
// is reached and then they will be flushed to storage files.
.DisableWal(1)
// This option is off by default, but just making sure that the C# wrapper
// doesn't change anything. The idea is that the DB won't wait for fsync to
// return before acknowledging the write as successful. This affects
// correctness, because a write may be ACKd before it is actually on disk,
// but it is much faster.
.SetSync(false);
var blockBasedTableOptions = new BlockBasedTableOptions()
// Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a
// ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which
// means its optimized for not having a key.
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
// Use a hash index in SST files to speed up point lookup.
.SetIndexType(BlockBasedTableIndexType.HashSearch)
// Whether to use the whole key or a prefix of it (obtained through the prefix extractor below).
// Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e.
// setting it to true).
.SetWholeKeyFiltering(true);
m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions()
#if PLATFORM_OSX
// As advised by the official documentation, LZ4 is the preferred compression algorithm, our RocksDB
// dynamic library has been compiled to support this on macOS. Fallback to Snappy on other systems (default).
.SetCompression(CompressionTypeEnum.rocksdb_lz4_compression)
#endif
.SetBlockBasedTableFactory(blockBasedTableOptions)
.SetPrefixExtractor(SliceTransform.CreateNoOp());
m_columns = new Dictionary <string, ColumnFamilyInfo>();
// The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor
HashSet <string> existingColumns;
try
{
existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory));
}
catch (RocksDbException)
{
// If there is no existing store, an exception will be thrown, ignore it
existingColumns = new HashSet <string>();
}
// In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families
if (arguments.ReadOnly)
{
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false);
}
else
{
// For read-write mode, column families may be added, so set up column families schema
var additionalColumns = arguments.AdditionalColumns ?? CollectionUtilities.EmptyArray <string>();
var columnsSchema = new HashSet <string>(additionalColumns);
// Default column
columnsSchema.Add(ColumnFamilies.DefaultName);
// For key-tracked column familiies, create two columns:
// 1: Normal column of { key : value }
// 2: Key-tracking column of { key : empty-value }
if (arguments.DefaultColumnKeyTracked)
{
// To be robust to the RocksDB-selected default column name changing,
// just name the default column's key-tracking column KeyColumnSuffix
columnsSchema.Add(KeyColumnSuffix);
}
var additionalKeyTrackedColumns = arguments.AdditionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>();
foreach (var name in additionalKeyTrackedColumns)
{
columnsSchema.Add(name);
columnsSchema.Add(name + KeyColumnSuffix);
}
// Figure out which columns are not part of the schema
var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema));
// RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns
// with the columns schema that was passed into the constructor
existingColumns.UnionWith(columnsSchema);
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies);
// Provide an opportunity to update the store to the new column family schema
if (arguments.DropMismatchingColumns)
{
foreach (var name in outsideSchemaColumns)
{
m_store.DropColumnFamily(name);
existingColumns.Remove(name);
}
}
}
var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix));
foreach (var name in userFacingColumns)
{
var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix);
m_columns.Add(name, new ColumnFamilyInfo()
{
Handle = m_store.GetColumnFamily(name),
UseKeyTracking = isKeyTracked,
KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null,
});
}
m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo);
}
public MetaDataCache(RocksDb db, byte key, ReadOptions?readOptions = null, WriteBatch?writeBatch = null, Func <T>?factory = null)
: this(db, new byte[] { key }, db.GetColumnFamily(METADATA_FAMILY), readOptions, writeBatch, factory)
{
}
public DataTracker(RocksDb db, string familyName)
{
this.db = db;
columnFamily = db.GetColumnFamily(familyName);
}
public DataCache(RocksDb db, string familyName, ReadOptions?readOptions = null, WriteBatch?writeBatch = null)
: this(db, db.GetColumnFamily(familyName), readOptions, writeBatch)
{
}
/// <summary>
/// Provides access to and/or creates a RocksDb persistent key-value store.
/// </summary>
/// <param name="storeDirectory">
/// The directory containing the key-value store.
/// </param>
/// <param name="defaultColumnKeyTracked">
/// Whether the default column should be key-tracked.
/// This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// </param>
/// <param name="additionalColumns">
/// The names of any additional column families in the key-value store.
/// If no additional column families are provided, all entries will be stored
/// in the default column.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="additionalKeyTrackedColumns">
/// The names of any additional column families in the key-value store that
/// should also be key-tracked. This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="readOnly">
/// Whether the database should be opened read-only. This prevents modifications and
/// creating unnecessary metadata files related to write sessions.
/// </param>
/// <param name="dropMismatchingColumns">
/// If a store already exists at the given directory, whether any columns that mismatch the the columns that were passed into the constructor
/// should be dropped. This will cause data loss and can only be applied in read-write mode.
/// </param>
public RocksDbStore(
string storeDirectory,
bool defaultColumnKeyTracked = false,
IEnumerable <string> additionalColumns = null,
IEnumerable <string> additionalKeyTrackedColumns = null,
bool readOnly = false,
bool dropMismatchingColumns = false)
{
m_storeDirectory = storeDirectory;
m_defaults.DbOptions = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
// Disable the write ahead log to reduce disk IO. The write ahead log
// is used to recover the store on crashes, so a crash will lose some writes.
// Writes will be made in-memory only until the write buffer size
// is reached and then they will be flushed to storage files.
m_defaults.WriteOptions = new WriteOptions().DisableWal(1);
m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions();
m_columns = new Dictionary <string, ColumnFamilyInfo>();
additionalColumns = additionalColumns ?? CollectionUtilities.EmptyArray <string>();
additionalKeyTrackedColumns = additionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>();
// The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor
HashSet <string> existingColumns;
try
{
existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory));
}
catch (RocksDbException)
{
// If there is no existing store, an exception will be thrown, ignore it
existingColumns = new HashSet <string>();
}
// In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families
if (readOnly)
{
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false);
}
else
{
// For read-write mode, column families may be added, so set up column families schema
var columnsSchema = new HashSet <string>(additionalColumns);
// Default column
columnsSchema.Add(ColumnFamilies.DefaultName);
// For key-tracked column familiies, create two columns:
// 1: Normal column of { key : value }
// 2: Key-tracking column of { key : empty-value }
if (defaultColumnKeyTracked)
{
// To be robust to the RocksDB-selected default column name changing,
// just name the default column's key-tracking column KeyColumnSuffix
columnsSchema.Add(KeyColumnSuffix);
}
foreach (var name in additionalKeyTrackedColumns)
{
columnsSchema.Add(name);
columnsSchema.Add(name + KeyColumnSuffix);
}
// Figure out which columns are not part of the schema
var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema));
// RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns
// with the columns schema that was passed into the constructor
existingColumns.UnionWith(columnsSchema);
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies);
// Provide an opportunity to update the store to the new column family schema
if (dropMismatchingColumns)
{
foreach (var name in outsideSchemaColumns)
{
m_store.DropColumnFamily(name);
existingColumns.Remove(name);
}
}
}
var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix));
foreach (var name in userFacingColumns)
{
var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix);
m_columns.Add(name, new ColumnFamilyInfo()
{
Handle = m_store.GetColumnFamily(name),
UseKeyTracking = isKeyTracked,
KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null,
});
}
m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo);
}
public override byte[] Get(byte prefix, byte[] key)
{
var columnFamily = db.GetColumnFamily(GENERAL_STORAGE_FAMILY);
return db.Get(GetKey(prefix, key), columnFamily);
}
private ColumnFamilyHandle GetPartition()
{
return(_db.GetColumnFamily(partitionName));
}
/// <summary>
/// A transactional (exactly once) processing loop that reads individual words and updates
/// the corresponding total count state.
///
/// When a rebalance occurs (including on startup), the total count state for all assigned
/// partitions is reloaded before the loop commences to update it.
/// </summary>
/// <remarks>
/// Refer to Processor_MapWords for more detailed comments.
/// </remarks>
public static void Processor_AggregateWords(string brokerList, string clientId, RocksDb db, CancellationToken ct)
{
var TxnCommitPeriod = TimeSpan.FromSeconds(10);
var cConfig = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = ConsumerGroup_Aggregate,
AutoOffsetReset = AutoOffsetReset.Earliest,
// This should be greater than the maximum amount of time required to read in
// existing count state.
MaxPollIntervalMs = 86400000,
EnableAutoCommit = false
};
ColumnFamilyHandle columnFamily = null;
var lastTxnCommit = DateTime.Now;
var producerState = new Dictionary <TopicPartition, ProducerState <string, int> >();
using (var consumer = new ConsumerBuilder <string, Null>(cConfig)
.SetPartitionsRevokedHandler((c, partitions) => {
// clear rocksdb state.
db.DropColumnFamily("counts");
db.CreateColumnFamily(new ColumnFamilyOptions(), "counts");
var tasks = new List <Task>();
foreach (var p in producerState.Values)
{
tasks.Add(Task.Run(() => {
p.Producer.AbortTransaction(DefaultTimeout); // Note: Not cancellable yet.
p.Producer.Dispose();
}, ct));
}
if (tasks.Count > 0)
{
Console.WriteLine("Aborting current AggregateWords transactions.");
}
Task.WaitAll(tasks.ToArray());
producerState.Clear();
})
.SetPartitionsAssignedHandler((c, partitions) => {
Console.WriteLine(
"** AggregateWords consumer group rebalanced. Partition assignment: [" +
string.Join(',', partitions.Select(p => p.Partition.Value)) +
"]");
Trace.Assert(producerState.Count == 0, "Unexpected producer state");
var tasks = new List <Task>();
foreach (var tp in partitions)
{
tasks.Add(Task.Run(() => {
var pConfig = new ProducerConfig
{
BootstrapServers = brokerList,
TransactionalId = TransactionalId_Aggregate + "-" + clientId + "-" + tp.Partition
};
var p = new ProducerBuilder <string, int>(pConfig).Build();
p.InitTransactions(DefaultTimeout); // Note: Not cancellable yet.
p.BeginTransaction();
lock (producerState)
{
producerState.Add(tp, new ProducerState <string, int> {
Producer = p, Offset = Offset.Unset
});
}
}, ct));
}
Task.WaitAll(tasks.ToArray());
columnFamily = db.GetColumnFamily("counts");
LoadCountState(db, brokerList, partitions.Select(p => p.Partition), columnFamily, ct);
})
.Build())
{
consumer.Subscribe(Topic_Words);
var wCount = 0;
while (true)
{
try
{
var cr = consumer.Consume(ct);
producerState[cr.TopicPartition].Offset = cr.Offset;
var kBytes = Encoding.UTF8.GetBytes(cr.Message.Key);
var vBytes = db.Get(kBytes, columnFamily);
var v = vBytes == null ? 0 : BitConverter.ToInt32(vBytes);
var updatedV = v + 1;
db.Put(kBytes, BitConverter.GetBytes(updatedV), columnFamily);
while (true)
{
try
{
producerState[cr.TopicPartition].Producer.Produce(
Topic_Counts, new Message <string, int> {
Key = cr.Message.Key, Value = updatedV
});
}
catch (KafkaException e)
{
if (e.Error.Code == ErrorCode.Local_QueueFull)
{
Thread.Sleep(TimeSpan.FromSeconds(1000));
continue;
}
throw;
}
break;
}
wCount += 1;
if (DateTime.Now > lastTxnCommit + TxnCommitPeriod)
{
// Execute the transaction commits for each producer in parallel.
var tasks = new List <Task>();
foreach (var state in producerState)
{
if (state.Value.Offset == Offset.Unset)
{
continue;
}
tasks.Add(Task.Run(() =>
{
state.Value.Producer.SendOffsetsToTransaction(
new List <TopicPartitionOffset> {
new TopicPartitionOffset(state.Key, state.Value.Offset + 1)
},
consumer.ConsumerGroupMetadata, DefaultTimeout);
state.Value.Producer.CommitTransaction(DefaultTimeout);
state.Value.Offset = Offset.Unset;
state.Value.Producer.BeginTransaction();
}));
}
Task.WaitAll(tasks.ToArray(), ct);
Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words.");
lastTxnCommit = DateTime.Now;
wCount = 0;
}
}
catch (Exception)
{
consumer.Close();
break;
}
}
}
}
