public Store(string path) { var families = new ColumnFamilies(); try { foreach (var family in RocksDb.ListColumnFamilies(Options.Default, Path.GetFullPath(path))) { families.Add(new ColumnFamilies.Descriptor(family, new ColumnFamilyOptions())); } } catch { } db = RocksDb.Open(Options.Default, Path.GetFullPath(path), families); ColumnFamilyHandle defaultFamily = db.GetDefaultColumnFamily(); byte[] value = db.Get(SYS_Version, defaultFamily, Options.ReadDefault); if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0")) { return; } if (value != null) { // Clean all families only if the version are different Parallel.For(0, byte.MaxValue + 1, (x) => db.DropColumnFamily(x.ToString())); _families.Clear(); } // Update version db.Put(SYS_Version, Encoding.ASCII.GetBytes(Assembly.GetExecutingAssembly().GetName().Version.ToString()), defaultFamily, Options.WriteDefault); }
/// <inheritdoc/> public override void DeleteChainId(Guid chainId) { ColumnFamilyHandle cf = GetColumnFamily(_chainDb, chainId); if (HasFork(chainId)) { _chainDb.Put(DeletedKey, new byte[0], cf); // We need only chain indexes, not tx nonces at this time because they already had // been copied on .ForkTxNonces(). // FIXME: We should remove this code after adjusting .ForkTxNonces(). using var batch = new WriteBatch(); foreach (Iterator k in IterateDb(_chainDb, TxNonceKeyPrefix, chainId)) { batch.Delete(k.Key(), cf); } _chainDb.Write(batch); return; } _logger.Debug($"Deleting chainID: {chainId}."); Guid?prevChain = GetPreviousChainInfo(cf)?.Item1; string cfName = chainId.ToString(); try { _chainDb.DropColumnFamily(cfName); if (prevChain is { } prevChainNotNull) { lock (_chainForkDeleteLock) { if (HasFork(prevChainNotNull)) { ColumnFamilyHandle prevCf = GetColumnFamily(_chainDb, prevChainNotNull); RemoveFork(prevCf, chainId); if (IsDeletionMarked(prevCf)) { DeleteChainId(prevChainNotNull); } } } } } catch (KeyNotFoundException) { // Do nothing according to the specification: DeleteChainId() should be idempotent. _logger.Debug($"No such chain ID in _chainDb: {cfName}.", cfName); } catch (Exception e) { LogUnexpectedException(nameof(DeleteChainId), e); } }
/// <inheritdoc/> public override void DeleteChainId(Guid chainId) { _logger.Debug($"Deleting chainID: {chainId}."); var cfName = chainId.ToString(); try { _chainDb.DropColumnFamily(cfName); } catch (KeyNotFoundException) { // Do nothing according to the specification: DeleteChainId() should be idempotent. _logger.Debug($"No such chain ID in _chainDb: {cfName}.", cfName); } }
/// <summary> /// Provides access to and/or creates a RocksDb persistent key-value store. /// </summary> /// <param name="storeDirectory"> /// The directory containing the key-value store. /// </param> /// <param name="defaultColumnKeyTracked"> /// Whether the default column should be key-tracked. /// This will create two columns for the same data, /// one with just keys and the other with key and value. /// </param> /// <param name="additionalColumns"> /// The names of any additional column families in the key-value store. /// If no additional column families are provided, all entries will be stored /// in the default column. /// Column families are analogous to tables in relational databases. /// </param> /// <param name="additionalKeyTrackedColumns"> /// The names of any additional column families in the key-value store that /// should also be key-tracked. This will create two columns for the same data, /// one with just keys and the other with key and value. /// Column families are analogous to tables in relational databases. /// </param> /// <param name="readOnly"> /// Whether the database should be opened read-only. This prevents modifications and /// creating unnecessary metadata files related to write sessions. /// </param> /// <param name="dropMismatchingColumns"> /// If a store already exists at the given directory, whether any columns that mismatch the the columns that were passed into the constructor /// should be dropped. This will cause data loss and can only be applied in read-write mode. /// </param> /// <param name="rotateLogs"> /// Have RocksDb rotate logs, useful for debugging performance issues. It will rotate logs every 12 hours, /// up to a maximum of 60 logs (i.e. 30 days). When the maximum amount of logs is reached, the oldest logs /// are overwritten in a circular fashion. /// /// Every time the RocksDb instance is open, the current log file is truncated, which means that if you /// open the DB more than once in a 12 hour period, you will only have partial information. /// </param> public RocksDbStore( string storeDirectory, bool defaultColumnKeyTracked = false, IEnumerable <string> additionalColumns = null, IEnumerable <string> additionalKeyTrackedColumns = null, bool readOnly = false, bool dropMismatchingColumns = false, bool rotateLogs = false) { m_storeDirectory = storeDirectory; m_defaults.DbOptions = new DbOptions() .SetCreateIfMissing(true) .SetCreateMissingColumnFamilies(true) // The background compaction threads run in low priority, so they should not hamper the rest of // the system. The number of cores in the system is what we want here according to official docs, // and we are setting this to the number of logical processors, which may be higher. .SetMaxBackgroundCompactions(Environment.ProcessorCount) .SetMaxBackgroundFlushes(1) .IncreaseParallelism(Environment.ProcessorCount / 2) // Ensure we have performance statistics for profiling .EnableStatistics(); // A small comment on things tested that did not work: // * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop // * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups, // it produces a dramatic performance drop otherwise. m_defaults.WriteOptions = new WriteOptions() // Disable the write ahead log to reduce disk IO. The write ahead log // is used to recover the store on crashes, so a crash will lose some writes. // Writes will be made in-memory only until the write buffer size // is reached and then they will be flushed to storage files. .DisableWal(1) // This option is off by default, but just making sure that the C# wrapper // doesn't change anything. The idea is that the DB won't wait for fsync to // return before acknowledging the write as successful. This affects // correctness, because a write may be ACKd before it is actually on disk, // but it is much faster. .SetSync(false); var blockBasedTableOptions = new BlockBasedTableOptions() // Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a // ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which // means its optimized for not having a key. .SetFilterPolicy(BloomFilterPolicy.Create(10, false)) // Use a hash index in SST files to speed up point lookup. .SetIndexType(BlockBasedTableIndexType.HashSearch) // Whether to use the whole key or a prefix of it (obtained through the prefix extractor below). // Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e. // setting it to true). .SetWholeKeyFiltering(true); m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions() .SetBlockBasedTableFactory(blockBasedTableOptions) .SetPrefixExtractor(SliceTransform.CreateNoOp()); if (rotateLogs) { // Maximum number of information log files m_defaults.DbOptions.SetKeepLogFileNum(60); // Do not rotate information logs based on file size m_defaults.DbOptions.SetMaxLogFileSize(0); // How long before we rotate the current information log file m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)TimeSpan.FromHours(12).Seconds); } m_columns = new Dictionary <string, ColumnFamilyInfo>(); additionalColumns = additionalColumns ?? CollectionUtilities.EmptyArray <string>(); additionalKeyTrackedColumns = additionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>(); // The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor HashSet <string> existingColumns; try { existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory)); } catch (RocksDbException) { // If there is no existing store, an exception will be thrown, ignore it existingColumns = new HashSet <string>(); } // In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families if (readOnly) { var columnFamilies = new ColumnFamilies(); foreach (var name in existingColumns) { columnFamilies.Add(name, m_defaults.ColumnFamilyOptions); } m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false); } else { // For read-write mode, column families may be added, so set up column families schema var columnsSchema = new HashSet <string>(additionalColumns); // Default column columnsSchema.Add(ColumnFamilies.DefaultName); // For key-tracked column familiies, create two columns: // 1: Normal column of { key : value } // 2: Key-tracking column of { key : empty-value } if (defaultColumnKeyTracked) { // To be robust to the RocksDB-selected default column name changing, // just name the default column's key-tracking column KeyColumnSuffix columnsSchema.Add(KeyColumnSuffix); } foreach (var name in additionalKeyTrackedColumns) { columnsSchema.Add(name); columnsSchema.Add(name + KeyColumnSuffix); } // Figure out which columns are not part of the schema var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema)); // RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns // with the columns schema that was passed into the constructor existingColumns.UnionWith(columnsSchema); var columnFamilies = new ColumnFamilies(); foreach (var name in existingColumns) { columnFamilies.Add(name, m_defaults.ColumnFamilyOptions); } m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies); // Provide an opportunity to update the store to the new column family schema if (dropMismatchingColumns) { foreach (var name in outsideSchemaColumns) { m_store.DropColumnFamily(name); existingColumns.Remove(name); } } } var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix)); foreach (var name in userFacingColumns) { var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix); m_columns.Add(name, new ColumnFamilyInfo() { Handle = m_store.GetColumnFamily(name), UseKeyTracking = isKeyTracked, KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null, }); } m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo); }
/// <summary> /// Provides access to and/or creates a RocksDb persistent key-value store. /// </summary> public RocksDbStore(RocksDbStoreArguments arguments) { m_storeDirectory = arguments.StoreDirectory; m_openBulkLoad = arguments.OpenBulkLoad; m_defaults.DbOptions = new DbOptions() .SetCreateIfMissing(true) .SetCreateMissingColumnFamilies(true) // The background compaction threads run in low priority, so they should not hamper the rest of // the system. The number of cores in the system is what we want here according to official docs, // and we are setting this to the number of logical processors, which may be higher. // See: https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide#parallelism-options #if !PLATFORM_OSX .SetMaxBackgroundCompactions(Environment.ProcessorCount) .SetMaxBackgroundFlushes(1) #else // The memtable uses significant chunks of available system memory on macOS, we increase the number // of background flushing threads (low priority) and set the DB write buffer size. This allows for // up to 128 MB in memtables across all column families before we flush to disk. .SetMaxBackgroundCompactions(Environment.ProcessorCount / 4) .SetMaxBackgroundFlushes(Environment.ProcessorCount / 4) .SetDbWriteBufferSize(128 << 20) #endif .IncreaseParallelism(Environment.ProcessorCount / 2); if (arguments.EnableStatistics) { m_defaults.DbOptions.EnableStatistics(); } if (arguments.OpenBulkLoad) { m_defaults.DbOptions.PrepareForBulkLoad(); } // Maximum number of information log files if (arguments.RotateLogsNumFiles != null) { m_defaults.DbOptions.SetKeepLogFileNum(arguments.RotateLogsNumFiles.Value); } // Do not rotate information logs based on file size if (arguments.RotateLogsMaxFileSizeBytes != null) { m_defaults.DbOptions.SetMaxLogFileSize(arguments.RotateLogsMaxFileSizeBytes.Value); } // How long before we rotate the current information log file if (arguments.RotateLogsMaxAge != null) { m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)arguments.RotateLogsMaxAge.Value.Seconds); } if (arguments.FastOpen) { // max_file_opening_threads is defaulted to 16, so no need to update here. RocksDbSharp.Native.Instance.rocksdb_options_set_skip_stats_update_on_db_open(m_defaults.DbOptions.Handle, true); } if (arguments.DisableAutomaticCompactions) { m_defaults.DbOptions.SetDisableAutoCompactions(1); } // A small comment on things tested that did not work: // * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop // * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups, // it produces a dramatic performance drop otherwise. m_defaults.WriteOptions = new WriteOptions() // Disable the write ahead log to reduce disk IO. The write ahead log // is used to recover the store on crashes, so a crash will lose some writes. // Writes will be made in-memory only until the write buffer size // is reached and then they will be flushed to storage files. .DisableWal(1) // This option is off by default, but just making sure that the C# wrapper // doesn't change anything. The idea is that the DB won't wait for fsync to // return before acknowledging the write as successful. This affects // correctness, because a write may be ACKd before it is actually on disk, // but it is much faster. .SetSync(false); var blockBasedTableOptions = new BlockBasedTableOptions() // Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a // ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which // means its optimized for not having a key. .SetFilterPolicy(BloomFilterPolicy.Create(10, false)) // Use a hash index in SST files to speed up point lookup. .SetIndexType(BlockBasedTableIndexType.HashSearch) // Whether to use the whole key or a prefix of it (obtained through the prefix extractor below). // Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e. // setting it to true). .SetWholeKeyFiltering(true); m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions() #if PLATFORM_OSX // As advised by the official documentation, LZ4 is the preferred compression algorithm, our RocksDB // dynamic library has been compiled to support this on macOS. Fallback to Snappy on other systems (default). .SetCompression(CompressionTypeEnum.rocksdb_lz4_compression) #endif .SetBlockBasedTableFactory(blockBasedTableOptions) .SetPrefixExtractor(SliceTransform.CreateNoOp()); m_columns = new Dictionary <string, ColumnFamilyInfo>(); // The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor HashSet <string> existingColumns; try { existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory)); } catch (RocksDbException) { // If there is no existing store, an exception will be thrown, ignore it existingColumns = new HashSet <string>(); } // In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families if (arguments.ReadOnly) { var columnFamilies = new ColumnFamilies(); foreach (var name in existingColumns) { columnFamilies.Add(name, m_defaults.ColumnFamilyOptions); } m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false); } else { // For read-write mode, column families may be added, so set up column families schema var additionalColumns = arguments.AdditionalColumns ?? CollectionUtilities.EmptyArray <string>(); var columnsSchema = new HashSet <string>(additionalColumns); // Default column columnsSchema.Add(ColumnFamilies.DefaultName); // For key-tracked column familiies, create two columns: // 1: Normal column of { key : value } // 2: Key-tracking column of { key : empty-value } if (arguments.DefaultColumnKeyTracked) { // To be robust to the RocksDB-selected default column name changing, // just name the default column's key-tracking column KeyColumnSuffix columnsSchema.Add(KeyColumnSuffix); } var additionalKeyTrackedColumns = arguments.AdditionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>(); foreach (var name in additionalKeyTrackedColumns) { columnsSchema.Add(name); columnsSchema.Add(name + KeyColumnSuffix); } // Figure out which columns are not part of the schema var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema)); // RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns // with the columns schema that was passed into the constructor existingColumns.UnionWith(columnsSchema); var columnFamilies = new ColumnFamilies(); foreach (var name in existingColumns) { columnFamilies.Add(name, m_defaults.ColumnFamilyOptions); } m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies); // Provide an opportunity to update the store to the new column family schema if (arguments.DropMismatchingColumns) { foreach (var name in outsideSchemaColumns) { m_store.DropColumnFamily(name); existingColumns.Remove(name); } } } var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix)); foreach (var name in userFacingColumns) { var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix); m_columns.Add(name, new ColumnFamilyInfo() { Handle = m_store.GetColumnFamily(name), UseKeyTracking = isKeyTracked, KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null, }); } m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo); }
/// <summary> /// Provides access to and/or creates a RocksDb persistent key-value store. /// </summary> /// <param name="storeDirectory"> /// The directory containing the key-value store. /// </param> /// <param name="defaultColumnKeyTracked"> /// Whether the default column should be key-tracked. /// This will create two columns for the same data, /// one with just keys and the other with key and value. /// </param> /// <param name="additionalColumns"> /// The names of any additional column families in the key-value store. /// If no additional column families are provided, all entries will be stored /// in the default column. /// Column families are analogous to tables in relational databases. /// </param> /// <param name="additionalKeyTrackedColumns"> /// The names of any additional column families in the key-value store that /// should also be key-tracked. This will create two columns for the same data, /// one with just keys and the other with key and value. /// Column families are analogous to tables in relational databases. /// </param> /// <param name="readOnly"> /// Whether the database should be opened read-only. This prevents modifications and /// creating unnecessary metadata files related to write sessions. /// </param> /// <param name="dropMismatchingColumns"> /// If a store already exists at the given directory, whether any columns that mismatch the the columns that were passed into the constructor /// should be dropped. This will cause data loss and can only be applied in read-write mode. /// </param> public RocksDbStore( string storeDirectory, bool defaultColumnKeyTracked = false, IEnumerable <string> additionalColumns = null, IEnumerable <string> additionalKeyTrackedColumns = null, bool readOnly = false, bool dropMismatchingColumns = false) { m_storeDirectory = storeDirectory; m_defaults.DbOptions = new DbOptions() .SetCreateIfMissing(true) .SetCreateMissingColumnFamilies(true); // Disable the write ahead log to reduce disk IO. The write ahead log // is used to recover the store on crashes, so a crash will lose some writes. // Writes will be made in-memory only until the write buffer size // is reached and then they will be flushed to storage files. m_defaults.WriteOptions = new WriteOptions().DisableWal(1); m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions(); m_columns = new Dictionary <string, ColumnFamilyInfo>(); additionalColumns = additionalColumns ?? CollectionUtilities.EmptyArray <string>(); additionalKeyTrackedColumns = additionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>(); // The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor HashSet <string> existingColumns; try { existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory)); } catch (RocksDbException) { // If there is no existing store, an exception will be thrown, ignore it existingColumns = new HashSet <string>(); } // In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families if (readOnly) { var columnFamilies = new ColumnFamilies(); foreach (var name in existingColumns) { columnFamilies.Add(name, m_defaults.ColumnFamilyOptions); } m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false); } else { // For read-write mode, column families may be added, so set up column families schema var columnsSchema = new HashSet <string>(additionalColumns); // Default column columnsSchema.Add(ColumnFamilies.DefaultName); // For key-tracked column familiies, create two columns: // 1: Normal column of { key : value } // 2: Key-tracking column of { key : empty-value } if (defaultColumnKeyTracked) { // To be robust to the RocksDB-selected default column name changing, // just name the default column's key-tracking column KeyColumnSuffix columnsSchema.Add(KeyColumnSuffix); } foreach (var name in additionalKeyTrackedColumns) { columnsSchema.Add(name); columnsSchema.Add(name + KeyColumnSuffix); } // Figure out which columns are not part of the schema var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema)); // RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns // with the columns schema that was passed into the constructor existingColumns.UnionWith(columnsSchema); var columnFamilies = new ColumnFamilies(); foreach (var name in existingColumns) { columnFamilies.Add(name, m_defaults.ColumnFamilyOptions); } m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies); // Provide an opportunity to update the store to the new column family schema if (dropMismatchingColumns) { foreach (var name in outsideSchemaColumns) { m_store.DropColumnFamily(name); existingColumns.Remove(name); } } } var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix)); foreach (var name in userFacingColumns) { var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix); m_columns.Add(name, new ColumnFamilyInfo() { Handle = m_store.GetColumnFamily(name), UseKeyTracking = isKeyTracked, KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null, }); } m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo); }
/// <summary> /// A transactional (exactly once) processing loop that reads individual words and updates /// the corresponding total count state. /// /// When a rebalance occurs (including on startup), the total count state for all assigned /// partitions is reloaded before the loop commences to update it. /// </summary> /// <remarks> /// Refer to Processor_MapWords for more detailed comments. /// </remarks> public static void Processor_AggregateWords(string brokerList, string clientId, RocksDb db, CancellationToken ct) { var TxnCommitPeriod = TimeSpan.FromSeconds(10); var cConfig = new ConsumerConfig { BootstrapServers = brokerList, GroupId = ConsumerGroup_Aggregate, AutoOffsetReset = AutoOffsetReset.Earliest, // This should be greater than the maximum amount of time required to read in // existing count state. MaxPollIntervalMs = 86400000, EnableAutoCommit = false }; ColumnFamilyHandle columnFamily = null; var lastTxnCommit = DateTime.Now; var producerState = new Dictionary <TopicPartition, ProducerState <string, int> >(); using (var consumer = new ConsumerBuilder <string, Null>(cConfig) .SetPartitionsRevokedHandler((c, partitions) => { // clear rocksdb state. db.DropColumnFamily("counts"); db.CreateColumnFamily(new ColumnFamilyOptions(), "counts"); var tasks = new List <Task>(); foreach (var p in producerState.Values) { tasks.Add(Task.Run(() => { p.Producer.AbortTransaction(DefaultTimeout); // Note: Not cancellable yet. p.Producer.Dispose(); }, ct)); } if (tasks.Count > 0) { Console.WriteLine("Aborting current AggregateWords transactions."); } Task.WaitAll(tasks.ToArray()); producerState.Clear(); }) .SetPartitionsAssignedHandler((c, partitions) => { Console.WriteLine( "** AggregateWords consumer group rebalanced. Partition assignment: [" + string.Join(',', partitions.Select(p => p.Partition.Value)) + "]"); Trace.Assert(producerState.Count == 0, "Unexpected producer state"); var tasks = new List <Task>(); foreach (var tp in partitions) { tasks.Add(Task.Run(() => { var pConfig = new ProducerConfig { BootstrapServers = brokerList, TransactionalId = TransactionalId_Aggregate + "-" + clientId + "-" + tp.Partition }; var p = new ProducerBuilder <string, int>(pConfig).Build(); p.InitTransactions(DefaultTimeout); // Note: Not cancellable yet. p.BeginTransaction(); lock (producerState) { producerState.Add(tp, new ProducerState <string, int> { Producer = p, Offset = Offset.Unset }); } }, ct)); } Task.WaitAll(tasks.ToArray()); columnFamily = db.GetColumnFamily("counts"); LoadCountState(db, brokerList, partitions.Select(p => p.Partition), columnFamily, ct); }) .Build()) { consumer.Subscribe(Topic_Words); var wCount = 0; while (true) { try { var cr = consumer.Consume(ct); producerState[cr.TopicPartition].Offset = cr.Offset; var kBytes = Encoding.UTF8.GetBytes(cr.Message.Key); var vBytes = db.Get(kBytes, columnFamily); var v = vBytes == null ? 0 : BitConverter.ToInt32(vBytes); var updatedV = v + 1; db.Put(kBytes, BitConverter.GetBytes(updatedV), columnFamily); while (true) { try { producerState[cr.TopicPartition].Producer.Produce( Topic_Counts, new Message <string, int> { Key = cr.Message.Key, Value = updatedV }); } catch (KafkaException e) { if (e.Error.Code == ErrorCode.Local_QueueFull) { Thread.Sleep(TimeSpan.FromSeconds(1000)); continue; } throw; } break; } wCount += 1; if (DateTime.Now > lastTxnCommit + TxnCommitPeriod) { // Execute the transaction commits for each producer in parallel. var tasks = new List <Task>(); foreach (var state in producerState) { if (state.Value.Offset == Offset.Unset) { continue; } tasks.Add(Task.Run(() => { state.Value.Producer.SendOffsetsToTransaction( new List <TopicPartitionOffset> { new TopicPartitionOffset(state.Key, state.Value.Offset + 1) }, consumer.ConsumerGroupMetadata, DefaultTimeout); state.Value.Producer.CommitTransaction(DefaultTimeout); state.Value.Offset = Offset.Unset; state.Value.Producer.BeginTransaction(); })); } Task.WaitAll(tasks.ToArray(), ct); Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words."); lastTxnCommit = DateTime.Now; wCount = 0; } } catch (Exception) { consumer.Close(); break; } } } }