protected override void PreStart()
{
#if DEBUG
Console.WriteLine("PreStart");
#endif
base.PreStart();
var producerSettings = ProducerSettings <string, byte[]>
.Create(settings.KafkaConfig, Serializers.Utf8, Serializers.ByteArray)
.WithBootstrapServers(settings.KafkaConfig.GetString("bootstrap.servers"));
materializer = Context.Materializer();
producer = producerSettings.CreateKafkaProducer();
topicMapper = settings.EventTopicMapper;
enabledTopicPartitions = new HashSet <TopicPartition>(settings.EnabledTopicPartitions);
#if DEBUG
Console.WriteLine("Gettings RocksDB");
#endif
var rocksDbSettings = settings.RocksDbSettings;
rocksDbReadOptions = new ReadOptions().SetVerifyChecksums(rocksDbSettings.Checksum);
rocksDbWriteOptions = new WriteOptions().SetSync(rocksDbSettings.FSync);
//retrieve offsets for the consumer to start reading; as rocksdb possible already knows about
database = RocksDb.Open(rocksDbOptions, rocksDbSettings.Path);
#if DEBUG
Console.WriteLine("Database opened");
#endif
idMap = ReadIdMap();
currentOffsets = ReadOffsetsForPartitions();
StartConsuming();
}
internal static RocksDb OpenRocksDb(
DbOptions options, string dbPath, ColumnFamilies columnFamilies = null)
{
return(columnFamilies is null
? RocksDb.Open(options, dbPath)
: RocksDb.Open(options, dbPath, columnFamilies));
}
public RocksDbKeyValueRepository(DataFolder dataFolder, DBreezeSerializer dataStoreSerializer)
{
Directory.CreateDirectory(dataFolder.KeyValueRepositoryPath);
this.dataStoreSerializer = dataStoreSerializer;
this.dbOptions = new DbOptions().SetCreateIfMissing(true);
this.rocksdb = RocksDb.Open(this.dbOptions, dataFolder.KeyValueRepositoryPath);
}
public Store(string path)
{
var families = new ColumnFamilies();
try
{
foreach (var family in RocksDb.ListColumnFamilies(Options.Default, Path.GetFullPath(path)))
{
families.Add(new ColumnFamilies.Descriptor(family, new ColumnFamilyOptions()));
}
}
catch { }
db = RocksDb.Open(Options.Default, Path.GetFullPath(path), families);
ColumnFamilyHandle defaultFamily = db.GetDefaultColumnFamily();
byte[] value = db.Get(SYS_Version, defaultFamily, Options.ReadDefault);
if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0"))
{
return;
}
if (value != null)
{
// Clean all families only if the version are different
Parallel.For(0, byte.MaxValue + 1, (x) => db.DropColumnFamily(x.ToString()));
_families.Clear();
}
// Update version
db.Put(SYS_Version, Encoding.ASCII.GetBytes(Assembly.GetExecutingAssembly().GetName().Version.ToString()), defaultFamily, Options.WriteDefault);
}
static void Main(string[] args)
{
var dbOptions = new DbOptions().SetCreateIfMissing().SetCreateMissingColumnFamilies();
var columnFamilies = new ColumnFamilies
{
{ "default", new ColumnFamilyOptions().OptimizeForPointLookup(256) },
{
"user", new ColumnFamilyOptions()
.SetMemtableHugePageSize(2 * 1024 * 1024)
.SetPrefixExtractor(SliceTransform.CreateFixedPrefix(5))
.SetBlockBasedTableFactory(
new BlockBasedTableOptions().SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false))
}
};
var db = RocksDb.Open(dbOptions, "my.db", columnFamilies);
var testFamily = db.GetColumnFamily("user");
db.Put("hello:hi", "hi");
for (var i = 0; i < 5; i++)
{
db.Put($"user:{Guid.NewGuid()}", Guid.NewGuid().ToString(), testFamily);
}
using (var it = db.NewIterator(testFamily))
{
it.Seek(Encoding.UTF8.GetBytes("user:"******"Done!" + db.Get("hello:hi"));
}
public void GetAsyncWithExistingBlocksReturnsBlocks()
{
string dir = CreateTestDir(this);
var blocks = new Block[10];
blocks[0] = this.Network.Consensus.ConsensusFactory.CreateBlock();
for (int i = 1; i < blocks.Length; i++)
{
blocks[i] = this.Network.Consensus.ConsensusFactory.CreateBlock();
blocks[i].Header.HashPrevBlock = blocks[i - 1].Header.GetHash();
}
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
for (int i = 0; i < blocks.Length; i++)
{
engine.Put(DBH.Key(RocksdbBlockRepository.BlockTableName, blocks[i].GetHash().ToBytes()), blocks[i].ToBytes());
}
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
List <Block> result = repository.GetBlocks(blocks.Select(b => b.GetHash()).ToList());
Assert.Equal(blocks.Length, result.Count);
for (int i = 0; i < 10; i++)
{
Assert.Equal(blocks[i].GetHash(), result[i].GetHash());
}
}
}
public DbOnTheRocks(string basePath, string dbPath, IDbConfig dbConfig, ILogManager logManager = null) // TODO: check column families
{
string fullPath = dbPath.GetApplicationResourcePath(basePath);
_logger = logManager?.GetClassLogger() ?? NullLogger.Instance;
if (!Directory.Exists(fullPath))
{
Directory.CreateDirectory(fullPath);
}
if (_logger.IsInfo)
{
_logger.Info($"Using database directory {fullPath}");
}
try
{
DbOptions options = BuildOptions(dbConfig);
_db = DbsByPath.GetOrAdd(fullPath, path => RocksDb.Open(options, path));
}
catch (DllNotFoundException e) when(e.Message.Contains("libdl"))
{
throw new ApplicationException($"Unable to load 'libdl' necessary to init the RocksDB database. Please run{Environment.NewLine}" +
"sudo apt update && sudo apt install libsnappy-dev libc6-dev libc6");
}
}
public async Task PutAsync_WritesProvenBlockHeaderAndSavesBlockHashAsync()
{
string folder = CreateTestDir(this);
ProvenBlockHeader provenBlockHeaderIn = CreateNewProvenBlockHeaderMock();
var blockHashHeightPair = new HashHeightPair(provenBlockHeaderIn.GetHash(), 0);
var items = new SortedDictionary <int, ProvenBlockHeader>()
{
{ 0, provenBlockHeaderIn }
};
using (IProvenBlockHeaderRepository repo = this.SetupRepository(folder))
{
await repo.PutAsync(items, blockHashHeightPair);
}
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(), folder))
{
var headerOut = this.dBreezeSerializer.Deserialize <ProvenBlockHeader>(engine.Get(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHashHeightPair.Height)));
var hashHeightPairOut = this.DBreezeSerializer.Deserialize <HashHeightPair>(engine.Get(BlockHashHeightTable, new byte[] { 1 }));
headerOut.Should().NotBeNull();
headerOut.GetHash().Should().Be(provenBlockHeaderIn.GetHash());
hashHeightPairOut.Should().NotBeNull();
hashHeightPairOut.Hash.Should().Be(provenBlockHeaderIn.GetHash());
}
}
public RocksDbStorage(DbOptions options, string path)
{
Path = path;
options.SetUint64addMergeOperator();
Database = RocksDb.Open(options, path);
AliasStore = new RocksDbAliasStore(this);
}
public FetchCoinsResponse FetchCoins(OutPoint[] utxos)
{
lock (this.locker)
{
FetchCoinsResponse res = new FetchCoinsResponse();
using (new StopwatchDisposable(o => this.performanceCounter.AddQueryTime(o)))
{
this.performanceCounter.AddQueriedEntities(utxos.Length);
using var rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder);
{
foreach (OutPoint outPoint in utxos)
{
byte[] row = rocksDb.Get(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray());
Coins outputs = row != null?this.dBreezeSerializer.Deserialize <Coins>(row) : null;
this.logger.Debug("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded");
res.UnspentOutputs.Add(outPoint, new UnspentOutput(outPoint, outputs));
}
}
}
return(res);
}
}
public void DoubleDisposableDoesNotThrow()
{
var testdir = Path.Combine(Path.GetTempPath(), "lifestyle_test");
var testdb = Path.Combine(testdir, "main");
var path = Environment.ExpandEnvironmentVariables(testdb);
if (Directory.Exists(testdir))
{
Directory.Delete(testdir, recursive: true);
}
Directory.CreateDirectory(testdir);
var options = new DbOptions().SetCreateIfMissing(true).EnableStatistics();
var db = RocksDb.Open(options, path);
db.Dispose();
// throws AccessViolationException, which on my machine crashed the process so hard that XUnit coulnd't cope...
//
db.Dispose();
//
// got this in Event Viewer though:
//
// Application: dotnet.exe
// CoreCLR Version: 4.6.28619.1
// Description: The process was terminated due to an internal error in the .NET Runtime at IP 00007FFF39BC5AA3 (00007FFF39A20000) with exit code c0000005.
//
}
private void SetBlockHash(HashHeightPair nextBlockHash)
{
this.persistedCoinviewTip = nextBlockHash;
using var rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder);
rocksDb.Put(new byte[] { blockTable }.Concat(blockHashKey).ToArray(), nextBlockHash.ToBytes());
}
static void Main(string[] args)
{
string temp = Path.GetTempPath();
var testdir = Path.Combine(temp, "functional_test");
var testdb = Path.Combine(testdir, "main");
var testcp = Path.Combine(testdir, "cp");
var path = Environment.ExpandEnvironmentVariables(testdb);
var cppath = Environment.ExpandEnvironmentVariables(testcp);
if (Directory.Exists(testdir))
{
Directory.Delete(testdir, true);
}
Directory.CreateDirectory(testdir);
var options = new DbOptions()
.SetCreateIfMissing(true);
using (var db = RocksDb.Open(options, path))
{
// Using strings below, but can also use byte arrays for both keys and values
// much care has been taken to minimize buffer copying
db.Put("key", "value");
string value = db.Get("key");
db.Remove("key");
}
Console.ReadKey();
}
static Utils()
{
LogFolder = Consts.LogsFolder;
LogFile = Consts.LogFile;
Consts.LocalAppDataPath.CreateDirectory();
Consts.LogsFolder.CreateDirectory();
var options = new DbOptions().SetCreateIfMissing(true);
_hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb"));
_startTime = DateTime.Now;
if (LogFile.Exists)
{
var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log");
LogFile.MoveToAsync(newPath, true).Wait();
}
var logFiles = LogFolder.EnumerateFiles(false).ToList();
if (logFiles.Count >= Consts.MaxOldLogs)
{
Log($"Maximum amount of old logs reached ({logFiles.Count} >= {Consts.MaxOldLogs})");
var filesToDelete = logFiles
.Where(f => f.IsFile)
.OrderBy(f => f.LastModified)
.Take(logFiles.Count - Consts.MaxOldLogs)
.ToList();
Log($"Found {filesToDelete.Count} old log files to delete");
var success = 0;
var failed = 0;
filesToDelete.Do(f =>
{
try
{
f.Delete();
success++;
}
catch (Exception e)
{
failed++;
Log($"Could not delete log at {f}!\n{e}");
}
});
Log($"Deleted {success} log files, failed to delete {failed} logs");
}
var watcher = new FileSystemWatcher((string)Consts.LocalAppDataPath);
AppLocalEvents = Observable.Merge(Observable.FromEventPattern <FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)),
Observable.FromEventPattern <FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)),
Observable.FromEventPattern <FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs)))
.ObserveOn(RxApp.TaskpoolScheduler);
watcher.EnableRaisingEvents = true;
InitPatches();
}
public void DeleteAsyncRemovesBlocksAndTransactions()
{
string dir = CreateTestDir(this);
Block block = this.Network.CreateBlock();
block.Transactions.Add(this.Network.CreateTransaction());
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
engine.Put(DBH.Key(RocksdbBlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes());
engine.Put(DBH.Key(RocksdbBlockRepository.TransactionTableName, block.Transactions[0].GetHash().ToBytes()), block.GetHash().ToBytes());
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true));
}
var tip = new HashHeightPair(new uint256(45), 100);
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
repository.Delete(tip, new List <uint256> {
block.GetHash()
});
}
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
byte[] blockHashKeyRow = engine.Get(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[0]));
Dictionary <byte[], byte[]> blockDict = engine.SelectDictionary(RocksdbBlockRepository.BlockTableName);
Dictionary <byte[], byte[]> transDict = engine.SelectDictionary(RocksdbBlockRepository.TransactionTableName);
Assert.Equal(tip, this.DataStoreSerializer.Deserialize <HashHeightPair>(blockHashKeyRow));
Assert.Empty(blockDict);
Assert.Empty(transDict);
}
}
private void CreateRocksDB(bool isCreateIfMissing = true)
{
var options = new DbOptions()
.SetCreateIfMissing(isCreateIfMissing);
_db = RocksDb.Open(options, _dbPath);
}
static void v1()
{
var options = new DbOptions()
.SetCreateIfMissing(true);
using (var db = RocksDb.Open(options, "db1"))
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var count = 0;
foreach (var p in PacketStreamUtilities.PacketsFromFileTree(DataDirectory))
{
var len = PacketAccessor.Length(p);
if (len < 5)
{
continue;
}
var timestamp = PacketAccessor.Time42(p);
var apid = PacketAccessor.APID(p);
var key = new PacketKey {
APID = (short)apid, Timestamp = timestamp
};
var keybuf = KeyToByteArray(key);
db.Put(keybuf, p);
count++;
}
stopwatch.Stop();
Console.WriteLine($"{count} packets; {count / (stopwatch.ElapsedMilliseconds / 1000d)} packets/sec");
}
}
private RocksDb GetOrCreateDB(string id)
{
if (string.IsNullOrEmpty(id))
{
throw new ArgumentException("message", nameof(id));
}
var db = GetDBOrNull(id);
if (db == null)
{
// Allow only one db to be created at a time to avoid conflicts
lock (_dbs) {
// Double check lock
if (!_dbs.TryGetValue(id, out db))
{
var dbPath = Path.Combine(_rootPath, id);
db = RocksDb.Open(_dbOptions, dbPath);
_logger.LogInformation($"created rocksdb at {dbPath}");
if (!_dbs.TryAdd(id, db))
{
// this should never happen
throw new Exception("unable to add db");
}
}
}
}
return(db);
}
public RocksDbHandler CreateRocksDb(bool isCreateIfMissing = true)
{
var options = new DbOptions()
.SetCreateIfMissing(isCreateIfMissing);
_rocksDb = RocksDb.Open(options, _dbPath);
return(this);
}
public void Setup()
{
var path = "rocksdb".xToPath();
var options = new DbOptions()
.SetCreateIfMissing(true);
_rocksDb = RocksDb.Open(options, path);
}
public DbOnTheRocks(string dbPath, IDbConfig dbConfig, ILogManager logManager = null) // TODO: check column families
{
ILogger logger = logManager?.GetClassLogger();
if (!Directory.Exists(dbPath))
{
Directory.CreateDirectory(dbPath);
}
if (logger != null)
{
if (logger.IsInfo)
{
logger.Info($"Using database directory {dbPath}");
}
}
if (dbPath.EndsWith(StateDbPath))
{
_dbInstance = DbInstance.State;
}
else if (dbPath.EndsWith(BlockInfosDbPath))
{
_dbInstance = DbInstance.BlockInfos;
}
else if (dbPath.EndsWith(HeadersPath))
{
_dbInstance = DbInstance.Headers;
}
else if (dbPath.EndsWith(BlocksDbPath))
{
_dbInstance = DbInstance.Blocks;
}
else if (dbPath.EndsWith(CodeDbPath))
{
_dbInstance = DbInstance.Code;
}
else if (dbPath.EndsWith(ReceiptsDbPath))
{
_dbInstance = DbInstance.Receipts;
}
else if (dbPath.EndsWith(PendingTxsDbPath))
{
_dbInstance = DbInstance.PendingTxs;
}
else if (dbPath.EndsWith(TraceDbPath))
{
_dbInstance = DbInstance.Trace;
}
else
{
_dbInstance = DbInstance.Other;
}
DbOptions options = BuildOptions(dbConfig, _dbInstance);
_db = DbsByPath.GetOrAdd(dbPath, path => RocksDb.Open(options, path));
}
static void Main(string[] args)
{
string temp = Path.GetTempPath();
string path = Environment.ExpandEnvironmentVariables(Path.Combine(temp, "rocksdb_cf_example"));
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var columnFamilies = new ColumnFamilies
{
{ "reverse", new ColumnFamilyOptions() },
};
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
db.Put("one", "uno");
db.Put("two", "dos");
db.Put("three", "tres");
db.Put("uno", "one", cf: reverse);
db.Put("dos", "two", cf: reverse);
db.Put("tres", "three", cf: reverse);
}
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
string uno = db.Get("one");
string one = db.Get("uno", cf: reverse);
string nada;
nada = db.Get("uno");
nada = db.Get("one", cf: reverse);
}
using (var db = RocksDb.Open(options, path, columnFamilies))
{
db.DropColumnFamily("reverse");
var reverse = db.CreateColumnFamily(new ColumnFamilyOptions(), "reverse");
var nada = db.Get("uno", cf: reverse);
db.Put("red", "rouge", cf: reverse);
}
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var reverse = db.GetColumnFamily("reverse");
var nada = db.Get("uno", cf: reverse);
var rouge = db.Get("red", cf: reverse);
}
using (var db = RocksDb.OpenReadOnly(options, path, columnFamilies, false))
{
string uno = db.Get("one");
}
}
public RocksDbKeyValueStore()
{
Directory.Delete("db.bin", true);
db = RocksDb.Open(
new DbOptions()
.SetCreateIfMissing()
.SetCompression(CompressionTypeEnum.rocksdb_snappy_compression),
"db.bin");
}
private RocksDb OpenRocksDb(
DbOptions options, string dbName, ColumnFamilies columnFamilies = null)
{
var dbPath = Path.Combine(_path, dbName);
return(columnFamilies is null
? RocksDb.Open(options, dbPath)
: RocksDb.Open(options, dbPath, columnFamilies));
}
public HttpExtractor(ILogger <HttpExtractor> logger, IMetricsServer metrics)
{
Logger = logger;
Metrics = metrics;
var options = new DbOptions().SetCreateIfMissing().EnableStatistics();
_db = RocksDb.Open(options,
Environment.ExpandEnvironmentVariables(Path.Combine(Environment.CurrentDirectory, "httpCall")));
}
private RocksDb OpenRocksDb(string path)
{
var options = new DbOptions().SetCreateIfMissing();
var db = RocksDb.Open(options, path);
db.Dispose();
return(_rocksDbService.Load(path));
}
public KeyValueRepository(string folder, DataStoreSerializer dataStoreSerializer)
{
Directory.CreateDirectory(folder);
this.dataStoreSerializer = dataStoreSerializer;
// Open a connection to a new DB and create if not found
var options = new DbOptions().SetCreateIfMissing(true);
this.rocksdb = RocksDb.Open(options, folder);
}
static void Main(string[] args)
{
string temp = Path.GetTempPath();
string path = Environment.ExpandEnvironmentVariables(Path.Combine(temp, "rocksdb_prefix_example"));
var bbto = new BlockBasedTableOptions()
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false)
;
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
;
var columnFamilies = new ColumnFamilies
{
{ "default", new ColumnFamilyOptions().OptimizeForPointLookup(256) },
{ "test", new ColumnFamilyOptions()
//.SetWriteBufferSize(writeBufferSize)
//.SetMaxWriteBufferNumber(maxWriteBufferNumber)
//.SetMinWriteBufferNumberToMerge(minWriteBufferNumberToMerge)
.SetMemtableHugePageSize(2 * 1024 * 1024)
.SetPrefixExtractor(SliceTransform.CreateFixedPrefix((ulong)8))
.SetBlockBasedTableFactory(bbto) },
};
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var cf = db.GetColumnFamily("test");
db.Put("00000000Zero", "", cf: cf);
db.Put("00000000One", "", cf: cf);
db.Put("00000000Two", "", cf: cf);
db.Put("00000000Three", "", cf: cf);
db.Put("00000001Red", "", cf: cf);
db.Put("00000001Green", "", cf: cf);
db.Put("00000001Black", "", cf: cf);
db.Put("00000002Apple", "", cf: cf);
db.Put("00000002Cranberry", "", cf: cf);
db.Put("00000002Banana", "", cf: cf);
var readOptions = new ReadOptions();
using (var iter = db.NewIterator(readOptions: readOptions, cf: cf))
{
GC.Collect();
GC.WaitForPendingFinalizers();
var b = Encoding.UTF8.GetBytes("00000001");
iter.Seek(b);
while (iter.Valid())
{
Console.WriteLine(iter.StringKey());
iter.Next();
}
}
}
Console.WriteLine("Done...");
}
public LocalStateRepository(ILogger <LocalStateRepository> logger,
IOptions <RuntimeDirectory> runtimeDirOptions)
{
_logger = logger;
_runtimeDirectory = runtimeDirOptions;
_formatter = new BinaryFormatter();
var options = new DbOptions().SetCreateIfMissing(true);
_dbInstance = RocksDb.Open(options, Path.Combine(_runtimeDirectory.Value.LocalStateRepositoryDb, "LocalOptions.db"));
}
public RocksDbCacheClient(RpcClient rpcClient, string cachePath)
{
this.rpcClient = rpcClient;
if (!Directory.Exists(cachePath))
{
Directory.CreateDirectory(cachePath);
}
var columnFamilies = GetColumnFamilies(cachePath);
db = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), cachePath, columnFamilies);