public void ParallelGetTransaction()
{
var path = Path.Combine(Path.GetTempPath(), $"rocksdb_test_{Guid.NewGuid()}");
var store = new RocksDBStore(path);
Transaction <DumbAction>[] txs = new[]
{
Fx.Transaction1,
Fx.Transaction2,
Fx.Transaction3,
};
try
{
store.PutTransaction(Fx.Transaction1);
store.PutTransaction(Fx.Transaction2);
store.PutTransaction(Fx.Transaction3);
store.Dispose();
store = new RocksDBStore(path);
Enumerable.Range(0, 3).AsParallel().ForAll(i =>
{
Assert.NotNull(store.GetTransaction <DumbAction>(txs[i].Id));
});
}
finally
{
store.Dispose();
Directory.Delete(path, true);
}
}
public void ReopenStoreAfterDispose()
{
var path = Path.Combine(Path.GetTempPath(), $"rocksdb_test_{Guid.NewGuid()}");
try
{
var store = new RocksDBStore(path);
var stateStore =
new TrieStateStore(new MemoryKeyValueStore(), new MemoryKeyValueStore());
var blocks = new BlockChain <DumbAction>(
new NullPolicy <DumbAction>(),
new VolatileStagePolicy <DumbAction>(),
store,
stateStore,
Fx.GenesisBlock
);
store.Dispose();
store = new RocksDBStore(path);
store.Dispose();
}
finally
{
Directory.Delete(path, true);
}
}
public void ParallelGetBlock()
{
var path = Path.Combine(Path.GetTempPath(), $"rocksdb_test_{Guid.NewGuid()}");
var store = new RocksDBStore(path);
try
{
Guid cid = Guid.NewGuid();
store.AppendIndex(cid, Fx.Block1.Hash);
store.AppendIndex(cid, Fx.Block2.Hash);
store.AppendIndex(cid, Fx.Block3.Hash);
store.PutBlock(Fx.Block1);
store.PutBlock(Fx.Block2);
store.PutBlock(Fx.Block3);
store.Dispose();
store = new RocksDBStore(path);
Enumerable.Range(0, 3).AsParallel().ForAll(i =>
{
var bHash = store.IndexBlockHash(cid, i).Value;
var block = store.GetBlock <DumbAction>(Fx.GetHashAlgorithm, bHash);
Assert.NotNull(block);
});
}
finally
{
store.Dispose();
Directory.Delete(path, true);
}
}
public void ListChainIds()
{
var path = Path.Combine(Path.GetTempPath(), $"rocksdb_test_{Guid.NewGuid()}");
var store = new RocksDBStore(path);
try
{
Guid cid = Guid.NewGuid();
store.AppendIndex(cid, Fx.Block1.Hash);
store.AppendIndex(cid, Fx.Block2.Hash);
store.AppendIndex(cid, Fx.Block3.Hash);
store.PutBlock(Fx.Block1);
store.PutBlock(Fx.Block2);
store.PutBlock(Fx.Block3);
Assert.Single(store.ListChainIds());
store.ForkBlockIndexes(cid, Guid.NewGuid(), Fx.Block3.Hash);
Assert.Equal(2, store.ListChainIds().Count());
}
finally
{
store.Dispose();
Directory.Delete(path, true);
}
}
public RocksDBStoreFixture()
{
Path = System.IO.Path.Combine(
System.IO.Path.GetTempPath(),
$"rocksdb_test_{Guid.NewGuid()}"
);
Store = new RocksDBStore(Path, blockCacheSize: 2, txCacheSize: 2);
}
public RocksDBStoreFixture(bool mpt = false)
{
Path = System.IO.Path.Combine(
System.IO.Path.GetTempPath(),
$"rocksdb_test_{Guid.NewGuid()}"
);
var store = new RocksDBStore(Path, blockCacheSize: 2, txCacheSize: 2);
Store = store;
StateStore = mpt ? LoadTrieStateStore(Path) : store;
}
) GetBlockChain(
ILogger logger,
string storePath,
Guid?chainId = null,
IKeyValueStore stateKeyValueStore = null
)
{
var policySource = new BlockPolicySource(logger);
IBlockPolicy <NCAction> policy = policySource.GetPolicy();
IStagePolicy <NCAction> stagePolicy = new VolatileStagePolicy <NCAction>();
IStore store = new RocksDBStore(storePath);
if (stateKeyValueStore is null)
{
stateKeyValueStore = new RocksDBKeyValueStore(Path.Combine(storePath, "states"));
}
IStateStore stateStore = new TrieStateStore(stateKeyValueStore);
Guid chainIdValue
= chainId ??
store.GetCanonicalChainId() ??
throw new CommandExitedException(
"No canonical chain ID. Available chain IDs:\n " +
string.Join <Guid>("\n ", store.ListChainIds()),
1);
BlockHash genesisBlockHash;
try
{
genesisBlockHash = store.IterateIndexes(chainIdValue).First();
}
catch (InvalidOperationException)
{
throw new CommandExitedException(
$"The chain {chainIdValue} seems empty; try with another chain ID:\n " +
string.Join <Guid>("\n ", store.ListChainIds()),
1
);
}
Block <NCAction> genesis = store.GetBlock <NCAction>(
policy.GetHashAlgorithm,
genesisBlockHash
);
BlockChain <NCAction> chain = new BlockChain <NCAction>(
policy,
stagePolicy,
store,
stateStore,
genesis
);
return(chain, store, stateKeyValueStore, stateStore);
}
public RocksDBStoreFixture()
{
Path = System.IO.Path.Combine(
System.IO.Path.GetTempPath(),
$"rocksdb_test_{Guid.NewGuid()}"
);
Scheme = "rocksdb://";
var store = new RocksDBStore(Path, blockCacheSize: 2, txCacheSize: 2);
Store = store;
StateStore = LoadTrieStateStore(Path);
}
public void TestRocksDb()
{
using var plugin = new RocksDBStore();
TestPersistenceDelete(plugin.GetStore(path_rocksdb));
// Test all with the same store
TestStorage(plugin.GetStore(path_rocksdb));
// Test with different storages
TestPersistenceWrite(plugin.GetStore(path_rocksdb));
TestPersistenceRead(plugin.GetStore(path_rocksdb), true);
TestPersistenceDelete(plugin.GetStore(path_rocksdb));
TestPersistenceRead(plugin.GetStore(path_rocksdb), false);
}
public void Migration(
[Option('o', Description = "Path to migration target root path.")]
string originRootPath,
[Option('d', Description = "Path to migrated dist root path.")]
string distRootPath,
[Option('c', Description = "Skip the copy from target store.")]
bool skipCopy,
[Option('b', Description = "Skip the block from target store.")]
bool skipBlock,
[Option('t', Description = "Skip the transaction from target store.")]
bool skipTransaction
)
{
if (!skipCopy)
{
Console.WriteLine("Copy Start!");
DirectoryCopy(originRootPath, distRootPath, true);
Directory.Delete(Path.Combine(distRootPath, "tx"), true);
Directory.Delete(Path.Combine(distRootPath, "block"), true);
Console.WriteLine("Copy Success!");
}
else
{
Console.WriteLine("Skip copy");
}
var originStore = new MonoRocksDBStore(originRootPath);
var distStore = new RocksDBStore(distRootPath);
var totalLength = originStore.CountBlocks();
if (!skipBlock)
{
Console.WriteLine("Start migrate block.");
foreach (var item in
originStore.IterateBlockHashes().Select((value, i) => new { i, value }))
{
Console.WriteLine($"block progress: {item.i}/{totalLength}");
Block <NCAction> block = originStore.GetBlock <NCAction>(
_ => HashAlgorithmType.Of <SHA256>(), // thunk getter
item.value
);
distStore.PutBlock(block);
}
Console.WriteLine("Finish migrate block.");
}
totalLength = originStore.CountTransactions();
if (!skipTransaction)
{
Console.WriteLine("Start migrate transaction.");
foreach (var item in
originStore.IterateTransactionIds()
.Select((value, i) => new { i, value }))
{
Console.WriteLine($"tx progress: {item.i}/{totalLength}");
distStore.PutTransaction(originStore.GetTransaction <NCAction>(item.value));
}
Console.WriteLine("Finish migrate transaction.");
}
originStore.Dispose();
distStore.Dispose();
Console.WriteLine("Migration Success!");
}
static void Main(string[] args)
{
if (args.Length < 2)
{
Console.Error.WriteLine("Too few arguments.");
Environment.Exit(1);
return;
}
string storePath = args[0];
int limit = int.Parse(args[1]);
int offset = 0;
if (args.Length >= 3)
{
offset = int.Parse(args[2]);
}
if (limit < 0)
{
Console.Error.WriteLine("Limit value must be greater than 0. Entered value: {0}", limit);
Environment.Exit(1);
return;
}
if (offset < 0)
{
Console.Error.WriteLine("Offset value must be greater than 0. Entered value: {0}", offset);
Environment.Exit(1);
return;
}
Log.Logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Console().CreateLogger();
Libplanet.Crypto.CryptoConfig.CryptoBackend = new Secp256K1CryptoBackend <SHA256>();
var policySource = new BlockPolicySource(Log.Logger, LogEventLevel.Verbose);
IBlockPolicy <NCAction> policy =
policySource.GetPolicy(
// Explicitly set to lowest possible difficulty.
minimumDifficulty: BlockPolicySource.DifficultyStability,
maxBlockBytesPolicy: null,
minTransactionsPerBlockPolicy: null,
maxTransactionsPerBlockPolicy: null,
maxTransactionsPerSignerPerBlockPolicy: null,
authorizedMinersPolicy: null,
permissionedMinersPolicy: null);
IStagePolicy <NCAction> stagePolicy = new VolatileStagePolicy <NCAction>();
var store = new RocksDBStore(storePath);
if (!(store.GetCanonicalChainId() is Guid chainId))
{
Console.Error.WriteLine("There is no canonical chain: {0}", storePath);
Environment.Exit(1);
return;
}
if (!(store.IndexBlockHash(chainId, 0) is { } gHash))
{
Console.Error.WriteLine("There is no genesis block: {0}", storePath);
Environment.Exit(1);
return;
}
DateTimeOffset started = DateTimeOffset.UtcNow;
Block <NCAction> genesis = store.GetBlock <NCAction>(policy.GetHashAlgorithm, gHash);
IKeyValueStore stateKeyValueStore = new RocksDBKeyValueStore(Path.Combine(storePath, "states"));
var stateStore = new TrieStateStore(stateKeyValueStore);
var chain = new BlockChain <NCAction>(policy, stagePolicy, store, stateStore, genesis);
long height = chain.Tip.Index;
if (offset + limit > (int)height)
{
Console.Error.WriteLine(
"The sum of the offset and limit is greater than the chain tip index: {0}", height);
Environment.Exit(1);
return;
}
BlockHash[] blockHashes = store.IterateIndexes(chain.Id, offset, limit).Select((value, i) => value).ToArray();
Console.Error.WriteLine(
"Executing {0} blocks: {1}-{2} (inclusive).",
blockHashes.Length,
blockHashes[0],
blockHashes.Last()
);
Block <NCAction>[] blocks = blockHashes.Select(h => chain[h]).ToArray();
DateTimeOffset blocksLoaded = DateTimeOffset.UtcNow;
long txs = 0;
long actions = 0;
foreach (Block <NCAction> block in blocks)
{
Console.Error.WriteLine(
"Block #{0} {1}; {2} txs",
block.Index,
block.Hash,
block.Transactions.Count()
);
IEnumerable <ActionEvaluation> blockEvals =
chain.ExecuteActions(block, StateCompleterSet <NCAction> .Reject);
SetStates(
chain.Id,
store,
stateStore,
block,
blockEvals.ToArray(),
buildStateReferences: true
);
txs += block.Transactions.LongCount();
actions += block.Transactions.Sum(tx => tx.Actions.LongCount()) + 1;
}
DateTimeOffset ended = DateTimeOffset.UtcNow;
Console.WriteLine("Loading blocks\t{0}", blocksLoaded - started);
long execActionsTotalMilliseconds = (long)(ended - blocksLoaded).TotalMilliseconds;
Console.WriteLine("Executing actions\t{0}ms", execActionsTotalMilliseconds);
Console.WriteLine("Average per block\t{0}ms", execActionsTotalMilliseconds / blockHashes.Length);
Console.WriteLine("Average per tx\t{0}ms", execActionsTotalMilliseconds / txs);
Console.WriteLine("Average per action\t{0}ms", execActionsTotalMilliseconds / actions);
Console.WriteLine("Total elapsed\t{0}", ended - started);
}
static void Main(string[] args)
{
if (args.Length < 2)
{
Console.Error.WriteLine("Too few arguments.");
Environment.Exit(1);
return;
}
string storePath = args[0];
int limit = int.Parse(args[1]);
ILogger logger = new LoggerConfiguration().CreateLogger();
Libplanet.Crypto.CryptoConfig.CryptoBackend = new Secp256K1CryptoBackend <SHA256>();
var policySource = new BlockPolicySource(logger, LogEventLevel.Verbose);
IBlockPolicy <NCAction> policy =
policySource.GetPolicy(BlockPolicySource.DifficultyBoundDivisor + 1, 0);
IStagePolicy <NCAction> stagePolicy = new VolatileStagePolicy <NCAction>();
var store = new RocksDBStore(storePath);
if (!(store.GetCanonicalChainId() is Guid chainId))
{
Console.Error.WriteLine("There is no canonical chain: {0}", storePath);
Environment.Exit(1);
return;
}
if (!(store.IndexBlockHash(chainId, 0) is HashDigest <SHA256> gHash))
{
Console.Error.WriteLine("There is no genesis block: {0}", storePath);
Environment.Exit(1);
return;
}
DateTimeOffset started = DateTimeOffset.UtcNow;
Block <NCAction> genesis = store.GetBlock <NCAction>(gHash);
IKeyValueStore stateRootKeyValueStore = new RocksDBKeyValueStore(Path.Combine(storePath, "state_hashes")),
stateKeyValueStore = new RocksDBKeyValueStore(Path.Combine(storePath, "states"));
IStateStore stateStore = new TrieStateStore(stateKeyValueStore, stateRootKeyValueStore);
var chain = new BlockChain <NCAction>(policy, stagePolicy, store, stateStore, genesis);
long height = chain.Tip.Index;
HashDigest <SHA256>[] blockHashes = limit < 0
? chain.BlockHashes.SkipWhile((_, i) => i < height + limit).ToArray()
: chain.BlockHashes.Take(limit).ToArray();
Console.Error.WriteLine(
"Executing {0} blocks: {1}-{2} (inclusive).",
blockHashes.Length,
blockHashes[0],
blockHashes.Last()
);
Block <NCAction>[] blocks = blockHashes.Select(h => chain[h]).ToArray();
DateTimeOffset blocksLoaded = DateTimeOffset.UtcNow;
long txs = 0;
long actions = 0;
foreach (Block <NCAction> block in blocks)
{
Console.Error.WriteLine(
"Block #{0} {1}; {2} txs",
block.Index,
block.Hash,
block.Transactions.Count()
);
IEnumerable <ActionEvaluation> blockEvals;
if (block.Index > 0)
{
blockEvals = block.Evaluate(
DateTimeOffset.UtcNow,
address => chain.GetState(address, block.Hash),
(address, currency) => chain.GetBalance(address, currency, block.Hash)
);
}
else
{
blockEvals = block.Evaluate(
DateTimeOffset.UtcNow,
_ => null,
((_, currency) => new FungibleAssetValue(currency))
);
}
SetStates(chain.Id, stateStore, block, blockEvals.ToArray(), buildStateReferences: true);
txs += block.Transactions.LongCount();
actions += block.Transactions.Sum(tx => tx.Actions.LongCount()) + 1;
}
DateTimeOffset ended = DateTimeOffset.UtcNow;
Console.WriteLine("Loading blocks\t{0}", blocksLoaded - started);
long execActionsTotalMilliseconds = (long)(ended - blocksLoaded).TotalMilliseconds;
Console.WriteLine("Executing actions\t{0}ms", execActionsTotalMilliseconds);
Console.WriteLine("Average per block\t{0}ms", execActionsTotalMilliseconds / blockHashes.Length);
Console.WriteLine("Average per tx\t{0}ms", execActionsTotalMilliseconds / txs);
Console.WriteLine("Average per action\t{0}ms", execActionsTotalMilliseconds / actions);
Console.WriteLine("Total elapsed\t{0}", ended - started);
}
public void PruneOutdatedChainsRocksDb()
{
var path = Path.Combine(Path.GetTempPath(), $"rocksdb_test_{Guid.NewGuid()}");
var store = new RocksDBStore(path);
RocksDb chainDb = null;
int KeysWithChainId(RocksDb db, Guid cid)
{
using (Iterator it = db.NewIterator())
{
byte[] key = cid.ToByteArray();
int count = 0;
for (it.SeekToFirst(); it.Valid(); it.Next())
{
if (!it.Key().Skip(1).ToArray().StartsWith(key))
{
continue;
}
count++;
}
return(count);
}
}
try
{
store.PutBlock(Fx.GenesisBlock);
store.PutBlock(Fx.Block1);
store.PutBlock(Fx.Block2);
store.PutBlock(Fx.Block3);
Guid cid1 = Guid.NewGuid();
int guidLength = cid1.ToByteArray().Length;
store.AppendIndex(cid1, Fx.GenesisBlock.Hash);
store.AppendIndex(cid1, Fx.Block1.Hash);
store.AppendIndex(cid1, Fx.Block2.Hash);
Assert.Single(store.ListChainIds());
Assert.Equal(
new[] { Fx.GenesisBlock.Hash, Fx.Block1.Hash, Fx.Block2.Hash },
store.IterateIndexes(cid1, 0, null));
Guid cid2 = Guid.NewGuid();
store.ForkBlockIndexes(cid1, cid2, Fx.Block1.Hash);
store.AppendIndex(cid2, Fx.Block2.Hash);
store.AppendIndex(cid2, Fx.Block3.Hash);
Assert.Equal(2, store.ListChainIds().Count());
Assert.Equal(
new[] { Fx.GenesisBlock.Hash, Fx.Block1.Hash, Fx.Block2.Hash, Fx.Block3.Hash },
store.IterateIndexes(cid2, 0, null));
Guid cid3 = Guid.NewGuid();
store.ForkBlockIndexes(cid1, cid3, Fx.Block2.Hash);
Assert.Equal(3, store.ListChainIds().Count());
Assert.Equal(
new[] { Fx.GenesisBlock.Hash, Fx.Block1.Hash, Fx.Block2.Hash },
store.IterateIndexes(cid3, 0, null));
Assert.Throws <InvalidOperationException>(() => store.PruneOutdatedChains());
store.PruneOutdatedChains(true);
store.SetCanonicalChainId(cid3);
store.PruneOutdatedChains();
Assert.Single(store.ListChainIds());
Assert.Equal(
new[] { Fx.GenesisBlock.Hash, Fx.Block1.Hash, Fx.Block2.Hash },
store.IterateIndexes(cid3, 0, null));
Assert.Equal(3, store.CountIndex(cid3));
store.Dispose();
store = null;
chainDb = RocksDb.Open(new DbOptions(), Path.Combine(path, "chain"));
Assert.Equal(0, KeysWithChainId(chainDb, cid1));
Assert.Equal(0, KeysWithChainId(chainDb, cid2));
Assert.NotEqual(0, KeysWithChainId(chainDb, cid3));
}
finally
{
store?.Dispose();
chainDb?.Dispose();
Directory.Delete(path, true);
}
}
