public void GetAsyncWithExistingBlocksReturnsBlocks()
{
string dir = CreateTestDir(this);
var blocks = new Block[10];
blocks[0] = this.Network.Consensus.ConsensusFactory.CreateBlock();
for (int i = 1; i < blocks.Length; i++)
{
blocks[i] = this.Network.Consensus.ConsensusFactory.CreateBlock();
blocks[i].Header.HashPrevBlock = blocks[i - 1].Header.GetHash();
}
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, dir))
{
for (int i = 0; i < blocks.Length; i++)
{
engine.Put(DBH.Key(BlockRepository.BlockTableName, blocks[i].GetHash().ToBytes()), blocks[i].ToBytes());
}
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
List <Block> result = repository.GetBlocks(blocks.Select(b => b.GetHash()).ToList());
Assert.Equal(blocks.Length, result.Count);
for (int i = 0; i < 10; i++)
{
Assert.Equal(blocks[i].GetHash(), result[i].GetHash());
}
}
}
public void SetTxIndexUpdatesTxIndex()
{
string dir = CreateTestDir(this);
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, dir))
{
engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true));
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
repository.SetTxIndex(false);
}
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, dir))
{
bool txIndexRow = BitConverter.ToBoolean(engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[1])));
Assert.False(txIndexRow);
}
}
public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash)
{
if (this.headers.TryGetValue(hash, out BlockHeader blockHeader))
{
return(blockHeader);
}
// TODO: Bring in uint256 span optimisations
byte[] bytes = hash.ToBytes();
lock (this.locker)
{
bytes = this.leveldb.Get(DBH.Key(HeaderTableName, bytes));
}
if (bytes == null)
{
throw new ApplicationException("Header must exist if requested");
}
blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader();
blockHeader.FromBytes(bytes, this.network.Consensus.ConsensusFactory);
// If the header is 500 blocks behind tip or 100 blocks ahead cache it.
if ((chainedHeader.Height > this.ChainIndexer.Height - 500) && (chainedHeader.Height <= this.ChainIndexer.Height + 100))
{
this.headers.AddOrUpdate(hash, blockHeader);
}
return(blockHeader);
}
public void DeleteAsyncRemovesBlocksAndTransactions()
{
string dir = CreateTestDir(this);
Block block = this.Network.CreateBlock();
block.Transactions.Add(this.Network.CreateTransaction());
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
engine.Put(DBH.Key(RocksdbBlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes());
engine.Put(DBH.Key(RocksdbBlockRepository.TransactionTableName, block.Transactions[0].GetHash().ToBytes()), block.GetHash().ToBytes());
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true));
}
var tip = new HashHeightPair(new uint256(45), 100);
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
repository.Delete(tip, new List <uint256> {
block.GetHash()
});
}
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
byte[] blockHashKeyRow = engine.Get(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[0]));
Dictionary <byte[], byte[]> blockDict = engine.SelectDictionary(RocksdbBlockRepository.BlockTableName);
Dictionary <byte[], byte[]> transDict = engine.SelectDictionary(RocksdbBlockRepository.TransactionTableName);
Assert.Equal(tip, this.DataStoreSerializer.Deserialize <HashHeightPair>(blockHashKeyRow));
Assert.Empty(blockDict);
Assert.Empty(transDict);
}
}
public async Task GetAsync_WithWrongBlockHeightReturnsNullAsync()
{
string folder = CreateTestDir(this);
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, folder))
{
engine.Put(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(1)), this.dataStoreSerializer.Serialize(CreateNewProvenBlockHeaderMock()));
engine.Put(DBH.Key(BlockHashHeightTable, new byte[0]), this.DataStoreSerializer.Serialize(new HashHeightPair(new uint256(), 1)));
}
using (LevelDbProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder))
{
// Select a different block height.
ProvenBlockHeader outHeader = await repo.GetAsync(2).ConfigureAwait(false);
outHeader.Should().BeNull();
// Select the original item inserted into the table
outHeader = await repo.GetAsync(1).ConfigureAwait(false);
outHeader.Should().NotBeNull();
}
}
public async Task GetAsync_ReadsProvenBlockHeaderAsync()
{
string folder = CreateTestDir(this);
ProvenBlockHeader headerIn = CreateNewProvenBlockHeaderMock();
int blockHeight = 1;
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, folder))
{
engine.Put(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight)), this.dataStoreSerializer.Serialize(headerIn));
}
// Query the repository for the item that was inserted in the above code.
using (LevelDbProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder))
{
var headerOut = await repo.GetAsync(blockHeight).ConfigureAwait(false);
headerOut.Should().NotBeNull();
uint256.Parse(headerOut.ToString()).Should().Be(headerOut.GetHash());
}
}
/// <inheritdoc />
public void PruneAndCompactDatabase(ChainedHeader blockRepositoryTip, Network network, bool nodeInitializing)
{
this.logger.LogInformation($"Pruning started...");
if (this.PrunedTip == null)
{
Block genesis = network.GetGenesis();
this.PrunedTip = new HashHeightPair(genesis.GetHash(), 0);
this.blockRepository.Leveldb.Put(DBH.Key(BlockRepository.CommonTableName, prunedTipKey), this.dataStoreSerializer.Serialize(this.PrunedTip));
}
if (nodeInitializing)
{
if (this.IsDatabasePruned())
{
return;
}
this.PrepareDatabaseForCompacting(blockRepositoryTip);
}
this.CompactDataBase();
this.logger.LogInformation($"Pruning complete.");
return;
}
/// <inheritdoc />
public uint256 GetBlockIdByTransactionId(uint256 trxid)
{
Guard.NotNull(trxid, nameof(trxid));
if (!this.TxIndex)
{
this.logger.LogTrace("(-)[NO_TXINDEX]:null");
return(default(uint256));
}
if (this.genesisTransactions.ContainsKey(trxid))
{
return(this.network.GenesisHash);
}
uint256 res = null;
lock (this.Locker)
{
byte[] transactionRow = this.leveldb.Get(DBH.Key(TransactionTableName, trxid.ToBytes()));
if (transactionRow != null)
{
res = new uint256(transactionRow);
}
}
return(res);
}
public bool PutHeader(BlockHeader blockHeader)
{
ConsensusFactory consensusFactory = this.network.Consensus.ConsensusFactory;
if (blockHeader is ProvenBlockHeader)
{
// If ProvenBlockHeader copy the header parameters.
BlockHeader newHeader = consensusFactory.CreateBlockHeader();
newHeader.Bits = blockHeader.Bits;
newHeader.Time = blockHeader.Time;
newHeader.Nonce = blockHeader.Nonce;
newHeader.Version = blockHeader.Version;
newHeader.HashMerkleRoot = blockHeader.HashMerkleRoot;
newHeader.HashPrevBlock = blockHeader.HashPrevBlock;
blockHeader = newHeader;
}
lock (this.locker)
{
this.leveldb.Put(DBH.Key(HeaderTableName, blockHeader.GetHash().ToBytes()), blockHeader.ToBytes(consensusFactory));
}
return(true);
}
public void GetTrxAsyncWithTransactionReturnsExistingTransaction()
{
string dir = CreateTestDir(this);
Transaction trans = this.Network.CreateTransaction();
trans.Version = 125;
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, dir))
{
Block block = this.Network.CreateBlock();
block.Header.GetHash();
block.Transactions.Add(trans);
engine.Put(DBH.Key(BlockRepository.BlockTableName, block.Header.GetHash().ToBytes()), block.ToBytes());
engine.Put(DBH.Key(BlockRepository.TransactionTableName, trans.GetHash().ToBytes()), block.Header.GetHash().ToBytes());
engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DataStoreSerializer.Serialize(new HashHeightPair(uint256.Zero, 1)));
engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true));
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
Assert.Equal((uint)125, repository.GetTransactionById(trans.GetHash()).Version);
}
}
public void DoesNotOverwriteExistingBlockAndTxIndexOnFirstLoad()
{
string dir = CreateTestDir(this);
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, dir))
{
engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DataStoreSerializer.Serialize(new HashHeightPair(new uint256(56), 1)));
engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true));
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
}
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, dir))
{
byte[] blockRow = engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[0]));
bool txIndexRow = BitConverter.ToBoolean(engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[1])));
Assert.Equal(new HashHeightPair(new uint256(56), 1), this.DataStoreSerializer.Deserialize <HashHeightPair>(blockRow));
Assert.True(txIndexRow);
}
}
/// <inheritdoc />
public void UpdatePrunedTip(ChainedHeader tip)
{
this.PrunedTip = new HashHeightPair(tip);
lock (this.blockRepository.Locker)
{
this.blockRepository.Leveldb.Put(DBH.Key(BlockRepository.CommonTableName, prunedTipKey), this.dataStoreSerializer.Serialize(this.PrunedTip));
}
}
/// <summary>
/// Set's the hash and height tip of the new <see cref="ProvenBlockHeader"/>.
/// </summary>
/// <param name="newTip"> Hash height pair of the new block tip.</param>
private void SetTip(HashHeightPair newTip)
{
Guard.NotNull(newTip, nameof(newTip));
lock (this.locker)
{
this.rocksDb.Put(DBH.Key(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey), this.dBreezeSerializer.Serialize(newTip));
}
}
/// <summary>
/// Set's the hash and height tip of the new <see cref="ProvenBlockHeader"/>.
/// </summary>
/// <param name="transaction"> Open DBreeze transaction.</param>
/// <param name="newTip"> Hash height pair of the new block tip.</param>
private void SetTip(HashHeightPair newTip)
{
Guard.NotNull(newTip, nameof(newTip));
lock (this.locker)
{
this.leveldb.Put(DBH.Key(blockHashHeightTable, blockHashHeightKey), this.dataStoreSerializer.Serialize(newTip));
}
}
/// <inheritdoc/>
public Transaction[] GetTransactionsByIds(uint256[] trxids, CancellationToken cancellation = default(CancellationToken))
{
if (!this.TxIndex)
{
this.logger.LogTrace("(-)[TX_INDEXING_DISABLED]:null");
return(null);
}
Transaction[] txes = new Transaction[trxids.Length];
lock (this.Locker)
{
for (int i = 0; i < trxids.Length; i++)
{
cancellation.ThrowIfCancellationRequested();
bool alreadyFetched = trxids.Take(i).Any(x => x == trxids[i]);
if (alreadyFetched)
{
this.logger.LogDebug("Duplicated transaction encountered. Tx id: '{0}'.", trxids[i]);
txes[i] = txes.First(x => x.GetHash() == trxids[i]);
continue;
}
if (this.genesisTransactions.TryGetValue(trxids[i], out Transaction genesisTransaction))
{
txes[i] = genesisTransaction;
continue;
}
byte[] transactionRow = this.leveldb.Get(DBH.Key(TransactionTableName, trxids[i].ToBytes()));
if (transactionRow == null)
{
this.logger.LogTrace("(-)[NO_TX_ROW]:null");
return(null);
}
byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, transactionRow));
if (blockRow != null)
{
this.logger.LogTrace("(-)[NO_BLOCK]:null");
return(null);
}
var block = this.dataStoreSerializer.Deserialize <Block>(blockRow);
Transaction tx = block.Transactions.FirstOrDefault(t => t.GetHash() == trxids[i]);
txes[i] = tx;
}
}
return(txes);
}
public bool PutHeader(BlockHeader blockHeader)
{
ConsensusFactory consensusFactory = this.network.Consensus.ConsensusFactory;
lock (this.locker)
{
this.leveldb.Put(DBH.Key(HeaderTableName, blockHeader.GetHash().ToReadOnlySpan()), blockHeader.ToBytes(consensusFactory));
}
return(true);
}
private void LoadPrunedTip(DB leveldb)
{
if (this.PrunedTip == null)
{
byte[] row = leveldb.Get(DBH.Key(BlockRepository.CommonTableName, prunedTipKey));
if (row != null)
{
this.PrunedTip = this.dataStoreSerializer.Deserialize <HashHeightPair>(row);
}
}
}
private void LoadPrunedTip(RocksDbSharp.RocksDb rocksdb)
{
if (this.PrunedTip == null)
{
lock (this.blockRepository.Locker)
{
byte[] row = rocksdb.Get(DBH.Key(BlockRepository.CommonTableName, prunedTipKey));
if (row != null)
{
this.PrunedTip = this.dataStoreSerializer.Deserialize <HashHeightPair>(row);
}
}
}
}
protected virtual void OnInsertBlocks(List <Block> blocks)
{
var transactions = new List <(Transaction, Block)>();
var byteListComparer = new ByteListComparer();
var blockDict = new Dictionary <uint256, Block>();
// Gather blocks.
foreach (Block block in blocks)
{
uint256 blockId = block.GetHash();
blockDict[blockId] = block;
}
// Sort blocks. Be consistent in always converting our keys to byte arrays using the ToBytes method.
List <KeyValuePair <uint256, Block> > blockList = blockDict.ToList();
blockList.Sort((pair1, pair2) => byteListComparer.Compare(pair1.Key.ToBytes(), pair2.Key.ToBytes()));
using (var batch = new WriteBatch())
{
// Index blocks.
foreach (KeyValuePair <uint256, Block> kv in blockList)
{
uint256 blockId = kv.Key;
Block block = kv.Value;
// If the block is already in store don't write it again.
byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, blockId.ToBytes()));
if (blockRow == null)
{
batch.Put(DBH.Key(BlockTableName, blockId.ToBytes()), this.dBreezeSerializer.Serialize(block));
if (this.TxIndex)
{
foreach (Transaction transaction in block.Transactions)
{
transactions.Add((transaction, block));
}
}
}
}
this.leveldb.Write(batch);
}
if (this.TxIndex)
{
this.OnInsertTransactions(transactions);
}
}
public void ExistAsyncWithExistingBlockReturnsTrue()
{
string dir = CreateTestDir(this);
Block block = this.Network.Consensus.ConsensusFactory.CreateBlock();
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
engine.Put(DBH.Key(RocksdbBlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes());
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
Assert.True(repository.Exist(block.GetHash()));
}
}
public void GetTrxAsyncWithoutTransactionIndexReturnsNewTransaction()
{
string dir = CreateTestDir(this);
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[0]), this.DataStoreSerializer.Serialize(new HashHeightPair(uint256.Zero, 1)));
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(false));
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
Assert.Equal(default(Transaction), repository.GetTransactionById(uint256.Zero));
}
}
/// <summary>
/// Inserts <see cref="ProvenBlockHeader"/> items into to the database.
/// </summary>
/// <param name="headers"> List of <see cref="ProvenBlockHeader"/> items to save.</param>
private void InsertHeaders(SortedDictionary <int, ProvenBlockHeader> headers)
{
using var batch = new WriteBatch();
{
foreach (KeyValuePair <int, ProvenBlockHeader> header in headers)
{
batch.Put(DBH.Key(BlockHeaderRepositoryConstants.ProvenBlockHeaderTable, BitConverter.GetBytes(header.Key)), this.dBreezeSerializer.Serialize(header.Value));
}
lock (this.locker)
{
this.rocksDb.Write(batch);
}
}
}
public void GetTrxBlockIdAsyncWithoutExistingTransactionReturnsNull()
{
string dir = CreateTestDir(this);
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[0]), this.DataStoreSerializer.Serialize(new HashHeightPair(uint256.Zero, 1)));
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true));
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
Assert.Null(repository.GetBlockIdByTransactionId(new uint256(26)));
}
}
public void PutChainData(IEnumerable <ChainDataItem> items)
{
using (var batch = new WriteBatch())
{
foreach (var item in items)
{
batch.Put(DBH.Key(ChainTableName, BitConverter.GetBytes(item.Height)), item.Data.ToBytes(this.network.Consensus.ConsensusFactory));
}
lock (this.locker)
{
this.leveldb.Write(batch);
}
}
}
/// <inheritdoc />
public void PrepareDatabase()
{
if (this.PrunedTip == null)
{
Block genesis = this.network.GetGenesis();
this.PrunedTip = new HashHeightPair(genesis.GetHash(), 0);
lock (this.blockRepository.Locker)
{
this.blockRepository.Leveldb.Put(DBH.Key(BlockRepository.CommonTableName, prunedTipKey), this.dataStoreSerializer.Serialize(this.PrunedTip));
}
}
return;
}
/// <summary>
/// Retrieves the current <see cref="HashHeightPair"/> tip from disk.
/// </summary>
/// <returns> Hash of blocks current tip.</returns>
private HashHeightPair GetTipHash()
{
HashHeightPair tipHash = null;
byte[] row = null;
lock (this.locker)
{
row = this.leveldb.Get(DBH.Key(blockHashHeightTable, blockHashHeightKey));
}
if (row != null)
{
tipHash = this.dataStoreSerializer.Deserialize <HashHeightPair>(row);
}
return(tipHash);
}
public void InitializesGenesisBlockAndTxIndexOnFirstLoad()
{
string dir = CreateTestDir(this);
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
}
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
byte[] blockRow = engine.Get(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[0]));
bool txIndexRow = BitConverter.ToBoolean(engine.Get(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[1])));
Assert.Equal(this.Network.GetGenesis().GetHash(), this.DataStoreSerializer.Deserialize <HashHeightPair>(blockRow).Hash);
Assert.False(txIndexRow);
}
}
/// <summary>
/// Retrieves the current <see cref="HashHeightPair"/> tip from disk.
/// </summary>
/// <returns> Hash of blocks current tip.</returns>
private HashHeightPair GetTipHash()
{
HashHeightPair tipHash = null;
byte[] row = null;
lock (this.locker)
{
row = this.rocksDb.Get(DBH.Key(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey));
}
if (row != null)
{
tipHash = this.dBreezeSerializer.Deserialize <HashHeightPair>(row);
}
return(tipHash);
}
public void GetAsyncWithExistingBlockReturnsBlock()
{
string dir = CreateTestDir(this);
Block block = this.Network.Consensus.ConsensusFactory.CreateBlock();
using (var engine = new DB(new Options()
{
CreateIfMissing = true
}, dir))
{
engine.Put(DBH.Key(BlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes());
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
Assert.Equal(block.GetHash(), repository.GetBlock(block.GetHash()).GetHash());
}
}
/// <summary>
/// Inserts <see cref="ProvenBlockHeader"/> items into to the database.
/// </summary>
/// <param name="headers"> List of <see cref="ProvenBlockHeader"/> items to save.</param>
private void InsertHeaders(SortedDictionary <int, ProvenBlockHeader> headers)
{
using (var batch = new WriteBatch())
{
foreach (KeyValuePair <int, ProvenBlockHeader> header in headers)
{
batch.Put(DBH.Key(provenBlockHeaderTable, BitConverter.GetBytes(header.Key)), this.dataStoreSerializer.Serialize(header.Value));
}
lock (this.locker)
{
this.leveldb.Write(batch);
}
}
// Store the latest ProvenBlockHeader in memory.
this.provenBlockHeaderTip = headers.Last().Value;
}