public CoreDaemon(ICoreRules rules, IStorageManager storageManager)
{
this.rules = rules;
this.storageManager = storageManager;
coreStorage = new CoreStorage(storageManager);
// create chain state builder
chainStateBuilder = new ChainStateBuilder(this.rules, coreStorage, this.storageManager);
// create unconfirmed txes builder
unconfirmedTxesBuilder = new UnconfirmedTxesBuilder(this, coreStorage, this.storageManager);
// create workers
targetChainWorker = new TargetChainWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(0), maxIdleTime: TimeSpan.FromSeconds(30)),
ChainParams, coreStorage);
chainStateWorker = new ChainStateWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(0), maxIdleTime: TimeSpan.FromSeconds(5)),
targetChainWorker, chainStateBuilder, this.rules, coreStorage);
unconfirmedTxesWorker = new UnconfirmedTxesWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(0), maxIdleTime: TimeSpan.FromSeconds(5)),
chainStateWorker, unconfirmedTxesBuilder, coreStorage);
pruningWorker = new PruningWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromSeconds(0), maxIdleTime: TimeSpan.FromMinutes(5)),
this, this.storageManager, chainStateWorker);
defragWorker = new DefragWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMinutes(5), maxIdleTime: TimeSpan.FromMinutes(5)),
this.storageManager);
gcWorker = new WorkerMethod("GC Worker", GcWorker,
initialNotify: true, minIdleTime: TimeSpan.FromMinutes(5), maxIdleTime: TimeSpan.FromMinutes(5));
utxoScanWorker = new WorkerMethod("UTXO Scan Worker", UtxoScanWorker,
initialNotify: true, minIdleTime: TimeSpan.FromSeconds(60), maxIdleTime: TimeSpan.FromSeconds(60));
statsWorker = new StatsWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMinutes(0), maxIdleTime: TimeSpan.MaxValue),
this);
// wire events
chainStateWorker.BlockMissed += HandleBlockMissed;
targetChainWorker.OnTargetChainChanged += HandleTargetChainChanged;
chainStateWorker.OnChainStateChanged += HandleChainStateChanged;
pruningWorker.OnWorkFinished += defragWorker.NotifyWork;
unconfirmedTxesBuilder.UnconfirmedTxAdded += RaiseUnconfirmedTxAdded;
unconfirmedTxesBuilder.TxesConfirmed += RaiseTxesConfirmed;
unconfirmedTxesBuilder.TxesUnconfirmed += RaiseTxesUnconfirmed;
}
public void TestPruneAllData()
{
// create genesis block
var genesisblock = CreateFakeBlock(1);
var genesisHeader = new ChainedHeader(genesisblock.Header, height: 0, totalWork: genesisblock.Header.CalculateWork().ToBigInteger(), dateSeen: DateTimeOffset.Now);
// create a block
var txCount = 100;
var block = CreateFakeBlock(txCount, genesisblock.Hash);
var chainedHeader = ChainedHeader.CreateFromPrev(genesisHeader, block.Header, dateSeen: DateTimeOffset.Now);
// create a long chain based off the block, to account for pruning buffer
var fakeHeaders = new FakeHeaders(new[] { genesisHeader, chainedHeader });
var chain = new ChainBuilder(Enumerable.Concat(new[] { genesisHeader, chainedHeader }, Enumerable.Range(0, 2000).Select(x => fakeHeaders.NextChained()))).ToImmutable();
// mock core daemon to return the chain
var coreDaemon = new Mock <ICoreDaemon>();
coreDaemon.Setup(x => x.CurrentChain).Returns(chain);
// create memory storage with the block
var storageManager = new MemoryStorageManager();
storageManager.BlockTxesStorage.TryAddBlockTransactions(block.Hash, block.BlockTxes);
// initialize the pruning worker
var workerConfig = new WorkerConfig(initialNotify: false, minIdleTime: TimeSpan.Zero, maxIdleTime: TimeSpan.MaxValue);
using (var pruningWorker = new PruningWorker(workerConfig, coreDaemon.Object, storageManager, null))
// get a chain state cursor
using (var handle = storageManager.OpenChainStateCursor())
{
var chainStateCursor = handle.Item;
// set the pruning worker to prune all data
pruningWorker.Mode = PruningMode.TxIndex | PruningMode.BlockSpentIndex | PruningMode.BlockTxesPreserveMerkle;
// wire event to wait for work
var workFinishedEvent = new AutoResetEvent(false);
pruningWorker.OnWorkFinished += () => workFinishedEvent.Set();
// wire event to track exceptions
Exception workException = null;
pruningWorker.OnWorkError += e => workException = e;
// start the worker
pruningWorker.Start();
// pick a random pruning order
var random = new Random();
var pruneOrderSource = Enumerable.Range(0, txCount).ToList();
var pruneOrder = new List <int>(txCount);
while (pruneOrderSource.Count > 0)
{
var randomIndex = random.Next(pruneOrderSource.Count);
pruneOrder.Add(pruneOrderSource[randomIndex]);
pruneOrderSource.RemoveAt(randomIndex);
}
// add an unspent tx for each transaction to storage
var unspentTxes = new UnspentTx[block.Transactions.Length];
chainStateCursor.BeginTransaction();
for (var txIndex = 0; txIndex < block.Transactions.Length; txIndex++)
{
var tx = block.Transactions[txIndex];
var unspentTx = new UnspentTx(tx.Hash, blockIndex: 1, txIndex: txIndex, txVersion: tx.Version, isCoinbase: txIndex == 0, outputStates: new OutputStates(1, OutputState.Spent));
unspentTxes[txIndex] = unspentTx;
chainStateCursor.TryAddUnspentTx(unspentTx);
}
chainStateCursor.CommitTransaction();
// create a memory pruning cursor to verify expected pruning results
var pruneCursor = new MemoryMerkleTreePruningCursor <BlockTxNode>(block.BlockTxes.Select(x => (BlockTxNode)x));
// prune each transaction in random order
var pruneHeight = 0;
foreach (var pruneTxIndex in pruneOrder)
{
// create a spent tx to prune the transaction
var pruneTx = block.Transactions[pruneTxIndex];
var spentTxes = BlockSpentTxes.CreateRange(new[] { unspentTxes[pruneTxIndex].ToSpentTx() });
// store the spent txes for the current pruning block
pruneHeight++;
chainStateCursor.BeginTransaction();
Assert.IsTrue(chainStateCursor.TryAddBlockSpentTxes(pruneHeight, spentTxes));
pruningWorker.PrunableHeight = pruneHeight;
// verify unspent tx is present before pruning
Assert.IsTrue(chainStateCursor.ContainsUnspentTx(pruneTx.Hash));
chainStateCursor.CommitTransaction();
// notify the pruning worker and wait
pruningWorker.NotifyWork();
workFinishedEvent.WaitOne();
// verify unspent tx is removed after pruning
chainStateCursor.BeginTransaction();
Assert.IsFalse(chainStateCursor.ContainsUnspentTx(pruneTx.Hash));
// verify unspent tx outputs are removed after pruning
for (var outputIndex = 0; outputIndex < pruneTx.Outputs.Length; outputIndex++)
{
Assert.IsFalse(chainStateCursor.ContainsUnspentTxOutput(new TxOutputKey(pruneTx.Hash, (uint)outputIndex)));
}
// verify the spent txes were removed
Assert.IsFalse(chainStateCursor.ContainsBlockSpentTxes(pruneHeight));
chainStateCursor.RollbackTransaction();
// prune to determine expected results
MerkleTree.PruneNode(pruneCursor, pruneTxIndex);
var expectedPrunedTxes = pruneCursor.ReadNodes().ToList();
// retrieve the actual transaction after pruning
IEnumerator <BlockTxNode> actualPrunedTxNodes;
Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out actualPrunedTxNodes));
// verify the actual pruned transactions match the expected results
CollectionAssert.AreEqual(expectedPrunedTxes, actualPrunedTxNodes.UsingAsEnumerable().ToList());
}
// verify all unspent txes were removed
chainStateCursor.BeginTransaction();
Assert.AreEqual(0, chainStateCursor.ReadUnspentTransactions().Count());
chainStateCursor.CommitTransaction();
// verify final block with all transactions pruned
IEnumerator <BlockTxNode> finalPrunedTxNodes;
Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out finalPrunedTxNodes));
var finalPrunedTxesList = finalPrunedTxNodes.UsingAsEnumerable().ToList();
Assert.AreEqual(1, finalPrunedTxesList.Count);
Assert.AreEqual(block.Header.MerkleRoot, finalPrunedTxesList.Single().Hash);
// verify no work exceptions occurred
Assert.IsNull(workException);
}
}
public CoreDaemon(Logger logger, IKernel kernel, IBlockchainRules rules, IStorageManager storageManager)
{
this.logger = logger;
this.shutdownToken = new CancellationTokenSource();
this.kernel = kernel;
this.rules = rules;
this.storageManager = storageManager;
this.coreStorage = new CoreStorage(storageManager, logger);
// write genesis block out to storage
this.coreStorage.AddGenesisBlock(this.rules.GenesisChainedHeader);
this.coreStorage.TryAddBlock(this.rules.GenesisBlock);
// create chain state builder
this.chainStateBuilder = new ChainStateBuilder(this.logger, this.rules, this.coreStorage);
// add genesis block to chain state, if needed
if (this.chainStateBuilder.Chain.Height < 0)
{
this.chainStateBuilder.AddBlock(this.rules.GenesisChainedHeader, this.rules.GenesisBlock.Transactions);
}
// create workers
this.targetChainWorker = new TargetChainWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(50), maxIdleTime: TimeSpan.FromSeconds(30)),
this.logger, this.rules, this.coreStorage);
this.chainStateWorker = new ChainStateWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(50), maxIdleTime: TimeSpan.FromSeconds(5)),
this.targetChainWorker, this.chainStateBuilder, this.logger, this.rules, this.coreStorage);
this.pruningWorker = new PruningWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromSeconds(60), maxIdleTime: TimeSpan.FromMinutes(15)),
this.coreStorage, this.chainStateWorker, this.chainStateBuilder, this.logger, this.rules);
this.defragWorker = new DefragWorker(
new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMinutes(5), maxIdleTime: TimeSpan.FromMinutes(5)),
this.coreStorage, this.logger);
// notify defrag worker after pruning
this.pruningWorker.OnWorkFinished += this.defragWorker.NotifyWork;
this.chainStateWorker.BlockMissed += HandleBlockMissed;
this.targetChainWorker.OnTargetChainChanged +=
() =>
{
var handler = this.OnTargetChainChanged;
if (handler != null)
{
handler(this, EventArgs.Empty);
}
};
this.chainStateWorker.OnChainStateChanged +=
() =>
{
this.pruningWorker.NotifyWork();
this.utxoScanWorker.NotifyWork();
var handler = this.OnChainStateChanged;
if (handler != null)
{
handler(this, EventArgs.Empty);
}
};
this.gcWorker = new WorkerMethod("GC Worker",
_ =>
{
this.logger.Info(
string.Join("\n",
new string('-', 80),
"GC Memory: {0,10:#,##0.00} MB",
"Process Memory: {1,10:#,##0.00} MB",
new string('-', 80)
)
.Format2
(
/*0*/ (float)GC.GetTotalMemory(false) / 1.MILLION(),
/*1*/ (float)Process.GetCurrentProcess().PrivateMemorySize64 / 1.MILLION()
));
}, initialNotify: true, minIdleTime: TimeSpan.FromSeconds(30), maxIdleTime: TimeSpan.FromSeconds(30), logger: this.logger);
this.utxoScanWorker = new WorkerMethod("UTXO Scan Worker",
_ =>
{
// time taking chain state snapshots
var stopwatch = Stopwatch.StartNew();
int chainStateHeight;
using (var chainState = this.GetChainState())
{
chainStateHeight = chainState.Chain.Height;
}
stopwatch.Stop();
this.logger.Info("GetChainState at {0:#,##0}: {1:#,##0.00}s".Format2(chainStateHeight, stopwatch.Elapsed.TotalSeconds));
// time enumerating chain state snapshots
stopwatch = Stopwatch.StartNew();
using (var chainState = this.GetChainState())
{
chainStateHeight = chainState.Chain.Height;
chainState.ReadUnspentTransactions().Count();
}
stopwatch.Stop();
this.logger.Info("Enumerate chain state at {0:#,##0}: {1:#,##0.00}s".Format2(chainStateHeight, stopwatch.Elapsed.TotalSeconds));
//using (var chainStateLocal = this.GetChainState())
//{
// new MethodTimer(this.logger).Time("UTXO Commitment: {0:#,##0}".Format2(chainStateLocal.UnspentTxCount), () =>
// {
// using (var utxoStream = new UtxoStream(this.logger, chainStateLocal.ReadUnspentTransactions()))
// {
// var sha256 = new SHA256Managed();
// var utxoHash = sha256.ComputeHash(utxoStream);
// this.logger.Info("UXO Commitment Hash: {0}".Format2(utxoHash.ToHexNumberString()));
// }
// });
// //new MethodTimer().Time("Full UTXO Scan: {0:#,##0}".Format2(chainStateLocal.Utxo.TransactionCount), () =>
// //{
// // var sha256 = new SHA256Managed();
// // foreach (var output in chainStateLocal.Utxo.GetUnspentTransactions())
// // {
// // }
// //});
//}
}, initialNotify: true, minIdleTime: TimeSpan.FromSeconds(60), maxIdleTime: TimeSpan.FromSeconds(60), logger: this.logger);
}