private void RestoreOutputs(RewindData rewindData) { foreach (UnspentOutputs unspentToRestore in rewindData.OutputsToRestore) { this.logger.LogTrace("Outputs of transaction ID '{0}' will be restored.", unspentToRestore.TransactionId); if (this.cachedUtxoItems.TryGetValue(unspentToRestore.TransactionId, out CacheItem cacheItem)) { cacheItem.UnspentOutputs = unspentToRestore; cacheItem.IsDirty = true; } else { this.logger.LogTrace("Outputs of transaction ID '{0}' not found in cache, inserting them.", unspentToRestore.TransactionId); cacheItem = new CacheItem { UnspentOutputs = unspentToRestore, IsDirty = true }; this.cachedUtxoItems.Add(unspentToRestore.TransactionId, cacheItem); } } }
private void RemoveTransactions(RewindData rewindData) { foreach (uint256 transactionToRemove in rewindData.TransactionsToRemove) { this.logger.LogTrace("Attempt to remove transaction with ID '{0}'.", transactionToRemove); this.cachedUtxoItems.Remove(transactionToRemove); } }
/// <inheritdoc /> public async Task <uint256> RewindAsync() { if (this.innerBlockHash == null) { this.innerBlockHash = await this.inner.GetTipHashAsync().ConfigureAwait(false); } using (await this.lockobj.LockAsync().ConfigureAwait(false)) { // Check if rewind data is available in local cache. If it is // we can rewind and there is no need to check underlying storage. if (this.cachedRewindDataIndex.Count > 0) { KeyValuePair <int, RewindData> lastRewindDataItem = this.cachedRewindDataIndex.Last(); RewindData lastRewindData = lastRewindDataItem.Value; this.RemoveTransactions(lastRewindData); this.RestoreOutputs(lastRewindData); // Change current block hash to the one from the rewind data. this.blockHash = lastRewindData.PreviousBlockHash; this.blockHeight = lastRewindDataItem.Key - 1; this.cachedRewindDataIndex.Remove(lastRewindDataItem.Key); if (this.rewindDataIndexCache != null) { await this.rewindDataIndexCache.Remove(this.blockHeight, this); } this.logger.LogTrace("(-)[REMOVED_FROM_BATCH]:'{0}'", this.blockHash); return(this.blockHash); } // Rewind data was not found in cache, try underlying storage. uint256 hash = await this.inner.RewindAsync().ConfigureAwait(false); // All the cached utxos are now on disk so we can clear the cached entry list. this.cachedUtxoItems.Clear(); this.innerBlockHash = hash; this.blockHash = hash; this.blockHeight -= 1; if (this.rewindDataIndexCache != null) { await this.rewindDataIndexCache.InitializeAsync(this.blockHeight, this); } return(hash); } }
/// <inheritdoc /> public async Task <uint256> Rewind() { this.logger.LogTrace("()"); if (this.innerBlockHash == null) { this.innerBlockHash = await this.inner.GetTipHashAsync().ConfigureAwait(false); } using (await this.lockobj.LockAsync().ConfigureAwait(false)) { // Check if rewind data is available in local cache. If it is // we can rewind and there is no need to check underlying storage. if (this.cachedRewindDataList.Count > 0) { RewindData lastRewindData = this.cachedRewindDataList.Last(); this.RemoveTransactions(lastRewindData); this.RestoreOutputs(lastRewindData); // Change current block hash to the one from the rewind data. this.blockHash = lastRewindData.PreviousBlockHash; this.cachedRewindDataList.RemoveAt(this.cachedRewindDataList.Count - 1); this.logger.LogTrace("(-)[REMOVED_FROM_BATCH]:'{0}'", this.blockHash); return(this.blockHash); } // Rewind data was not found in cache, try underlying storage. uint256 hash = await this.inner.Rewind().ConfigureAwait(false); // All the cached utxos are now on disk so we can clear the cached entry list. this.cachedUtxoItems.Clear(); this.innerBlockHash = hash; this.blockHash = hash; this.logger.LogTrace("(-):'{0}'", hash); return(hash); } }
/// <inheritdoc /> public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null) { Guard.NotNull(oldBlockHash, nameof(oldBlockHash)); Guard.NotNull(nextBlockHash, nameof(nextBlockHash)); Guard.NotNull(outputs, nameof(outputs)); lock (this.lockobj) { if ((this.blockHash != null) && (oldBlockHash != this.blockHash)) { this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash); this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.blockHash = nextBlockHash; long utxoSkipDisk = 0; var rewindData = new RewindData(oldBlockHash); Dictionary <OutPoint, int> indexItems = null; if (this.rewindDataIndexCache != null) { indexItems = new Dictionary <OutPoint, int>(); } foreach (UnspentOutput output in outputs) { if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem)) { // Add outputs to cache, this will happen for two cases // 1. if a chaced item was evicted // 2. for new outputs that are added if (output.CreatedFromBlock) { // if the output is indicate that it was added from a block // There is no need to spend an extra call to disk. this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint); cacheItem = new CacheItem() { ExistInInner = false, IsDirty = false, OutPoint = output.OutPoint, Coins = null }; } else { // This can happen if the cashe item was evicted while // the block was being processed, fetch the outut again from disk. this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint); FetchCoinsResponse result = this.inner.FetchCoins(new[] { output.OutPoint }); this.performanceCounter.AddMissCount(1); UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value; cacheItem = new CacheItem() { ExistInInner = unspentOutput.Coins != null, IsDirty = false, OutPoint = unspentOutput.OutPoint, Coins = unspentOutput.Coins }; } this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem); this.cacheSizeBytes += cacheItem.GetSize; this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint); } // If output.Coins is null this means the utxo needs to be deleted // otherwise this is a new utxo and we store it to cache. if (output.Coins == null) { // DELETE COINS // In cases of an output spent in the same block // it wont exist in cash or in disk so its safe to remove it if (cacheItem.Coins == null) { if (cacheItem.ExistInInner) { throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint)); } } else { // Handle rewind data this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint); rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins)); rewindData.TotalSize += cacheItem.GetSize; if (this.rewindDataIndexCache != null && indexItems != null) { indexItems[cacheItem.OutPoint] = this.blockHash.Height; } } // If a spent utxo never made it to disk then no need to keep it in memory. if (!cacheItem.ExistInInner) { this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint); this.cachedUtxoItems.Remove(cacheItem.OutPoint); this.cacheSizeBytes -= cacheItem.GetSize; utxoSkipDisk++; if (cacheItem.IsDirty) { this.dirtyCacheCount--; } } else { // Now modify the cached items with the mutated data. this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint); this.cacheSizeBytes -= cacheItem.GetScriptSize; cacheItem.Coins = null; // Delete output from cache but keep a the cache // item reference so it will get deleted form disk cacheItem.IsDirty = true; this.dirtyCacheCount++; } } else { // ADD COINS if (cacheItem.Coins != null) { // Allow overrides. // See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94 bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null; if (!allowOverride) { throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint)); } this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint); // Deduct the crurrent script size form the // total cache size, it will be added again later. this.cacheSizeBytes -= cacheItem.GetScriptSize; // Clear this in order to calculate the cache sie // this will get set later when overriden cacheItem.Coins = null; } // Handle rewind data // New trx so it needs to be deleted if a rewind happens. this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint); rewindData.OutputsToRemove.Add(cacheItem.OutPoint); rewindData.TotalSize += cacheItem.GetSize; // Put in the cache the new UTXOs. this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint); cacheItem.Coins = output.Coins; this.cacheSizeBytes += cacheItem.GetScriptSize; // Mark the cache item as dirty so it get persisted // to disk and not evicted form cache cacheItem.IsDirty = true; this.dirtyCacheCount++; } } this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk); if (this.rewindDataIndexCache != null && indexItems.Any()) { this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems); } // Add the most recent rewind data to the cache. this.cachedRewindData.Add(this.blockHash.Height, rewindData); this.rewindDataSizeBytes += rewindData.TotalSize; // Remove rewind data form the back of a moving window. // The closer we get to the tip we keep a longer rewind data window. // Anything bellow last checkpoint we keep the minimal of 10 // (random low number) rewind data items. // Beyond last checkpoint: // - For POS we keep a window of MaxReorg. // - For POW we keep 100 items (possibly better is an algo that grows closer to tip) // A moving window of information needed to rewind the node to a previous block. // When cache is flushed the rewind data will allow to rewind the node up to the // number of rewind blocks. // TODO: move rewind data to use block store. // Rewind data can go away all togetehr if the node uses the blocks in block store // to get the rewind information, blockstore persists much more frequent then coin cache // So using block store for rewinds is not entirely impossible. uint rewindDataWindow = 10; if (this.blockHash.Height >= this.lastCheckpointHeight) { if (this.network.Consensus.MaxReorgLength != 0) { rewindDataWindow = this.network.Consensus.MaxReorgLength + 1; } else { // TODO: make the rewind data window a configuration // parameter of evern a network parameter. // For POW assume BTC where a rewind data of 100 is more then enough. rewindDataWindow = 100; } } int rewindToRemove = this.blockHash.Height - (int)rewindDataWindow; if (this.cachedRewindData.TryGetValue(rewindToRemove, out RewindData delete)) { this.logger.LogDebug("Remove rewind data height '{0}' from cache.", rewindToRemove); this.cachedRewindData.Remove(rewindToRemove); this.rewindDataSizeBytes -= delete.TotalSize; } } }
/// <inheritdoc /> public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash) { this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs?.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash); RewindData rewindData = originalOutputs != null ? new RewindData(oldBlockHash) : null; int insertedEntities = 0; List <UnspentOutputs> all = unspentOutputs.ToList(); Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count); using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o))) { if (originalOutputs != null) { IEnumerator <TxOut[]> originalEnumerator = originalOutputs.GetEnumerator(); foreach (UnspentOutputs output in all) { originalEnumerator.MoveNext(); unspentToOriginal.Add(output.TransactionId, originalEnumerator.Current); } } } Task task = Task.Run(() => { this.logger.LogTrace("()"); using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) { transaction.ValuesLazyLoadingIsOn = false; transaction.SynchronizeTables("BlockHash", "Coins", "Rewind"); transaction.Technical_SetTable_OverwriteIsNotAllowed("Coins"); using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o))) { uint256 current = this.GetCurrentHash(transaction); if (current != oldBlockHash) { this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.SetBlockHash(transaction, nextBlockHash); all.Sort(UnspentOutputsComparer.Instance); foreach (UnspentOutputs coin in all) { this.logger.LogTrace("Outputs of transaction ID '{0}' are {1} and will be {2} to the database.", coin.TransactionId, coin.IsPrunable ? "PRUNABLE" : "NOT PRUNABLE", coin.IsPrunable ? "removed" : "inserted"); if (coin.IsPrunable) { transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false)); } else { transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins()); } if (originalOutputs != null) { TxOut[] original = null; unspentToOriginal.TryGetValue(coin.TransactionId, out original); if (original == null) { // This one haven't existed before, if we rewind, delete it. rewindData.TransactionsToRemove.Add(coin.TransactionId); } else { // We'll need to restore the original outputs. UnspentOutputs clone = coin.Clone(); clone._Outputs = original.ToArray(); rewindData.OutputsToRestore.Add(clone); } } } if (rewindData != null) { int nextRewindIndex = this.GetRewindIndex(transaction) + 1; this.logger.LogTrace("Rewind state #{0} created.", nextRewindIndex); transaction.Insert("Rewind", nextRewindIndex, rewindData); } insertedEntities += all.Count; transaction.Commit(); } } this.PerformanceCounter.AddInsertedEntities(insertedEntities); this.logger.LogTrace("(-)"); }); this.logger.LogTrace("(-)"); return(task); }
/// <inheritdoc /> public void SaveChanges(IList <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, int height, List <RewindData> rewindDataList = null) { Guard.NotNull(oldBlockHash, nameof(oldBlockHash)); Guard.NotNull(nextBlockHash, nameof(nextBlockHash)); Guard.NotNull(unspentOutputs, nameof(unspentOutputs)); lock (this.lockobj) { if ((this.blockHash != null) && (oldBlockHash != this.blockHash)) { this.logger.LogTrace("{0}:'{1}'", nameof(this.blockHash), this.blockHash); this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.blockHeight = height; this.blockHash = nextBlockHash; var rewindData = new RewindData(oldBlockHash); var indexItems = new Dictionary <OutPoint, int>(); foreach (UnspentOutputs unspent in unspentOutputs) { if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem)) { // This can happen very rarely in the case where we fetch items from // disk but immediately call the Evict method which then removes the cached item(s). this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId); FetchCoinsResponse result = this.inner.FetchCoins(new[] { unspent.TransactionId }); UnspentOutputs unspentOutput = result.UnspentOutputs[0]; cacheItem = new CacheItem(); cacheItem.ExistInInner = unspentOutput != null; cacheItem.IsDirty = false; cacheItem.UnspentOutputs = unspentOutput?.Clone(); this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem); this.logger.LogTrace("CacheItem added to the cache during save '{0}'.", cacheItem.UnspentOutputs); } // If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk, // that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache // otherwise we store to cache only the UTXO that have been spent. if (cacheItem.UnspentOutputs != null) { // To handle rewind we'll need to restore the original outputs, // so we clone it and save it in rewind data. UnspentOutputs clone = unspent.Clone(); // We take the original items that are in cache and put them in rewind data. clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray(); this.logger.LogTrace("Modifying transaction '{0}' in OutputsToRestore rewind data.", unspent.TransactionId); rewindData.OutputsToRestore.Add(clone); this.logger.LogTrace("Cache item before spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs); // Now modify the cached items with the mutated data. cacheItem.UnspentOutputs.Spend(unspent); this.logger.LogTrace("Cache item after spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs); } else { // New trx so it needs to be deleted if a rewind happens. this.logger.LogTrace("Adding transaction '{0}' to TransactionsToRemove rewind data.", unspent.TransactionId); rewindData.TransactionsToRemove.Add(unspent.TransactionId); // Put in the cache the new UTXOs. this.logger.LogTrace("Setting {0} to {1}: '{2}'.", nameof(cacheItem.UnspentOutputs), nameof(unspent), unspent); cacheItem.UnspentOutputs = unspent; } cacheItem.IsDirty = true; if (this.rewindDataIndexCache != null) { for (int i = 0; i < unspent.Outputs.Length; i++) { var key = new OutPoint(unspent.TransactionId, i); indexItems[key] = this.blockHeight; } } // Inner does not need to know pruned unspent that it never saw. if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner) { this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId); this.cachedUtxoItems.Remove(unspent.TransactionId); } } if (this.rewindDataIndexCache != null && indexItems.Any()) { this.rewindDataIndexCache.Save(indexItems); this.rewindDataIndexCache.Flush(this.blockHeight); } this.cachedRewindDataIndex.Add(this.blockHeight, rewindData); } }
/// <inheritdoc /> public async Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, List <RewindData> rewindDataList = null) { Guard.NotNull(oldBlockHash, nameof(oldBlockHash)); Guard.NotNull(nextBlockHash, nameof(nextBlockHash)); Guard.NotNull(unspentOutputs, nameof(unspentOutputs)); this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash); using (await this.lockobj.LockAsync().ConfigureAwait(false)) { if ((this.blockHash != null) && (oldBlockHash != this.blockHash)) { this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.blockHash = nextBlockHash; var rewindData = new RewindData(oldBlockHash); foreach (UnspentOutputs unspent in unspentOutputs) { if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem)) { // This can happen very rarely in the case where we fetch items from // disk but immediately call the Evict method which then removes the cached item(s). this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId); FetchCoinsResponse result = await this.inner.FetchCoinsAsync(new[] { unspent.TransactionId }).ConfigureAwait(false); UnspentOutputs unspentOutput = result.UnspentOutputs[0]; cacheItem = new CacheItem(); cacheItem.ExistInInner = unspentOutput != null; cacheItem.IsDirty = false; cacheItem.UnspentOutputs = unspentOutput?.Clone(); this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem); } else { this.logger.LogTrace("Outputs of transaction ID '{0}' are in cache already, updating them.", unspent.TransactionId); } // If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk, // that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache // otherwise we store to cache only the UTXO that have been spent. if (cacheItem.UnspentOutputs != null) { // To handle rewind we'll need to restore the original outputs, // so we clone it and save it in rewind data. UnspentOutputs clone = unspent.Clone(); // We take the original items that are in cache and put them in rewind data. clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray(); rewindData.OutputsToRestore.Add(clone); // Now modify the cached items with the mutated data. cacheItem.UnspentOutputs.Spend(unspent); } else { // New trx so it needs to be deleted if a rewind happens. rewindData.TransactionsToRemove.Add(unspent.TransactionId); // Put in the cache the new UTXOs. cacheItem.UnspentOutputs = unspent; } cacheItem.IsDirty = true; // Inner does not need to know pruned unspent that it never saw. if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner) { this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId); this.cachedUtxoItems.Remove(unspent.TransactionId); } } this.cachedRewindDataList.Add(rewindData); } this.logger.LogTrace("(-)"); }
/// <inheritdoc /> public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash) { return(this.session.Execute(() => { this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs?.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash); RewindData rewindData = originalOutputs == null ? null : new RewindData(oldBlockHash); int insertedEntities = 0; using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o))) { uint256 current = this.GetCurrentHash(); if (current != oldBlockHash) { this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.SetBlockHash(nextBlockHash); List <UnspentOutputs> all = unspentOutputs.ToList(); Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count); if (originalOutputs != null) { IEnumerator <TxOut[]> originalEnumerator = originalOutputs.GetEnumerator(); foreach (UnspentOutputs output in all) { originalEnumerator.MoveNext(); unspentToOriginal.Add(output.TransactionId, originalEnumerator.Current); } } all.Sort(UnspentOutputsComparer.Instance); foreach (UnspentOutputs coin in all) { this.logger.LogTrace("Outputs of transaction ID '{0}' are {1} and will be {2} to the database.", coin.TransactionId, coin.IsPrunable ? "PRUNABLE" : "NOT PRUNABLE", coin.IsPrunable ? "removed" : "inserted"); if (coin.IsPrunable) { this.session.Transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false)); } else { this.session.Transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins()); } if (originalOutputs != null) { TxOut[] original = null; unspentToOriginal.TryGetValue(coin.TransactionId, out original); if (original == null) { // This one haven't existed before, if we rewind, delete it. rewindData.TransactionsToRemove.Add(coin.TransactionId); } else { // We'll need to restore the original outputs. UnspentOutputs clone = coin.Clone(); // TODO: Can we remove this line? before is not used anywhere and clone.UnspentCount does not seem to have side effects? int before = clone.UnspentCount; clone._Outputs = original.ToArray(); rewindData.OutputsToRestore.Add(clone); } } } if (rewindData != null) { int nextRewindIndex = this.GetRewindIndex() + 1; this.logger.LogTrace("Rewind state #{0} created.", nextRewindIndex); this.session.Transaction.Insert("Rewind", nextRewindIndex, rewindData); } insertedEntities += all.Count; this.session.Transaction.Commit(); } this.PerformanceCounter.AddInsertedEntities(insertedEntities); this.logger.LogTrace("(-)"); })); }
public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash) { return(this.session.Execute(() => { RewindData rewindData = originalOutputs == null ? null : new RewindData(oldBlockHash); int insertedEntities = 0; using (new StopWatch().Start(o => this.PerformanceCounter.AddInsertTime(o))) { var current = this.GetCurrentHash(); if (current != oldBlockHash) { throw new InvalidOperationException("Invalid oldBlockHash"); } this.SetBlockHash(nextBlockHash); var all = unspentOutputs.ToList(); Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count); if (originalOutputs != null) { var originalEnumerator = originalOutputs.GetEnumerator(); foreach (var u in all) { originalEnumerator.MoveNext(); unspentToOriginal.Add(u.TransactionId, originalEnumerator.Current); } } all.Sort(UnspentOutputsComparer.Instance); foreach (var coin in all) { if (coin.IsPrunable) { this.session.Transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false)); } else { this.session.Transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins()); } if (originalOutputs != null) { TxOut[] original = null; unspentToOriginal.TryGetValue(coin.TransactionId, out original); if (original == null) { //This one did not existed before, if we rewind, delete it rewindData.TransactionsToRemove.Add(coin.TransactionId); } else { //We'll need to restore the original outputs var clone = coin.Clone(); var before = clone.UnspentCount; clone._Outputs = original.ToArray(); rewindData.OutputsToRestore.Add(clone); } } } if (rewindData != null) { int nextRewindIndex = this.GetRewindIndex() + 1; this.session.Transaction.Insert <int, RewindData>("Rewind", nextRewindIndex, rewindData); } insertedEntities += all.Count; this.session.Transaction.Commit(); } this.PerformanceCounter.AddInsertedEntities(insertedEntities); })); }
/// <inheritdoc /> public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null) { Guard.NotNull(oldBlockHash, nameof(oldBlockHash)); Guard.NotNull(nextBlockHash, nameof(nextBlockHash)); Guard.NotNull(outputs, nameof(outputs)); lock (this.lockobj) { if ((this.blockHash != null) && (oldBlockHash != this.blockHash)) { this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash); this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.blockHash = nextBlockHash; long utxoSkipDisk = 0; var rewindData = new RewindData(oldBlockHash); Dictionary <OutPoint, int> indexItems = null; if (this.rewindDataIndexCache != null) { indexItems = new Dictionary <OutPoint, int>(); } foreach (UnspentOutput output in outputs) { if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem)) { // Add outputs to cache, this will happen for two cases // 1. if a chaced item was evicted // 2. for new outputs that are added if (output.CreatedFromBlock) { // if the output is indicate that it was added from a block // There is no need to spend an extra call to disk. this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint); cacheItem = new CacheItem() { ExistInInner = false, IsDirty = false, OutPoint = output.OutPoint, Coins = null }; } else { // This can happen if the cashe item was evicted while // the block was being processed, fetch the outut again from disk. this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint); FetchCoinsResponse result = this.coindb.FetchCoins(new[] { output.OutPoint }); this.performanceCounter.AddMissCount(1); UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value; cacheItem = new CacheItem() { ExistInInner = unspentOutput.Coins != null, IsDirty = false, OutPoint = unspentOutput.OutPoint, Coins = unspentOutput.Coins }; } this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem); this.cacheSizeBytes += cacheItem.GetSize; this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint); } // If output.Coins is null this means the utxo needs to be deleted // otherwise this is a new utxo and we store it to cache. if (output.Coins == null) { // DELETE COINS // In cases of an output spent in the same block // it wont exist in cash or in disk so its safe to remove it if (cacheItem.Coins == null) { if (cacheItem.ExistInInner) { throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint)); } } else { // Handle rewind data this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint); rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins)); rewindData.TotalSize += cacheItem.GetSize; if (this.rewindDataIndexCache != null && indexItems != null) { indexItems[cacheItem.OutPoint] = this.blockHash.Height; } } // If a spent utxo never made it to disk then no need to keep it in memory. if (!cacheItem.ExistInInner) { this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint); this.cachedUtxoItems.Remove(cacheItem.OutPoint); this.cacheSizeBytes -= cacheItem.GetSize; utxoSkipDisk++; if (cacheItem.IsDirty) { this.dirtyCacheCount--; } } else { // Now modify the cached items with the mutated data. this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint); this.cacheSizeBytes -= cacheItem.GetScriptSize; cacheItem.Coins = null; // Delete output from cache but keep a the cache // item reference so it will get deleted form disk cacheItem.IsDirty = true; this.dirtyCacheCount++; } } else { // ADD COINS if (cacheItem.Coins != null) { // Allow overrides. // See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94 bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null; if (!allowOverride) { throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint)); } this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint); // Deduct the crurrent script size form the // total cache size, it will be added again later. this.cacheSizeBytes -= cacheItem.GetScriptSize; // Clear this in order to calculate the cache sie // this will get set later when overriden cacheItem.Coins = null; } // Handle rewind data // New trx so it needs to be deleted if a rewind happens. this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint); rewindData.OutputsToRemove.Add(cacheItem.OutPoint); rewindData.TotalSize += cacheItem.GetSize; // Put in the cache the new UTXOs. this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint); cacheItem.Coins = output.Coins; this.cacheSizeBytes += cacheItem.GetScriptSize; // Mark the cache item as dirty so it get persisted // to disk and not evicted form cache cacheItem.IsDirty = true; this.dirtyCacheCount++; } } this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk); if (this.rewindDataIndexCache != null && indexItems.Any()) { this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems); } // Add the most recent rewind data to the cache. this.cachedRewindData.Add(this.blockHash.Height, rewindData); this.rewindDataSizeBytes += rewindData.TotalSize; } }