/// <inheritdoc /> public FetchCoinsResponse FetchCoins(OutPoint[] utxos) { Guard.NotNull(utxos, nameof(utxos)); var result = new FetchCoinsResponse(); var missedOutpoint = new List <OutPoint>(); lock (this.lockobj) { foreach (OutPoint outPoint in utxos) { if (!this.cachedUtxoItems.TryGetValue(outPoint, out CacheItem cache)) { this.logger.LogDebug("Utxo '{0}' not found in cache.", outPoint); missedOutpoint.Add(outPoint); } else { this.logger.LogDebug("Utxo '{0}' found in cache, UTXOs:'{1}'.", outPoint, cache.Coins); result.UnspentOutputs.Add(outPoint, new UnspentOutput(outPoint, cache.Coins)); } } this.performanceCounter.AddMissCount(missedOutpoint.Count); this.performanceCounter.AddHitCount(utxos.Length - missedOutpoint.Count); if (missedOutpoint.Count > 0) { this.logger.LogDebug("{0} cache missed transaction needs to be loaded from underlying CoinView.", missedOutpoint.Count); FetchCoinsResponse fetchedCoins = this.coindb.FetchCoins(missedOutpoint.ToArray()); foreach (var unspentOutput in fetchedCoins.UnspentOutputs) { result.UnspentOutputs.Add(unspentOutput.Key, unspentOutput.Value); var cache = new CacheItem() { ExistInInner = unspentOutput.Value.Coins != null, IsDirty = false, OutPoint = unspentOutput.Key, Coins = unspentOutput.Value.Coins }; this.logger.LogDebug("CacheItem added to the cache, UTXO '{0}', Coin:'{1}'.", cache.OutPoint, cache.Coins); this.cachedUtxoItems.Add(cache.OutPoint, cache); this.cacheSizeBytes += cache.GetSize; } } // Check if we need to evict items form the cache. // This happens every time data is fetched fomr coindb this.TryEvictCacheLocked(); } return(result); }
/// <inheritdoc /> public FetchCoinsResponse FetchCoins(OutPoint[] txIds) { Guard.NotNull(txIds, nameof(txIds)); using (this.lockobj.LockRead()) { var result = new FetchCoinsResponse(); for (int i = 0; i < txIds.Length; i++) { var output = this.unspents.TryGet(txIds[i]); if (output != null) { result.UnspentOutputs.Add(output.OutPoint, output); } } return(result); } }
/// <inheritdoc /> public void CacheCoins(OutPoint[] utxos) { lock (this.lockobj) { var missedOutpoint = new List <OutPoint>(); foreach (OutPoint outPoint in utxos) { if (!this.cachedUtxoItems.TryGetValue(outPoint, out CacheItem cache)) { this.logger.LogDebug("Prefetch Utxo '{0}' not found in cache.", outPoint); missedOutpoint.Add(outPoint); } } this.performanceCounter.AddCacheMissCount(missedOutpoint.Count); this.performanceCounter.AddCacheHitCount(utxos.Length - missedOutpoint.Count); if (missedOutpoint.Count > 0) { FetchCoinsResponse fetchedCoins = this.coindb.FetchCoins(missedOutpoint.ToArray()); foreach (var unspentOutput in fetchedCoins.UnspentOutputs) { var cache = new CacheItem() { ExistInInner = unspentOutput.Value.Coins != null, IsDirty = false, OutPoint = unspentOutput.Key, Coins = unspentOutput.Value.Coins }; this.logger.LogDebug("Prefetch CacheItem added to the cache, UTXO: '{0}', Coin:'{1}'.", cache.OutPoint, cache.Coins); this.cachedUtxoItems.Add(cache.OutPoint, cache); this.cacheSizeBytes += cache.GetSize; } } } }
/// <inheritdoc /> public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null) { Guard.NotNull(oldBlockHash, nameof(oldBlockHash)); Guard.NotNull(nextBlockHash, nameof(nextBlockHash)); Guard.NotNull(outputs, nameof(outputs)); lock (this.lockobj) { if ((this.blockHash != null) && (oldBlockHash != this.blockHash)) { this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash); this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.blockHash = nextBlockHash; long utxoSkipDisk = 0; var rewindData = new RewindData(oldBlockHash); Dictionary <OutPoint, int> indexItems = null; if (this.rewindDataIndexCache != null) { indexItems = new Dictionary <OutPoint, int>(); } foreach (UnspentOutput output in outputs) { if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem)) { // Add outputs to cache, this will happen for two cases // 1. if a cached item was evicted // 2. for new outputs that are added if (output.CreatedFromBlock) { // if the output is indicate that it was added from a block // There is no need to spend an extra call to disk. this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint); cacheItem = new CacheItem() { ExistInInner = false, IsDirty = false, OutPoint = output.OutPoint, Coins = null }; } else { // This can happen if the cached item was evicted while // the block was being processed, fetch the output again from disk. this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint); FetchCoinsResponse result = this.coindb.FetchCoins(new[] { output.OutPoint }); this.performanceCounter.AddMissCount(1); UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value; cacheItem = new CacheItem() { ExistInInner = unspentOutput.Coins != null, IsDirty = false, OutPoint = unspentOutput.OutPoint, Coins = unspentOutput.Coins }; } this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem); this.cacheSizeBytes += cacheItem.GetSize; this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint); } // If output.Coins is null this means the utxo needs to be deleted // otherwise this is a new utxo and we store it to cache. if (output.Coins == null) { // DELETE COINS // In cases of an output spent in the same block // it wont exist in cash or in disk so its safe to remove it if (cacheItem.Coins == null) { if (cacheItem.ExistInInner) { throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint)); } } else { // Handle rewind data this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint); rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins)); rewindData.TotalSize += cacheItem.GetSize; if (this.rewindDataIndexCache != null && indexItems != null) { indexItems[cacheItem.OutPoint] = this.blockHash.Height; } } // If a spent utxo never made it to disk then no need to keep it in memory. if (!cacheItem.ExistInInner) { this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint); this.cachedUtxoItems.Remove(cacheItem.OutPoint); this.cacheSizeBytes -= cacheItem.GetSize; utxoSkipDisk++; if (cacheItem.IsDirty) { this.dirtyCacheCount--; } } else { // Now modify the cached items with the mutated data. this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint); this.cacheSizeBytes -= cacheItem.GetScriptSize; cacheItem.Coins = null; // Delete output from cache but keep a the cache // item reference so it will get deleted form disk cacheItem.IsDirty = true; this.dirtyCacheCount++; } } else { // ADD COINS if (cacheItem.Coins != null) { // Allow overrides. // See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94 bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null; if (!allowOverride) { throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint)); } this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint); // Deduct the current script size form the // total cache size, it will be added again later. this.cacheSizeBytes -= cacheItem.GetScriptSize; // Clear this in order to calculate the cache size // this will get set later when overridden cacheItem.Coins = null; } // Handle rewind data // New trx so it needs to be deleted if a rewind happens. this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint); rewindData.OutputsToRemove.Add(cacheItem.OutPoint); rewindData.TotalSize += cacheItem.GetSize; // Put in the cache the new UTXOs. this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint); cacheItem.Coins = output.Coins; this.cacheSizeBytes += cacheItem.GetScriptSize; // Mark the cache item as dirty so it get persisted // to disk and not evicted form cache cacheItem.IsDirty = true; this.dirtyCacheCount++; } } this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk); if (this.rewindDataIndexCache != null && indexItems.Any()) { this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems); } // Add the most recent rewind data to the cache. this.cachedRewindData.Add(this.blockHash.Height, rewindData); this.rewindDataSizeBytes += rewindData.TotalSize; // Remove rewind data form the back of a moving window. // The closer we get to the tip we keep a longer rewind data window. // Anything bellow last checkpoint we keep the minimal of 10 // (random low number) rewind data items. // Beyond last checkpoint: // - For POS we keep a window of MaxReorg. // - For POW we keep 100 items (possibly better is an algo that grows closer to tip) // A moving window of information needed to rewind the node to a previous block. // When cache is flushed the rewind data will allow to rewind the node up to the // number of rewind blocks. // TODO: move rewind data to use block store. // Rewind data can go away all together if the node uses the blocks in block store // to get the rewind information, blockstore persists much more frequent then coin cache // So using block store for rewinds is not entirely impossible. int rewindDataWindow = this.CalculateRewindWindow(); int rewindToRemove = this.blockHash.Height - (int)rewindDataWindow; if (this.cachedRewindData.TryGetValue(rewindToRemove, out RewindData delete)) { this.logger.LogDebug("Remove rewind data height '{0}' from cache.", rewindToRemove); this.cachedRewindData.Remove(rewindToRemove); this.rewindDataSizeBytes -= delete.TotalSize; } } }