public async Task TestRewindAsync() { uint256 tip = await this.cachedCoinView.GetTipHashAsync(); Assert.Equal(this.concurrentChain.Genesis.HashBlock, tip); int currentHeight = 0; // Create a lot of new coins. List <UnspentOutputs> outputsList = this.CreateOutputsList(currentHeight + 1, 100); await this.SaveChangesAsync(outputsList, new List <TxOut[]>(), currentHeight + 1); currentHeight++; await this.cachedCoinView.FlushAsync(true); uint256 tipAfterOriginalCoinsCreation = await this.cachedCoinView.GetTipHashAsync(); // Collection that will be used as a coinview that we will update in parallel. Needed to verify that actual coinview is ok. List <OutPoint> outPoints = this.ConvertToListOfOutputPoints(outputsList); // Copy of current state to later rewind and verify against it. List <OutPoint> copyOfOriginalOutPoints = new List <OutPoint>(outPoints); List <OutPoint> copyAfterHalfOfAdditions = new List <OutPoint>(); uint256 coinviewTipAfterHalf = null; int addChangesTimes = 500; // Spend some coins in the next N saves. for (int i = 0; i < addChangesTimes; ++i) { uint256 txId = outPoints[this.random.Next(0, outPoints.Count)].Hash; List <OutPoint> txPoints = outPoints.Where(x => x.Hash == txId).ToList(); this.Shuffle(txPoints); List <OutPoint> txPointsToSpend = txPoints.Take(txPoints.Count / 2).ToList(); // First spend in cached coinview FetchCoinsResponse response = await this.cachedCoinView.FetchCoinsAsync(new[] { txId }); Assert.Single(response.UnspentOutputs); UnspentOutputs coins = response.UnspentOutputs[0]; UnspentOutputs unchangedClone = coins.Clone(); foreach (OutPoint outPointToSpend in txPointsToSpend) { coins.Spend(outPointToSpend.N); } // Spend from outPoints. outPoints.RemoveAll(x => txPointsToSpend.Contains(x)); // Save coinview await this.SaveChangesAsync(new List <UnspentOutputs>() { coins }, new List <TxOut[]>() { unchangedClone.Outputs }, currentHeight + 1); currentHeight++; if (i == addChangesTimes / 2) { copyAfterHalfOfAdditions = new List <OutPoint>(outPoints); coinviewTipAfterHalf = await this.cachedCoinView.GetTipHashAsync(); } } await this.ValidateCoinviewIntegrityAsync(outPoints); for (int i = 0; i < addChangesTimes; i++) { await this.cachedCoinView.RewindAsync(); uint256 currentTip = await this.cachedCoinView.GetTipHashAsync(); if (currentTip == coinviewTipAfterHalf) { await this.ValidateCoinviewIntegrityAsync(copyAfterHalfOfAdditions); } } Assert.Equal(tipAfterOriginalCoinsCreation, await this.cachedCoinView.GetTipHashAsync()); await this.ValidateCoinviewIntegrityAsync(copyOfOriginalOutPoints); }
/// <inheritdoc /> public void SaveChanges(IList <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, int height, List <RewindData> rewindDataList = null) { Guard.NotNull(oldBlockHash, nameof(oldBlockHash)); Guard.NotNull(nextBlockHash, nameof(nextBlockHash)); Guard.NotNull(unspentOutputs, nameof(unspentOutputs)); lock (this.lockobj) { if ((this.blockHash != null) && (oldBlockHash != this.blockHash)) { this.logger.LogTrace("{0}:'{1}'", nameof(this.blockHash), this.blockHash); this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.blockHeight = height; this.blockHash = nextBlockHash; var rewindData = new RewindData(oldBlockHash); var indexItems = new Dictionary <OutPoint, int>(); foreach (UnspentOutputs unspent in unspentOutputs) { if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem)) { // This can happen very rarely in the case where we fetch items from // disk but immediately call the Evict method which then removes the cached item(s). this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId); FetchCoinsResponse result = this.inner.FetchCoins(new[] { unspent.TransactionId }); UnspentOutputs unspentOutput = result.UnspentOutputs[0]; cacheItem = new CacheItem(); cacheItem.ExistInInner = unspentOutput != null; cacheItem.IsDirty = false; cacheItem.UnspentOutputs = unspentOutput?.Clone(); this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem); this.logger.LogTrace("CacheItem added to the cache during save '{0}'.", cacheItem.UnspentOutputs); } // If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk, // that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache // otherwise we store to cache only the UTXO that have been spent. if (cacheItem.UnspentOutputs != null) { // To handle rewind we'll need to restore the original outputs, // so we clone it and save it in rewind data. UnspentOutputs clone = unspent.Clone(); // We take the original items that are in cache and put them in rewind data. clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray(); this.logger.LogTrace("Modifying transaction '{0}' in OutputsToRestore rewind data.", unspent.TransactionId); rewindData.OutputsToRestore.Add(clone); this.logger.LogTrace("Cache item before spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs); // Now modify the cached items with the mutated data. cacheItem.UnspentOutputs.Spend(unspent); this.logger.LogTrace("Cache item after spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs); } else { // New trx so it needs to be deleted if a rewind happens. this.logger.LogTrace("Adding transaction '{0}' to TransactionsToRemove rewind data.", unspent.TransactionId); rewindData.TransactionsToRemove.Add(unspent.TransactionId); // Put in the cache the new UTXOs. this.logger.LogTrace("Setting {0} to {1}: '{2}'.", nameof(cacheItem.UnspentOutputs), nameof(unspent), unspent); cacheItem.UnspentOutputs = unspent; } cacheItem.IsDirty = true; if (this.rewindDataIndexCache != null) { for (int i = 0; i < unspent.Outputs.Length; i++) { var key = new OutPoint(unspent.TransactionId, i); indexItems[key] = this.blockHeight; } } // Inner does not need to know pruned unspent that it never saw. if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner) { this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId); this.cachedUtxoItems.Remove(unspent.TransactionId); } } if (this.rewindDataIndexCache != null && indexItems.Any()) { this.rewindDataIndexCache.Save(indexItems); this.rewindDataIndexCache.Flush(this.blockHeight); } this.cachedRewindDataIndex.Add(this.blockHeight, rewindData); } }
/// <inheritdoc /> public async Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, List <RewindData> rewindDataList = null) { Guard.NotNull(oldBlockHash, nameof(oldBlockHash)); Guard.NotNull(nextBlockHash, nameof(nextBlockHash)); Guard.NotNull(unspentOutputs, nameof(unspentOutputs)); this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash); using (await this.lockobj.LockAsync().ConfigureAwait(false)) { if ((this.blockHash != null) && (oldBlockHash != this.blockHash)) { this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); throw new InvalidOperationException("Invalid oldBlockHash"); } this.blockHash = nextBlockHash; var rewindData = new RewindData(oldBlockHash); foreach (UnspentOutputs unspent in unspentOutputs) { if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem)) { // This can happen very rarely in the case where we fetch items from // disk but immediately call the Evict method which then removes the cached item(s). this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId); FetchCoinsResponse result = await this.inner.FetchCoinsAsync(new[] { unspent.TransactionId }).ConfigureAwait(false); UnspentOutputs unspentOutput = result.UnspentOutputs[0]; cacheItem = new CacheItem(); cacheItem.ExistInInner = unspentOutput != null; cacheItem.IsDirty = false; cacheItem.UnspentOutputs = unspentOutput?.Clone(); this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem); } else { this.logger.LogTrace("Outputs of transaction ID '{0}' are in cache already, updating them.", unspent.TransactionId); } // If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk, // that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache // otherwise we store to cache only the UTXO that have been spent. if (cacheItem.UnspentOutputs != null) { // To handle rewind we'll need to restore the original outputs, // so we clone it and save it in rewind data. UnspentOutputs clone = unspent.Clone(); // We take the original items that are in cache and put them in rewind data. clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray(); rewindData.OutputsToRestore.Add(clone); // Now modify the cached items with the mutated data. cacheItem.UnspentOutputs.Spend(unspent); } else { // New trx so it needs to be deleted if a rewind happens. rewindData.TransactionsToRemove.Add(unspent.TransactionId); // Put in the cache the new UTXOs. cacheItem.UnspentOutputs = unspent; } cacheItem.IsDirty = true; // Inner does not need to know pruned unspent that it never saw. if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner) { this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId); this.cachedUtxoItems.Remove(unspent.TransactionId); } } this.cachedRewindDataList.Add(rewindData); } this.logger.LogTrace("(-)"); }
/// <inheritdoc /> public FetchCoinsResponse FetchCoins(uint256[] txIds, CancellationToken cancellationToken = default(CancellationToken)) { Guard.NotNull(txIds, nameof(txIds)); FetchCoinsResponse result = null; var outputs = new UnspentOutputs[txIds.Length]; var miss = new List <int>(); var missedTxIds = new List <uint256>(); lock (this.lockobj) { for (int i = 0; i < txIds.Length; i++) { CacheItem cache; if (!this.cachedUtxoItems.TryGetValue(txIds[i], out cache)) { this.logger.LogTrace("Transaction '{0}' not found in cache.", txIds[i]); miss.Add(i); missedTxIds.Add(txIds[i]); } else { this.logger.LogTrace("Transaction '{0}' found in cache, UTXOs:'{1}'.", txIds[i], cache.UnspentOutputs); outputs[i] = cache.UnspentOutputs == null ? null : cache.UnspentOutputs.IsPrunable ? null : cache.UnspentOutputs.Clone(); } } this.performanceCounter.AddMissCount(miss.Count); this.performanceCounter.AddHitCount(txIds.Length - miss.Count); FetchCoinsResponse fetchedCoins = null; if (missedTxIds.Count > 0 || this.blockHash == null) { this.logger.LogTrace("{0} cache missed transaction needs to be loaded from underlying CoinView.", missedTxIds.Count); fetchedCoins = this.Inner.FetchCoins(missedTxIds.ToArray(), cancellationToken); } if (this.blockHash == null) { uint256 innerblockHash = fetchedCoins.BlockHash; Debug.Assert(this.cachedUtxoItems.Count == 0); this.innerBlockHash = innerblockHash; this.blockHash = this.innerBlockHash; } for (int i = 0; i < miss.Count; i++) { int index = miss[i]; UnspentOutputs unspent = fetchedCoins.UnspentOutputs[i]; outputs[index] = unspent; var cache = new CacheItem(); cache.ExistInInner = unspent != null; cache.IsDirty = false; cache.UnspentOutputs = unspent?.Clone(); this.logger.LogTrace("CacheItem added to the cache, Transaction Id '{0}', UTXO:'{1}'.", txIds[index], cache.UnspentOutputs); this.cachedUtxoItems.TryAdd(txIds[index], cache); } result = new FetchCoinsResponse(outputs, this.blockHash); int cacheEntryCount = this.cacheEntryCount; if (cacheEntryCount > this.MaxItems) { this.logger.LogTrace("Cache is full now with {0} entries, evicting.", cacheEntryCount); this.EvictLocked(); } } return(result); }
/// <inheritdoc /> public async Task <FetchCoinsResponse> FetchCoinsAsync(uint256[] txIds, CancellationToken cancellationToken = default(CancellationToken)) { Guard.NotNull(txIds, nameof(txIds)); this.logger.LogTrace("({0}.{1}:{2})", nameof(txIds), nameof(txIds.Length), txIds.Length); FetchCoinsResponse result = null; var outputs = new UnspentOutputs[txIds.Length]; var miss = new List <int>(); var missedTxIds = new List <uint256>(); using (await this.lockobj.LockAsync(cancellationToken).ConfigureAwait(false)) { for (int i = 0; i < txIds.Length; i++) { CacheItem cache; if (!this.cachedUtxoItems.TryGetValue(txIds[i], out cache)) { this.logger.LogTrace("Cache missed for transaction ID '{0}'.", txIds[i]); miss.Add(i); missedTxIds.Add(txIds[i]); } else { this.logger.LogTrace("Cache hit for transaction ID '{0}'.", txIds[i]); outputs[i] = cache.UnspentOutputs == null ? null : cache.UnspentOutputs.IsPrunable ? null : cache.UnspentOutputs.Clone(); } } this.PerformanceCounter.AddMissCount(miss.Count); this.PerformanceCounter.AddHitCount(txIds.Length - miss.Count); } this.logger.LogTrace("{0} cache missed transaction needs to be loaded from underlying CoinView.", missedTxIds.Count); FetchCoinsResponse fetchedCoins = await this.Inner.FetchCoinsAsync(missedTxIds.ToArray(), cancellationToken).ConfigureAwait(false); using (await this.lockobj.LockAsync(cancellationToken).ConfigureAwait(false)) { uint256 innerblockHash = fetchedCoins.BlockHash; if (this.blockHash == null) { Debug.Assert(this.cachedUtxoItems.Count == 0); this.innerBlockHash = innerblockHash; this.blockHash = this.innerBlockHash; } for (int i = 0; i < miss.Count; i++) { int index = miss[i]; UnspentOutputs unspent = fetchedCoins.UnspentOutputs[i]; outputs[index] = unspent; var cache = new CacheItem(); cache.ExistInInner = unspent != null; cache.IsDirty = false; cache.UnspentOutputs = unspent?.Clone(); this.cachedUtxoItems.TryAdd(txIds[index], cache); } result = new FetchCoinsResponse(outputs, this.blockHash); } int cacheEntryCount = this.CacheEntryCount; if (cacheEntryCount > this.MaxItems) { this.logger.LogTrace("Cache is full now with {0} entries, evicting.", cacheEntryCount); await this.EvictAsync().ConfigureAwait(false); } this.logger.LogTrace("(-):*.{0}='{1}',*.{2}.{3}={4}", nameof(result.BlockHash), result.BlockHash, nameof(result.UnspentOutputs), nameof(result.UnspentOutputs.Length), result.UnspentOutputs.Length); return(result); }