Beispiel #1
0
        private void RestoreOutputs(RewindData rewindData)
        {
            foreach (UnspentOutputs unspentToRestore in rewindData.OutputsToRestore)
            {
                this.logger.LogTrace("Outputs of transaction ID '{0}' will be restored.", unspentToRestore.TransactionId);

                if (this.cachedUtxoItems.TryGetValue(unspentToRestore.TransactionId, out CacheItem cacheItem))
                {
                    cacheItem.UnspentOutputs = unspentToRestore;
                    cacheItem.IsDirty        = true;
                }
                else
                {
                    this.logger.LogTrace("Outputs of transaction ID '{0}' not found in cache, inserting them.", unspentToRestore.TransactionId);

                    cacheItem = new CacheItem
                    {
                        UnspentOutputs = unspentToRestore,
                        IsDirty        = true
                    };

                    this.cachedUtxoItems.Add(unspentToRestore.TransactionId, cacheItem);
                }
            }
        }
Beispiel #2
0
 private void RemoveTransactions(RewindData rewindData)
 {
     foreach (uint256 transactionToRemove in rewindData.TransactionsToRemove)
     {
         this.logger.LogTrace("Attempt to remove transaction with ID '{0}'.", transactionToRemove);
         this.cachedUtxoItems.Remove(transactionToRemove);
     }
 }
        /// <inheritdoc />
        public async Task <uint256> RewindAsync()
        {
            if (this.innerBlockHash == null)
            {
                this.innerBlockHash = await this.inner.GetTipHashAsync().ConfigureAwait(false);
            }

            using (await this.lockobj.LockAsync().ConfigureAwait(false))
            {
                // Check if rewind data is available in local cache. If it is
                // we can rewind and there is no need to check underlying storage.
                if (this.cachedRewindDataIndex.Count > 0)
                {
                    KeyValuePair <int, RewindData> lastRewindDataItem = this.cachedRewindDataIndex.Last();
                    RewindData lastRewindData = lastRewindDataItem.Value;

                    this.RemoveTransactions(lastRewindData);
                    this.RestoreOutputs(lastRewindData);

                    // Change current block hash to the one from the rewind data.
                    this.blockHash   = lastRewindData.PreviousBlockHash;
                    this.blockHeight = lastRewindDataItem.Key - 1;

                    this.cachedRewindDataIndex.Remove(lastRewindDataItem.Key);

                    if (this.rewindDataIndexCache != null)
                    {
                        await this.rewindDataIndexCache.Remove(this.blockHeight, this);
                    }

                    this.logger.LogTrace("(-)[REMOVED_FROM_BATCH]:'{0}'", this.blockHash);
                    return(this.blockHash);
                }

                // Rewind data was not found in cache, try underlying storage.
                uint256 hash = await this.inner.RewindAsync().ConfigureAwait(false);

                // All the cached utxos are now on disk so we can clear the cached entry list.
                this.cachedUtxoItems.Clear();

                this.innerBlockHash = hash;
                this.blockHash      = hash;
                this.blockHeight   -= 1;

                if (this.rewindDataIndexCache != null)
                {
                    await this.rewindDataIndexCache.InitializeAsync(this.blockHeight, this);
                }

                return(hash);
            }
        }
Beispiel #4
0
        /// <inheritdoc />
        public async Task <uint256> Rewind()
        {
            this.logger.LogTrace("()");

            if (this.innerBlockHash == null)
            {
                this.innerBlockHash = await this.inner.GetTipHashAsync().ConfigureAwait(false);
            }

            using (await this.lockobj.LockAsync().ConfigureAwait(false))
            {
                // Check if rewind data is available in local cache. If it is
                // we can rewind and there is no need to check underlying storage.
                if (this.cachedRewindDataList.Count > 0)
                {
                    RewindData lastRewindData = this.cachedRewindDataList.Last();

                    this.RemoveTransactions(lastRewindData);
                    this.RestoreOutputs(lastRewindData);

                    // Change current block hash to the one from the rewind data.
                    this.blockHash = lastRewindData.PreviousBlockHash;

                    this.cachedRewindDataList.RemoveAt(this.cachedRewindDataList.Count - 1);
                    this.logger.LogTrace("(-)[REMOVED_FROM_BATCH]:'{0}'", this.blockHash);
                    return(this.blockHash);
                }

                // Rewind data was not found in cache, try underlying storage.
                uint256 hash = await this.inner.Rewind().ConfigureAwait(false);

                // All the cached utxos are now on disk so we can clear the cached entry list.
                this.cachedUtxoItems.Clear();

                this.innerBlockHash = hash;
                this.blockHash      = hash;

                this.logger.LogTrace("(-):'{0}'", hash);
                return(hash);
            }
        }
        /// <inheritdoc />
        public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
        {
            Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
            Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
            Guard.NotNull(outputs, nameof(outputs));

            lock (this.lockobj)
            {
                if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
                {
                    this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
                    this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
                    throw new InvalidOperationException("Invalid oldBlockHash");
                }

                this.blockHash = nextBlockHash;
                long utxoSkipDisk = 0;

                var rewindData = new RewindData(oldBlockHash);
                Dictionary <OutPoint, int> indexItems = null;
                if (this.rewindDataIndexCache != null)
                {
                    indexItems = new Dictionary <OutPoint, int>();
                }

                foreach (UnspentOutput output in outputs)
                {
                    if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem))
                    {
                        // Add outputs to cache, this will happen for two cases
                        // 1. if a chaced item was evicted
                        // 2. for new outputs that are added

                        if (output.CreatedFromBlock)
                        {
                            // if the output is indicate that it was added from a block
                            // There is no need to spend an extra call to disk.

                            this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint);

                            cacheItem = new CacheItem()
                            {
                                ExistInInner = false,
                                IsDirty      = false,
                                OutPoint     = output.OutPoint,
                                Coins        = null
                            };
                        }
                        else
                        {
                            // This can happen if the cashe item was evicted while
                            // the block was being processed, fetch the outut again from disk.

                            this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint);

                            FetchCoinsResponse result = this.inner.FetchCoins(new[] { output.OutPoint });
                            this.performanceCounter.AddMissCount(1);

                            UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value;

                            cacheItem = new CacheItem()
                            {
                                ExistInInner = unspentOutput.Coins != null,
                                IsDirty      = false,
                                OutPoint     = unspentOutput.OutPoint,
                                Coins        = unspentOutput.Coins
                            };
                        }

                        this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem);
                        this.cacheSizeBytes += cacheItem.GetSize;
                        this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint);
                    }

                    // If output.Coins is null this means the utxo needs to be deleted
                    // otherwise this is a new utxo and we store it to cache.

                    if (output.Coins == null)
                    {
                        // DELETE COINS

                        // In cases of an output spent in the same block
                        // it wont exist in cash or in disk so its safe to remove it
                        if (cacheItem.Coins == null)
                        {
                            if (cacheItem.ExistInInner)
                            {
                                throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint));
                            }
                        }
                        else
                        {
                            // Handle rewind data
                            this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint);
                            rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins));
                            rewindData.TotalSize += cacheItem.GetSize;

                            if (this.rewindDataIndexCache != null && indexItems != null)
                            {
                                indexItems[cacheItem.OutPoint] = this.blockHash.Height;
                            }
                        }

                        // If a spent utxo never made it to disk then no need to keep it in memory.
                        if (!cacheItem.ExistInInner)
                        {
                            this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint);
                            this.cachedUtxoItems.Remove(cacheItem.OutPoint);
                            this.cacheSizeBytes -= cacheItem.GetSize;
                            utxoSkipDisk++;
                            if (cacheItem.IsDirty)
                            {
                                this.dirtyCacheCount--;
                            }
                        }
                        else
                        {
                            // Now modify the cached items with the mutated data.
                            this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint);

                            this.cacheSizeBytes -= cacheItem.GetScriptSize;
                            cacheItem.Coins      = null;

                            // Delete output from cache but keep a the cache
                            // item reference so it will get deleted form disk

                            cacheItem.IsDirty = true;
                            this.dirtyCacheCount++;
                        }
                    }
                    else
                    {
                        // ADD COINS

                        if (cacheItem.Coins != null)
                        {
                            // Allow overrides.
                            // See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94

                            bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null;

                            if (!allowOverride)
                            {
                                throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint));
                            }

                            this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint);

                            // Deduct the crurrent script size form the
                            // total cache size, it will be added again later.
                            this.cacheSizeBytes -= cacheItem.GetScriptSize;

                            // Clear this in order to calculate the cache sie
                            // this will get set later when overriden
                            cacheItem.Coins = null;
                        }

                        // Handle rewind data
                        // New trx so it needs to be deleted if a rewind happens.
                        this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint);
                        rewindData.OutputsToRemove.Add(cacheItem.OutPoint);
                        rewindData.TotalSize += cacheItem.GetSize;

                        // Put in the cache the new UTXOs.
                        this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint);

                        cacheItem.Coins      = output.Coins;
                        this.cacheSizeBytes += cacheItem.GetScriptSize;

                        // Mark the cache item as dirty so it get persisted
                        // to disk and not evicted form cache

                        cacheItem.IsDirty = true;
                        this.dirtyCacheCount++;
                    }
                }

                this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk);

                if (this.rewindDataIndexCache != null && indexItems.Any())
                {
                    this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems);
                }

                // Add the most recent rewind data to the cache.
                this.cachedRewindData.Add(this.blockHash.Height, rewindData);
                this.rewindDataSizeBytes += rewindData.TotalSize;

                // Remove rewind data form the back of a moving window.
                // The closer we get to the tip we keep a longer rewind data window.
                // Anything bellow last checkpoint we keep the minimal of 10
                // (random low number) rewind data items.
                // Beyond last checkpoint:
                // - For POS we keep a window of MaxReorg.
                // - For POW we keep 100 items (possibly better is an algo that grows closer to tip)

                // A moving window of information needed to rewind the node to a previous block.
                // When cache is flushed the rewind data will allow to rewind the node up to the
                // number of rewind blocks.
                // TODO: move rewind data to use block store.
                // Rewind data can go away all togetehr if the node uses the blocks in block store
                // to get the rewind information, blockstore persists much more frequent then coin cache
                // So using block store for rewinds is not entirely impossible.

                uint rewindDataWindow = 10;

                if (this.blockHash.Height >= this.lastCheckpointHeight)
                {
                    if (this.network.Consensus.MaxReorgLength != 0)
                    {
                        rewindDataWindow = this.network.Consensus.MaxReorgLength + 1;
                    }
                    else
                    {
                        // TODO: make the rewind data window a configuration
                        // parameter of evern a network parameter.

                        // For POW assume BTC where a rewind data of 100 is more then enough.
                        rewindDataWindow = 100;
                    }
                }

                int rewindToRemove = this.blockHash.Height - (int)rewindDataWindow;

                if (this.cachedRewindData.TryGetValue(rewindToRemove, out RewindData delete))
                {
                    this.logger.LogDebug("Remove rewind data height '{0}' from cache.", rewindToRemove);
                    this.cachedRewindData.Remove(rewindToRemove);
                    this.rewindDataSizeBytes -= delete.TotalSize;
                }
            }
        }
Beispiel #6
0
        /// <inheritdoc />
        public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash)
        {
            this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs?.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash);

            RewindData rewindData       = originalOutputs != null ? new RewindData(oldBlockHash) : null;
            int        insertedEntities = 0;

            List <UnspentOutputs>         all = unspentOutputs.ToList();
            Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count);

            using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o)))
            {
                if (originalOutputs != null)
                {
                    IEnumerator <TxOut[]> originalEnumerator = originalOutputs.GetEnumerator();
                    foreach (UnspentOutputs output in all)
                    {
                        originalEnumerator.MoveNext();
                        unspentToOriginal.Add(output.TransactionId, originalEnumerator.Current);
                    }
                }
            }

            Task task = Task.Run(() =>
            {
                this.logger.LogTrace("()");

                using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction())
                {
                    transaction.ValuesLazyLoadingIsOn = false;
                    transaction.SynchronizeTables("BlockHash", "Coins", "Rewind");
                    transaction.Technical_SetTable_OverwriteIsNotAllowed("Coins");

                    using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o)))
                    {
                        uint256 current = this.GetCurrentHash(transaction);
                        if (current != oldBlockHash)
                        {
                            this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
                            throw new InvalidOperationException("Invalid oldBlockHash");
                        }

                        this.SetBlockHash(transaction, nextBlockHash);

                        all.Sort(UnspentOutputsComparer.Instance);
                        foreach (UnspentOutputs coin in all)
                        {
                            this.logger.LogTrace("Outputs of transaction ID '{0}' are {1} and will be {2} to the database.", coin.TransactionId, coin.IsPrunable ? "PRUNABLE" : "NOT PRUNABLE", coin.IsPrunable ? "removed" : "inserted");
                            if (coin.IsPrunable)
                            {
                                transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false));
                            }
                            else
                            {
                                transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins());
                            }

                            if (originalOutputs != null)
                            {
                                TxOut[] original = null;
                                unspentToOriginal.TryGetValue(coin.TransactionId, out original);
                                if (original == null)
                                {
                                    // This one haven't existed before, if we rewind, delete it.
                                    rewindData.TransactionsToRemove.Add(coin.TransactionId);
                                }
                                else
                                {
                                    // We'll need to restore the original outputs.
                                    UnspentOutputs clone = coin.Clone();
                                    clone._Outputs       = original.ToArray();
                                    rewindData.OutputsToRestore.Add(clone);
                                }
                            }
                        }

                        if (rewindData != null)
                        {
                            int nextRewindIndex = this.GetRewindIndex(transaction) + 1;
                            this.logger.LogTrace("Rewind state #{0} created.", nextRewindIndex);
                            transaction.Insert("Rewind", nextRewindIndex, rewindData);
                        }

                        insertedEntities += all.Count;
                        transaction.Commit();
                    }
                }

                this.PerformanceCounter.AddInsertedEntities(insertedEntities);
                this.logger.LogTrace("(-)");
            });

            this.logger.LogTrace("(-)");
            return(task);
        }
Beispiel #7
0
        /// <inheritdoc />
        public void SaveChanges(IList <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, int height, List <RewindData> rewindDataList = null)
        {
            Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
            Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
            Guard.NotNull(unspentOutputs, nameof(unspentOutputs));

            lock (this.lockobj)
            {
                if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
                {
                    this.logger.LogTrace("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
                    this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
                    throw new InvalidOperationException("Invalid oldBlockHash");
                }

                this.blockHeight = height;
                this.blockHash   = nextBlockHash;
                var rewindData = new RewindData(oldBlockHash);
                var indexItems = new Dictionary <OutPoint, int>();

                foreach (UnspentOutputs unspent in unspentOutputs)
                {
                    if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem))
                    {
                        // This can happen very rarely in the case where we fetch items from
                        // disk but immediately call the Evict method which then removes the cached item(s).

                        this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId);

                        FetchCoinsResponse result = this.inner.FetchCoins(new[] { unspent.TransactionId });

                        UnspentOutputs unspentOutput = result.UnspentOutputs[0];

                        cacheItem = new CacheItem();
                        cacheItem.ExistInInner   = unspentOutput != null;
                        cacheItem.IsDirty        = false;
                        cacheItem.UnspentOutputs = unspentOutput?.Clone();

                        this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem);
                        this.logger.LogTrace("CacheItem added to the cache during save '{0}'.", cacheItem.UnspentOutputs);
                    }

                    // If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk,
                    // that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache
                    // otherwise we store to cache only the UTXO that have been spent.

                    if (cacheItem.UnspentOutputs != null)
                    {
                        // To handle rewind we'll need to restore the original outputs,
                        // so we clone it and save it in rewind data.
                        UnspentOutputs clone = unspent.Clone();

                        // We take the original items that are in cache and put them in rewind data.
                        clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray();

                        this.logger.LogTrace("Modifying transaction '{0}' in OutputsToRestore rewind data.", unspent.TransactionId);
                        rewindData.OutputsToRestore.Add(clone);

                        this.logger.LogTrace("Cache item before spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs);

                        // Now modify the cached items with the mutated data.
                        cacheItem.UnspentOutputs.Spend(unspent);

                        this.logger.LogTrace("Cache item after spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs);
                    }
                    else
                    {
                        // New trx so it needs to be deleted if a rewind happens.
                        this.logger.LogTrace("Adding transaction '{0}' to TransactionsToRemove rewind data.", unspent.TransactionId);
                        rewindData.TransactionsToRemove.Add(unspent.TransactionId);

                        // Put in the cache the new UTXOs.
                        this.logger.LogTrace("Setting {0} to {1}: '{2}'.", nameof(cacheItem.UnspentOutputs), nameof(unspent), unspent);
                        cacheItem.UnspentOutputs = unspent;
                    }

                    cacheItem.IsDirty = true;

                    if (this.rewindDataIndexCache != null)
                    {
                        for (int i = 0; i < unspent.Outputs.Length; i++)
                        {
                            var key = new OutPoint(unspent.TransactionId, i);
                            indexItems[key] = this.blockHeight;
                        }
                    }

                    // Inner does not need to know pruned unspent that it never saw.
                    if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner)
                    {
                        this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId);
                        this.cachedUtxoItems.Remove(unspent.TransactionId);
                    }
                }

                if (this.rewindDataIndexCache != null && indexItems.Any())
                {
                    this.rewindDataIndexCache.Save(indexItems);
                    this.rewindDataIndexCache.Flush(this.blockHeight);
                }

                this.cachedRewindDataIndex.Add(this.blockHeight, rewindData);
            }
        }
Beispiel #8
0
        /// <inheritdoc />
        public async Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, List <RewindData> rewindDataList = null)
        {
            Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
            Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
            Guard.NotNull(unspentOutputs, nameof(unspentOutputs));
            this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash);

            using (await this.lockobj.LockAsync().ConfigureAwait(false))
            {
                if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
                {
                    this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
                    throw new InvalidOperationException("Invalid oldBlockHash");
                }

                this.blockHash = nextBlockHash;
                var rewindData = new RewindData(oldBlockHash);

                foreach (UnspentOutputs unspent in unspentOutputs)
                {
                    if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem))
                    {
                        // This can happen very rarely in the case where we fetch items from
                        // disk but immediately call the Evict method which then removes the cached item(s).

                        this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId);

                        FetchCoinsResponse result = await this.inner.FetchCoinsAsync(new[] { unspent.TransactionId }).ConfigureAwait(false);

                        UnspentOutputs unspentOutput = result.UnspentOutputs[0];

                        cacheItem = new CacheItem();
                        cacheItem.ExistInInner = unspentOutput != null;
                        cacheItem.IsDirty      = false;

                        cacheItem.UnspentOutputs = unspentOutput?.Clone();

                        this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem);
                    }
                    else
                    {
                        this.logger.LogTrace("Outputs of transaction ID '{0}' are in cache already, updating them.", unspent.TransactionId);
                    }

                    // If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk,
                    // that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache
                    // otherwise we store to cache only the UTXO that have been spent.

                    if (cacheItem.UnspentOutputs != null)
                    {
                        // To handle rewind we'll need to restore the original outputs,
                        // so we clone it and save it in rewind data.
                        UnspentOutputs clone = unspent.Clone();

                        // We take the original items that are in cache and put them in rewind data.
                        clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray();
                        rewindData.OutputsToRestore.Add(clone);

                        // Now modify the cached items with the mutated data.
                        cacheItem.UnspentOutputs.Spend(unspent);
                    }
                    else
                    {
                        // New trx so it needs to be deleted if a rewind happens.
                        rewindData.TransactionsToRemove.Add(unspent.TransactionId);

                        // Put in the cache the new UTXOs.
                        cacheItem.UnspentOutputs = unspent;
                    }

                    cacheItem.IsDirty = true;

                    // Inner does not need to know pruned unspent that it never saw.
                    if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner)
                    {
                        this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId);
                        this.cachedUtxoItems.Remove(unspent.TransactionId);
                    }
                }

                this.cachedRewindDataList.Add(rewindData);
            }

            this.logger.LogTrace("(-)");
        }
        /// <inheritdoc />
        public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash)
        {
            return(this.session.Execute(() =>
            {
                this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs?.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash);

                RewindData rewindData = originalOutputs == null ? null : new RewindData(oldBlockHash);
                int insertedEntities = 0;
                using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o)))
                {
                    uint256 current = this.GetCurrentHash();
                    if (current != oldBlockHash)
                    {
                        this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
                        throw new InvalidOperationException("Invalid oldBlockHash");
                    }

                    this.SetBlockHash(nextBlockHash);
                    List <UnspentOutputs> all = unspentOutputs.ToList();
                    Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count);
                    if (originalOutputs != null)
                    {
                        IEnumerator <TxOut[]> originalEnumerator = originalOutputs.GetEnumerator();
                        foreach (UnspentOutputs output in all)
                        {
                            originalEnumerator.MoveNext();
                            unspentToOriginal.Add(output.TransactionId, originalEnumerator.Current);
                        }
                    }

                    all.Sort(UnspentOutputsComparer.Instance);
                    foreach (UnspentOutputs coin in all)
                    {
                        this.logger.LogTrace("Outputs of transaction ID '{0}' are {1} and will be {2} to the database.", coin.TransactionId, coin.IsPrunable ? "PRUNABLE" : "NOT PRUNABLE", coin.IsPrunable ? "removed" : "inserted");
                        if (coin.IsPrunable)
                        {
                            this.session.Transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false));
                        }
                        else
                        {
                            this.session.Transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins());
                        }

                        if (originalOutputs != null)
                        {
                            TxOut[] original = null;
                            unspentToOriginal.TryGetValue(coin.TransactionId, out original);
                            if (original == null)
                            {
                                // This one haven't existed before, if we rewind, delete it.
                                rewindData.TransactionsToRemove.Add(coin.TransactionId);
                            }
                            else
                            {
                                // We'll need to restore the original outputs.
                                UnspentOutputs clone = coin.Clone();
                                // TODO: Can we remove this line? before is not used anywhere and clone.UnspentCount does not seem to have side effects?
                                int before = clone.UnspentCount;
                                clone._Outputs = original.ToArray();
                                rewindData.OutputsToRestore.Add(clone);
                            }
                        }
                    }

                    if (rewindData != null)
                    {
                        int nextRewindIndex = this.GetRewindIndex() + 1;
                        this.logger.LogTrace("Rewind state #{0} created.", nextRewindIndex);
                        this.session.Transaction.Insert("Rewind", nextRewindIndex, rewindData);
                    }

                    insertedEntities += all.Count;
                    this.session.Transaction.Commit();
                }

                this.PerformanceCounter.AddInsertedEntities(insertedEntities);
                this.logger.LogTrace("(-)");
            }));
        }
 public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash)
 {
     return(this.session.Execute(() =>
     {
         RewindData rewindData = originalOutputs == null ? null : new RewindData(oldBlockHash);
         int insertedEntities = 0;
         using (new StopWatch().Start(o => this.PerformanceCounter.AddInsertTime(o)))
         {
             var current = this.GetCurrentHash();
             if (current != oldBlockHash)
             {
                 throw new InvalidOperationException("Invalid oldBlockHash");
             }
             this.SetBlockHash(nextBlockHash);
             var all = unspentOutputs.ToList();
             Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count);
             if (originalOutputs != null)
             {
                 var originalEnumerator = originalOutputs.GetEnumerator();
                 foreach (var u in all)
                 {
                     originalEnumerator.MoveNext();
                     unspentToOriginal.Add(u.TransactionId, originalEnumerator.Current);
                 }
             }
             all.Sort(UnspentOutputsComparer.Instance);
             foreach (var coin in all)
             {
                 if (coin.IsPrunable)
                 {
                     this.session.Transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false));
                 }
                 else
                 {
                     this.session.Transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins());
                 }
                 if (originalOutputs != null)
                 {
                     TxOut[] original = null;
                     unspentToOriginal.TryGetValue(coin.TransactionId, out original);
                     if (original == null)
                     {
                         //This one did not existed before, if we rewind, delete it
                         rewindData.TransactionsToRemove.Add(coin.TransactionId);
                     }
                     else
                     {
                         //We'll need to restore the original outputs
                         var clone = coin.Clone();
                         var before = clone.UnspentCount;
                         clone._Outputs = original.ToArray();
                         rewindData.OutputsToRestore.Add(clone);
                     }
                 }
             }
             if (rewindData != null)
             {
                 int nextRewindIndex = this.GetRewindIndex() + 1;
                 this.session.Transaction.Insert <int, RewindData>("Rewind", nextRewindIndex, rewindData);
             }
             insertedEntities += all.Count;
             this.session.Transaction.Commit();
         }
         this.PerformanceCounter.AddInsertedEntities(insertedEntities);
     }));
 }
        /// <inheritdoc />
        public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
        {
            Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
            Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
            Guard.NotNull(outputs, nameof(outputs));

            lock (this.lockobj)
            {
                if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
                {
                    this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
                    this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
                    throw new InvalidOperationException("Invalid oldBlockHash");
                }

                this.blockHash = nextBlockHash;
                long utxoSkipDisk = 0;

                var rewindData = new RewindData(oldBlockHash);
                Dictionary <OutPoint, int> indexItems = null;
                if (this.rewindDataIndexCache != null)
                {
                    indexItems = new Dictionary <OutPoint, int>();
                }

                foreach (UnspentOutput output in outputs)
                {
                    if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem))
                    {
                        // Add outputs to cache, this will happen for two cases
                        // 1. if a chaced item was evicted
                        // 2. for new outputs that are added

                        if (output.CreatedFromBlock)
                        {
                            // if the output is indicate that it was added from a block
                            // There is no need to spend an extra call to disk.

                            this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint);

                            cacheItem = new CacheItem()
                            {
                                ExistInInner = false,
                                IsDirty      = false,
                                OutPoint     = output.OutPoint,
                                Coins        = null
                            };
                        }
                        else
                        {
                            // This can happen if the cashe item was evicted while
                            // the block was being processed, fetch the outut again from disk.

                            this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint);

                            FetchCoinsResponse result = this.coindb.FetchCoins(new[] { output.OutPoint });
                            this.performanceCounter.AddMissCount(1);

                            UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value;

                            cacheItem = new CacheItem()
                            {
                                ExistInInner = unspentOutput.Coins != null,
                                IsDirty      = false,
                                OutPoint     = unspentOutput.OutPoint,
                                Coins        = unspentOutput.Coins
                            };
                        }

                        this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem);
                        this.cacheSizeBytes += cacheItem.GetSize;
                        this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint);
                    }

                    // If output.Coins is null this means the utxo needs to be deleted
                    // otherwise this is a new utxo and we store it to cache.

                    if (output.Coins == null)
                    {
                        // DELETE COINS

                        // In cases of an output spent in the same block
                        // it wont exist in cash or in disk so its safe to remove it
                        if (cacheItem.Coins == null)
                        {
                            if (cacheItem.ExistInInner)
                            {
                                throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint));
                            }
                        }
                        else
                        {
                            // Handle rewind data
                            this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint);
                            rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins));
                            rewindData.TotalSize += cacheItem.GetSize;

                            if (this.rewindDataIndexCache != null && indexItems != null)
                            {
                                indexItems[cacheItem.OutPoint] = this.blockHash.Height;
                            }
                        }

                        // If a spent utxo never made it to disk then no need to keep it in memory.
                        if (!cacheItem.ExistInInner)
                        {
                            this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint);
                            this.cachedUtxoItems.Remove(cacheItem.OutPoint);
                            this.cacheSizeBytes -= cacheItem.GetSize;
                            utxoSkipDisk++;
                            if (cacheItem.IsDirty)
                            {
                                this.dirtyCacheCount--;
                            }
                        }
                        else
                        {
                            // Now modify the cached items with the mutated data.
                            this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint);

                            this.cacheSizeBytes -= cacheItem.GetScriptSize;
                            cacheItem.Coins      = null;

                            // Delete output from cache but keep a the cache
                            // item reference so it will get deleted form disk

                            cacheItem.IsDirty = true;
                            this.dirtyCacheCount++;
                        }
                    }
                    else
                    {
                        // ADD COINS

                        if (cacheItem.Coins != null)
                        {
                            // Allow overrides.
                            // See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94

                            bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null;

                            if (!allowOverride)
                            {
                                throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint));
                            }

                            this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint);

                            // Deduct the crurrent script size form the
                            // total cache size, it will be added again later.
                            this.cacheSizeBytes -= cacheItem.GetScriptSize;

                            // Clear this in order to calculate the cache sie
                            // this will get set later when overriden
                            cacheItem.Coins = null;
                        }

                        // Handle rewind data
                        // New trx so it needs to be deleted if a rewind happens.
                        this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint);
                        rewindData.OutputsToRemove.Add(cacheItem.OutPoint);
                        rewindData.TotalSize += cacheItem.GetSize;

                        // Put in the cache the new UTXOs.
                        this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint);

                        cacheItem.Coins      = output.Coins;
                        this.cacheSizeBytes += cacheItem.GetScriptSize;

                        // Mark the cache item as dirty so it get persisted
                        // to disk and not evicted form cache

                        cacheItem.IsDirty = true;
                        this.dirtyCacheCount++;
                    }
                }

                this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk);

                if (this.rewindDataIndexCache != null && indexItems.Any())
                {
                    this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems);
                }

                // Add the most recent rewind data to the cache.
                this.cachedRewindData.Add(this.blockHash.Height, rewindData);
                this.rewindDataSizeBytes += rewindData.TotalSize;
            }
        }