private void RestoreOutputs(RewindData rewindData)
{
foreach (UnspentOutputs unspentToRestore in rewindData.OutputsToRestore)
{
this.logger.LogTrace("Outputs of transaction ID '{0}' will be restored.", unspentToRestore.TransactionId);
if (this.cachedUtxoItems.TryGetValue(unspentToRestore.TransactionId, out CacheItem cacheItem))
{
cacheItem.UnspentOutputs = unspentToRestore;
cacheItem.IsDirty = true;
}
else
{
this.logger.LogTrace("Outputs of transaction ID '{0}' not found in cache, inserting them.", unspentToRestore.TransactionId);
cacheItem = new CacheItem
{
UnspentOutputs = unspentToRestore,
IsDirty = true
};
this.cachedUtxoItems.Add(unspentToRestore.TransactionId, cacheItem);
}
}
}
private void RemoveTransactions(RewindData rewindData)
{
foreach (uint256 transactionToRemove in rewindData.TransactionsToRemove)
{
this.logger.LogTrace("Attempt to remove transaction with ID '{0}'.", transactionToRemove);
this.cachedUtxoItems.Remove(transactionToRemove);
}
}
/// <inheritdoc />
public async Task <uint256> RewindAsync()
{
if (this.innerBlockHash == null)
{
this.innerBlockHash = await this.inner.GetTipHashAsync().ConfigureAwait(false);
}
using (await this.lockobj.LockAsync().ConfigureAwait(false))
{
// Check if rewind data is available in local cache. If it is
// we can rewind and there is no need to check underlying storage.
if (this.cachedRewindDataIndex.Count > 0)
{
KeyValuePair <int, RewindData> lastRewindDataItem = this.cachedRewindDataIndex.Last();
RewindData lastRewindData = lastRewindDataItem.Value;
this.RemoveTransactions(lastRewindData);
this.RestoreOutputs(lastRewindData);
// Change current block hash to the one from the rewind data.
this.blockHash = lastRewindData.PreviousBlockHash;
this.blockHeight = lastRewindDataItem.Key - 1;
this.cachedRewindDataIndex.Remove(lastRewindDataItem.Key);
if (this.rewindDataIndexCache != null)
{
await this.rewindDataIndexCache.Remove(this.blockHeight, this);
}
this.logger.LogTrace("(-)[REMOVED_FROM_BATCH]:'{0}'", this.blockHash);
return(this.blockHash);
}
// Rewind data was not found in cache, try underlying storage.
uint256 hash = await this.inner.RewindAsync().ConfigureAwait(false);
// All the cached utxos are now on disk so we can clear the cached entry list.
this.cachedUtxoItems.Clear();
this.innerBlockHash = hash;
this.blockHash = hash;
this.blockHeight -= 1;
if (this.rewindDataIndexCache != null)
{
await this.rewindDataIndexCache.InitializeAsync(this.blockHeight, this);
}
return(hash);
}
}
/// <inheritdoc />
public async Task <uint256> Rewind()
{
this.logger.LogTrace("()");
if (this.innerBlockHash == null)
{
this.innerBlockHash = await this.inner.GetTipHashAsync().ConfigureAwait(false);
}
using (await this.lockobj.LockAsync().ConfigureAwait(false))
{
// Check if rewind data is available in local cache. If it is
// we can rewind and there is no need to check underlying storage.
if (this.cachedRewindDataList.Count > 0)
{
RewindData lastRewindData = this.cachedRewindDataList.Last();
this.RemoveTransactions(lastRewindData);
this.RestoreOutputs(lastRewindData);
// Change current block hash to the one from the rewind data.
this.blockHash = lastRewindData.PreviousBlockHash;
this.cachedRewindDataList.RemoveAt(this.cachedRewindDataList.Count - 1);
this.logger.LogTrace("(-)[REMOVED_FROM_BATCH]:'{0}'", this.blockHash);
return(this.blockHash);
}
// Rewind data was not found in cache, try underlying storage.
uint256 hash = await this.inner.Rewind().ConfigureAwait(false);
// All the cached utxos are now on disk so we can clear the cached entry list.
this.cachedUtxoItems.Clear();
this.innerBlockHash = hash;
this.blockHash = hash;
this.logger.LogTrace("(-):'{0}'", hash);
return(hash);
}
}
/// <inheritdoc />
public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(outputs, nameof(outputs));
lock (this.lockobj)
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHash = nextBlockHash;
long utxoSkipDisk = 0;
var rewindData = new RewindData(oldBlockHash);
Dictionary <OutPoint, int> indexItems = null;
if (this.rewindDataIndexCache != null)
{
indexItems = new Dictionary <OutPoint, int>();
}
foreach (UnspentOutput output in outputs)
{
if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem))
{
// Add outputs to cache, this will happen for two cases
// 1. if a chaced item was evicted
// 2. for new outputs that are added
if (output.CreatedFromBlock)
{
// if the output is indicate that it was added from a block
// There is no need to spend an extra call to disk.
this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint);
cacheItem = new CacheItem()
{
ExistInInner = false,
IsDirty = false,
OutPoint = output.OutPoint,
Coins = null
};
}
else
{
// This can happen if the cashe item was evicted while
// the block was being processed, fetch the outut again from disk.
this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint);
FetchCoinsResponse result = this.inner.FetchCoins(new[] { output.OutPoint });
this.performanceCounter.AddMissCount(1);
UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value;
cacheItem = new CacheItem()
{
ExistInInner = unspentOutput.Coins != null,
IsDirty = false,
OutPoint = unspentOutput.OutPoint,
Coins = unspentOutput.Coins
};
}
this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem);
this.cacheSizeBytes += cacheItem.GetSize;
this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint);
}
// If output.Coins is null this means the utxo needs to be deleted
// otherwise this is a new utxo and we store it to cache.
if (output.Coins == null)
{
// DELETE COINS
// In cases of an output spent in the same block
// it wont exist in cash or in disk so its safe to remove it
if (cacheItem.Coins == null)
{
if (cacheItem.ExistInInner)
{
throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint));
}
}
else
{
// Handle rewind data
this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins));
rewindData.TotalSize += cacheItem.GetSize;
if (this.rewindDataIndexCache != null && indexItems != null)
{
indexItems[cacheItem.OutPoint] = this.blockHash.Height;
}
}
// If a spent utxo never made it to disk then no need to keep it in memory.
if (!cacheItem.ExistInInner)
{
this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint);
this.cachedUtxoItems.Remove(cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetSize;
utxoSkipDisk++;
if (cacheItem.IsDirty)
{
this.dirtyCacheCount--;
}
}
else
{
// Now modify the cached items with the mutated data.
this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetScriptSize;
cacheItem.Coins = null;
// Delete output from cache but keep a the cache
// item reference so it will get deleted form disk
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
else
{
// ADD COINS
if (cacheItem.Coins != null)
{
// Allow overrides.
// See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94
bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null;
if (!allowOverride)
{
throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint));
}
this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint);
// Deduct the crurrent script size form the
// total cache size, it will be added again later.
this.cacheSizeBytes -= cacheItem.GetScriptSize;
// Clear this in order to calculate the cache sie
// this will get set later when overriden
cacheItem.Coins = null;
}
// Handle rewind data
// New trx so it needs to be deleted if a rewind happens.
this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRemove.Add(cacheItem.OutPoint);
rewindData.TotalSize += cacheItem.GetSize;
// Put in the cache the new UTXOs.
this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint);
cacheItem.Coins = output.Coins;
this.cacheSizeBytes += cacheItem.GetScriptSize;
// Mark the cache item as dirty so it get persisted
// to disk and not evicted form cache
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk);
if (this.rewindDataIndexCache != null && indexItems.Any())
{
this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems);
}
// Add the most recent rewind data to the cache.
this.cachedRewindData.Add(this.blockHash.Height, rewindData);
this.rewindDataSizeBytes += rewindData.TotalSize;
// Remove rewind data form the back of a moving window.
// The closer we get to the tip we keep a longer rewind data window.
// Anything bellow last checkpoint we keep the minimal of 10
// (random low number) rewind data items.
// Beyond last checkpoint:
// - For POS we keep a window of MaxReorg.
// - For POW we keep 100 items (possibly better is an algo that grows closer to tip)
// A moving window of information needed to rewind the node to a previous block.
// When cache is flushed the rewind data will allow to rewind the node up to the
// number of rewind blocks.
// TODO: move rewind data to use block store.
// Rewind data can go away all togetehr if the node uses the blocks in block store
// to get the rewind information, blockstore persists much more frequent then coin cache
// So using block store for rewinds is not entirely impossible.
uint rewindDataWindow = 10;
if (this.blockHash.Height >= this.lastCheckpointHeight)
{
if (this.network.Consensus.MaxReorgLength != 0)
{
rewindDataWindow = this.network.Consensus.MaxReorgLength + 1;
}
else
{
// TODO: make the rewind data window a configuration
// parameter of evern a network parameter.
// For POW assume BTC where a rewind data of 100 is more then enough.
rewindDataWindow = 100;
}
}
int rewindToRemove = this.blockHash.Height - (int)rewindDataWindow;
if (this.cachedRewindData.TryGetValue(rewindToRemove, out RewindData delete))
{
this.logger.LogDebug("Remove rewind data height '{0}' from cache.", rewindToRemove);
this.cachedRewindData.Remove(rewindToRemove);
this.rewindDataSizeBytes -= delete.TotalSize;
}
}
}
/// <inheritdoc />
public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash)
{
this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs?.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash);
RewindData rewindData = originalOutputs != null ? new RewindData(oldBlockHash) : null;
int insertedEntities = 0;
List <UnspentOutputs> all = unspentOutputs.ToList();
Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count);
using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o)))
{
if (originalOutputs != null)
{
IEnumerator <TxOut[]> originalEnumerator = originalOutputs.GetEnumerator();
foreach (UnspentOutputs output in all)
{
originalEnumerator.MoveNext();
unspentToOriginal.Add(output.TransactionId, originalEnumerator.Current);
}
}
}
Task task = Task.Run(() =>
{
this.logger.LogTrace("()");
using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction())
{
transaction.ValuesLazyLoadingIsOn = false;
transaction.SynchronizeTables("BlockHash", "Coins", "Rewind");
transaction.Technical_SetTable_OverwriteIsNotAllowed("Coins");
using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o)))
{
uint256 current = this.GetCurrentHash(transaction);
if (current != oldBlockHash)
{
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.SetBlockHash(transaction, nextBlockHash);
all.Sort(UnspentOutputsComparer.Instance);
foreach (UnspentOutputs coin in all)
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are {1} and will be {2} to the database.", coin.TransactionId, coin.IsPrunable ? "PRUNABLE" : "NOT PRUNABLE", coin.IsPrunable ? "removed" : "inserted");
if (coin.IsPrunable)
{
transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false));
}
else
{
transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins());
}
if (originalOutputs != null)
{
TxOut[] original = null;
unspentToOriginal.TryGetValue(coin.TransactionId, out original);
if (original == null)
{
// This one haven't existed before, if we rewind, delete it.
rewindData.TransactionsToRemove.Add(coin.TransactionId);
}
else
{
// We'll need to restore the original outputs.
UnspentOutputs clone = coin.Clone();
clone._Outputs = original.ToArray();
rewindData.OutputsToRestore.Add(clone);
}
}
}
if (rewindData != null)
{
int nextRewindIndex = this.GetRewindIndex(transaction) + 1;
this.logger.LogTrace("Rewind state #{0} created.", nextRewindIndex);
transaction.Insert("Rewind", nextRewindIndex, rewindData);
}
insertedEntities += all.Count;
transaction.Commit();
}
}
this.PerformanceCounter.AddInsertedEntities(insertedEntities);
this.logger.LogTrace("(-)");
});
this.logger.LogTrace("(-)");
return(task);
}
/// <inheritdoc />
public void SaveChanges(IList <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, int height, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(unspentOutputs, nameof(unspentOutputs));
lock (this.lockobj)
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogTrace("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHeight = height;
this.blockHash = nextBlockHash;
var rewindData = new RewindData(oldBlockHash);
var indexItems = new Dictionary <OutPoint, int>();
foreach (UnspentOutputs unspent in unspentOutputs)
{
if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem))
{
// This can happen very rarely in the case where we fetch items from
// disk but immediately call the Evict method which then removes the cached item(s).
this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId);
FetchCoinsResponse result = this.inner.FetchCoins(new[] { unspent.TransactionId });
UnspentOutputs unspentOutput = result.UnspentOutputs[0];
cacheItem = new CacheItem();
cacheItem.ExistInInner = unspentOutput != null;
cacheItem.IsDirty = false;
cacheItem.UnspentOutputs = unspentOutput?.Clone();
this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem);
this.logger.LogTrace("CacheItem added to the cache during save '{0}'.", cacheItem.UnspentOutputs);
}
// If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk,
// that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache
// otherwise we store to cache only the UTXO that have been spent.
if (cacheItem.UnspentOutputs != null)
{
// To handle rewind we'll need to restore the original outputs,
// so we clone it and save it in rewind data.
UnspentOutputs clone = unspent.Clone();
// We take the original items that are in cache and put them in rewind data.
clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray();
this.logger.LogTrace("Modifying transaction '{0}' in OutputsToRestore rewind data.", unspent.TransactionId);
rewindData.OutputsToRestore.Add(clone);
this.logger.LogTrace("Cache item before spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs);
// Now modify the cached items with the mutated data.
cacheItem.UnspentOutputs.Spend(unspent);
this.logger.LogTrace("Cache item after spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs);
}
else
{
// New trx so it needs to be deleted if a rewind happens.
this.logger.LogTrace("Adding transaction '{0}' to TransactionsToRemove rewind data.", unspent.TransactionId);
rewindData.TransactionsToRemove.Add(unspent.TransactionId);
// Put in the cache the new UTXOs.
this.logger.LogTrace("Setting {0} to {1}: '{2}'.", nameof(cacheItem.UnspentOutputs), nameof(unspent), unspent);
cacheItem.UnspentOutputs = unspent;
}
cacheItem.IsDirty = true;
if (this.rewindDataIndexCache != null)
{
for (int i = 0; i < unspent.Outputs.Length; i++)
{
var key = new OutPoint(unspent.TransactionId, i);
indexItems[key] = this.blockHeight;
}
}
// Inner does not need to know pruned unspent that it never saw.
if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner)
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId);
this.cachedUtxoItems.Remove(unspent.TransactionId);
}
}
if (this.rewindDataIndexCache != null && indexItems.Any())
{
this.rewindDataIndexCache.Save(indexItems);
this.rewindDataIndexCache.Flush(this.blockHeight);
}
this.cachedRewindDataIndex.Add(this.blockHeight, rewindData);
}
}
/// <inheritdoc />
public async Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(unspentOutputs, nameof(unspentOutputs));
this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash);
using (await this.lockobj.LockAsync().ConfigureAwait(false))
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHash = nextBlockHash;
var rewindData = new RewindData(oldBlockHash);
foreach (UnspentOutputs unspent in unspentOutputs)
{
if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem))
{
// This can happen very rarely in the case where we fetch items from
// disk but immediately call the Evict method which then removes the cached item(s).
this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId);
FetchCoinsResponse result = await this.inner.FetchCoinsAsync(new[] { unspent.TransactionId }).ConfigureAwait(false);
UnspentOutputs unspentOutput = result.UnspentOutputs[0];
cacheItem = new CacheItem();
cacheItem.ExistInInner = unspentOutput != null;
cacheItem.IsDirty = false;
cacheItem.UnspentOutputs = unspentOutput?.Clone();
this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem);
}
else
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are in cache already, updating them.", unspent.TransactionId);
}
// If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk,
// that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache
// otherwise we store to cache only the UTXO that have been spent.
if (cacheItem.UnspentOutputs != null)
{
// To handle rewind we'll need to restore the original outputs,
// so we clone it and save it in rewind data.
UnspentOutputs clone = unspent.Clone();
// We take the original items that are in cache and put them in rewind data.
clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray();
rewindData.OutputsToRestore.Add(clone);
// Now modify the cached items with the mutated data.
cacheItem.UnspentOutputs.Spend(unspent);
}
else
{
// New trx so it needs to be deleted if a rewind happens.
rewindData.TransactionsToRemove.Add(unspent.TransactionId);
// Put in the cache the new UTXOs.
cacheItem.UnspentOutputs = unspent;
}
cacheItem.IsDirty = true;
// Inner does not need to know pruned unspent that it never saw.
if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner)
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId);
this.cachedUtxoItems.Remove(unspent.TransactionId);
}
}
this.cachedRewindDataList.Add(rewindData);
}
this.logger.LogTrace("(-)");
}
/// <inheritdoc />
public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash)
{
return(this.session.Execute(() =>
{
this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs?.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash);
RewindData rewindData = originalOutputs == null ? null : new RewindData(oldBlockHash);
int insertedEntities = 0;
using (new StopwatchDisposable(o => this.PerformanceCounter.AddInsertTime(o)))
{
uint256 current = this.GetCurrentHash();
if (current != oldBlockHash)
{
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.SetBlockHash(nextBlockHash);
List <UnspentOutputs> all = unspentOutputs.ToList();
Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count);
if (originalOutputs != null)
{
IEnumerator <TxOut[]> originalEnumerator = originalOutputs.GetEnumerator();
foreach (UnspentOutputs output in all)
{
originalEnumerator.MoveNext();
unspentToOriginal.Add(output.TransactionId, originalEnumerator.Current);
}
}
all.Sort(UnspentOutputsComparer.Instance);
foreach (UnspentOutputs coin in all)
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are {1} and will be {2} to the database.", coin.TransactionId, coin.IsPrunable ? "PRUNABLE" : "NOT PRUNABLE", coin.IsPrunable ? "removed" : "inserted");
if (coin.IsPrunable)
{
this.session.Transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false));
}
else
{
this.session.Transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins());
}
if (originalOutputs != null)
{
TxOut[] original = null;
unspentToOriginal.TryGetValue(coin.TransactionId, out original);
if (original == null)
{
// This one haven't existed before, if we rewind, delete it.
rewindData.TransactionsToRemove.Add(coin.TransactionId);
}
else
{
// We'll need to restore the original outputs.
UnspentOutputs clone = coin.Clone();
// TODO: Can we remove this line? before is not used anywhere and clone.UnspentCount does not seem to have side effects?
int before = clone.UnspentCount;
clone._Outputs = original.ToArray();
rewindData.OutputsToRestore.Add(clone);
}
}
}
if (rewindData != null)
{
int nextRewindIndex = this.GetRewindIndex() + 1;
this.logger.LogTrace("Rewind state #{0} created.", nextRewindIndex);
this.session.Transaction.Insert("Rewind", nextRewindIndex, rewindData);
}
insertedEntities += all.Count;
this.session.Transaction.Commit();
}
this.PerformanceCounter.AddInsertedEntities(insertedEntities);
this.logger.LogTrace("(-)");
}));
}
public override Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash)
{
return(this.session.Execute(() =>
{
RewindData rewindData = originalOutputs == null ? null : new RewindData(oldBlockHash);
int insertedEntities = 0;
using (new StopWatch().Start(o => this.PerformanceCounter.AddInsertTime(o)))
{
var current = this.GetCurrentHash();
if (current != oldBlockHash)
{
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.SetBlockHash(nextBlockHash);
var all = unspentOutputs.ToList();
Dictionary <uint256, TxOut[]> unspentToOriginal = new Dictionary <uint256, TxOut[]>(all.Count);
if (originalOutputs != null)
{
var originalEnumerator = originalOutputs.GetEnumerator();
foreach (var u in all)
{
originalEnumerator.MoveNext();
unspentToOriginal.Add(u.TransactionId, originalEnumerator.Current);
}
}
all.Sort(UnspentOutputsComparer.Instance);
foreach (var coin in all)
{
if (coin.IsPrunable)
{
this.session.Transaction.RemoveKey("Coins", coin.TransactionId.ToBytes(false));
}
else
{
this.session.Transaction.Insert("Coins", coin.TransactionId.ToBytes(false), coin.ToCoins());
}
if (originalOutputs != null)
{
TxOut[] original = null;
unspentToOriginal.TryGetValue(coin.TransactionId, out original);
if (original == null)
{
//This one did not existed before, if we rewind, delete it
rewindData.TransactionsToRemove.Add(coin.TransactionId);
}
else
{
//We'll need to restore the original outputs
var clone = coin.Clone();
var before = clone.UnspentCount;
clone._Outputs = original.ToArray();
rewindData.OutputsToRestore.Add(clone);
}
}
}
if (rewindData != null)
{
int nextRewindIndex = this.GetRewindIndex() + 1;
this.session.Transaction.Insert <int, RewindData>("Rewind", nextRewindIndex, rewindData);
}
insertedEntities += all.Count;
this.session.Transaction.Commit();
}
this.PerformanceCounter.AddInsertedEntities(insertedEntities);
}));
}
/// <inheritdoc />
public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(outputs, nameof(outputs));
lock (this.lockobj)
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHash = nextBlockHash;
long utxoSkipDisk = 0;
var rewindData = new RewindData(oldBlockHash);
Dictionary <OutPoint, int> indexItems = null;
if (this.rewindDataIndexCache != null)
{
indexItems = new Dictionary <OutPoint, int>();
}
foreach (UnspentOutput output in outputs)
{
if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem))
{
// Add outputs to cache, this will happen for two cases
// 1. if a chaced item was evicted
// 2. for new outputs that are added
if (output.CreatedFromBlock)
{
// if the output is indicate that it was added from a block
// There is no need to spend an extra call to disk.
this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint);
cacheItem = new CacheItem()
{
ExistInInner = false,
IsDirty = false,
OutPoint = output.OutPoint,
Coins = null
};
}
else
{
// This can happen if the cashe item was evicted while
// the block was being processed, fetch the outut again from disk.
this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint);
FetchCoinsResponse result = this.coindb.FetchCoins(new[] { output.OutPoint });
this.performanceCounter.AddMissCount(1);
UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value;
cacheItem = new CacheItem()
{
ExistInInner = unspentOutput.Coins != null,
IsDirty = false,
OutPoint = unspentOutput.OutPoint,
Coins = unspentOutput.Coins
};
}
this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem);
this.cacheSizeBytes += cacheItem.GetSize;
this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint);
}
// If output.Coins is null this means the utxo needs to be deleted
// otherwise this is a new utxo and we store it to cache.
if (output.Coins == null)
{
// DELETE COINS
// In cases of an output spent in the same block
// it wont exist in cash or in disk so its safe to remove it
if (cacheItem.Coins == null)
{
if (cacheItem.ExistInInner)
{
throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint));
}
}
else
{
// Handle rewind data
this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins));
rewindData.TotalSize += cacheItem.GetSize;
if (this.rewindDataIndexCache != null && indexItems != null)
{
indexItems[cacheItem.OutPoint] = this.blockHash.Height;
}
}
// If a spent utxo never made it to disk then no need to keep it in memory.
if (!cacheItem.ExistInInner)
{
this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint);
this.cachedUtxoItems.Remove(cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetSize;
utxoSkipDisk++;
if (cacheItem.IsDirty)
{
this.dirtyCacheCount--;
}
}
else
{
// Now modify the cached items with the mutated data.
this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetScriptSize;
cacheItem.Coins = null;
// Delete output from cache but keep a the cache
// item reference so it will get deleted form disk
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
else
{
// ADD COINS
if (cacheItem.Coins != null)
{
// Allow overrides.
// See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94
bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null;
if (!allowOverride)
{
throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint));
}
this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint);
// Deduct the crurrent script size form the
// total cache size, it will be added again later.
this.cacheSizeBytes -= cacheItem.GetScriptSize;
// Clear this in order to calculate the cache sie
// this will get set later when overriden
cacheItem.Coins = null;
}
// Handle rewind data
// New trx so it needs to be deleted if a rewind happens.
this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRemove.Add(cacheItem.OutPoint);
rewindData.TotalSize += cacheItem.GetSize;
// Put in the cache the new UTXOs.
this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint);
cacheItem.Coins = output.Coins;
this.cacheSizeBytes += cacheItem.GetScriptSize;
// Mark the cache item as dirty so it get persisted
// to disk and not evicted form cache
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk);
if (this.rewindDataIndexCache != null && indexItems.Any())
{
this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems);
}
// Add the most recent rewind data to the cache.
this.cachedRewindData.Add(this.blockHash.Height, rewindData);
this.rewindDataSizeBytes += rewindData.TotalSize;
}
}
