/// <inheritdoc />
public FetchCoinsResponse FetchCoins(OutPoint[] utxos)
{
Guard.NotNull(utxos, nameof(utxos));
var result = new FetchCoinsResponse();
var missedOutpoint = new List <OutPoint>();
lock (this.lockobj)
{
foreach (OutPoint outPoint in utxos)
{
if (!this.cachedUtxoItems.TryGetValue(outPoint, out CacheItem cache))
{
this.logger.LogDebug("Utxo '{0}' not found in cache.", outPoint);
missedOutpoint.Add(outPoint);
}
else
{
this.logger.LogDebug("Utxo '{0}' found in cache, UTXOs:'{1}'.", outPoint, cache.Coins);
result.UnspentOutputs.Add(outPoint, new UnspentOutput(outPoint, cache.Coins));
}
}
this.performanceCounter.AddMissCount(missedOutpoint.Count);
this.performanceCounter.AddHitCount(utxos.Length - missedOutpoint.Count);
if (missedOutpoint.Count > 0)
{
this.logger.LogDebug("{0} cache missed transaction needs to be loaded from underlying CoinView.", missedOutpoint.Count);
FetchCoinsResponse fetchedCoins = this.coindb.FetchCoins(missedOutpoint.ToArray());
foreach (var unspentOutput in fetchedCoins.UnspentOutputs)
{
result.UnspentOutputs.Add(unspentOutput.Key, unspentOutput.Value);
var cache = new CacheItem()
{
ExistInInner = unspentOutput.Value.Coins != null,
IsDirty = false,
OutPoint = unspentOutput.Key,
Coins = unspentOutput.Value.Coins
};
this.logger.LogDebug("CacheItem added to the cache, UTXO '{0}', Coin:'{1}'.", cache.OutPoint, cache.Coins);
this.cachedUtxoItems.Add(cache.OutPoint, cache);
this.cacheSizeBytes += cache.GetSize;
}
}
// Check if we need to evict items form the cache.
// This happens every time data is fetched fomr coindb
this.TryEvictCacheLocked();
}
return(result);
}
/// <inheritdoc />
public FetchCoinsResponse FetchCoins(OutPoint[] txIds)
{
Guard.NotNull(txIds, nameof(txIds));
using (this.lockobj.LockRead())
{
var result = new FetchCoinsResponse();
for (int i = 0; i < txIds.Length; i++)
{
var output = this.unspents.TryGet(txIds[i]);
if (output != null)
{
result.UnspentOutputs.Add(output.OutPoint, output);
}
}
return(result);
}
}
/// <inheritdoc />
public void CacheCoins(OutPoint[] utxos)
{
lock (this.lockobj)
{
var missedOutpoint = new List <OutPoint>();
foreach (OutPoint outPoint in utxos)
{
if (!this.cachedUtxoItems.TryGetValue(outPoint, out CacheItem cache))
{
this.logger.LogDebug("Prefetch Utxo '{0}' not found in cache.", outPoint);
missedOutpoint.Add(outPoint);
}
}
this.performanceCounter.AddCacheMissCount(missedOutpoint.Count);
this.performanceCounter.AddCacheHitCount(utxos.Length - missedOutpoint.Count);
if (missedOutpoint.Count > 0)
{
FetchCoinsResponse fetchedCoins = this.coindb.FetchCoins(missedOutpoint.ToArray());
foreach (var unspentOutput in fetchedCoins.UnspentOutputs)
{
var cache = new CacheItem()
{
ExistInInner = unspentOutput.Value.Coins != null,
IsDirty = false,
OutPoint = unspentOutput.Key,
Coins = unspentOutput.Value.Coins
};
this.logger.LogDebug("Prefetch CacheItem added to the cache, UTXO: '{0}', Coin:'{1}'.", cache.OutPoint, cache.Coins);
this.cachedUtxoItems.Add(cache.OutPoint, cache);
this.cacheSizeBytes += cache.GetSize;
}
}
}
}
/// <inheritdoc />
public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(outputs, nameof(outputs));
lock (this.lockobj)
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHash = nextBlockHash;
long utxoSkipDisk = 0;
var rewindData = new RewindData(oldBlockHash);
Dictionary <OutPoint, int> indexItems = null;
if (this.rewindDataIndexCache != null)
{
indexItems = new Dictionary <OutPoint, int>();
}
foreach (UnspentOutput output in outputs)
{
if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem))
{
// Add outputs to cache, this will happen for two cases
// 1. if a cached item was evicted
// 2. for new outputs that are added
if (output.CreatedFromBlock)
{
// if the output is indicate that it was added from a block
// There is no need to spend an extra call to disk.
this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint);
cacheItem = new CacheItem()
{
ExistInInner = false,
IsDirty = false,
OutPoint = output.OutPoint,
Coins = null
};
}
else
{
// This can happen if the cached item was evicted while
// the block was being processed, fetch the output again from disk.
this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint);
FetchCoinsResponse result = this.coindb.FetchCoins(new[] { output.OutPoint });
this.performanceCounter.AddMissCount(1);
UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value;
cacheItem = new CacheItem()
{
ExistInInner = unspentOutput.Coins != null,
IsDirty = false,
OutPoint = unspentOutput.OutPoint,
Coins = unspentOutput.Coins
};
}
this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem);
this.cacheSizeBytes += cacheItem.GetSize;
this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint);
}
// If output.Coins is null this means the utxo needs to be deleted
// otherwise this is a new utxo and we store it to cache.
if (output.Coins == null)
{
// DELETE COINS
// In cases of an output spent in the same block
// it wont exist in cash or in disk so its safe to remove it
if (cacheItem.Coins == null)
{
if (cacheItem.ExistInInner)
{
throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint));
}
}
else
{
// Handle rewind data
this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins));
rewindData.TotalSize += cacheItem.GetSize;
if (this.rewindDataIndexCache != null && indexItems != null)
{
indexItems[cacheItem.OutPoint] = this.blockHash.Height;
}
}
// If a spent utxo never made it to disk then no need to keep it in memory.
if (!cacheItem.ExistInInner)
{
this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint);
this.cachedUtxoItems.Remove(cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetSize;
utxoSkipDisk++;
if (cacheItem.IsDirty)
{
this.dirtyCacheCount--;
}
}
else
{
// Now modify the cached items with the mutated data.
this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetScriptSize;
cacheItem.Coins = null;
// Delete output from cache but keep a the cache
// item reference so it will get deleted form disk
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
else
{
// ADD COINS
if (cacheItem.Coins != null)
{
// Allow overrides.
// See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94
bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null;
if (!allowOverride)
{
throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint));
}
this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint);
// Deduct the current script size form the
// total cache size, it will be added again later.
this.cacheSizeBytes -= cacheItem.GetScriptSize;
// Clear this in order to calculate the cache size
// this will get set later when overridden
cacheItem.Coins = null;
}
// Handle rewind data
// New trx so it needs to be deleted if a rewind happens.
this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRemove.Add(cacheItem.OutPoint);
rewindData.TotalSize += cacheItem.GetSize;
// Put in the cache the new UTXOs.
this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint);
cacheItem.Coins = output.Coins;
this.cacheSizeBytes += cacheItem.GetScriptSize;
// Mark the cache item as dirty so it get persisted
// to disk and not evicted form cache
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk);
if (this.rewindDataIndexCache != null && indexItems.Any())
{
this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems);
}
// Add the most recent rewind data to the cache.
this.cachedRewindData.Add(this.blockHash.Height, rewindData);
this.rewindDataSizeBytes += rewindData.TotalSize;
// Remove rewind data form the back of a moving window.
// The closer we get to the tip we keep a longer rewind data window.
// Anything bellow last checkpoint we keep the minimal of 10
// (random low number) rewind data items.
// Beyond last checkpoint:
// - For POS we keep a window of MaxReorg.
// - For POW we keep 100 items (possibly better is an algo that grows closer to tip)
// A moving window of information needed to rewind the node to a previous block.
// When cache is flushed the rewind data will allow to rewind the node up to the
// number of rewind blocks.
// TODO: move rewind data to use block store.
// Rewind data can go away all together if the node uses the blocks in block store
// to get the rewind information, blockstore persists much more frequent then coin cache
// So using block store for rewinds is not entirely impossible.
int rewindDataWindow = this.CalculateRewindWindow();
int rewindToRemove = this.blockHash.Height - (int)rewindDataWindow;
if (this.cachedRewindData.TryGetValue(rewindToRemove, out RewindData delete))
{
this.logger.LogDebug("Remove rewind data height '{0}' from cache.", rewindToRemove);
this.cachedRewindData.Remove(rewindToRemove);
this.rewindDataSizeBytes -= delete.TotalSize;
}
}
}