/// <inheritdoc />
public VerboseAddressBalancesResult GetVerboseAddressBalancesData(string[] addresses)
{
(bool isQueryable, string reason) = this.IsQueryable();
if (!isQueryable)
{
return(VerboseAddressBalancesResult.RequestFailed(reason));
}
var result = new VerboseAddressBalancesResult();
lock (this.lockObject)
{
foreach (var address in addresses)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
var copy = new AddressIndexerData()
{
Address = indexData.Address,
BalanceChanges = new List <AddressBalanceChange>(indexData.BalanceChanges)
};
result.BalancesData.Add(copy);
}
}
result.ConsensusTipHeight = this.consensusManager.Tip.Height;
return(result);
}
/// <summary>Adds a new balance change entry to to the <see cref="addressIndexRepository"/>.</summary>
/// <remarks>Should be protected by <see cref="lockObject"/>.</remarks>
private void ProcessBalanceChangeLocked(int height, string address, Money amount, bool deposited)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
// Record new balance change into the address index data.
indexData.BalanceChanges.Add(new AddressBalanceChange()
{
BalanceChangedHeight = height,
Satoshi = amount.Satoshi,
Deposited = deposited
});
// Anything less than that should be compacted.
int heightThreshold = this.consensusManager.Tip.Height - this.compactionTriggerDistance;
bool compact = (indexData.BalanceChanges.Count > CompactingThreshold) &&
(indexData.BalanceChanges[1].BalanceChangedHeight < heightThreshold);
if (!compact)
{
this.logger.LogTrace("(-)[TOO_FEW_CHANGE_RECORDS]");
return;
}
var compacted = new List <AddressBalanceChange>(CompactingThreshold / 2)
{
new AddressBalanceChange()
{
BalanceChangedHeight = 0,
Satoshi = 0,
Deposited = true
}
};
foreach (AddressBalanceChange change in indexData.BalanceChanges)
{
if (change.BalanceChangedHeight < heightThreshold)
{
this.logger.LogDebug("Balance change: {0} was selected for compaction. Compacted balance now: {1}.", change, compacted[0].Satoshi);
if (change.Deposited)
{
compacted[0].Satoshi += change.Satoshi;
}
else
{
compacted[0].Satoshi -= change.Satoshi;
}
this.logger.LogDebug("New compacted balance: {0}.", compacted[0].Satoshi);
}
else
{
compacted.Add(change);
}
}
indexData.BalanceChanges = compacted;
this.addressIndexRepository.AddOrUpdate(indexData.Address, indexData, indexData.BalanceChanges.Count + 1);
}
/// <inheritdoc />
public AddressBalancesResult GetAddressBalances(string[] addresses, int minConfirmations = 1)
{
var(isQueryable, reason) = this.IsQueryable();
if (!isQueryable)
{
return(AddressBalancesResult.RequestFailed(reason));
}
var result = new AddressBalancesResult();
lock (this.lockObject)
{
foreach (var address in addresses)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
int maxAllowedHeight = this.consensusManager.Tip.Height - minConfirmations + 1;
long balance = indexData.BalanceChanges.Where(x => x.BalanceChangedHeight <= maxAllowedHeight).CalculateBalance();
this.logger.LogTrace("Address: {0}, balance: {1}.", address, balance);
result.Balances.Add(new AddressBalanceResult(address, new Money(balance)));
}
return(result);
}
}
private void RewindAndSave(ChainedHeader rewindToHeader)
{
lock (this.lockObject)
{
// The cache doesn't really lend itself to handling a reorg very well.
// Therefore, we leverage LiteDb's indexing capabilities to tell us
// which records are for the affected blocks.
List <string> affectedAddresses = this.addressIndexRepository.GetAddressesHigherThanHeight(rewindToHeader.Height);
foreach (string address in affectedAddresses)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
indexData.BalanceChanges.RemoveAll(x => x.BalanceChangedHeight > rewindToHeader.Height);
}
this.logger.LogDebug("Rewinding changes for {0} addresses.", affectedAddresses.Count);
// Rewind all the way back to the fork point.
this.outpointsRepository.RewindDataAboveHeight(rewindToHeader.Height);
this.IndexerTip = rewindToHeader;
this.SaveAll();
}
}
/// <inheritdoc />
public Money GetReceivedByAddress(string address, int minConfirmations = 1)
{
if (this.addressIndexRepository == null)
{
this.logger.LogTrace("(-)[NOT_INITIALIZED]");
throw new IndexerNotInitializedException();
}
if (!this.IsSynced())
{
this.logger.LogTrace("(-)[NOT_SYNCED]");
throw new OutOfSyncException();
}
lock (this.lockObject)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
if (indexData == null)
{
this.logger.LogTrace("(-)[NOT_FOUND]");
return(null);
}
int maxAllowedHeight = this.consensusManager.Tip.Height - minConfirmations + 1;
long deposited = indexData.BalanceChanges.Where(x => x.Deposited && x.BalanceChangedHeight <= maxAllowedHeight).Sum(x => x.Satoshi);
return(new Money(deposited));
}
}
/// <summary>Adds a new balance change entry to to the <see cref="addressIndexRepository"/>.</summary>
/// <remarks>Should be protected by <see cref="lockObject"/>.</remarks>
private void ProcessBalanceChangeLocked(int height, string address, Money amount, bool deposited)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
// Record new balance change into the address index data.
indexData.BalanceChanges.Add(new AddressBalanceChange()
{
BalanceChangedHeight = height,
Satoshi = amount.Satoshi,
Deposited = deposited
});
// Anything less than that should be compacted.
int heightThreshold = this.consensusManager.Tip.Height - (int)this.network.Consensus.MaxReorgLength;
bool compact = (this.network.Consensus.MaxReorgLength != 0) &&
(indexData.BalanceChanges.Count > CompactingThreshold) &&
(indexData.BalanceChanges[1].BalanceChangedHeight < heightThreshold);
if (!compact)
{
this.logger.LogTrace("(-)[TOO_FEW_CHANGE_RECORDS]");
return;
}
var compacted = new List <AddressBalanceChange>(CompactingThreshold / 2)
{
new AddressBalanceChange()
{
BalanceChangedHeight = 0,
Satoshi = 0,
Deposited = true
}
};
for (int i = 0; i < indexData.BalanceChanges.Count; i++)
{
AddressBalanceChange change = indexData.BalanceChanges[i];
if (change.BalanceChangedHeight < heightThreshold)
{
if (change.Deposited)
{
compacted[0].Satoshi += change.Satoshi;
}
else
{
compacted[0].Satoshi -= change.Satoshi;
}
}
else if (i < indexData.BalanceChanges.Count - 1)
{
compacted.AddRange(indexData.BalanceChanges.Skip(i + 1));
break;
}
}
indexData.BalanceChanges = compacted;
this.addressIndexRepository.AddOrUpdate(indexData.Address, indexData, indexData.BalanceChanges.Count + 1);
}
public void Initialize()
{
if (!this.storeSettings.TxIndex || !this.storeSettings.AddressIndex)
{
this.logger.LogTrace("(-)[DISABLED]");
return;
}
string dbPath = Path.Combine(this.dataFolder.RootPath, "addressindex.litedb");
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath, Mode = fileMode
});
this.logger.LogDebug("TxIndexing is enabled.");
this.dataStore = this.db.GetCollection <AddressIndexerData>(DbKey);
lock (this.lockObject)
{
this.addressesIndex = this.dataStore.FindAll().FirstOrDefault();
if (this.addressesIndex == null)
{
this.logger.LogDebug("Tip was not found, initializing with genesis.");
this.addressesIndex = new AddressIndexerData()
{
TipHashBytes = this.network.GenesisHash.ToBytes(),
AddressChanges = new Dictionary <string, List <AddressBalanceChange> >()
};
this.dataStore.Insert(this.addressesIndex);
}
this.IndexerTip = this.consensusManager.Tip.FindAncestorOrSelf(new uint256(this.addressesIndex.TipHashBytes));
}
if (this.IndexerTip == null)
{
this.IndexerTip = this.consensusManager.Tip.GetAncestor(0);
}
this.indexingTask = Task.Run(async() => await this.IndexAddressesContinuouslyAsync().ConfigureAwait(false));
this.nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, 400);
}
/// <inheritdoc />
public VerboseAddressBalancesResult GetAddressIndexerState(string[] addresses)
{
// If the containing feature is not initialized then wait a bit.
this.InitializingFeature?.WaitInitialized();
var result = new VerboseAddressBalancesResult(this.consensusManager.Tip.Height);
if (addresses.Length == 0)
{
return(result);
}
if (!this.storeSettings.AddressIndex)
{
throw new NotSupportedException("Address indexing is not enabled.");
}
(bool isQueryable, string reason) = this.IsQueryable();
if (!isQueryable)
{
return(VerboseAddressBalancesResult.RequestFailed(reason));
}
lock (this.lockObject)
{
foreach (var address in addresses)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
var copy = new AddressIndexerData()
{
Address = indexData.Address,
BalanceChanges = new List <AddressBalanceChange>(indexData.BalanceChanges)
};
result.BalancesData.Add(copy);
}
}
return(result);
}
/// <inheritdoc />
public Money GetAddressBalance(string address, int minConfirmations = 1)
{
if (this.addressIndexRepository == null)
{
this.logger.LogTrace("(-)[NOT_INITIALIZED]");
throw new IndexerNotInitializedException();
}
if (!this.IsSynced())
{
this.logger.LogTrace("(-)[NOT_SYNCED]");
throw new OutOfSyncException();
}
lock (this.lockObject)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
if (indexData == null)
{
this.logger.LogTrace("(-)[NOT_FOUND]");
return(null);
}
long balance = 0;
int maxAllowedHeight = this.consensusManager.Tip.Height - minConfirmations + 1;
foreach (AddressBalanceChange change in indexData.BalanceChanges.Where(x => x.BalanceChangedHeight <= maxAllowedHeight))
{
if (change.Deposited)
{
balance += change.Satoshi;
}
else
{
balance -= change.Satoshi;
}
}
return(new Money(balance));
}
}
private async Task IndexAddressesContinuouslyAsync()
{
Stopwatch watch = Stopwatch.StartNew();
while (!this.cancellation.IsCancellationRequested)
{
if (DateTime.Now - this.lastFlushTime > this.flushChangesInterval)
{
this.logger.LogDebug("Flushing changes.");
lock (this.lockObject)
{
this.addressIndexRepository.SaveAllItems();
this.outpointsRepository.SaveAllItems();
this.tipDataStore.Update(this.tipData);
}
this.lastFlushTime = DateTime.Now;
this.logger.LogDebug("Flush completed.");
}
if (this.cancellation.IsCancellationRequested)
{
break;
}
ChainedHeader nextHeader = this.consensusManager.Tip.GetAncestor(this.IndexerTip.Height + 1);
if (nextHeader == null)
{
this.logger.LogDebug("Next header wasn't found. Waiting.");
try
{
await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
continue;
}
if (nextHeader.Previous.HashBlock != this.IndexerTip.HashBlock)
{
ChainedHeader lastCommonHeader = nextHeader.FindFork(this.IndexerTip);
this.logger.LogDebug("Reorganization detected. Rewinding till '{0}'.", lastCommonHeader);
lock (this.lockObject)
{
// The cache doesn't really lend itself to handling a reorg very well.
// Therefore, we leverage LiteDb's indexing capabilities to tell us
// which records are for the affected blocks.
// TODO: May also be efficient to run ProcessBlocks with inverted deposit flags instead, depending on size of reorg
List <string> affectedAddresses = this.addressIndexRepository.GetAddressesHigherThanHeight(lastCommonHeader.Height);
foreach (string address in affectedAddresses)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
indexData.BalanceChanges.RemoveAll(x => x.BalanceChangedHeight > lastCommonHeader.Height);
}
// Rewind all the way back to the fork point.
while (this.IndexerTip.HashBlock != lastCommonHeader.HashBlock)
{
this.outpointsRepository.Rewind(this.IndexerTip.HashBlock);
this.IndexerTip = this.IndexerTip.Previous;
}
this.tipData.TipHashBytes = this.IndexerTip.HashBlock.ToBytes();
this.tipData.Height = this.IndexerTip.Height;
}
continue;
}
// First try to see if it's prefetched.
ChainedHeaderBlock prefetchedBlock = this.prefetchingTask == null ? null : await this.prefetchingTask.ConfigureAwait(false);
Block blockToProcess;
if (prefetchedBlock != null && prefetchedBlock.ChainedHeader == nextHeader)
{
blockToProcess = prefetchedBlock.Block;
}
else
{
blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock).Block;
}
if (blockToProcess == null)
{
this.logger.LogDebug("Next block wasn't found. Waiting.");
try
{
await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
continue;
}
// Schedule prefetching of the next block;
ChainedHeader headerToPrefetch = this.consensusManager.Tip.GetAncestor(nextHeader.Height + 1);
if (headerToPrefetch != null)
{
this.prefetchingTask = Task.Run(() => this.consensusManager.GetBlockData(headerToPrefetch.HashBlock));
}
watch.Restart();
bool success = this.ProcessBlock(blockToProcess, nextHeader);
watch.Stop();
this.averageTimePerBlock.AddSample(watch.Elapsed.TotalMilliseconds);
if (!success)
{
this.logger.LogDebug("Failed to process next block. Waiting.");
try
{
await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
continue;
}
this.IndexerTip = nextHeader;
lock (this.lockObject)
{
this.tipData.TipHashBytes = this.IndexerTip.HashBlock.ToBytes();
this.tipData.Height = this.IndexerTip.Height;
}
}
lock (this.lockObject)
{
this.addressIndexRepository.SaveAllItems();
this.outpointsRepository.SaveAllItems();
this.tipDataStore.Update(this.tipData);
}
}
public void Initialize()
{
if (!this.storeSettings.TxIndex || !this.storeSettings.IndexAddresses)
{
this.logger.LogTrace("(-)[DISABLED]");
return;
}
string dbPath = Path.Combine(this.dataFolder.RootPath, "addressindex.litedb");
this.db = new LiteDatabase(new ConnectionString()
{
Filename = dbPath
});
this.logger.LogDebug("TxIndexing is enabled.");
this.dataStore = this.db.GetCollection <AddressIndexerData>(DbKey);
this.addressesIndex = this.dataStore.FindAll().FirstOrDefault();
if (this.addressesIndex == null)
{
this.logger.LogDebug("Tip was not found, initializing with genesis.");
this.addressesIndex = new AddressIndexerData()
{
TipHash = this.network.GenesisHash.ToString(),
AddressIndexDatas = new List <AddressIndexData>()
};
this.dataStore.Insert(this.addressesIndex);
}
this.blockReceivedQueue = new AsyncQueue <KeyValuePair <bool, ChainedHeaderBlock> >(this.OnEnqueueAsync);
// Subscribe to events.
this.blockConnectedSubscription = this.signals.Subscribe <BlockConnected>(blockConnectedData =>
{
while (this.blockReceivedQueue.Count > 100)
{
this.logger.LogWarning("Address indexing is slowing down the consensus.");
Thread.Sleep(5000);
}
this.blockReceivedQueue.Enqueue(new KeyValuePair <bool, ChainedHeaderBlock>(true, blockConnectedData.ConnectedBlock));
});
this.blockDisconnectedSubscription = this.signals.Subscribe <BlockDisconnected>(blockDisconnectedData =>
{
this.blockReceivedQueue.Enqueue(new KeyValuePair <bool, ChainedHeaderBlock>(false, blockDisconnectedData.DisconnectedBlock));
});
this.nodeStats.RegisterStats(this.AddComponentStats, StatsType.Component, 400);
if ((this.consensusManager.Tip.HashBlock.ToString() != this.addressesIndex.TipHash))
{
const string message = "TransactionIndexer is in inconsistent state. This can happen if you've enabled txindex on an already synced or partially synced node. " +
"Remove everything from the data folder and run the node with -txindex=true.";
this.logger.LogCritical(message);
this.logger.LogTrace("(-)[INCONSISTENT_STATE]");
throw new Exception(message);
}
}
public LastBalanceDecreaseTransactionModel GetLastBalanceDecreaseTransaction(string address)
{
if (address == null)
{
return(null);
}
(bool isQueryable, string reason) = this.IsQueryable();
if (!isQueryable)
{
return(null);
}
int lastBalanceHeight;
lock (this.lockObject)
{
AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address);
AddressBalanceChange lastBalanceUpdate = indexData.BalanceChanges.Where(a => !a.Deposited).OrderByDescending(b => b.BalanceChangedHeight).FirstOrDefault();
if (lastBalanceUpdate == null)
{
return(null);
}
lastBalanceHeight = lastBalanceUpdate.BalanceChangedHeight;
}
// Height 0 is used as a placeholder height for compacted address balance records, so ignore them if they are the only record.
if (lastBalanceHeight == 0)
{
return(null);
}
ChainedHeader header = this.chainIndexer.GetHeader(lastBalanceHeight);
if (header == null)
{
return(null);
}
Block block = this.consensusManager.GetBlockData(header.HashBlock).Block;
if (block == null)
{
return(null);
}
// Get the UTXO snapshot as of one block lower than the last balance change, so that we are definitely able to look up the inputs of each transaction in the next block.
ReconstructedCoinviewContext utxos = this.utxoIndexer.GetCoinviewAtHeight(lastBalanceHeight - 1);
Transaction foundTransaction = null;
foreach (Transaction transaction in block.Transactions)
{
if (transaction.IsCoinBase)
{
continue;
}
foreach (TxIn txIn in transaction.Inputs)
{
Transaction prevTx = utxos.Transactions[txIn.PrevOut.Hash];
foreach (TxOut txOut in prevTx.Outputs)
{
if (this.scriptAddressReader.GetAddressFromScriptPubKey(this.network, txOut.ScriptPubKey) == address)
{
foundTransaction = transaction;
}
}
}
}
return(foundTransaction == null ? null : new LastBalanceDecreaseTransactionModel()
{
BlockHeight = lastBalanceHeight, Transaction = new TransactionVerboseModel(foundTransaction, this.network)
});
}
