private SyncPoolTransactions FindPoolInternal(SyncConnection connection, SyncingBlocks syncingBlocks)
{
watch.Restart();
BitcoinClient client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
IEnumerable <string> memPool = client.GetRawMemPool();
var currentMemoryPool = new HashSet <string>(memPool);
var currentTable = new HashSet <string>(syncingBlocks.CurrentPoolSyncing);
var newTransactions = currentMemoryPool.Except(currentTable).ToList();
var deleteTransaction = currentTable.Except(currentMemoryPool).ToList();
//var newTransactionsLimited = newTransactions.Count() < 1000 ? newTransactions : newTransactions.Take(1000).ToList();
syncingBlocks.CurrentPoolSyncing.AddRange(newTransactions);
deleteTransaction.ForEach(t => syncingBlocks.CurrentPoolSyncing.Remove(t));
watch.Stop();
log.LogDebug($"SyncPool: Seconds = {watch.Elapsed.TotalSeconds} - New Transactions = {newTransactions.Count()}");
return(new SyncPoolTransactions {
Transactions = newTransactions
});
}
private SyncPoolTransactions FindPoolInternal(SyncConnection connection, SyncingBlocks syncingBlocks)
{
var stoper = Stopwatch.Start();
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var memPool = client.GetRawMemPool();
var currentMemoryPool = new HashSet <string>(memPool);
var currentTable = new HashSet <string>(syncingBlocks.CurrentPoolSyncing);
var newTransactions = currentMemoryPool.Except(currentTable).ToList();
var deleteTransaction = currentTable.Except(currentMemoryPool).ToList();
//var newTransactionsLimited = newTransactions.Count() < 1000 ? newTransactions : newTransactions.Take(1000).ToList();
syncingBlocks.CurrentPoolSyncing.AddRange(newTransactions);
deleteTransaction.ForEach(t => syncingBlocks.CurrentPoolSyncing.Remove(t));
stoper.Stop();
this.tracer.DetailedTrace("SyncPool", string.Format("Seconds = {0} - New Transactions = {1}", stoper.Elapsed.TotalSeconds, newTransactions.Count()));
return(new SyncPoolTransactions {
Transactions = newTransactions
});
}
public async Task <List <PeerInfo> > Peers()
{
var connection = this.syncConnection;
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
return((await client.GetPeerInfo()).ToList());
}
public async Task <StatsConnection> StatsConnection()
{
var connection = this.syncConnection;
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var clientConnection = await client.GetConnectionCountAsync();
return(new StatsConnection {
Connections = clientConnection
});
}
public async Task <string> SendTransaction(string transactionHex)
{
// todo: consider adding support for retries.
// todo: check how a failure is porpageted
var connection = this.syncConnection;
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var trxid = await client.SentRawTransactionAsync(transactionHex);
return(trxid);
}
private SyncBlockTransactionsOperation SyncPoolInternal(SyncConnection connection, SyncPoolTransactions poolTransactions)
{
var stoper = Stopwatch.Start();
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var returnBlock = this.SyncBlockTransactions(client, connection, poolTransactions.Transactions, false);
stoper.Stop();
this.tracer.DetailedTrace("SyncPool", string.Format("Seconds = {0} - Transactions = {1}", stoper.Elapsed.TotalSeconds, returnBlock.Transactions.Count()));
return(returnBlock);
}
private SyncBlockTransactionsOperation SyncBlockInternal(SyncConnection connection, BlockInfo block)
{
var stoper = Stopwatch.Start();
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var returnBlock = this.SyncBlockTransactions(client, connection, block.Transactions, true);
returnBlock.BlockInfo = block;
stoper.Stop();
this.tracer.DetailedTrace("SyncBlock", string.Format("Seconds = {0} - Transactions = {1} - BlockIndex = {2}", stoper.Elapsed.TotalSeconds, returnBlock.Transactions.Count(), returnBlock.BlockInfo.Height));
return(returnBlock);
}
public async Task <Statistics> Statistics()
{
var connection = this.syncConnection;
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var stats = new Statistics {
CoinTag = this.syncConnection.CoinTag
};
stats.ClientInfo = await client.GetInfoAsync();
stats.TransactionsInPool = this.storage.GetMemoryTransactions().Count();
stats.SyncBlockIndex = this.storage.BlockGetBlockCount(1).First().Height;
stats.Progress = $"{stats.SyncBlockIndex}/{stats.ClientInfo.Blocks} - {stats.ClientInfo.Blocks - stats.SyncBlockIndex}";
return(stats);
}
public SyncTransactionItems TransactionItemsGet(string transactionId)
{
var client = CryptoClientFactory.Create(this.syncConnection.ServerDomain, this.syncConnection.RpcAccessPort, this.syncConnection.User, this.syncConnection.Password, this.syncConnection.Secure);
var res = client.GetRawTransactionAsync(transactionId, 1).Result;
return(new SyncTransactionItems
{
Inputs = res.VIn.Select(v => new SyncTransactionItemInput {
PreviousTransactionHash = v.TxId, PreviousIndex = v.VOut, InputCoinBase = v.CoinBase
}).ToList(),
Outputs = res.VOut.Where(v => v.ScriptPubKey != null && v.ScriptPubKey.Addresses != null).Select(v => new SyncTransactionItemOutput {
Address = v.ScriptPubKey.Addresses.FirstOrDefault(), Index = v.N, Value = v.Value, OutputType = v.ScriptPubKey.Type
}).ToList()
});
}
private SyncBlockTransactionsOperation SyncPoolInternal(SyncConnection connection, SyncPoolTransactions poolTransactions)
{
watch.Restart();
BitcoinClient client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
SyncBlockTransactionsOperation returnBlock = SyncBlockTransactions(client, connection, poolTransactions.Transactions, false);
watch.Stop();
int transactionCount = returnBlock.Transactions.Count();
double totalSeconds = watch.Elapsed.TotalSeconds;
log.LogDebug($"SyncPool: Seconds = {watch.Elapsed.TotalSeconds} - Transactions = {transactionCount}");
return(returnBlock);
}
private SyncBlockTransactionsOperation SyncPoolInternal(SyncConnection connection, SyncPoolTransactions poolTransactions)
{
var watch = Stopwatch.Start();
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var returnBlock = this.SyncBlockTransactions(client, connection, poolTransactions.Transactions, false);
watch.Stop();
var transactionCount = returnBlock.Transactions.Count();
var totalSeconds = watch.Elapsed.TotalSeconds;
this.log.LogDebug($"SyncPool: Seconds = {watch.Elapsed.TotalSeconds} - Transactions = {transactionCount}");
return(returnBlock);
}
private SyncBlockOperation FindBlockInternal(SyncConnection connection, SyncingBlocks syncingBlocks)
{
watch.Restart();
BitcoinClient client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
syncingBlocks.LastClientBlockIndex = GetBlockCount(client);
SyncBlockOperation blockToSync = GetNextBlockToSync(client, connection, syncingBlocks.LastClientBlockIndex, syncingBlocks);
if (blockToSync != null && blockToSync.BlockInfo != null)
{
syncingBlocks.CurrentSyncing.TryAdd(blockToSync.BlockInfo.Hash, blockToSync.BlockInfo);
}
watch.Stop();
return(blockToSync);
}
private SyncBlockOperation FindBlockInternal(SyncConnection connection, SyncingBlocks syncingBlocks)
{
var stoper = Stopwatch.Start();
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var lastCryptoBlockIndex = client.GetBlockCount();
var blockToSync = this.GetNextBlockToSync(client, connection, lastCryptoBlockIndex, syncingBlocks);
if (blockToSync != null && blockToSync.BlockInfo != null)
{
syncingBlocks.CurrentSyncing.TryAdd(blockToSync.BlockInfo.Hash, blockToSync.BlockInfo);
}
stoper.Stop();
return(blockToSync);
}
public async Task <Statistics> Statistics()
{
SyncConnection connection = syncConnection;
BitcoinClient client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var stats = new Statistics {
Symbol = syncConnection.Symbol
};
try
{
stats.BlockchainInfo = await client.GetBlockchainInfo();
stats.NetworkInfo = await client.GetNetworkInfo();
}
catch (Exception ex)
{
stats.Error = ex.Message;
return(stats);
}
stats.TransactionsInPool = storage.GetMemoryTransactions().Count();
try
{
stats.SyncBlockIndex = storage.BlockGetBlockCount(1).First().BlockIndex;
stats.Progress = $"{stats.SyncBlockIndex}/{stats.BlockchainInfo.Blocks} - {stats.BlockchainInfo.Blocks - stats.SyncBlockIndex}";
double totalSeconds = syncConnection.RecentItems.Sum(s => s.Duration.TotalSeconds);
stats.AvgBlockPersistInSeconds = Math.Round(totalSeconds / syncConnection.RecentItems.Count, 2);
long totalSize = syncConnection.RecentItems.Sum(s => s.Size);
stats.AvgBlockSizeKb = Math.Round((double)totalSize / syncConnection.RecentItems.Count, 0);
stats.BlocksPerMinute = syncConnection.RecentItems.Count(w => w.Inserted > DateTime.UtcNow.AddMinutes(-1));
}
catch (Exception ex)
{
stats.Progress = ex.Message;
}
return(stats);
}
public async Task <List <PeerInfo> > Peers()
{
SyncConnection connection = syncConnection;
BitcoinClient client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var res = (await client.GetPeerInfo()).ToList();
res.ForEach(p =>
{
if (TryParse(p.Addr, out IPEndPoint ipe))
{
string addr = ipe.Address.ToString();
if (ipe.Address.IsIPv4MappedToIPv6)
{
addr = ipe.Address.MapToIPv4().ToString();
}
p.Addr = $"{addr}:{ipe.Port}";
}
});
return(res);
}
public async Task CheckBlockReorganization(SyncConnection connection)
{
while (true)
{
Storage.Types.SyncBlockInfo block = storage.BlockGetBlockCount(1).FirstOrDefault();
if (block == null)
{
break;
}
BitcoinClient client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
string currentHash = await client.GetblockHashAsync(block.BlockIndex);
if (currentHash == block.BlockHash)
{
break;
}
log.LogInformation($"SyncOperations: Deleting block {block.BlockIndex}");
storage.DeleteBlock(block.BlockHash);
}
}
public async Task CheckBlockReorganization(SyncConnection connection)
{
while (true)
{
var block = this.storage.BlockGetBlockCount(1).FirstOrDefault();
if (block == null)
{
break;
}
var client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
var currentHash = await client.GetblockHashAsync(block.BlockIndex);
if (currentHash == block.BlockHash)
{
break;
}
this.tracer.Trace("SyncOperations", string.Format("Deleting block {0}", block.BlockIndex));
this.storage.DeleteBlock(block.BlockHash);
}
}
private SyncBlockTransactionsOperation SyncBlockInternal(SyncConnection connection, BlockInfo block)
{
System.Diagnostics.Stopwatch watch = Stopwatch.Start();
BitcoinClient client = CryptoClientFactory.Create(connection.ServerDomain, connection.RpcAccessPort, connection.User, connection.Password, connection.Secure);
string hex = client.GetBlockHex(block.Hash);
var blockItem = Block.Parse(hex, connection.Network.Consensus.ConsensusFactory);
foreach (Transaction blockItemTransaction in blockItem.Transactions)
{
blockItemTransaction.PrecomputeHash(false, true);
}
//var blockItem = Block.Load(Encoders.Hex.DecodeData(hex), consensusFactory);
var returnBlock = new SyncBlockTransactionsOperation {
BlockInfo = block, Transactions = blockItem.Transactions
}; //this.SyncBlockTransactions(client, connection, block.Transactions, true);
watch.Stop();
return(returnBlock);
}
public SyncTransactionItems TransactionItemsGet(string transactionId)
{
NBitcoin.Transaction transaction;
// Try to find the trx in disk
SyncRawTransaction rawtrx = TransactionGetByHash(transactionId);
if (rawtrx == null)
{
BitcoinClient client = CryptoClientFactory.Create(syncConnection.ServerDomain, syncConnection.RpcAccessPort, syncConnection.User, syncConnection.Password, syncConnection.Secure);
Client.Types.DecodedRawTransaction res = client.GetRawTransactionAsync(transactionId, 0).Result;
if (res.Hex == null)
{
return(null);
}
transaction = syncConnection.Network.Consensus.ConsensusFactory.CreateTransaction(res.Hex);
transaction.PrecomputeHash(false, true);
}
else
{
transaction = syncConnection.Network.Consensus.ConsensusFactory.CreateTransaction(rawtrx.RawTransaction);
transaction.PrecomputeHash(false, true);
}
var ret = new SyncTransactionItems
{
RBF = transaction.RBF,
LockTime = transaction.LockTime.ToString(),
Version = transaction.Version,
IsCoinbase = transaction.IsCoinBase,
IsCoinstake = syncConnection.Network.Consensus.IsProofOfStake && transaction.IsCoinStake,
Inputs = transaction.Inputs.Select(v => new SyncTransactionItemInput
{
PreviousTransactionHash = v.PrevOut.Hash.ToString(),
PreviousIndex = (int)v.PrevOut.N,
WitScript = v.WitScript.ToScript().ToHex(),
ScriptSig = v.ScriptSig.ToHex(),
SequenceLock = v.Sequence.ToString(),
}).ToList(),
Outputs = transaction.Outputs.Select((output, index) => new SyncTransactionItemOutput
{
Address = ScriptToAddressParser.GetAddress(syncConnection.Network, output.ScriptPubKey)?.FirstOrDefault(),
Index = index,
Value = output.Value,
OutputType = StandardScripts.GetTemplateFromScriptPubKey(output.ScriptPubKey)?.Type.ToString(),
ScriptPubKey = output.ScriptPubKey.ToHex()
}).ToList()
};
// try to fetch spent outputs
foreach (SyncTransactionItemOutput output in ret.Outputs)
{
output.SpentInTransaction = GetSpendingTransaction(transactionId, output.Index);
}
return(ret);
}
public InsertStats InsertTransactions(SyncBlockTransactionsOperation item)
{
var stats = new InsertStats {
Items = new List <MapTransactionAddress>()
};
if (item.BlockInfo != null)
{
// remove all transactions from the memory pool
item.Transactions.ForEach(t =>
{
DecodedRawTransaction outer;
this.data.MemoryTransactions.TryRemove(t.TxId, out outer);
});
// break the work in to batches transactions
var queue = new Queue <DecodedRawTransaction>(item.Transactions);
do
{
var transactions = this.GetBatch(this.configuration.MongoBatchSize, queue).ToList();
var bitcoinClient = CryptoClientFactory.Create(
syncConnection.ServerDomain,
syncConnection.RpcAccessPort,
syncConnection.User,
syncConnection.Password,
syncConnection.Secure);
try
{
if (item.BlockInfo != null)
{
var inserts = new List <MapTransactionBlock>();
var insertDetails = new List <MapTransactionDetail>();
foreach (var tx in transactions)
{
var isCoinBase = !string.IsNullOrWhiteSpace(tx.VIn.First().CoinBase);
var syncVin = tx.VIn.Select(vin =>
{
var previousTransaction = isCoinBase ?
null : bitcoinClient.GetRawTransaction(vin.TxId, 1);
return(new SyncVin
{
TxId = vin.TxId,
CoinBase = vin.CoinBase,
IsCoinBase = !string.IsNullOrWhiteSpace(vin.CoinBase),
ScriptSig = vin.ScriptSig,
Sequence = vin.Sequence,
VOut = vin.VOut,
PreviousVout = previousTransaction?.VOut.First(o => o.N == vin.VOut)
});
}).ToList();
var totalVout = tx.VOut.Sum(o => o.Value);
var totalVin = syncVin.Sum(i => i.PreviousVout?.Value);
inserts.Add(new MapTransactionBlock
{
BlockIndex = item.BlockInfo.Height,
TransactionId = tx.TxId,
TotalVout = totalVout,
TotalVin = totalVin,
Time = tx.Time,
BlockHash = tx.BlockHash,
BlockTime = tx.BlockTime,
Locktime = tx.Locktime,
Version = tx.Version,
IsCoinBase = isCoinBase
});
insertDetails.Add(new MapTransactionDetail
{
TransactionId = tx.TxId,
Vin = syncVin,
Vout = tx.VOut
});
}
stats.Transactions += inserts.Count();
this.data.MapTransactionBlock.InsertMany(inserts, new InsertManyOptions {
IsOrdered = false
});
this.data.MapTransactionDetails.InsertMany(insertDetails, new InsertManyOptions {
IsOrdered = false
});
}
}
catch (MongoBulkWriteException mbwex)
{
if (!mbwex.Message.Contains("E11000 duplicate key error collection"))
{
throw;
}
}
// insert inputs and add to the list for later to use on the notification task.
var inputs = this.CreateInputs(item.BlockInfo.Height, transactions).ToList();
var queueInner = new Queue <MapTransactionAddress>(inputs);
do
{
try
{
var itemsInner = this.GetBatch(this.configuration.MongoBatchSize, queueInner).ToList();
if (itemsInner.Any())
{
stats.Inputs += itemsInner.Count();
stats.Items.AddRange(itemsInner);
this.data.MapTransactionAddress.InsertMany(itemsInner, new InsertManyOptions {
IsOrdered = false
});
}
}
catch (MongoBulkWriteException mbwex)
{
if (!mbwex.Message.Contains("E11000 duplicate key error collection"))
{
throw;
}
}
}while (queueInner.Any());
// insert outputs
var outputs = this.CreateOutputs(transactions).ToList();
stats.Outputs += outputs.Count();
outputs.ForEach(outp => this.data.MarkOutput(outp.InputTransactionId, outp.InputIndex, outp.TransactionId));
}while (queue.Any());
// mark the block as synced.
this.CompleteBlock(item.BlockInfo);
}
else
{
// memory transaction push in to the pool.
item.Transactions.ForEach(t =>
{
this.data.MemoryTransactions.TryAdd(t.TxId, t);
});
stats.Transactions = this.data.MemoryTransactions.Count();
// todo: for accuracy - remove transactions from the mongo memory pool that are not anymore in the syncing pool
// remove all transactions from the memory pool
// this can be done using the SyncingBlocks objects - see method SyncOperations.FindPoolInternal()
// add to the list for later to use on the notification task.
var inputs = this.CreateInputs(-1, item.Transactions).ToList();
stats.Items.AddRange(inputs);
}
return(stats);
}
