/// <inheritdoc />
        public override async Task <bool> OnExecute()
        {
            if (TryDequeue(out SyncBlockTransactionsOperation item))
            {
                watch.Restart();

                storageOperations.ValidateBlock(item);

                InsertStats count = storageOperations.InsertTransactions(item);

                if (item.BlockInfo != null)
                {
                    if (!Runner.SyncingBlocks.CurrentSyncing.TryRemove(item.BlockInfo.Hash, out BlockInfo blockInfo))
                    {
                        throw new Exception(string.Format("Failed to remove block hash {0} from collection", item.BlockInfo.Hash));
                    }

                    syncConnection.RecentItems.Add((DateTime.UtcNow, watch.Elapsed, item.BlockInfo.Size));
                }

                var notifications = new AddressNotifications {
                    Addresses = count.Items.Where(ad => ad.Addresses != null).SelectMany(s => s.Addresses).Distinct().ToList()
                };
                Runner.Get <Notifier>().Enqueue(notifications);

                watch.Stop();

                string message = item.BlockInfo != null?
                                 string.Format("Seconds = {0} - BlockIndex = {1} - TotalItems = {2} - Size = {3} kb", watch.Elapsed.TotalSeconds, item.BlockInfo.Height, count.Transactions + count.InputsOutputs, item.BlockInfo.Size) :
                                     string.Format("Seconds = {0} - PoolSync - TotalItems = {1}", watch.Elapsed.TotalSeconds, count.Transactions + count.InputsOutputs);


                log.LogDebug(message);

                return(await Task.FromResult(true));
            }

            return(await Task.FromResult(false));
        }
Beispiel #2
0
        public InsertStats InsertTransactions(SyncBlockTransactionsOperation item)
        {
            var stats = new InsertStats {
                Items = new List <MapTransactionAddress>()
            };

            if (item.BlockInfo != null)
            {
                // remove all transactions from the memory pool
                item.Transactions.ForEach(t =>
                {
                    DecodedRawTransaction outer;
                    this.data.MemoryTransactions.TryRemove(t.TxId, out outer);
                });

                // break the work in to batches transactions
                var queue = new Queue <DecodedRawTransaction>(item.Transactions);
                do
                {
                    var items = this.GetBatch(this.configuration.MongoBatchSize, queue).ToList();

                    try
                    {
                        if (item.BlockInfo != null)
                        {
                            var inserts = items.Select(s => new MapTransactionBlock {
                                BlockIndex = item.BlockInfo.Height, TransactionId = s.TxId
                            }).ToList();
                            stats.Transactions += inserts.Count();
                            this.data.MapTransactionBlock.InsertMany(inserts, new InsertManyOptions {
                                IsOrdered = false
                            });
                        }
                    }
                    catch (MongoBulkWriteException mbwex)
                    {
                        if (!mbwex.Message.Contains("E11000 duplicate key error collection"))
                        {
                            throw;
                        }
                    }

                    // insert inputs and add to the list for later to use on the notification task.
                    var inputs     = this.CreateInputs(item.BlockInfo.Height, items).ToList();
                    var queueInner = new Queue <MapTransactionAddress>(inputs);
                    do
                    {
                        try
                        {
                            var itemsInner = this.GetBatch(this.configuration.MongoBatchSize, queueInner).ToList();
                            if (itemsInner.Any())
                            {
                                stats.Inputs += itemsInner.Count();
                                stats.Items.AddRange(itemsInner);
                                this.data.MapTransactionAddress.InsertMany(itemsInner, new InsertManyOptions {
                                    IsOrdered = false
                                });
                            }
                        }
                        catch (MongoBulkWriteException mbwex)
                        {
                            if (!mbwex.Message.Contains("E11000 duplicate key error collection"))
                            {
                                throw;
                            }
                        }
                    }while (queueInner.Any());

                    // insert outputs
                    var outputs = this.CreateOutputs(items).ToList();
                    stats.Outputs += outputs.Count();
                    outputs.ForEach(outp => this.data.MarkOutput(outp.InputTransactionId, outp.InputIndex, outp.TransactionId));
                }while (queue.Any());

                // mark the block as synced.
                this.CompleteBlock(item.BlockInfo);
            }
            else
            {
                // memory transaction push in to the pool.
                item.Transactions.ForEach(t =>
                {
                    this.data.MemoryTransactions.TryAdd(t.TxId, t);
                });

                stats.Transactions = this.data.MemoryTransactions.Count();

                // todo: for accuracy - remove transactions from the mongo memory pool that are not anymore in the syncing pool
                // remove all transactions from the memory pool
                // this can be done using the SyncingBlocks objects - see method SyncOperations.FindPoolInternal()

                // add to the list for later to use on the notification task.
                var inputs = this.CreateInputs(-1, item.Transactions).ToList();
                stats.Items.AddRange(inputs);
            }

            return(stats);
        }
Beispiel #3
0
        public InsertStats InsertTransactions(SyncBlockTransactionsOperation item)
        {
            var stats = new InsertStats {
                Items = new List <MapTransactionAddress>()
            };

            if (item.BlockInfo != null)
            {
                // remove all transactions from the memory pool
                item.Transactions.ForEach(t =>
                {
                    NBitcoin.Transaction outer;
                    this.data.MemoryTransactions.TryRemove(t.GetHash().ToString(), out outer);
                });

                // break the work in to batches of transactions
                var queue = new Queue <NBitcoin.Transaction>(item.Transactions);
                do
                {
                    var items = this.GetBatch(this.configuration.MongoBatchSize, queue).ToList();

                    try
                    {
                        if (item.BlockInfo != null)
                        {
                            var inserts = items.Select(s => new MapTransactionBlock {
                                BlockIndex = item.BlockInfo.Height, TransactionId = s.GetHash().ToString()
                            }).ToList();
                            stats.Transactions += inserts.Count;
                            this.data.MapTransactionBlock.InsertMany(inserts, new InsertManyOptions {
                                IsOrdered = false
                            });
                        }
                    }
                    catch (MongoBulkWriteException mbwex)
                    {
                        if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection"))
                        {
                            throw;
                        }
                    }

                    // insert inputs and add to the list for later to use on the notification task.
                    var inputs  = this.CreateInputs(item.BlockInfo.Height, items).ToList();
                    var outputs = this.CreateOutputs(items, item.BlockInfo.Height).ToList();
                    inputs.AddRange(outputs);
                    var queueInner = new Queue <MapTransactionAddress>(inputs);

                    do
                    {
                        try
                        {
                            var itemsInner   = this.GetBatch(this.configuration.MongoBatchSize, queueInner).ToList();
                            var ops          = new Dictionary <string, WriteModel <MapTransactionAddress> >();
                            var writeOptions = new BulkWriteOptions()
                            {
                                IsOrdered = false
                            };

                            foreach (var mapTransactionAddress in itemsInner)
                            {
                                if (mapTransactionAddress.SpendingTransactionId == null)
                                {
                                    ops.Add(mapTransactionAddress.Id, new InsertOneModel <MapTransactionAddress>(mapTransactionAddress));
                                }
                                else
                                {
                                    if (ops.TryGetValue(mapTransactionAddress.Id, out WriteModel <MapTransactionAddress> mta))
                                    {
                                        // in case a utxo is spent in the same block
                                        // we just modify the inserted item directly

                                        var imta = mta as InsertOneModel <MapTransactionAddress>;
                                        imta.Document.SpendingTransactionId = mapTransactionAddress.SpendingTransactionId;
                                        imta.Document.SpendingBlockIndex    = mapTransactionAddress.SpendingBlockIndex;
                                    }
                                    else
                                    {
                                        var filter = Builders <MapTransactionAddress> .Filter.Eq(addr => addr.Id, mapTransactionAddress.Id);

                                        var update = Builders <MapTransactionAddress> .Update
                                                     .Set(blockInfo => blockInfo.SpendingTransactionId, mapTransactionAddress.SpendingTransactionId)
                                                     .Set(blockInfo => blockInfo.SpendingBlockIndex, mapTransactionAddress.SpendingBlockIndex);

                                        ops.Add(mapTransactionAddress.Id, new UpdateOneModel <MapTransactionAddress>(filter, update));
                                    }
                                }
                            }

                            if (itemsInner.Any())
                            {
                                stats.Items.AddRange(itemsInner);
                                stats.InputsOutputs += ops.Count;
                                this.data.MapTransactionAddress.BulkWrite(ops.Values, writeOptions);
                            }
                        }
                        catch (MongoBulkWriteException mbwex)
                        {
                            if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection"))
                            {
                                throw;
                            }
                        }
                    }while (queueInner.Any());

                    // If insert trx supported then push trx in batches.
                    if (this.configuration.StoreRawTransactions)
                    {
                        try
                        {
                            var inserts = items.Select(t => new MapTransaction {
                                TransactionId = t.GetHash().ToString(), RawTransaction = t.ToBytes(syncConnection.Network.Consensus.ConsensusFactory)
                            }).ToList();
                            stats.RawTransactions = inserts.Count;
                            this.data.MapTransaction.InsertMany(inserts, new InsertManyOptions {
                                IsOrdered = false
                            });
                        }
                        catch (MongoBulkWriteException mbwex)
                        {
                            if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection"))
                            {
                                throw;
                            }
                        }
                    }
                }while (queue.Any());

                // mark the block as synced.
                this.CompleteBlock(item.BlockInfo);
            }
            else
            {
                // memory transaction push in to the pool.
                item.Transactions.ForEach(t =>
                {
                    this.data.MemoryTransactions.TryAdd(t.GetHash().ToString(), t);
                });

                stats.Transactions = this.data.MemoryTransactions.Count();

                // todo: for accuracy - remove transactions from the mongo memory pool that are not anymore in the syncing pool
                // remove all transactions from the memory pool
                // this can be done using the SyncingBlocks objects - see method SyncOperations.FindPoolInternal()

                // add to the list for later to use on the notification task.
                var inputs = this.CreateInputs(-1, item.Transactions).ToList();
                stats.Items.AddRange(inputs);
            }

            return(stats);
        }
        public InsertStats InsertTransactions(SyncBlockTransactionsOperation item)
        {
            var stats = new InsertStats {
                Items = new List <MapTransactionAddress>()
            };

            if (item.BlockInfo != null)
            {
                // remove all transactions from the memory pool
                item.Transactions.ForEach(t =>
                {
                    DecodedRawTransaction outer;
                    this.data.MemoryTransactions.TryRemove(t.TxId, out outer);
                });

                // break the work in to batches transactions
                var queue = new Queue <DecodedRawTransaction>(item.Transactions);
                do
                {
                    var transactions  = this.GetBatch(this.configuration.MongoBatchSize, queue).ToList();
                    var bitcoinClient = CryptoClientFactory.Create(
                        syncConnection.ServerDomain,
                        syncConnection.RpcAccessPort,
                        syncConnection.User,
                        syncConnection.Password,
                        syncConnection.Secure);
                    try
                    {
                        if (item.BlockInfo != null)
                        {
                            var inserts       = new List <MapTransactionBlock>();
                            var insertDetails = new List <MapTransactionDetail>();
                            foreach (var tx in transactions)
                            {
                                var isCoinBase = !string.IsNullOrWhiteSpace(tx.VIn.First().CoinBase);
                                var syncVin    = tx.VIn.Select(vin =>
                                {
                                    var previousTransaction = isCoinBase ?
                                                              null : bitcoinClient.GetRawTransaction(vin.TxId, 1);

                                    return(new SyncVin
                                    {
                                        TxId = vin.TxId,
                                        CoinBase = vin.CoinBase,
                                        IsCoinBase = !string.IsNullOrWhiteSpace(vin.CoinBase),
                                        ScriptSig = vin.ScriptSig,
                                        Sequence = vin.Sequence,
                                        VOut = vin.VOut,
                                        PreviousVout = previousTransaction?.VOut.First(o => o.N == vin.VOut)
                                    });
                                }).ToList();
                                var totalVout = tx.VOut.Sum(o => o.Value);
                                var totalVin  = syncVin.Sum(i => i.PreviousVout?.Value);

                                inserts.Add(new MapTransactionBlock
                                {
                                    BlockIndex    = item.BlockInfo.Height,
                                    TransactionId = tx.TxId,
                                    TotalVout     = totalVout,
                                    TotalVin      = totalVin,
                                    Time          = tx.Time,
                                    BlockHash     = tx.BlockHash,
                                    BlockTime     = tx.BlockTime,
                                    Locktime      = tx.Locktime,
                                    Version       = tx.Version,
                                    IsCoinBase    = isCoinBase
                                });

                                insertDetails.Add(new MapTransactionDetail
                                {
                                    TransactionId = tx.TxId,
                                    Vin           = syncVin,
                                    Vout          = tx.VOut
                                });
                            }

                            stats.Transactions += inserts.Count();
                            this.data.MapTransactionBlock.InsertMany(inserts, new InsertManyOptions {
                                IsOrdered = false
                            });
                            this.data.MapTransactionDetails.InsertMany(insertDetails, new InsertManyOptions {
                                IsOrdered = false
                            });
                        }
                    }
                    catch (MongoBulkWriteException mbwex)
                    {
                        if (!mbwex.Message.Contains("E11000 duplicate key error collection"))
                        {
                            throw;
                        }
                    }

                    // insert inputs and add to the list for later to use on the notification task.
                    var inputs     = this.CreateInputs(item.BlockInfo.Height, transactions).ToList();
                    var queueInner = new Queue <MapTransactionAddress>(inputs);
                    do
                    {
                        try
                        {
                            var itemsInner = this.GetBatch(this.configuration.MongoBatchSize, queueInner).ToList();
                            if (itemsInner.Any())
                            {
                                stats.Inputs += itemsInner.Count();
                                stats.Items.AddRange(itemsInner);
                                this.data.MapTransactionAddress.InsertMany(itemsInner, new InsertManyOptions {
                                    IsOrdered = false
                                });
                            }
                        }
                        catch (MongoBulkWriteException mbwex)
                        {
                            if (!mbwex.Message.Contains("E11000 duplicate key error collection"))
                            {
                                throw;
                            }
                        }
                    }while (queueInner.Any());

                    // insert outputs
                    var outputs = this.CreateOutputs(transactions).ToList();
                    stats.Outputs += outputs.Count();
                    outputs.ForEach(outp => this.data.MarkOutput(outp.InputTransactionId, outp.InputIndex, outp.TransactionId));
                }while (queue.Any());

                // mark the block as synced.
                this.CompleteBlock(item.BlockInfo);
            }
            else
            {
                // memory transaction push in to the pool.
                item.Transactions.ForEach(t =>
                {
                    this.data.MemoryTransactions.TryAdd(t.TxId, t);
                });

                stats.Transactions = this.data.MemoryTransactions.Count();

                // todo: for accuracy - remove transactions from the mongo memory pool that are not anymore in the syncing pool
                // remove all transactions from the memory pool
                // this can be done using the SyncingBlocks objects - see method SyncOperations.FindPoolInternal()

                // add to the list for later to use on the notification task.
                var inputs = this.CreateInputs(-1, item.Transactions).ToList();
                stats.Items.AddRange(inputs);
            }

            return(stats);
        }