예제 #1
0
        /// <summary>
        /// Parses one Bitcoin block.
        /// </summary>
        /// <param name="blockchainFileName">
        /// The name of the blockchain file that contains the block being parsed.
        /// </param>
        /// <param name="binaryReader">
        /// Provides access to a Bitcoin blockchain file.
        /// </param>
        private ParserBlock ParseBlockchainFile(string blockchainFileName, BinaryReader binaryReader)
        {
            // There are some rare situations where a block is preceded by a section containing zero bytes.
            if (binaryReader.SkipZeroBytes() == false)
            {
                // We reached the end of the file. There is no block to be parsed.
                return(null);
            }

            UInt32 blockId = binaryReader.ReadUInt32();

            if (blockId != this.blockMagicId)
            {
                throw new InvalidBlockchainContentException(string.Format(CultureInfo.InvariantCulture, "Invalid block Id: {0:X}. Expected: {1:X}", blockId, this.blockMagicId));
            }

            int blockLength = (int)binaryReader.ReadUInt32();

            byte[] blockBuffer = binaryReader.ReadBytes(blockLength);

            using (BlockMemoryStreamReader blockMemoryStreamReader = new BlockMemoryStreamReader(blockBuffer))
            {
                return(BlockchainParser.InternalParseBlockchainFile(blockBuffer,
                                                                    blockchainFileName, blockMemoryStreamReader));
            }
        }
        /// <summary>
        /// Parses one Bitcoin block except for a few fields before the actual block header.
        /// </summary>
        /// <param name="blockchainFileName">
        /// The name of the blockchain file that contains the block being parsed.
        /// </param>
        /// <param name="blockMemoryStreamReader">
        /// Provides access to a section of the Bitcoin blockchain file.
        /// </param>
        private static Block InternalParseBlockchainFile(string blockchainFileName, BlockMemoryStreamReader blockMemoryStreamReader)
        {
            BlockHeader blockHeader = BlockchainParser.ParseBlockHeader(blockMemoryStreamReader);

            Block block = new Block(blockchainFileName, blockHeader);

            int blockTransactionCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger();

            for (int transactionIndex = 0; transactionIndex < blockTransactionCount; transactionIndex++)
            {
                Transaction transaction = BlockchainParser.ParseTransaction(blockMemoryStreamReader);
                block.AddTransaction(transaction);
            }

            return(block);
        }
        /// <summary>
        /// Parses a Bitcoin transaction.
        /// </summary>
        /// <param name="blockMemoryStreamReader">
        /// Provides access to a section of the Bitcoin blockchain file.
        /// </param>
        /// <returns>
        /// The Bitcoin transaction that was parsed.
        /// </returns>
        private static Transaction ParseTransaction(BlockMemoryStreamReader blockMemoryStreamReader)
        {
            Transaction transaction = new Transaction();

            int positionInBaseStreamAtTransactionStart = (int)blockMemoryStreamReader.BaseStream.Position;

            transaction.TransactionVersion = blockMemoryStreamReader.ReadUInt32();

            int inputsCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger();

            for (int inputIndex = 0; inputIndex < inputsCount; inputIndex++)
            {
                TransactionInput transactionInput = BlockchainParser.ParseTransactionInput(blockMemoryStreamReader);
                transaction.AddInput(transactionInput);
            }

            int outputsCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger();

            for (int outputIndex = 0; outputIndex < outputsCount; outputIndex++)
            {
                TransactionOutput transactionOutput = BlockchainParser.ParseTransactionOutput(blockMemoryStreamReader);
                transaction.AddOutput(transactionOutput);
            }

            // TODO: Need to find out more details about the semantic of TransactionLockTime.
            transaction.TransactionLockTime = blockMemoryStreamReader.ReadUInt32();

            int positionInBaseStreamAfterTransactionEnd = (int)blockMemoryStreamReader.BaseStream.Position;

            using (SHA256Managed sha256 = new SHA256Managed())
            {
                //// We need to calculate the double SHA256 hash of this transaction.
                //// We need to access the buffer that contains the transaction that we jut read through.
                //// Here we take advantage of the fact that the entire block was loaded as an in-memory buffer.
                //// The base stream of blockMemoryStreamReader is that in-memory buffer.

                byte[] baseBuffer            = blockMemoryStreamReader.GetBuffer();
                int    transactionBufferSize = positionInBaseStreamAfterTransactionEnd - positionInBaseStreamAtTransactionStart;

                byte[] hash1 = sha256.ComputeHash(baseBuffer, positionInBaseStreamAtTransactionStart, transactionBufferSize);
                transaction.TransactionHash = new ByteArray(sha256.ComputeHash(hash1).ReverseByteArray());
            }

            return(transaction);
        }
예제 #4
0
        private static void ParseBlockchainFiles(string pathToBlockchain)
        {
            BlockchainStatistics overallStatistics = new BlockchainStatistics();
            BlockchainStatistics blockFileStatistics = new BlockchainStatistics();

            string currentBlockchainFile = null;

            // Instantiate a BlockchainParser. We will pass the path to the blockchain files
            // to its constructor.
            // TIP: Class IBlockchainParser provides several constructors that are useful
            //      in different scenarios.
            IBlockchainParser blockchainParser = new BlockchainParser(pathToBlockchain);

            // Start the parsing process by calling blockchainParser.ParseBlockchain() and
            // process each block that is returned by the parser.
            // The parser exposes the blocks it parses via an "IEnumerable<Block>".
            // TIPS:
            // 1. An instance of type BitcoinBlockchain.Data.Block holds information
            //    about all its transactions, inputs and outputs and it can use a lot of memory.
            //    After you are done processing a block do not keep it around in memory.
            //    For example do not simply collect all instances of type BitcoinBlockchain.Data.Block
            //    in a list. That would consume huge amounts of memory.
            //
            // 2. To improve the performance of your application you may want to dispatch the processing
            //    of a block on a background thread.
            //    If you do that however you need to account for the fact that multiple blocks will
            //    be processed concurrently. You have to be prepared to deal with various multi-threading
            //    aspects. For example a transaction input may end up being processed before the output
            //    it links to. You may want to consider a hybrid approach where some of the processing
            //    for a block is done on the main thread and some of the processing is dispatched on a
            //    background thread.
            //
            // 3. If during processing you need to store so much information that you expect to
            //    exceed 2 GB of memory, build your tool for the x64 configuration.
            //
            // 4. Make sure that you are aware of the concept of stale blocks.
            //    Depending on what your processing does, not accounting for stale blocks could
            //    lead to incorrect results. The parser has no way to know that a block is stale
            //    when it encounters it. It will enumerate it to you and you will have the chance
            //    to detect the stale blocks once the parsing of all blocks is complete.
            //    See:
            //          https://bitcoin.org/en/developer-guide#orphan-blocks
            //          https://bitcoin.org/en/glossary/stale-block
            //          https://bitcoin.org/en/glossary/orphan-block
            //          http://bitcoin.stackexchange.com/questions/5859/what-are-orphaned-and-stale-blocks
            foreach (Block block in blockchainParser.ParseBlockchain())
            {
                if (currentBlockchainFile != block.BlockchainFileName)
                {
                    if (currentBlockchainFile != null)
                    {
                        ReportBlockChainStatistics(blockFileStatistics);
                        blockFileStatistics.Reset();
                    }

                    currentBlockchainFile = block.BlockchainFileName;

                    Console.WriteLine("Parsing file: {0}", block.BlockchainFileName);
                }

                blockFileStatistics.AddStatistics(1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount);
                overallStatistics.AddStatistics(1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount);
            }

            ReportBlockChainStatistics(blockFileStatistics);

            Console.WriteLine("=================================================");
            Console.WriteLine("Overall statistics:");
            ReportBlockChainStatistics(overallStatistics);
        }
        /// <summary>
        /// Parses a Bitcoin transaction.
        /// </summary>
        /// <param name="blockMemoryStreamReader">
        /// Provides access to a section of the Bitcoin blockchain file.
        /// </param>
        /// <returns>
        /// The Bitcoin transaction that was parsed.
        /// </returns>
        private static Transaction ParseTransaction(BlockMemoryStreamReader blockMemoryStreamReader)
        {
            Transaction transaction = new Transaction();

            int positionInBaseStreamAtTransactionStart = (int)blockMemoryStreamReader.BaseStream.Position;

            transaction.TransactionVersion = blockMemoryStreamReader.ReadUInt32();

            int inputsCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger();


            bool isSegWit = false;

            if (inputsCount == 0)
            {
                byte flag = blockMemoryStreamReader.ReadByte();
                if (flag != 0x01)
                {
                    throw new InvalidBlockchainContentException(string.Format(CultureInfo.InvariantCulture,
                                                                              "Unknown transaction serialization. No input transactions, but SegWit flag was {0} instead of 1.", flag));
                }
                inputsCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger();
                isSegWit    = true;
            }


            for (int inputIndex = 0; inputIndex < inputsCount; inputIndex++)
            {
                TransactionInput transactionInput = BlockchainParser.ParseTransactionInput(blockMemoryStreamReader);
                transaction.AddInput(transactionInput);
            }

            int outputsCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger();

            for (int outputIndex = 0; outputIndex < outputsCount; outputIndex++)
            {
                TransactionOutput transactionOutput = BlockchainParser.ParseTransactionOutput(blockMemoryStreamReader);
                transaction.AddOutput(transactionOutput);
            }


            int positionInBaseStreamAfterTxOuts = (int)blockMemoryStreamReader.BaseStream.Position;

            if (isSegWit)
            {
                for (int inputIndex = 0; inputIndex < inputsCount; inputIndex++)
                {
                    Witness witness = BlockchainParser.ParseWitness(blockMemoryStreamReader);
                    transaction.AddWitness(witness);
                }
            }

            // TODO: Need to find out more details about the semantic of TransactionLockTime.
            transaction.TransactionLockTime = blockMemoryStreamReader.ReadUInt32();

            int positionInBaseStreamAfterTransactionEnd = (int)blockMemoryStreamReader.BaseStream.Position;

            using (SHA256Managed sha256 = new SHA256Managed())
            {
                //// We need to calculate the double SHA256 hash of this transaction.
                //// We need to access the buffer that contains the transaction that we jut read through.
                //// Here we take advantage of the fact that the entire block was loaded as an in-memory buffer.
                //// The base stream of blockMemoryStreamReader is that in-memory buffer.

                //byte[] baseBuffer = blockMemoryStreamReader.GetBuffer();
                //int transactionBufferSize = positionInBaseStreamAfterTransactionEnd - positionInBaseStreamAtTransactionStart;
                byte[] baseBuffer = blockMemoryStreamReader.GetBuffer(), hash1 = null;

                if (isSegWit)
                {
                    using (SHA256Managed innerSHA256 = new SHA256Managed())
                    {
                        //// SegWit transactions are still identified by their txid, which is double SHA256 of the old
                        //// serialization format (i.e. no marker, flag, or witness). So, we need to calculate the txid by
                        //// recreating the old format as the input to the hash algorithm.

                        // First, the version number
                        innerSHA256.TransformBlock(baseBuffer, positionInBaseStreamAtTransactionStart, 4, baseBuffer, positionInBaseStreamAtTransactionStart);

                        // Skip the marker and flag (each one byte), then read in txins and txouts (starting with txin count)
                        int txStart = positionInBaseStreamAtTransactionStart + 6;
                        int txSize  = positionInBaseStreamAfterTxOuts - txStart;
                        innerSHA256.TransformBlock(baseBuffer, txStart, txSize, baseBuffer, txStart);

                        ///// After the transactions comes the segregated witness data, which is not included in the txid.
                        ///// The only thing left to add to calcualte the txid is nLockTime located in the last 4 bytes
                        int lockTimeStart = positionInBaseStreamAfterTransactionEnd - 4;
                        innerSHA256.TransformFinalBlock(baseBuffer, lockTimeStart, 4);
                        hash1 = innerSHA256.Hash;
                    }
                }
                else
                {
                    int transactionBufferSize = positionInBaseStreamAfterTransactionEnd - positionInBaseStreamAtTransactionStart;
                    hash1 = sha256.ComputeHash(baseBuffer, positionInBaseStreamAtTransactionStart, transactionBufferSize);
                }



                // byte[] hash1 = sha256.ComputeHash(baseBuffer, positionInBaseStreamAtTransactionStart, transactionBufferSize);
                transaction.TransactionHash = new ByteArray(sha256.ComputeHash(hash1).ReverseByteArray());
            }

            return(transaction);
        }
        private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase)
        {
            DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager();
            TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads); // What if we use 1 thread now that we use bulk copy?

            IBlockchainParser blockchainParser;
            if (this.blockchainParserFactory == null)
            {
                blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName);
            }
            else
            {
                blockchainParser = this.blockchainParserFactory();
            }

            if (this.parameters.BlockId != null)
            {
                blockchainParser.SetBlockId(this.parameters.BlockId.Value);
            }

            this.processingStatistics.ProcessingBlockchainStarting();

            Stopwatch currentBlockchainFileStopwatch = new Stopwatch();
            currentBlockchainFileStopwatch.Start();

            SourceDataPipeline sourceDataPipeline = new SourceDataPipeline();

            int blockFileId = -1;

            foreach (ParserData.Block block in blockchainParser.ParseBlockchain())
            {
                if (this.currentBlockchainFile != block.BlockchainFileName)
                {
                    if (this.currentBlockchainFile != null)
                    {
                        this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
                        currentBlockchainFileStopwatch.Restart();
                    }

                    this.lastReportedPercentage = -1;

                    blockFileId = databaseIdManager.GetNextBlockchainFileId(1);
                    this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName);
                    this.currentBlockchainFile = block.BlockchainFileName;
                }

                this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile);

                // We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that 
                // the database primary keys are generated in a certain order. The primary keys in our tables will be 
                // in the same order as the corresponding entities appear in the blockchain. For example, with the 
                // current implementation, the block ID will be the block depth as reported by http://blockchain.info/. 
                DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount);

                this.processingStatistics.AddBlocksCount(1);
                this.processingStatistics.AddTransactionsCount(block.Transactions.Count);
                this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount);
                this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount);

                int blockFileId2 = blockFileId;
                ParserData.Block block2 = block;

                // Dispatch the work of "filling the source pipeline" to an available background thread.
                // Note: The await awaits only until the work is dispatched and not until the work is completed. 
                //       Dispatching the work itself may take a while if all available background threads are busy. 
                await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager));

                await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
            }

            // Wait for the last remaining background tasks if any that are still executing 
            // sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish.
            await taskDispatcher.WaitForAllWorkToComplete();

            // Instruct sourceDataPipeline to transfer all remaining data to the available data queue.
            // IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline.
            sourceDataPipeline.Flush();

            // Now trigger the SQL bulk copy for the data that remains.
            await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);

            // Wait for the last remaining background tasks if any that are still executing 
            // the SQL bulk copy to finish.
            await taskDispatcher.WaitForAllWorkToComplete();

            this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
        }