public Block GenerateBlock( ByteArray blockHash, ByteArray transactionHash, InputInfo[] inputs, OutputInfo[] outputs) { BlockHeader blockHeader = new BlockHeader() { BlockHash = blockHash, BlockNonce = 0, BlockTargetDifficulty = 0, BlockTimestamp = this.GetCurrentBlockTimeStamp(), BlockTimestampUnix = 0, BlockVersion = 1, MerkleRootHash = ByteArray.Empty, PreviousBlockHash = this.previousBlockHash, }; this.blockTimeStamp = this.blockTimeStamp.AddDays(1); string expectedFileName = string.Format(CultureInfo.InvariantCulture, "blk{0:00000}.dat", this.GetCurentBlockchainFileIndex()); Block block = new Block(expectedFileName, blockHeader); Transaction transaction = new Transaction() { TransactionHash = transactionHash, TransactionLockTime = 0, TransactionVersion = 1, }; foreach (InputInfo inputInfo in inputs) { TransactionInput transactionInput = new TransactionInput() { InputScript = ByteArray.Empty, SourceTransactionHash = inputInfo.SourceTransactionHash, SourceTransactionOutputIndex = inputInfo.SourceTransactionOutputIndex, }; transaction.AddInput(transactionInput); } foreach (OutputInfo outputInfo in outputs) { TransactionOutput transactionOutput = new TransactionOutput() { OutputScript = ByteArray.Empty, OutputValueSatoshi = (ulong)(outputInfo.OutputValueSatoshi * DatabaseGenerator.BtcToSatoshi), }; transaction.AddOutput(transactionOutput); } block.AddTransaction(transaction); this.previousBlockHash = block.BlockHeader.BlockHash; return block; }
private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase) { DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager(); TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads); IBlockchainParser blockchainParser; if (this.blockchainParserFactory == null) { blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName); } else { blockchainParser = this.blockchainParserFactory(); } if (this.parameters.BlockId != null) { blockchainParser.SetBlockId(this.parameters.BlockId.Value); } this.processingStatistics.ProcessingBlockchainStarting(); Stopwatch currentBlockchainFileStopwatch = new Stopwatch(); currentBlockchainFileStopwatch.Start(); SourceDataPipeline sourceDataPipeline = new SourceDataPipeline(); int blockFileId = -1; try { foreach (ParserData.Block block in blockchainParser.ParseBlockchain()) { if (this.currentBlockchainFile != block.BlockchainFileName) { if (this.currentBlockchainFile != null) { this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch); currentBlockchainFileStopwatch.Restart(); } this.lastReportedPercentage = -1; blockFileId = databaseIdManager.GetNextBlockchainFileId(1); this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName); this.currentBlockchainFile = block.BlockchainFileName; } this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile); // We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that // the database primary keys are generated in a certain order. The primary keys in our tables will be // in the same order as the corresponding entities appear in the blockchain. For example, with the // current implementation, the block ID will be the block depth as reported by http://blockchain.info/. DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount); this.processingStatistics.AddBlocksCount(1); this.processingStatistics.AddTransactionsCount(block.Transactions.Count); this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount); this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount); int blockFileId2 = blockFileId; ParserData.Block block2 = block; // Dispatch the work of "filling the source pipeline" to an available background thread. // Note: The await awaits only until the work is dispatched and not until the work is completed. // Dispatching the work itself may take a while if all available background threads are busy. await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager)); await this.TransferAvailableData(taskDispatcher, sourceDataPipeline); } } finally { // Whatever we have in the pipeline we'll push to the DB. We do this in a finally block. // Otherwise an exception that occurs in blockchain file 100 may prevent data that was // collected in blockchain file 99 to be saved to DB. // Wait for the last remaining background tasks if any that are still executing // sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish. await taskDispatcher.WaitForAllWorkToComplete(); // Instruct sourceDataPipeline to transfer all remaining data to the available data queue. // IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline. sourceDataPipeline.Flush(); // Now trigger the SQL bulk copy for the data that remains. await this.TransferAvailableData(taskDispatcher, sourceDataPipeline); // Wait for the last remaining background tasks if any that are still executing // the SQL bulk copy to finish. await taskDispatcher.WaitForAllWorkToComplete(); } this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch); }
public void FillBlockchainPipeline(int currentBlockchainFileId, ParserData.Block parserBlock, DatabaseIdSegmentManager databaseIdSegmentManager) { long blockId = databaseIdSegmentManager.GetNextBlockId(); lock (this.blockLockObject) { this.blockDataSet.Block.AddBlockRow( blockId, currentBlockchainFileId, (int)parserBlock.BlockHeader.BlockVersion, parserBlock.BlockHeader.BlockHash.ToArray(), parserBlock.BlockHeader.PreviousBlockHash.ToArray(), parserBlock.TransactionsCount, parserBlock.BlockHeader.MerkleRootHash.ToArray(), parserBlock.BlockHeader.BlockTimestamp); if (this.MakeDataTableAvailableIfLarge(this.blockDataSet.Block)) { this.blockDataSet = new BlockDataSet(); } } lock (this.bitcoinTransactionLockObject) { foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions) { long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId(); this.bitcoinTransactionDataSetBuffer.BitcoinTransaction.AddBitcoinTransactionRow( bitcoinTransactionId, blockId, parserTransaction.TransactionHash.ToArray(), parserTransaction.TransactionInputsCount, parserTransaction.TransactionOutputsCount, (int)parserTransaction.TransactionVersion, (int)parserTransaction.TransactionLockTime, parserTransaction.IsSegWit); } if (this.MakeDataTableAvailableIfLarge(this.bitcoinTransactionDataSetBuffer.BitcoinTransaction)) { this.bitcoinTransactionDataSetBuffer = new BitcoinTransactionDataSet(); } } lock (this.transactionInputLockObject) { databaseIdSegmentManager.ResetNextTransactionId(); foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions) { long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId(); foreach (ParserData.TransactionInput parserTransactionInput in parserTransaction.Inputs) { long transactionInput = databaseIdSegmentManager.GetNextTransactionInputId(); this.transactionInputDataSetBuffer.TransactionInput.AddTransactionInputRow( transactionInput, bitcoinTransactionId, parserTransactionInput.SourceTransactionHash.ToArray(), (int)parserTransactionInput.SourceTransactionOutputIndex, DBData.TransactionInput.SourceTransactionOutputIdUnknown); } if (this.MakeDataTableAvailableIfLarge(this.transactionInputDataSetBuffer.TransactionInput)) { this.transactionInputDataSetBuffer = new TransactionInputDataSet(); } } } lock (this.transactionOutputLockObject) { databaseIdSegmentManager.ResetNextTransactionId(); foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions) { long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId(); for (int outputIndex = 0; outputIndex < parserTransaction.Outputs.Count; outputIndex++) { ParserData.TransactionOutput parserTransactionOutput = parserTransaction.Outputs[outputIndex]; byte[] temp; temp = parserTransactionOutput.OutputAddress.ToArray(); string str = string.Empty; for (int i = 0; i < temp.Length; i++) { str += (char)temp[i]; } long transactionOutputId = databaseIdSegmentManager.GetNextTransactionOutputId(); this.transactionOutputDataSetBuffer.TransactionOutput.AddTransactionOutputRow( transactionOutputId, bitcoinTransactionId, outputIndex, (decimal)parserTransactionOutput.OutputValueSatoshi / DatabaseGenerator.BtcToSatoshi, parserTransactionOutput.OutputScript.ToArray(), str, DBData.TransactionOutput.OutputAddressIdUnknown); } } if (this.MakeDataTableAvailableIfLarge(this.transactionOutputDataSetBuffer.TransactionOutput)) { this.transactionOutputDataSetBuffer = new TransactionOutputDataSet(); } } }
public void FillBlockchainPipeline(int currentBlockchainFileId, ParserData.Block parserBlock, DatabaseIdSegmentManager databaseIdSegmentManager) { long blockId = databaseIdSegmentManager.GetNextBlockId(); lock (this.blockLockObject) { this.blockDataSet.Block.AddBlockRow( blockId, currentBlockchainFileId, (int)parserBlock.BlockHeader.BlockVersion, parserBlock.BlockHeader.BlockHash.ToArray(), parserBlock.BlockHeader.PreviousBlockHash.ToArray(), parserBlock.BlockHeader.BlockTimestamp); if (this.MakeDataTableAvailableIfLarge(this.blockDataSet.Block)) { this.blockDataSet = new BlockDataSet(); } } lock (this.bitcoinTransactionLockObject) { foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions) { long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId(); this.bitcoinTransactionDataSetBuffer.BitcoinTransaction.AddBitcoinTransactionRow( bitcoinTransactionId, blockId, parserTransaction.TransactionHash.ToArray(), (int)parserTransaction.TransactionVersion, (int)parserTransaction.TransactionLockTime); } if (this.MakeDataTableAvailableIfLarge(this.bitcoinTransactionDataSetBuffer.BitcoinTransaction)) { this.bitcoinTransactionDataSetBuffer = new BitcoinTransactionDataSet(); } } lock (this.transactionInputLockObject) { databaseIdSegmentManager.ResetNextTransactionId(); foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions) { long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId(); long inputScriptId = 0; foreach (ParserData.TransactionInput parserTransactionInput in parserTransaction.Inputs) { long transactionInput = databaseIdSegmentManager.GetNextTransactionInputId(); this.transactionInputDataSetBuffer.TransactionInput.AddTransactionInputRow( transactionInput, inputScriptId++, bitcoinTransactionId, DBData.TransactionInput.SourceTransactionOutputIdUnknown); this.transactionInputSourceDataSetBuffer.TransactionInputSource.AddTransactionInputSourceRow( transactionInput, parserTransactionInput.SourceTransactionHash.ToArray(), (int)parserTransactionInput.SourceTransactionOutputIndex, parserTransactionInput.InputScript.ToArray() ); } if (this.MakeDataTableAvailableIfLarge(this.transactionInputDataSetBuffer.TransactionInput)) { this.transactionInputDataSetBuffer = new TransactionInputDataSet(); } if (this.MakeDataTableAvailableIfLarge(this.transactionInputSourceDataSetBuffer.TransactionInputSource)) { this.transactionInputSourceDataSetBuffer = new TransactionInputSourceDataSet(); } } } lock (this.transactionOutputLockObject) { databaseIdSegmentManager.ResetNextTransactionId(); foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions) { long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId(); for (int outputIndex = 0; outputIndex < parserTransaction.Outputs.Count; outputIndex++) { ParserData.TransactionOutput parserTransactionOutput = parserTransaction.Outputs[outputIndex]; long transactionOutputId = databaseIdSegmentManager.GetNextTransactionOutputId(); this.transactionOutputDataSetBuffer.TransactionOutput.AddTransactionOutputRow( transactionOutputId, bitcoinTransactionId, outputIndex, (decimal)parserTransactionOutput.OutputValueSatoshi / DatabaseGenerator.BtcToSatoshi, parserTransactionOutput.OutputScript.ToArray()); } } if (this.MakeDataTableAvailableIfLarge(this.transactionOutputDataSetBuffer.TransactionOutput)) { this.transactionOutputDataSetBuffer = new TransactionOutputDataSet(); } } }
/// <summary> /// Initializes a new instance of the <see cref="BlockProcessedEventArgs" /> class. /// </summary> /// <param name="block"> /// Contains data describing the block that was processed. /// </param> public BlockProcessedEventArgs(Block block) { this.Block = block; }
/// <summary> /// Parses one Bitcoin block except for a few fields before the actual block header. /// </summary> /// <param name="blockchainFileName"> /// The name of the blockchain file that contains the block being parsed. /// </param> /// <param name="blockMemoryStreamReader"> /// Provides access to a section of the Bitcoin blockchain file. /// </param> private static Block InternalParseBlockchainFile(string blockchainFileName, BlockMemoryStreamReader blockMemoryStreamReader) { BlockHeader blockHeader = BlockchainParser.ParseBlockHeader(blockMemoryStreamReader); Block block = new Block(blockchainFileName, blockHeader); int blockTransactionCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger(); for (int transactionIndex = 0; transactionIndex < blockTransactionCount; transactionIndex++) { Transaction transaction = BlockchainParser.ParseTransaction(blockMemoryStreamReader); block.AddTransaction(transaction); } return block; }