//j.恢复区块池上下文状态 public void restore_blockReadBlocksFromFile(List <FileStatusItem_Class> fileStatusList) { foreach (FileStatusItem_Class fileStatusItemObject in fileStatusList) { blockCountOfFile.Add(fileStatusItemObject.totalBlocks); if (fileStatusItemObject.unusedBlocks == 0) { for (int i = 0; i < fileStatusItemObject.totalBlocks; i++) { blockReadBlocksFromFile.Add(null); } } else { IBlockchainParser blockFileParser = new BlockchainParser(blockchainFilePath, fileStatusItemObject.fileName, 1); List <ParserBlock> blocksOfFile = blockFileParser.ParseBlockchain().ToList(); foreach (ParserBlock block in blocksOfFile) { if (fileStatusItemObject.unusedBlockHash.Contains(block.Header.GetHash().ToString())) { blockReadBlocksFromFile.Add(block); } else { blockReadBlocksFromFile.Add(null); } } } } }
//g.获取创世区块 public ParserBlock get_GenesisBlock(bool displayMark) { if (Directory.Exists(blockchainFilePath)) { Console.WriteLine("正在获取创世区块........."); string blockfileName = get_filename(0); string blockchainFile = Path.Combine(blockchainFilePath, blockfileName); if (File.Exists(blockchainFile)) { IBlockchainParser blockFileParser = new BlockchainParser(blockchainFilePath, blockfileName, 1); List <ParserBlock> blocksOfFirstBlockFile = blockFileParser.ParseBlockchain().ToList(); ParserBlock genesisBlock = blocksOfFirstBlockFile[0]; if (displayMark) { Console.WriteLine("创世区块前一个区块hash:" + genesisBlock.Header.HashPrevBlock); Console.WriteLine("创世区块time:" + genesisBlock.Header.BlockTime); Console.WriteLine("创世区块hash:" + genesisBlock.Header.GetHash()); Console.WriteLine("**********************************************************"); } return(genesisBlock); } else { Console.WriteLine("区块文件:" + blockchainFile + " 不存在!!!"); return(null); } } else { Console.WriteLine("区块链文件路径不存在!!!"); return(null); } }
//b.加载下一个区块文件到区块池 public void load_NextFile_ToBlockPooling() { if (Directory.Exists(blockchainFilePath)) { currentLoadFileNumber++; string blockfileName = get_filename(currentLoadFileNumber); string blockfile = Path.Combine(blockchainFilePath, blockfileName); Console.WriteLine("正在加载:" + blockfile); if (File.Exists(blockfile)) { IBlockchainParser blockFileParser = new BlockchainParser(blockchainFilePath, blockfileName, 1); List <ParserBlock> blocksOfOneBlockFile = blockFileParser.ParseBlockchain().ToList(); blockReadBlocksFromFile.AddRange(blocksOfOneBlockFile); blockCountOfFile.Add(blocksOfOneBlockFile.Count); } else { missingBlockFile = true; Console.WriteLine("区块文件:" + blockfile + " 不存在!!!"); } } else { Console.WriteLine("区块链文件路径不存在!!!"); } }
//(弃)(可以直接读取一个文件) //5.读取一个编号blk*****文件夹下的blk00000.dat文件 public List <Block> load_one_blockfile(string blockFilePath) { List <Block> oneFileBlockData = new List <Block>(); IBlockchainParser blockFileParser = new BlockchainParser(blockFilePath); IEnumerable <Block> blocks = blockFileParser.ParseBlockchain(); oneFileBlockData = blocks.ToList(); return(oneFileBlockData); }
//(改成增量统计Bitcoin Core 同步数据路径下每个区块文件的区块数量) //4.增量统计分装路径下每个区块文件的区块数量(待改) public void get_blockfiles_count_increment(bool printInfoMark) { Console.WriteLine("开始增量统计文件中的区块........."); int maxFolderNumber = 0; if (get_lastrecord_foldernumber(out maxFolderNumber)) { Dictionary <string, int> blockFileCountDictionary = new Dictionary <string, int>(); if (Directory.Exists(blockFileDestinationPath)) { DirectoryInfo directoryInfo = new DirectoryInfo(blockFileDestinationPath); if (directoryInfo.GetDirectories().Length != 0) { foreach (DirectoryInfo dirInfo in directoryInfo.GetDirectories().OrderBy(v => Convert.ToInt32(v.Name.Substring(3, v.Name.Length - 3)))) { if (dirInfo.Name.StartsWith("blk") && Convert.ToInt32(dirInfo.Name.Substring(3, dirInfo.Name.Length - 3)) > maxFolderNumber) { string blockFileFolder = Path.Combine(blockFileDestinationPath, dirInfo.Name); Console.WriteLine("正在计算" + dirInfo.Name + "中的区块数量........."); List <Block> blocks = new BlockchainParser(blockFileFolder).ParseBlockchain().ToList(); blockFileCountDictionary.Add(dirInfo.Name, blocks.Count); if (printInfoMark) { Console.WriteLine(dirInfo.Name + ":" + blocks.Count); } File.AppendAllText(blockFileDestinationPath + "\\blockFileCount.txt", dirInfo.Name + "_" + blocks.Count + "|"); } } } else { Console.WriteLine("分装文件夹为空"); } } else { Console.WriteLine("分装路径不正确或不存在"); } } else { Console.WriteLine("没有在blockFileCount.txt中找到最大的区块文件编号"); } }
//(改成统计Bitcoin Core 同步数据路径下每个区块文件的区块数量) //3.统计分装路径下每个区块文件的区块数量 public void get_blockfiles_count(bool printInfoMark) { File.WriteAllText(blockFileDestinationPath + "\\blockFileCount.txt", string.Empty); Dictionary <string, int> blockFileCountDictionary = new Dictionary <string, int>(); if (Directory.Exists(blockFileDestinationPath)) { DirectoryInfo directoryInfo = new DirectoryInfo(blockFileDestinationPath); if (directoryInfo.GetDirectories().Length != 0) { foreach (DirectoryInfo dirInfo in directoryInfo.GetDirectories().OrderBy(v => Convert.ToInt32(v.Name.Substring(3, v.Name.Length - 3)))) { if (dirInfo.Name.StartsWith("blk")) { string blockFileFolder = Path.Combine(blockFileDestinationPath, dirInfo.Name); Console.WriteLine("正在计算" + dirInfo.Name + "中的区块数量........."); List <Block> blocks = new BlockchainParser(blockFileFolder).ParseBlockchain().ToList(); blockFileCountDictionary.Add(dirInfo.Name, blocks.Count); if (printInfoMark) { Console.WriteLine(dirInfo.Name + ":" + blocks.Count); } File.AppendAllText(blockFileDestinationPath + "\\blockFileCount.txt", dirInfo.Name + "_" + blocks.Count + "|"); Console.WriteLine(dirInfo.Name + "中的区块数量记录完成"); } else { Console.WriteLine("没有合法的分装文件夹"); } } } else { Console.WriteLine("分装文件夹为空"); } } else { Console.WriteLine("分装路径不正确或不存在"); } Console.WriteLine("执行结束"); }
private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase) { DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager(); TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads); IBlockchainParser blockchainParser; if (this.blockchainParserFactory == null) { blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName); } else { blockchainParser = this.blockchainParserFactory(); } if (this.parameters.BlockId != null) { blockchainParser.SetBlockId(this.parameters.BlockId.Value); } this.processingStatistics.ProcessingBlockchainStarting(); Stopwatch currentBlockchainFileStopwatch = new Stopwatch(); currentBlockchainFileStopwatch.Start(); SourceDataPipeline sourceDataPipeline = new SourceDataPipeline(); int blockFileId = -1; try { foreach (ParserData.Block block in blockchainParser.ParseBlockchain()) { if (this.currentBlockchainFile != block.BlockchainFileName) { if (this.currentBlockchainFile != null) { this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch); currentBlockchainFileStopwatch.Restart(); } this.lastReportedPercentage = -1; blockFileId = databaseIdManager.GetNextBlockchainFileId(1); this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName); this.currentBlockchainFile = block.BlockchainFileName; } this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile); // We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that // the database primary keys are generated in a certain order. The primary keys in our tables will be // in the same order as the corresponding entities appear in the blockchain. For example, with the // current implementation, the block ID will be the block depth as reported by http://blockchain.info/. DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount); this.processingStatistics.AddBlocksCount(1); this.processingStatistics.AddTransactionsCount(block.Transactions.Count); this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount); this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount); int blockFileId2 = blockFileId; ParserData.Block block2 = block; // Dispatch the work of "filling the source pipeline" to an available background thread. // Note: The await awaits only until the work is dispatched and not until the work is completed. // Dispatching the work itself may take a while if all available background threads are busy. await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager)); await this.TransferAvailableData(taskDispatcher, sourceDataPipeline); } } finally { // Whatever we have in the pipeline we'll push to the DB. We do this in a finally block. // Otherwise an exception that occurs in blockchain file 100 may prevent data that was // collected in blockchain file 99 to be saved to DB. // Wait for the last remaining background tasks if any that are still executing // sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish. await taskDispatcher.WaitForAllWorkToComplete(); // Instruct sourceDataPipeline to transfer all remaining data to the available data queue. // IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline. sourceDataPipeline.Flush(); // Now trigger the SQL bulk copy for the data that remains. await this.TransferAvailableData(taskDispatcher, sourceDataPipeline); // Wait for the last remaining background tasks if any that are still executing // the SQL bulk copy to finish. await taskDispatcher.WaitForAllWorkToComplete(); } this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch); }
private static void ParseBlockchainFiles(string pathToBlockchain) { BlockchainStatistics overallStatistics = new BlockchainStatistics(); BlockchainStatistics blockFileStatistics = new BlockchainStatistics(); string currentBlockchainFile = null; // Instantiate a BlockchainParser. We will pass the path to the blockchain files // to its constructor. // TIP: Class IBlockchainParser provides several constructors that are useful // in different scenarios. IBlockchainParser blockchainParser = new BlockchainParser(pathToBlockchain, "blk00000.dat", 1); // Call blockchainParser.SetBlockId is the blockchain uses a value different than the standard one // for the "BlockId" field of each block. For example on testnet / testnet3. // blockchainParser.SetBlockId(0x0709110B); // Start the parsing process by calling blockchainParser.ParseBlockchain() and // process each block that is returned by the parser. // The parser exposes the blocks it parses via an "IEnumerable<Block>". // TIPS: // 1. An instance of type BitcoinBlockchain.Data.Block holds information // about all its transactions, inputs and outputs and it can use a lot of memory. // After you are done processing a block do not keep it around in memory. // For example do not simply collect all instances of type BitcoinBlockchain.Data.Block // in a list. That would consume huge amounts of memory. // // 2. To improve the performance of your application you may want to dispatch the processing // of a block on a background thread. // If you do that however you need to account for the fact that multiple blocks will // be processed concurrently. You have to be prepared to deal with various multi-threading // aspects. For example a transaction input may end up being processed before the output // it links to. You may want to consider a hybrid approach where some of the processing // for a block is done on the main thread and some of the processing is dispatched on a // background thread. // // 3. If during processing you need to store so much information that you expect to // exceed 2 GB of memory, build your tool for the x64 configuration. // // 4. Make sure that you are aware of the concept of stale blocks. // Depending on what your processing does, not accounting for stale blocks could // lead to incorrect results. The parser has no way to know that a block is stale // when it encounters it. It will enumerate it to you and you will have the chance // to detect the stale blocks once the parsing of all blocks is complete. // See: // https://bitcoin.org/en/developer-guide#orphan-blocks // https://bitcoin.org/en/glossary/stale-block // https://bitcoin.org/en/glossary/orphan-block // http://bitcoin.stackexchange.com/questions/5859/what-are-orphaned-and-stale-blocks //NBitcoin.Block b = new NBitcoin.Block(); IEnumerable <ParserBlock> blocks = blockchainParser.ParseBlockchain(); foreach (ParserBlock block in blocks) { int blocklength = block.BlockLength; string filename = block.BlockchainFileName; if (currentBlockchainFile != block.BlockchainFileName) { if (currentBlockchainFile != null) { ReportBlockChainStatistics(blockFileStatistics); blockFileStatistics.Reset(); } currentBlockchainFile = block.BlockchainFileName; Console.WriteLine("Parsing file: {0}", block.BlockchainFileName); } blockFileStatistics.AddStatistics(1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount); overallStatistics.AddStatistics(1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount); } ReportBlockChainStatistics(blockFileStatistics); Console.WriteLine("================================================="); Console.WriteLine("Overall statistics:"); ReportBlockChainStatistics(overallStatistics); }
private static void ParseBlockchainFiles(string pathToBlockchain, string firstBlock) { BlockchainStatistics overallStatistics = new BlockchainStatistics(); BlockchainStatistics blockFileStatistics = new BlockchainStatistics(); string currentBlockchainFile = null; // Instantiate a BlockchainParser. We will pass the path to the blockchain files // to its constructor. // TIP: Class IBlockchainParser provides several constructors that are useful // in different scenarios. IBlockchainParser blockchainParser = new BlockchainParser(pathToBlockchain, firstBlock); // Call blockchainParser.SetBlockId is the blockchain uses a value different than the standard one // for the "BlockId" field of each block. For example on testnet / testnet3. // blockchainParser.SetBlockId(0x0709110B); // Start the parsing process by calling blockchainParser.ParseBlockchain() and // process each block that is returned by the parser. // The parser exposes the blocks it parses via an "IEnumerable<Block>". // TIPS: // 1. An instance of type BitcoinBlockchain.Data.Block holds information // about all its transactions, inputs and outputs and it can use a lot of memory. // After you are done processing a block do not keep it around in memory. // For example do not simply collect all instances of type BitcoinBlockchain.Data.Block // in a list. That would consume huge amounts of memory. // // 2. To improve the performance of your application you may want to dispatch the processing // of a block on a background thread. // If you do that however you need to account for the fact that multiple blocks will // be processed concurrently. You have to be prepared to deal with various multi-threading // aspects. For example a transaction input may end up being processed before the output // it links to. You may want to consider a hybrid approach where some of the processing // for a block is done on the main thread and some of the processing is dispatched on a // background thread. // // 3. If during processing you need to store so much information that you expect to // exceed 2 GB of memory, build your tool for the x64 configuration. // // 4. Make sure that you are aware of the concept of stale blocks. // Depending on what your processing does, not accounting for stale blocks could // lead to incorrect results. The parser has no way to know that a block is stale // when it encounters it. It will enumerate it to you and you will have the chance // to detect the stale blocks once the parsing of all blocks is complete. // See: // https://bitcoin.org/en/developer-guide#orphan-blocks // https://bitcoin.org/en/glossary/stale-block // https://bitcoin.org/en/glossary/orphan-block // http://bitcoin.stackexchange.com/questions/5859/what-are-orphaned-and-stale-blocks long[,] TXInCount = new long[10, 10]; long[,] TXOutCount = new long[10, 10]; long[] TransactionCounts = new long[10]; for (int i = 0; i < 10; i++) { for (int j = 0; j < 10; j++) { TXInCount[i, j] = 0; TXOutCount[i, j] = 0; } TransactionCounts[i] = 0; } foreach (Block block in blockchainParser.ParseBlockchain()) { if (currentBlockchainFile != block.BlockchainFileName) { if (currentBlockchainFile != null) { ReportBlockChainStatistics(blockFileStatistics); blockFileStatistics.Reset(); } currentBlockchainFile = block.BlockchainFileName; Console.WriteLine("Parsing file: {0}", block.BlockchainFileName); } blockFileStatistics.AddStatistics(1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount); overallStatistics.AddStatistics(1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount); int year = block.BlockHeader.BlockTimestamp.Year; TransactionCounts[year - 2009] += block.TransactionsCount; foreach (Transaction transaction in block.Transactions) { if (transaction.TransactionInputsCount < 10 && transaction.TransactionOutputsCount < 10) { TXInCount[year - 2009, transaction.TransactionInputsCount]++; TXOutCount[year - 2009, transaction.TransactionOutputsCount]++; } else { TXInCount[year - 2009, 0]++; TXOutCount[year - 2009, 0]++; } } } ReportBlockChainStatistics(blockFileStatistics); Console.WriteLine("================================================="); Console.WriteLine("Overall statistics:"); ReportBlockChainStatistics(overallStatistics); string filePath = @"F:\Statistic.csv"; try { StreamWriter sw = File.CreateText(filePath); sw.Write("NUM/ITEMS"); for (int i = 0; i < 10; i++) { sw.Write(", {0}_TXIn_NumStatistic, {0}_TXIn_NumStatisticInPercent, {0}_TXOut_NumStatistic, {0}_TXOut_NumStatisticInPercent", 2009 + i); } sw.WriteLine(); for (int i = 1; i < 10; i++) { sw.Write("{0}", i); for (int j = 0; j < 10; j++) { sw.Write(", {0}, {1:000}%, {2:000}, {3}%", TXInCount[j, i], (float)(TXInCount[j, i] * 100.0 / TransactionCounts[j]), TXOutCount[j, i], (float)(TXOutCount[j, i] * 100.0 / TransactionCounts[j])); } sw.WriteLine(); } sw.Write("Other"); for (int i = 0; i < 10; i++) { sw.Write(", {0}, {1:000}%, {2:000}, {3}%", TXInCount[i, 0], (float)(TXInCount[i, 0] * 100.0 / TransactionCounts[i]), TXOutCount[i, 0], (float)(TXOutCount[i, 0] * 100.0 / TransactionCounts[i])); } sw.WriteLine(); sw.Write("Total"); for (int i = 0; i < 10; i++) { sw.Write(", {0}, 1, {0}, 1", TransactionCounts[i]); } sw.Close(); } catch (Exception e) { Console.WriteLine(e.Message); } }