private async Task TransferAvailableData(TaskDispatcher taskDispatcher, SourceDataPipeline sourceDataPipeline) { DataTable availableDataTable; while (sourceDataPipeline.TryGetNextAvailableDataSource(out availableDataTable)) { DataTable availableDataTable2 = availableDataTable; // Dispatch the work of transferring data from the "source pipeline" int the database to an available background thread. // Note: The await awaits only until the work is dispatched and not until the work is completed. // Dispatching the work itself may take a while if all available background threads are busy. await taskDispatcher.DispatchWorkAsync(() => this.TransferTable(availableDataTable2)); } }
private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase) { DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager(); TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads); // What if we use 1 thread now that we use bulk copy? IBlockchainParser blockchainParser; if (this.blockchainParserFactory == null) { blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName); } else { blockchainParser = this.blockchainParserFactory(); } if (this.parameters.BlockId != null) { blockchainParser.SetBlockId(this.parameters.BlockId.Value); } this.processingStatistics.ProcessingBlockchainStarting(); Stopwatch currentBlockchainFileStopwatch = new Stopwatch(); currentBlockchainFileStopwatch.Start(); SourceDataPipeline sourceDataPipeline = new SourceDataPipeline(); int blockFileId = -1; foreach (ParserData.Block block in blockchainParser.ParseBlockchain()) { if (this.currentBlockchainFile != block.BlockchainFileName) { if (this.currentBlockchainFile != null) { this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch); currentBlockchainFileStopwatch.Restart(); } this.lastReportedPercentage = -1; blockFileId = databaseIdManager.GetNextBlockchainFileId(1); this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName); this.currentBlockchainFile = block.BlockchainFileName; } this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile); // We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that // the database primary keys are generated in a certain order. The primary keys in our tables will be // in the same order as the corresponding entities appear in the blockchain. For example, with the // current implementation, the block ID will be the block depth as reported by http://blockchain.info/. DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount); this.processingStatistics.AddBlocksCount(1); this.processingStatistics.AddTransactionsCount(block.Transactions.Count); this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount); this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount); int blockFileId2 = blockFileId; ParserData.Block block2 = block; // Dispatch the work of "filling the source pipeline" to an available background thread. // Note: The await awaits only until the work is dispatched and not until the work is completed. // Dispatching the work itself may take a while if all available background threads are busy. await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager)); await this.TransferAvailableData(taskDispatcher, sourceDataPipeline); } // Wait for the last remaining background tasks if any that are still executing // sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish. await taskDispatcher.WaitForAllWorkToComplete(); // Instruct sourceDataPipeline to transfer all remaining data to the available data queue. // IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline. sourceDataPipeline.Flush(); // Now trigger the SQL bulk copy for the data that remains. await this.TransferAvailableData(taskDispatcher, sourceDataPipeline); // Wait for the last remaining background tasks if any that are still executing // the SQL bulk copy to finish. await taskDispatcher.WaitForAllWorkToComplete(); this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch); }