private async Task TransferAvailableData(TaskDispatcher taskDispatcher, SourceDataPipeline sourceDataPipeline)
{
DataTable availableDataTable;
while (sourceDataPipeline.TryGetNextAvailableDataSource(out availableDataTable))
{
DataTable availableDataTable2 = availableDataTable;
// Dispatch the work of transferring data from the "source pipeline" int the database to an available background thread.
// Note: The await awaits only until the work is dispatched and not until the work is completed.
// Dispatching the work itself may take a while if all available background threads are busy.
await taskDispatcher.DispatchWorkAsync(() => this.TransferTable(availableDataTable2));
}
}
private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase)
{
DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager();
TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads);
IBlockchainParser blockchainParser;
if (this.blockchainParserFactory == null)
{
blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName);
}
else
{
blockchainParser = this.blockchainParserFactory();
}
if (this.parameters.BlockId != null)
{
blockchainParser.SetBlockId(this.parameters.BlockId.Value);
}
this.processingStatistics.ProcessingBlockchainStarting();
Stopwatch currentBlockchainFileStopwatch = new Stopwatch();
currentBlockchainFileStopwatch.Start();
SourceDataPipeline sourceDataPipeline = new SourceDataPipeline();
int blockFileId = -1;
try
{
foreach (ParserData.Block block in blockchainParser.ParseBlockchain())
{
if (this.currentBlockchainFile != block.BlockchainFileName)
{
if (this.currentBlockchainFile != null)
{
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
currentBlockchainFileStopwatch.Restart();
}
this.lastReportedPercentage = -1;
blockFileId = databaseIdManager.GetNextBlockchainFileId(1);
this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName);
this.currentBlockchainFile = block.BlockchainFileName;
}
this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile);
// We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that
// the database primary keys are generated in a certain order. The primary keys in our tables will be
// in the same order as the corresponding entities appear in the blockchain. For example, with the
// current implementation, the block ID will be the block depth as reported by http://blockchain.info/.
DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount);
this.processingStatistics.AddBlocksCount(1);
this.processingStatistics.AddTransactionsCount(block.Transactions.Count);
this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount);
this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount);
int blockFileId2 = blockFileId;
ParserData.Block block2 = block;
// Dispatch the work of "filling the source pipeline" to an available background thread.
// Note: The await awaits only until the work is dispatched and not until the work is completed.
// Dispatching the work itself may take a while if all available background threads are busy.
await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager));
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
}
}
finally
{
// Whatever we have in the pipeline we'll push to the DB. We do this in a finally block.
// Otherwise an exception that occurs in blockchain file 100 may prevent data that was
// collected in blockchain file 99 to be saved to DB.
// Wait for the last remaining background tasks if any that are still executing
// sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
// Instruct sourceDataPipeline to transfer all remaining data to the available data queue.
// IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline.
sourceDataPipeline.Flush();
// Now trigger the SQL bulk copy for the data that remains.
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
// Wait for the last remaining background tasks if any that are still executing
// the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
}
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
}
private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase)
{
DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager();
TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads); // What if we use 1 thread now that we use bulk copy?
IBlockchainParser blockchainParser;
if (this.blockchainParserFactory == null)
{
blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName);
}
else
{
blockchainParser = this.blockchainParserFactory();
}
if (this.parameters.BlockId != null)
{
blockchainParser.SetBlockId(this.parameters.BlockId.Value);
}
this.processingStatistics.ProcessingBlockchainStarting();
Stopwatch currentBlockchainFileStopwatch = new Stopwatch();
currentBlockchainFileStopwatch.Start();
SourceDataPipeline sourceDataPipeline = new SourceDataPipeline();
int blockFileId = -1;
foreach (ParserData.Block block in blockchainParser.ParseBlockchain())
{
if (this.currentBlockchainFile != block.BlockchainFileName)
{
if (this.currentBlockchainFile != null)
{
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
currentBlockchainFileStopwatch.Restart();
}
this.lastReportedPercentage = -1;
blockFileId = databaseIdManager.GetNextBlockchainFileId(1);
this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName);
this.currentBlockchainFile = block.BlockchainFileName;
}
this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile);
// We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that
// the database primary keys are generated in a certain order. The primary keys in our tables will be
// in the same order as the corresponding entities appear in the blockchain. For example, with the
// current implementation, the block ID will be the block depth as reported by http://blockchain.info/.
DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount);
this.processingStatistics.AddBlocksCount(1);
this.processingStatistics.AddTransactionsCount(block.Transactions.Count);
this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount);
this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount);
int blockFileId2 = blockFileId;
ParserData.Block block2 = block;
// Dispatch the work of "filling the source pipeline" to an available background thread.
// Note: The await awaits only until the work is dispatched and not until the work is completed.
// Dispatching the work itself may take a while if all available background threads are busy.
await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager));
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
}
// Wait for the last remaining background tasks if any that are still executing
// sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
// Instruct sourceDataPipeline to transfer all remaining data to the available data queue.
// IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline.
sourceDataPipeline.Flush();
// Now trigger the SQL bulk copy for the data that remains.
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
// Wait for the last remaining background tasks if any that are still executing
// the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
}
