public void FillBlockchainPipeline(int currentBlockchainFileId, ParserData.Block parserBlock, DatabaseIdSegmentManager databaseIdSegmentManager)
{
long blockId = databaseIdSegmentManager.GetNextBlockId();
lock (this.blockLockObject)
{
this.blockDataSet.Block.AddBlockRow(
blockId,
currentBlockchainFileId,
(int)parserBlock.BlockHeader.BlockVersion,
parserBlock.BlockHeader.BlockHash.ToArray(),
parserBlock.BlockHeader.PreviousBlockHash.ToArray(),
parserBlock.BlockHeader.BlockTimestamp,
(decimal)parserBlock.TransparentSpent / DatabaseGenerator.BtcToSatoshi,
(decimal)parserBlock.ShieldedIn / DatabaseGenerator.BtcToSatoshi,
(decimal)parserBlock.ShieldedOut / DatabaseGenerator.BtcToSatoshi,
(decimal)parserBlock.ShieldedDiff / DatabaseGenerator.BtcToSatoshi,
(decimal)parserBlock.BlockReward / DatabaseGenerator.BtcToSatoshi);
if (this.MakeDataTableAvailableIfLarge(this.blockDataSet.Block))
{
this.blockDataSet = new BlockDataSet();
}
}
lock (this.bitcoinTransactionLockObject)
{
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
this.bitcoinTransactionDataSetBuffer.BitcoinTransaction.AddBitcoinTransactionRow(
bitcoinTransactionId,
blockId,
parserTransaction.TransactionHash.ToArray(),
(int)parserTransaction.TransactionVersion,
(int)parserTransaction.TransactionLockTime);
}
if (this.MakeDataTableAvailableIfLarge(this.bitcoinTransactionDataSetBuffer.BitcoinTransaction))
{
this.bitcoinTransactionDataSetBuffer = new BitcoinTransactionDataSet();
}
}
lock (this.transactionInputLockObject)
{
databaseIdSegmentManager.ResetNextTransactionId();
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
foreach (ParserData.TransactionInput parserTransactionInput in parserTransaction.Inputs)
{
long transactionInput = databaseIdSegmentManager.GetNextTransactionInputId();
this.transactionInputDataSetBuffer.TransactionInput.AddTransactionInputRow(
transactionInput,
bitcoinTransactionId,
DBData.TransactionInput.SourceTransactionOutputIdUnknown);
this.transactionInputSourceDataSetBuffer.TransactionInputSource.AddTransactionInputSourceRow(
transactionInput,
parserTransactionInput.SourceTransactionHash.ToArray(),
(int)parserTransactionInput.SourceTransactionOutputIndex);
}
if (this.MakeDataTableAvailableIfLarge(this.transactionInputDataSetBuffer.TransactionInput))
{
this.transactionInputDataSetBuffer = new TransactionInputDataSet();
}
if (this.MakeDataTableAvailableIfLarge(this.transactionInputSourceDataSetBuffer.TransactionInputSource))
{
this.transactionInputSourceDataSetBuffer = new TransactionInputSourceDataSet();
}
}
}
lock (this.transactionOutputLockObject)
{
databaseIdSegmentManager.ResetNextTransactionId();
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
for (int outputIndex = 0; outputIndex < parserTransaction.Outputs.Count; outputIndex++)
{
ParserData.TransactionOutput parserTransactionOutput = parserTransaction.Outputs[outputIndex];
long transactionOutputId = databaseIdSegmentManager.GetNextTransactionOutputId();
this.transactionOutputDataSetBuffer.TransactionOutput.AddTransactionOutputRow(
transactionOutputId,
bitcoinTransactionId,
outputIndex,
(decimal)parserTransactionOutput.OutputValueSatoshi / DatabaseGenerator.BtcToSatoshi,
parserTransactionOutput.Address);
}
}
if (this.MakeDataTableAvailableIfLarge(this.transactionOutputDataSetBuffer.TransactionOutput))
{
this.transactionOutputDataSetBuffer = new TransactionOutputDataSet();
}
}
lock (this.joinSplitLockObject)
{
databaseIdSegmentManager.ResetNextTransactionId();
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
for (int joinSplitIndex = 0; joinSplitIndex < parserTransaction.JoinSplits.Count; ++joinSplitIndex)
{
ParserData.JoinSplit parserJoinSplit = parserTransaction.JoinSplits[joinSplitIndex];
long joinSplitId = databaseIdSegmentManager.GetNextJoinSplitId();
this.JoinSplitDataSetBuffer.JoinSplit.AddJoinSplitRow(
joinSplitId,
bitcoinTransactionId,
joinSplitIndex,
(decimal)parserJoinSplit.AmountIn / DatabaseGenerator.BtcToSatoshi,
(decimal)parserJoinSplit.AmountOut / DatabaseGenerator.BtcToSatoshi);
}
}
if (this.MakeDataTableAvailableIfLarge(this.JoinSplitDataSetBuffer.JoinSplit))
{
this.JoinSplitDataSetBuffer = new JoinSplitDataSet();
}
}
}
private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase)
{
DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager();
TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads); // What if we use 1 thread now that we use bulk copy?
IBlockchainParser blockchainParser;
if (this.blockchainParserFactory == null)
{
blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName);
}
else
{
blockchainParser = this.blockchainParserFactory();
}
if (this.parameters.BlockId != null)
{
blockchainParser.SetBlockId(this.parameters.BlockId.Value);
}
this.processingStatistics.ProcessingBlockchainStarting();
Stopwatch currentBlockchainFileStopwatch = new Stopwatch();
currentBlockchainFileStopwatch.Start();
SourceDataPipeline sourceDataPipeline = new SourceDataPipeline();
int blockFileId = -1;
try
{
foreach (ParserData.Block block in blockchainParser.ParseBlockchain())
{
if (this.currentBlockchainFile != block.BlockchainFileName)
{
if (this.currentBlockchainFile != null)
{
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
currentBlockchainFileStopwatch.Restart();
}
this.lastReportedPercentage = -1;
blockFileId = databaseIdManager.GetNextBlockchainFileId(1);
this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName);
this.currentBlockchainFile = block.BlockchainFileName;
}
this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile);
// We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that
// the database primary keys are generated in a certain order. The primary keys in our tables will be
// in the same order as the corresponding entities appear in the blockchain. For example, with the
// current implementation, the block ID will be the block depth as reported by http://blockchain.info/.
DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount, block.JoinSplitsCount);
this.processingStatistics.AddBlocksCount(1);
this.processingStatistics.AddTransactionsCount(block.Transactions.Count);
this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount);
this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount);
this.processingStatistics.AddJoinSplitsCount(block.JoinSplitsCount);
int blockFileId2 = blockFileId;
ParserData.Block block2 = block;
// Dispatch the work of "filling the source pipeline" to an available background thread.
// Note: The await awaits only until the work is dispatched and not until the work is completed.
// Dispatching the work itself may take a while if all available background threads are busy.
await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager));
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
}
}
finally
{
// Whatever we have in the pipeline we'll push to the DB. We do this in a finally block.
// Otherwise an exception that occurs in blockchain file 100 may prevent data that was
// collected in blockchain file 99 to be saved to DB.
// Wait for the last remaining background tasks if any that are still executing
// sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
// Instruct sourceDataPipeline to transfer all remaining data to the available data queue.
// IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline.
sourceDataPipeline.Flush();
// Now trigger the SQL bulk copy for the data that remains.
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
// Wait for the last remaining background tasks if any that are still executing
// the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
}
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
}
