public Block GenerateBlock(
ByteArray blockHash,
ByteArray transactionHash,
InputInfo[] inputs,
OutputInfo[] outputs)
{
BlockHeader blockHeader = new BlockHeader()
{
BlockHash = blockHash,
BlockNonce = 0,
BlockTargetDifficulty = 0,
BlockTimestamp = this.GetCurrentBlockTimeStamp(),
BlockTimestampUnix = 0,
BlockVersion = 1,
MerkleRootHash = ByteArray.Empty,
PreviousBlockHash = this.previousBlockHash,
};
this.blockTimeStamp = this.blockTimeStamp.AddDays(1);
string expectedFileName = string.Format(CultureInfo.InvariantCulture, "blk{0:00000}.dat", this.GetCurentBlockchainFileIndex());
Block block = new Block(expectedFileName, blockHeader);
Transaction transaction = new Transaction()
{
TransactionHash = transactionHash,
TransactionLockTime = 0,
TransactionVersion = 1,
};
foreach (InputInfo inputInfo in inputs)
{
TransactionInput transactionInput = new TransactionInput()
{
InputScript = ByteArray.Empty,
SourceTransactionHash = inputInfo.SourceTransactionHash,
SourceTransactionOutputIndex = inputInfo.SourceTransactionOutputIndex,
};
transaction.AddInput(transactionInput);
}
foreach (OutputInfo outputInfo in outputs)
{
TransactionOutput transactionOutput = new TransactionOutput()
{
OutputScript = ByteArray.Empty,
OutputValueSatoshi = (ulong)(outputInfo.OutputValueSatoshi * DatabaseGenerator.BtcToSatoshi),
};
transaction.AddOutput(transactionOutput);
}
block.AddTransaction(transaction);
this.previousBlockHash = block.BlockHeader.BlockHash;
return block;
}
private async Task TransferBlockchainDataAsync(string lastKnownBlockchainFileName, bool newDatabase)
{
DatabaseIdManager databaseIdManager = this.GetDatabaseIdManager();
TaskDispatcher taskDispatcher = new TaskDispatcher(this.parameters.Threads);
IBlockchainParser blockchainParser;
if (this.blockchainParserFactory == null)
{
blockchainParser = new BlockchainParser(this.parameters.BlockchainPath, lastKnownBlockchainFileName);
}
else
{
blockchainParser = this.blockchainParserFactory();
}
if (this.parameters.BlockId != null)
{
blockchainParser.SetBlockId(this.parameters.BlockId.Value);
}
this.processingStatistics.ProcessingBlockchainStarting();
Stopwatch currentBlockchainFileStopwatch = new Stopwatch();
currentBlockchainFileStopwatch.Start();
SourceDataPipeline sourceDataPipeline = new SourceDataPipeline();
int blockFileId = -1;
try
{
foreach (ParserData.Block block in blockchainParser.ParseBlockchain())
{
if (this.currentBlockchainFile != block.BlockchainFileName)
{
if (this.currentBlockchainFile != null)
{
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
currentBlockchainFileStopwatch.Restart();
}
this.lastReportedPercentage = -1;
blockFileId = databaseIdManager.GetNextBlockchainFileId(1);
this.ProcessBlockchainFile(blockFileId, block.BlockchainFileName);
this.currentBlockchainFile = block.BlockchainFileName;
}
this.ReportProgressReport(block.BlockchainFileName, block.PercentageOfCurrentBlockchainFile);
// We instantiate databaseIdSegmentManager on the main thread and by doing this we'll guarantee that
// the database primary keys are generated in a certain order. The primary keys in our tables will be
// in the same order as the corresponding entities appear in the blockchain. For example, with the
// current implementation, the block ID will be the block depth as reported by http://blockchain.info/.
DatabaseIdSegmentManager databaseIdSegmentManager = new DatabaseIdSegmentManager(databaseIdManager, 1, block.Transactions.Count, block.TransactionInputsCount, block.TransactionOutputsCount);
this.processingStatistics.AddBlocksCount(1);
this.processingStatistics.AddTransactionsCount(block.Transactions.Count);
this.processingStatistics.AddTransactionInputsCount(block.TransactionInputsCount);
this.processingStatistics.AddTransactionOutputsCount(block.TransactionOutputsCount);
int blockFileId2 = blockFileId;
ParserData.Block block2 = block;
// Dispatch the work of "filling the source pipeline" to an available background thread.
// Note: The await awaits only until the work is dispatched and not until the work is completed.
// Dispatching the work itself may take a while if all available background threads are busy.
await taskDispatcher.DispatchWorkAsync(() => sourceDataPipeline.FillBlockchainPipeline(blockFileId2, block2, databaseIdSegmentManager));
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
}
}
finally
{
// Whatever we have in the pipeline we'll push to the DB. We do this in a finally block.
// Otherwise an exception that occurs in blockchain file 100 may prevent data that was
// collected in blockchain file 99 to be saved to DB.
// Wait for the last remaining background tasks if any that are still executing
// sourceDataPipeline.FillBlockchainPipeline or the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
// Instruct sourceDataPipeline to transfer all remaining data to the available data queue.
// IMPORTANT: do not call this while there could still be threads executing sourceDataPipeline.FillBlockchainPipeline.
sourceDataPipeline.Flush();
// Now trigger the SQL bulk copy for the data that remains.
await this.TransferAvailableData(taskDispatcher, sourceDataPipeline);
// Wait for the last remaining background tasks if any that are still executing
// the SQL bulk copy to finish.
await taskDispatcher.WaitForAllWorkToComplete();
}
this.FinalizeBlockchainFileProcessing(currentBlockchainFileStopwatch);
}
public void FillBlockchainPipeline(int currentBlockchainFileId, ParserData.Block parserBlock, DatabaseIdSegmentManager databaseIdSegmentManager)
{
long blockId = databaseIdSegmentManager.GetNextBlockId();
lock (this.blockLockObject)
{
this.blockDataSet.Block.AddBlockRow(
blockId,
currentBlockchainFileId,
(int)parserBlock.BlockHeader.BlockVersion,
parserBlock.BlockHeader.BlockHash.ToArray(),
parserBlock.BlockHeader.PreviousBlockHash.ToArray(),
parserBlock.TransactionsCount,
parserBlock.BlockHeader.MerkleRootHash.ToArray(),
parserBlock.BlockHeader.BlockTimestamp);
if (this.MakeDataTableAvailableIfLarge(this.blockDataSet.Block))
{
this.blockDataSet = new BlockDataSet();
}
}
lock (this.bitcoinTransactionLockObject)
{
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
this.bitcoinTransactionDataSetBuffer.BitcoinTransaction.AddBitcoinTransactionRow(
bitcoinTransactionId,
blockId,
parserTransaction.TransactionHash.ToArray(),
parserTransaction.TransactionInputsCount,
parserTransaction.TransactionOutputsCount,
(int)parserTransaction.TransactionVersion,
(int)parserTransaction.TransactionLockTime,
parserTransaction.IsSegWit);
}
if (this.MakeDataTableAvailableIfLarge(this.bitcoinTransactionDataSetBuffer.BitcoinTransaction))
{
this.bitcoinTransactionDataSetBuffer = new BitcoinTransactionDataSet();
}
}
lock (this.transactionInputLockObject)
{
databaseIdSegmentManager.ResetNextTransactionId();
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
foreach (ParserData.TransactionInput parserTransactionInput in parserTransaction.Inputs)
{
long transactionInput = databaseIdSegmentManager.GetNextTransactionInputId();
this.transactionInputDataSetBuffer.TransactionInput.AddTransactionInputRow(
transactionInput,
bitcoinTransactionId,
parserTransactionInput.SourceTransactionHash.ToArray(),
(int)parserTransactionInput.SourceTransactionOutputIndex,
DBData.TransactionInput.SourceTransactionOutputIdUnknown);
}
if (this.MakeDataTableAvailableIfLarge(this.transactionInputDataSetBuffer.TransactionInput))
{
this.transactionInputDataSetBuffer = new TransactionInputDataSet();
}
}
}
lock (this.transactionOutputLockObject)
{
databaseIdSegmentManager.ResetNextTransactionId();
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
for (int outputIndex = 0; outputIndex < parserTransaction.Outputs.Count; outputIndex++)
{
ParserData.TransactionOutput parserTransactionOutput = parserTransaction.Outputs[outputIndex];
byte[] temp;
temp = parserTransactionOutput.OutputAddress.ToArray();
string str = string.Empty;
for (int i = 0; i < temp.Length; i++)
{
str += (char)temp[i];
}
long transactionOutputId = databaseIdSegmentManager.GetNextTransactionOutputId();
this.transactionOutputDataSetBuffer.TransactionOutput.AddTransactionOutputRow(
transactionOutputId,
bitcoinTransactionId,
outputIndex,
(decimal)parserTransactionOutput.OutputValueSatoshi / DatabaseGenerator.BtcToSatoshi,
parserTransactionOutput.OutputScript.ToArray(),
str,
DBData.TransactionOutput.OutputAddressIdUnknown);
}
}
if (this.MakeDataTableAvailableIfLarge(this.transactionOutputDataSetBuffer.TransactionOutput))
{
this.transactionOutputDataSetBuffer = new TransactionOutputDataSet();
}
}
}
public void FillBlockchainPipeline(int currentBlockchainFileId, ParserData.Block parserBlock, DatabaseIdSegmentManager databaseIdSegmentManager)
{
long blockId = databaseIdSegmentManager.GetNextBlockId();
lock (this.blockLockObject)
{
this.blockDataSet.Block.AddBlockRow(
blockId,
currentBlockchainFileId,
(int)parserBlock.BlockHeader.BlockVersion,
parserBlock.BlockHeader.BlockHash.ToArray(),
parserBlock.BlockHeader.PreviousBlockHash.ToArray(),
parserBlock.BlockHeader.BlockTimestamp);
if (this.MakeDataTableAvailableIfLarge(this.blockDataSet.Block))
{
this.blockDataSet = new BlockDataSet();
}
}
lock (this.bitcoinTransactionLockObject)
{
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
this.bitcoinTransactionDataSetBuffer.BitcoinTransaction.AddBitcoinTransactionRow(
bitcoinTransactionId,
blockId,
parserTransaction.TransactionHash.ToArray(),
(int)parserTransaction.TransactionVersion,
(int)parserTransaction.TransactionLockTime);
}
if (this.MakeDataTableAvailableIfLarge(this.bitcoinTransactionDataSetBuffer.BitcoinTransaction))
{
this.bitcoinTransactionDataSetBuffer = new BitcoinTransactionDataSet();
}
}
lock (this.transactionInputLockObject)
{
databaseIdSegmentManager.ResetNextTransactionId();
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
long inputScriptId = 0;
foreach (ParserData.TransactionInput parserTransactionInput in parserTransaction.Inputs)
{
long transactionInput = databaseIdSegmentManager.GetNextTransactionInputId();
this.transactionInputDataSetBuffer.TransactionInput.AddTransactionInputRow(
transactionInput,
inputScriptId++,
bitcoinTransactionId,
DBData.TransactionInput.SourceTransactionOutputIdUnknown);
this.transactionInputSourceDataSetBuffer.TransactionInputSource.AddTransactionInputSourceRow(
transactionInput,
parserTransactionInput.SourceTransactionHash.ToArray(),
(int)parserTransactionInput.SourceTransactionOutputIndex,
parserTransactionInput.InputScript.ToArray()
);
}
if (this.MakeDataTableAvailableIfLarge(this.transactionInputDataSetBuffer.TransactionInput))
{
this.transactionInputDataSetBuffer = new TransactionInputDataSet();
}
if (this.MakeDataTableAvailableIfLarge(this.transactionInputSourceDataSetBuffer.TransactionInputSource))
{
this.transactionInputSourceDataSetBuffer = new TransactionInputSourceDataSet();
}
}
}
lock (this.transactionOutputLockObject)
{
databaseIdSegmentManager.ResetNextTransactionId();
foreach (ParserData.Transaction parserTransaction in parserBlock.Transactions)
{
long bitcoinTransactionId = databaseIdSegmentManager.GetNextTransactionId();
for (int outputIndex = 0; outputIndex < parserTransaction.Outputs.Count; outputIndex++)
{
ParserData.TransactionOutput parserTransactionOutput = parserTransaction.Outputs[outputIndex];
long transactionOutputId = databaseIdSegmentManager.GetNextTransactionOutputId();
this.transactionOutputDataSetBuffer.TransactionOutput.AddTransactionOutputRow(
transactionOutputId,
bitcoinTransactionId,
outputIndex,
(decimal)parserTransactionOutput.OutputValueSatoshi / DatabaseGenerator.BtcToSatoshi,
parserTransactionOutput.OutputScript.ToArray());
}
}
if (this.MakeDataTableAvailableIfLarge(this.transactionOutputDataSetBuffer.TransactionOutput))
{
this.transactionOutputDataSetBuffer = new TransactionOutputDataSet();
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="BlockProcessedEventArgs" /> class.
/// </summary>
/// <param name="block">
/// Contains data describing the block that was processed.
/// </param>
public BlockProcessedEventArgs(Block block)
{
this.Block = block;
}
/// <summary>
/// Parses one Bitcoin block except for a few fields before the actual block header.
/// </summary>
/// <param name="blockchainFileName">
/// The name of the blockchain file that contains the block being parsed.
/// </param>
/// <param name="blockMemoryStreamReader">
/// Provides access to a section of the Bitcoin blockchain file.
/// </param>
private static Block InternalParseBlockchainFile(string blockchainFileName, BlockMemoryStreamReader blockMemoryStreamReader)
{
BlockHeader blockHeader = BlockchainParser.ParseBlockHeader(blockMemoryStreamReader);
Block block = new Block(blockchainFileName, blockHeader);
int blockTransactionCount = (int)blockMemoryStreamReader.ReadVariableLengthInteger();
for (int transactionIndex = 0; transactionIndex < blockTransactionCount; transactionIndex++)
{
Transaction transaction = BlockchainParser.ParseTransaction(blockMemoryStreamReader);
block.AddTransaction(transaction);
}
return block;
}
