public async Task GetBlocks_Test()
{
var block = new BlockWithTransactions
{
Header = _osTestHelper.GenerateBlock(HashHelper.ComputeFrom("PreBlockHash"), 100).Header
};
var blockList = new BlockList();
blockList.Blocks.Add(block);
var mockClient = new Mock <PeerService.PeerServiceClient>();
mockClient.Setup(c =>
c.RequestBlocksAsync(It.IsAny <BlocksRequest>(), It.IsAny <Metadata>(), null,
CancellationToken.None))
.Returns(MockAsyncUnaryCall(blockList));
var grpcPeer = CreatePeer(mockClient.Object);
var result = await grpcPeer.GetBlocksAsync(block.Header.PreviousBlockHash, 1);
result.ShouldBe(blockList.Blocks);
var metrics = grpcPeer.GetRequestMetrics();
metrics["GetBlocks"].Count.ShouldBe(1);
metrics["GetBlocks"][0].MethodName.ShouldContain("GetBlocks");
metrics["GetBlocks"][0].Info.ShouldContain("Get blocks for");
}
public async Task BroadcastBlock_OnePeerKnowsBlock_Test()
{
var blockHeader = OsCoreTestHelper.CreateFakeBlockHeader(1, 2);
var blockWithTransactions = new BlockWithTransactions {
Header = blockHeader
};
_peerPool.FindPeerByPublicKey("Pubkey0").TryAddKnownBlock(blockWithTransactions.GetHash());
await _networkService.BroadcastBlockWithTransactionsAsync(blockWithTransactions);
foreach (var peer in _testContext.MockedPeers)
{
peer.Verify(p => p.TryAddKnownBlock(blockHeader.GetHash()), Times.Once());
}
_testContext.MockedPeers[0].Verify(
p => p.EnqueueBlock(blockWithTransactions, It.IsAny <Action <NetworkException> >()),
Times.Never);
_testContext.MockedPeers[1].Verify(
p => p.EnqueueBlock(blockWithTransactions, It.IsAny <Action <NetworkException> >()),
Times.Once());
_testContext.MockedPeers[2].Verify(
p => p.EnqueueBlock(blockWithTransactions, It.IsAny <Action <NetworkException> >()),
Times.Once());
}
private long InsertOnlyBlock(BlockWithTransactions block, long firstTransactionId)
{
var timestamp = block.Timestamp.ToLong();
var cmd = _insertBlock;
cmd.Reset();
cmd.Parameters.Clear();
cmd.Parameters.Add(new SQLiteParameter("hash", block.BlockHash));
cmd.Parameters.Add(new SQLiteParameter("previous_hash", block.ParentHash));
cmd.Parameters.Add(new SQLiteParameter("miner", _cache.Encode(block.Miner)));
cmd.Parameters.Add(new SQLiteParameter("timestamp", timestamp));
cmd.Parameters.Add(new SQLiteParameter("first_transaction_id", firstTransactionId));
cmd.Parameters.Add(new SQLiteParameter("transaction_count", block.Transactions.LongLength));
cmd.Parameters.Add(new SQLiteParameter("average_gasprice", null));
cmd.ExecuteNonQuery();
_timestampIndex.AddBlock(new TimestampIndex.TimestampItem
{
Timestamp = timestamp,
TxBegin = firstTransactionId,
TxEnd = firstTransactionId + block.Transactions.LongLength,
});
return(_conn.LastInsertRowId);
}
public Task BroadcastBlockWithTransactionsAsync(BlockWithTransactions blockWithTransactions)
{
if (!TryAddKnownBlock(blockWithTransactions.Header))
{
return(Task.CompletedTask);
}
if (IsOldBlock(blockWithTransactions.Header))
{
return(Task.CompletedTask);
}
_taskQueueManager.Enqueue(async() =>
{
foreach (var peer in _peerPool.GetPeers())
{
try
{
await peer.SendBlockAsync(blockWithTransactions);
}
catch (NetworkException ex)
{
Logger.LogError(ex, $"Error while broadcasting block to {peer}.");
await HandleNetworkException(peer, ex);
}
}
}, NetworkConstants.BlockBroadcastQueueName);
return(Task.CompletedTask);
}
private async Task <bool> ValidateTransactionAsync(BlockWithTransactions blockWithTransactions)
{
foreach (var transaction in blockWithTransactions.Transactions)
{
if (!transaction.VerifyExpiration(blockWithTransactions.Height - 1))
{
Logger.LogWarning($"Transaction {transaction.GetHash()} expired.");
return(false);
}
// No need to validate again if this tx already in local database.
if (await _blockchainService.HasTransactionAsync(transaction.GetHash()))
{
continue;
}
if (!await _transactionValidationService.ValidateTransactionAsync(transaction))
{
return(false);
}
var constrainedTransactionValidationResult =
_transactionValidationService.ValidateConstrainedTransaction(transaction,
blockWithTransactions.GetHash());
_transactionValidationService.ClearConstrainedTransactionValidationProvider(blockWithTransactions
.GetHash());
if (!constrainedTransactionValidationResult)
{
Logger.LogWarning($"Transaction {transaction} validation failed for constraint.");
return(false);
}
}
return(true);
}
public Block(HexBigInteger blockNumber, Web3 web3)
{
BlockWithTransactions blockWithTransactions = web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(blockNumber).Result;
while (blockWithTransactions == null)
{
blockWithTransactions = web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(blockNumber).Result;
}
var resultTask = Task.Run(() =>
{
return(blockWithTransactions.Transactions.
AsParallel().
Select(txHash => new Result(web3, txHash)).
AsEnumerable().
OrderBy(tx => tx.TransactionIndex).
ToList());
});
Number = blockWithTransactions.Number;
Timestamp = ConvertUNIXTimestampToLocalDateTime(blockWithTransactions.Timestamp.HexValue.ToString());
GasUsedPercent = Math.Round((decimal)blockWithTransactions.GasUsed.Value / (decimal)blockWithTransactions.GasLimit.Value * 100, 3);
Results = resultTask.Result;
}
private async Task <bool> ValidateTransactionAsync(BlockWithTransactions blockWithTransactions)
{
foreach (var transaction in blockWithTransactions.Transactions)
{
// No need to validate again if this tx already in local database.
var tx = await _transactionManager.GetTransactionAsync(transaction.GetHash());
if (tx != null)
{
continue;
}
if (!await _transactionValidationService.ValidateTransactionAsync(transaction))
{
return(false);
}
var constrainedTransactionValidationResult =
_transactionValidationService.ValidateConstrainedTransaction(transaction,
blockWithTransactions.GetHash());
_transactionValidationService.ClearConstrainedTransactionValidationProvider(blockWithTransactions
.GetHash());
if (!constrainedTransactionValidationResult)
{
return(false);
}
}
return(true);
}
public async Task AttachBlockWithTransactions_Test()
{
var chain = await _blockchainService.GetChainAsync();
var transactions = await _osTestHelper.GenerateTransferTransactions(2);
var block = _osTestHelper.GenerateBlock(chain.BestChainHash, chain.BestChainHeight, transactions);
var executedBlock = (await _blockExecutingService.ExecuteBlockAsync(block.Header, transactions)).Block;
var blockWithTransactions = new BlockWithTransactions
{
Header = executedBlock.Header, Transactions = { transactions }
};
var attachFinished = false;
await _blockSyncAttachService.AttachBlockWithTransactionsAsync(blockWithTransactions, "pubkey",
() =>
{
attachFinished = true;
return(Task.CompletedTask);
});
chain = await _blockchainService.GetChainAsync();
chain.BestChainHash.ShouldBe(blockWithTransactions.GetHash());
chain.BestChainHeight.ShouldBe(blockWithTransactions.Height);
attachFinished.ShouldBeTrue();
var txs = await _blockchainService.GetTransactionsAsync(transactions.Select(t => t.GetHash()));
txs.Count.ShouldBe(2);
}
public async Task BroadcastBlockWithTransactionsAsync(BlockWithTransactions blockWithTransactions)
{
if (IsOldBlock(blockWithTransactions.Header))
{
return;
}
var nextMinerPubkey = await GetNextMinerPubkey(blockWithTransactions.Header);
var nextPeer = _peerPool.FindPeerByPublicKey(nextMinerPubkey);
if (nextPeer != null)
{
EnqueueBlock(nextPeer, blockWithTransactions);
}
foreach (var peer in _peerPool.GetPeers())
{
if (nextPeer != null && peer.Info.Pubkey == nextPeer.Info.Pubkey)
{
continue;
}
EnqueueBlock(peer, blockWithTransactions);
}
}
private async Task <SaveResult> SaveTransactions(AkromaContext context, BlockWithTransactions unsavedBlock)
{
if (!unsavedBlock.Transaction.Any())
{
return(SaveResult.Success());
}
foreach (var transaction in unsavedBlock.Transaction)
{
var aka = UnitConversion.Convert.FromWeiToBigDecimal(transaction.Value);
var toSave = new TransactionEntity
{
Hash = transaction.Hash,
Nonce = transaction.Nonce.HexValue,
BlockHash = transaction.BlockHash,
BlockNumber = int.Parse(transaction.BlockNumber.Value.ToString()),
TransactionIndex = int.Parse(transaction.TransactionIndex.Value.ToString()),
From = transaction.From,
To = transaction.To,
Value = decimal.Parse(aka.ToString()),
Gas = transaction.Gas.HexValue,
GasPrice = transaction.GasPrice.HexValue,
Timestamp = long.Parse(unsavedBlock.Timestamp.Value.ToString()),
Input = transaction.Input
};
context.Transactions.Add(toSave);
await context.SaveChangesAsync();
}
return(SaveResult.Success());
}
private void EnqueueBlock(IPeer peer, BlockWithTransactions blockWithTransactions)
{
try
{
var blockHash = blockWithTransactions.GetHash();
if (peer.KnowsBlock(blockHash))
{
return; // block already known to this peer
}
peer.EnqueueBlock(blockWithTransactions, async ex =>
{
peer.TryAddKnownBlock(blockHash);
if (ex != null)
{
Logger.LogError(ex, $"Error while broadcasting block to {peer}.");
await HandleNetworkException(peer, ex);
}
});
}
catch (NetworkException ex)
{
Logger.LogError(ex, $"Error while broadcasting block to {peer}.");
}
}
public async Task BlockWithTransactions_Test()
{
var chain = await _blockchainService.GetChainAsync();
var transactions = new List <Transaction>();
for (var i = 0; i < 5; i++)
{
transactions.Add(_kernelTestHelper.GenerateTransaction());
}
var block = _kernelTestHelper.GenerateBlock(chain.BestChainHeight, chain.BestChainHash, transactions);
var blockWithTransactions = new BlockWithTransactions
{
Header = block.Header,
Transactions = { transactions }
};
blockWithTransactions.FullTransactionList.ShouldBe(transactions);
blockWithTransactions.TransactionIds.ShouldBe(transactions.Select(o => o.GetHash()));
blockWithTransactions.Height.ShouldBe(block.Height);
blockWithTransactions.Body.ShouldBe(new BlockBody
{
TransactionIds = { transactions.Select(tx => tx.GetHash()).ToList() }
});
}
private void EnqueueBlock(IPeer peer, BlockWithTransactions blockWithTransactions)
{
try
{
var blockHash = blockWithTransactions.GetHash();
if (peer.KnowsBlock(blockHash))
{
return; // block already known to this peer
}
peer.EnqueueBlock(blockWithTransactions, async ex =>
{
peer.TryAddKnownBlock(blockHash);
if (ex != null)
{
Logger.LogWarning(ex, $"Could not broadcast block to {peer} - status {peer.ConnectionStatus}.");
await HandleNetworkException(peer, ex);
}
});
}
catch (NetworkException ex)
{
Logger.LogWarning(ex, $"Could not enqueue block to {peer} - status {peer.ConnectionStatus}.");
}
}
public async Task AttachBlockWithTransactionsAsync(BlockWithTransactions blockWithTransactions,
Func <Task> attachFinishedCallback = null)
{
var valid = await _validationService.ValidateBlockBeforeAttachAsync(blockWithTransactions);
if (!valid)
{
throw new InvalidOperationException(
$"The block was invalid, block hash: {blockWithTransactions}.");
}
await _blockchainService.AddTransactionsAsync(blockWithTransactions.Transactions);
var block = blockWithTransactions.ToBlock();
await _blockchainService.AddBlockAsync(block);
_blockSyncQueueService.Enqueue(async() =>
{
try
{
await _blockAttachService.AttachBlockAsync(block);
}
finally
{
if (attachFinishedCallback != null)
{
await attachFinishedCallback();
}
}
},
KernelConstants.UpdateChainQueueName);
}
public async Task BroadcastBlock_ThrowNetworkException_Test()
{
_testContext.MockedPeers[0]
.Setup(p => p.EnqueueBlock(It.IsAny <BlockWithTransactions>(), It.IsAny <Action <NetworkException> >()))
.Callback <BlockWithTransactions, Action <NetworkException> >((block, action) =>
action.Invoke(new NetworkException()));
_testContext.MockedPeers[1]
.Setup(p => p.EnqueueBlock(It.IsAny <BlockWithTransactions>(), It.IsAny <Action <NetworkException> >()))
.Callback <BlockWithTransactions, Action <NetworkException> >((block, action) =>
action.Invoke(new NetworkException("Unrecoverable", NetworkExceptionType.Unrecoverable)));
_testContext.MockedPeers[2]
.Setup(p => p.EnqueueBlock(It.IsAny <BlockWithTransactions>(), It.IsAny <Action <NetworkException> >()))
.Callback <BlockWithTransactions, Action <NetworkException> >((block, action) =>
action.Invoke(new NetworkException("PeerUnstable", NetworkExceptionType.PeerUnstable)));
var blockHeader = OsCoreTestHelper.CreateFakeBlockHeader(chainId: 1, height: 2);
var blockWithTx = new BlockWithTransactions {
Header = blockHeader
};
await _networkService.BroadcastBlockWithTransactionsAsync(blockWithTx);
foreach (var peer in _testContext.MockedPeers)
{
peer.Verify(p => p.EnqueueBlock(blockWithTx, It.IsAny <Action <NetworkException> >()), Times.Once());
peer.Verify(p => p.TryAddKnownBlock(blockHeader.GetHash()), Times.Once());
}
_testContext.MockAElfNetworkServer.Verify(s => s.TrySchedulePeerReconnectionAsync(It.IsAny <IPeer>()),
Times.Once());
}
public override void ConfigureServices(ServiceConfigurationContext context)
{
context.Services.AddSingleton(o =>
{
var networkServiceMock = new Mock <INetworkService>();
networkServiceMock
.Setup(p => p.GetBlockByHashAsync(It.IsAny <Hash>(), It.IsAny <string>()))
.Returns <Hash, int>((hash, peer) =>
{
BlockWithTransactions result = null;
if (hash != Hash.Empty)
{
var blockchainService = context.Services.GetServiceLazy <IBlockchainService>().Value;
var chain = AsyncHelper.RunSync(() => blockchainService.GetChainAsync());
result = new BlockWithTransactions {
Header = _peerBlockList[chain.BestChainHash].Header
};
}
return(Task.FromResult(new Response <BlockWithTransactions>(result)));
});
networkServiceMock
.Setup(p => p.GetBlocksAsync(It.IsAny <Hash>(), It.IsAny <int>(),
It.IsAny <string>()))
.Returns <Hash, int, string>((previousBlockHash, count, peerPubkey) =>
{
var result = new List <BlockWithTransactions>();
var hash = previousBlockHash;
while (result.Count < count && _peerBlockList.TryGetValue(hash, out var block))
{
result.Add(new BlockWithTransactions {
Header = block.Header
});
hash = block.GetHash();
}
return(Task.FromResult(new Response <List <BlockWithTransactions> >(result)));
});
networkServiceMock.Setup(p => p.GetPeerByPubkey(It.IsAny <string>())).Returns <string>(
(peerPubkey) =>
{
if (peerPubkey == "RemovedPeer")
{
return(null);
}
return(new PeerInfo());
});
return(networkServiceMock.Object);
});
context.Services.AddSingleton <PeerConnectedEventHandler>();
}
private void EnqueueAttachBlockJob(BlockWithTransactions blockWithTransactions, string senderPubkey)
{
_blockSyncQueueService.Enqueue(async() =>
{
Logger.LogDebug($"Block sync: sync block, block: {blockWithTransactions}.");
await _blockSyncAttachService.AttachBlockWithTransactionsAsync(blockWithTransactions, senderPubkey);
}, OSConstants.BlockSyncAttachQueueName);
}
private void EnqueueSyncBlockJob(BlockWithTransactions blockWithTransactions)
{
_blockSyncQueueService.Enqueue(async() =>
{
Logger.LogTrace($"Block sync: sync block, block: {blockWithTransactions}.");
await _blockSyncAttachService.AttachBlockWithTransactionsAsync(blockWithTransactions);
}, OSConstants.BlockSyncAttachQueueName);
}
public async Task CrawlAndExec(IClientSessionHandle session, BigInteger execBlockNumber)
{
JArray logs = new JArray();
//先判断有没有要预先入库的
var contractNeedPending = contractNeedModel.Find(new BsonDocument("type", EnumExecuteType.PENDING), new BsonDocument(), 0, 1);
if (contractNeedPending.Count != 0 && contractNeedPending[0].type == EnumExecuteType.PENDING)
{
var _logs = await Helper.GetLogsFromAmber(contractNeedPending[0].contractHash);
logs.Merge(_logs);
BsonDocument filter = new BsonDocument("type", EnumExecuteType.PENDING);
filter["contractHash"] = contractNeedPending[0].contractHash;
await contractNeedModel.Update(session, filter, new ContractNeed()
{
type = EnumExecuteType.EXECUTE, contractHash = contractNeedPending[0].contractHash
});
}
//爬取链上数据
BlockWithTransactions blockWithTransactions = await web3Manager.Current.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new HexBigInteger(execBlockNumber));
Transaction[] transactions = blockWithTransactions.Transactions;
contractNeedModel.FindAllByContractHash();
List <Task> actions = transactions.Select <Transaction, Task>(async(tran) =>
{
try
{
TransactionReceipt transactionR = await web3Manager.Current.Eth.Transactions.GetTransactionReceipt.SendRequestAsync(tran.TransactionHash);
if (transactionR.Logs.Count != 0)
{
var _logs = transactionR.Logs.Where(l =>
{
string contractHash = (string)l["address"];
if (contractNeedModel.ContractHash_ALL.ContainsKey(contractHash))
{
return(true);
}
return(false);
}).ToList();
if (_logs.Count != 0)
{
logs.Merge(_logs);
}
}
//Console.WriteLine(string.Format("foreach 高度{0}中", execBlockNumber));
}
catch (Exception e)
{
Console.WriteLine(string.Format("高度{0}获取交易receipt报错了", execBlockNumber));
throw e;
}
}).ToList();
await Task.WhenAll(actions);
Console.WriteLine("处理到了这里");
ExecLogs(logs);
await Commit(session);
}
private BlockDto MapBlockToDto(BlockWithTransactions b)
{
return(new BlockDto
{
BlockHeight = (long)b.Number.Value,
BlockHash = b.BlockHash,
TimeStamp = b.Timestamp.ToString(),
TransactionCount = b.TransactionCount()
});
}
public async Task <bool> ValidateBlockAsync(Chain chain, BlockWithTransactions blockWithTransactions, string senderPubKey)
{
if (blockWithTransactions.Height <= chain.LastIrreversibleBlockHeight)
{
Logger.LogWarning($"Receive lower block {blockWithTransactions} ignore.");
return(false);
}
return(true);
}
public static Block ToBlock(this BlockWithTransactions block)
{
return(new Block
{
Header = block.Header,
Body = new BlockBody
{
TransactionIds = { block.Transactions.Select(tx => tx.GetHash()).ToList() }
}
});
}
public static BlockFoundEvent Create(BlockWithTransactions block)
{
return(new()
{
BlockNumber = block.Number.ToUlong(),
Difficulty = block.Difficulty.ToUlong(),
Timestamp = block.Timestamp.ToUlong(),
UncleCount = (ulong)block.Uncles.Length,
TransactionCount = (ulong)block.Transactions.Length
});
}
public void BlockWithTransactions_TransactionCount_Returns_Number_Of_Transactions()
{
var blockWithTransactions = new BlockWithTransactions {
Transactions = new []
{
new Transaction()
}
};
Assert.Equal(blockWithTransactions.Transactions.Length, blockWithTransactions.TransactionCount());
}
public void EnqueueBlock(BlockWithTransactions blockWithTransactions, Action <NetworkException> sendCallback)
{
if (!IsReady)
{
throw new NetworkException($"Dropping block, peer is not ready - {this}.",
NetworkExceptionType.NotConnected);
}
_sendBlockJobs.Post(new StreamJob {
BlockWithTransactions = blockWithTransactions, SendCallback = sendCallback
});
}
protected virtual async Task ProcessTransactions(BlockWithTransactions block)
{
foreach (var txn in block.Transactions)
{
if (!HasAlreadyBeenProcessed(txn))
{
await TransactionProcessor.ProcessTransactionAsync(block, txn);
MarkAsProcessed(txn);
}
}
}
private TransactionDto MapTransactionToDto(Transaction t, BlockWithTransactions b, TransactionReceipt r)
{
return(new TransactionDto
{
BlockHeight = (long)t.BlockNumber.Value,
From = t.From,
To = t.To,
Quantity = t.Value.Value.ToString(),
Status = r != null?r.Status.ToString() : "",
TimeStamp = b.Timestamp.ToString(),
TransactionHash = t.TransactionHash
});
}
public async Task <List <CustomEventLog> > GetFullEventLogs(ERC20Token contract, int lastBlockNumber, int blocknum = 1)
{
var cont = web3.Eth.GetContract(Constants.Strings.ABI.Abi, contract.Address);
var transEvent = cont.GetEvent("Transfer");
var logs = new List <EventLog <Transfer> >();
var events = new List <CustomEventLog>();
for (var i = blocknum; i <= lastBlockNumber; i += 100)
{
if ((i + 99) > lastBlockNumber)
{
break;
}
try
{
var filter = transEvent.CreateFilterInput(new BlockParameter((ulong)i), new BlockParameter((ulong)(i + 99)));
var log = transEvent.GetAllChanges <Transfer>(filter).Result;
logs.AddRange(log);
}
catch (Exception e)
{
i -= 100;
}
}
BlockWithTransactions block = null;
foreach (var log in logs)
{
if (block == null || log.Log.BlockNumber.Value != block.Number.Value)
{
block = web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(
new HexBigInteger(log.Log.BlockNumber)).Result;
}
events.Add(new CustomEventLog()
{
ERC20TokenId = contract.Id,
From = log.Event.AddressFrom,
To = log.Event.AddressTo,
AmountOfToken = Web3.Convert.FromWei(log.Event.Value, contract.DecimalPlaces),
WhenDateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0).AddSeconds(
(long)(block.Timestamp.Value)),
BlockNumber = (int)log.Log.BlockNumber.Value
});
}
return(events);
}
private void Initialize(Web3 web3, Task <BlockWithTransactions> requestBlockTask)
{
requestBlockTask.Wait();
_block = requestBlockTask.Result;
if (Configuration.Get().RecordBinaryBuffers)
{
var receiptTasks =
_block.Transactions.Select(
x => web3.Eth.Transactions.GetTransactionReceipt.SendRequestAsync(x.TransactionHash))
.ToArray();
Task.WaitAll(receiptTasks);
_receipts = receiptTasks.Select(x => x.Result).ToArray();
}
}
public async Task <bool> ValidateBlockBeforeAttachAsync(BlockWithTransactions blockWithTransactions)
{
if (!await _blockValidationService.ValidateBlockBeforeAttachAsync(blockWithTransactions))
{
return(false);
}
if (!await ValidateTransactionAsync(blockWithTransactions))
{
return(false);
}
return(true);
}
