public async Task AnEventAndItsTransaction()
{
const string EVENT_INDEX_NAME = "transfer-logs-related";
const string FUNCTION_INDEX_NAME = "related-transfer-functions";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
var transferEventIndex = await azureSearchService.CreateIndexForEventLogAsync <TransferEvent_ERC20>(EVENT_INDEX_NAME);
var transferFunctionIndex = await azureSearchService.CreateIndexForFunctionMessageAsync <TransferFunction>(FUNCTION_INDEX_NAME);
var transferIndexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20>(transferEventIndex.Name);
var transferFunctionIndexer = azureSearchService.CreateIndexerForFunctionMessage <TransferFunction>(transferFunctionIndex.Name);
//this handler ensures the transaction is a Transfer and invokes the indexer
var transferFunctionProcessorHandler = transferFunctionIndexer.CreateProcessorHandler();
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>(async(log) => {
await transferIndexer.IndexAsync(log);
var transactionWithReceipt = await web3.Eth.GetTransactionReceiptVO(log.Log.BlockNumber, log.Log.TransactionHash).ConfigureAwait(false);
await transferFunctionProcessorHandler.ExecuteAsync(transactionWithReceipt);
});
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(19, await azureSearchService.CountDocumentsAsync(EVENT_INDEX_NAME));
Assert.Equal(3, await azureSearchService.CountDocumentsAsync(FUNCTION_INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(EVENT_INDEX_NAME);
await azureSearchService.DeleteIndexAsync(FUNCTION_INDEX_NAME);
}
}
}
public async Task OneEvent()
{
const string INDEX_NAME = "transfer-logs";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
var index = await azureSearchService.CreateIndexForEventLogAsync <TransferEvent_ERC20>(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20>(index.Name, documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>((transfer) => indexer.IndexAsync(transfer));
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(19, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task FilterLogsWithCriteria()
{
const string INDEX_NAME = "filter-logs-with-criteria";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
// create an index - if an existing index is required: azureSearchService.GetIndexAsync()
var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForLog(
index.Name,
documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor(
action: log => indexer.IndexAsync(log),
criteria: log => AddressUtil.Current.AreAddressesTheSame(log.Address, "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831"));
var cancellationTokenSource = new CancellationTokenSource();
await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684);
await Task.Delay(5000); // allow time to index
Assert.Equal(2, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task FilterLogs()
{
const string INDEX_NAME = "filter-logs";
//surround with "using" so that anything in a buffer is sent on dispose
//to clear the buffer manually - searchIndexProcessor.EventIndexer.SubmitPendingItemsAsync()
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
// create an index - if an existing index is required: azureSearchService.GetIndexAsync()
var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForLog(index.Name, documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor(log => indexer.IndexAsync(log));
var cancellationTokenSource = new CancellationTokenSource();
await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684);
await Task.Delay(5000); // allow time to index
Assert.Equal(25, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task OneEventWithMapping()
{
const string INDEX_NAME = "one-event-with-mapping";
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
//create a definition of an index -
//it describes the fields, key and data types
//at this stage - it is only an in-memory schema
var index = CreateAzureIndexDefinition(INDEX_NAME);
//create the index in azure
index = await azureSearchService.CreateIndexAsync(index);
// create a processor for a specific event and map to a custom DTO for the search
var indexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20, CustomTransferSearchDocumentDto>(
index.Name,
(e) =>
new CustomTransferSearchDocumentDto
{
From = e.Event.From,
To = e.Event.To,
Value = e.Event.Value.ToString(),
BlockNumber = e.Log.BlockNumber.Value.ToString(),
TxHash = e.Log.TransactionHash,
LogAddress = e.Log.Address,
LogIndex = (int)e.Log.LogIndex.Value,
DocumentKey = $"{e.Log.TransactionHash}_{e.Log.LogIndex.Value}"
},
documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>(
transfer => indexer.IndexAsync(transfer));
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(19, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task PendingItemsBuffer()
{
const string INDEX_NAME = "filter-logs-clearing-buffer";
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
// create an index - if an existing index is required: azureSearchService.GetIndexAsync()
var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForLog(INDEX_NAME, documentsPerBatch: 10);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor(log => indexer.IndexAsync(log));
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684);
//as the indexer processes in batches and we've dictated a size of 10 items per batch
//we should have a buffer of items pending submission
//these are processed on disposal - but we can force this process manually
Assert.Equal(5, indexer.PendingDocumentCount);
Assert.Equal(20, indexer.Indexed);
//process the pending items
await indexer.IndexPendingDocumentsAsync();
//the buffer should be clear now
Assert.Equal(0, indexer.PendingDocumentCount);
Assert.Equal(25, indexer.Indexed);
await Task.Delay(5000); // allow time for Azure to index
Assert.Equal(25, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task WritingEventsToASearchIndex()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);
//Requires: Nethereum.BlockchainStore.Search
// Load config
// - this will contain the secrets and connection strings we don't want to hard code
var config = TestConfiguration.LoadConfig();
string ApiKeyName = "AzureSearchApiKey";
string AzureSearchServiceName = "blockchainsearch";
var apiKey = config[ApiKeyName];
//cancellation token to enable the listener to be stopped
//passing in a time limit as a safety valve for the unit test
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1));
//initialise the processor
//within "using" block so that the processor cleans up the search resources it creates
using (var processor = web3.Eth.LogsProcessor <TransferEventDto>()
.SetBlocksPerBatch(1) //optional: restrict batches to one block at a time
.SetMinimumBlockNumber(7540103) //optional: default is to start at current block on chain
.OnBatchProcessed((args) => cancellationTokenSource.Cancel())
.AddToSearchIndexAsync <TransferEventDto>(AzureSearchServiceName, apiKey, "sep-transfers")
.Result
.Build())
{
//run the processor for a while
var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token);
}
await Task.Delay(5000); //give azure time to update
using (var searchService = new AzureSearchService(AzureSearchServiceName, apiKey))
{
Assert.Equal((long)13, await searchService.CountDocumentsAsync("sep-transfers"));
await searchService.DeleteIndexAsync("sep-transfers");
}
}
public async Task IndexingTransferFunctions()
{
const string INDEX_NAME = "transfer-functions";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
var index = await azureSearchService.CreateIndexForFunctionMessageAsync <TransferFunction>(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForFunctionMessage <TransferFunction>(index.Name, documentsPerBatch: 1);
var processorHandler = indexer.CreateProcessorHandler();
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Blocks.CreateBlockProcessor((steps) => {
steps.TransactionStep.SetMatchCriteria(tx => tx.Transaction.IsFrom("0xf47a8bb5c9ff814d39509591281ae31c0c7c2f38"));
steps.TransactionReceiptStep.AddProcessorHandler(processorHandler);
});
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146689, cancellationTokenSource.Token, 3146689);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(1, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}