public async Task IndexingTransferFunctions()
{
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
using (var azureFunctionMessageIndexer =
await azureSearchService.GetOrCreateFunctionIndex <TransferFunction>(indexName: AzureTransferIndexName))
{
var transferHandler =
new FunctionIndexTransactionHandler <TransferFunction>(azureFunctionMessageIndexer);
var web3 = new Web3.Web3("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60");
var blockchainProxy = new BlockchainProxyService(web3);
var handlers = new HandlerContainer {
TransactionHandler = transferHandler
};
var blockProcessor = BlockProcessorFactory.Create(blockchainProxy, handlers);
var processingStrategy = new ProcessingStrategy(blockProcessor);
var blockchainProcessor = new BlockchainProcessor(processingStrategy);
var blockRange = new BlockRange(3146684, 3146694);
await blockchainProcessor.ProcessAsync(blockRange);
await Task.Delay(TimeSpan.FromSeconds(5));
//ensure we have written the expected docs to the index
Assert.Equal(3, await azureFunctionMessageIndexer.DocumentCountAsync());
}
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
}
}
public async Task AnEventAndItsTransaction()
{
const string EVENT_INDEX_NAME = "transfer-logs-related";
const string FUNCTION_INDEX_NAME = "related-transfer-functions";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
var transferEventIndex = await azureSearchService.CreateIndexForEventLogAsync <TransferEvent_ERC20>(EVENT_INDEX_NAME);
var transferFunctionIndex = await azureSearchService.CreateIndexForFunctionMessageAsync <TransferFunction>(FUNCTION_INDEX_NAME);
var transferIndexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20>(transferEventIndex.Name);
var transferFunctionIndexer = azureSearchService.CreateIndexerForFunctionMessage <TransferFunction>(transferFunctionIndex.Name);
//this handler ensures the transaction is a Transfer and invokes the indexer
var transferFunctionProcessorHandler = transferFunctionIndexer.CreateProcessorHandler();
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>(async(log) => {
await transferIndexer.IndexAsync(log);
var transactionWithReceipt = await web3.Eth.GetTransactionReceiptVO(log.Log.BlockNumber, log.Log.TransactionHash).ConfigureAwait(false);
await transferFunctionProcessorHandler.ExecuteAsync(transactionWithReceipt);
});
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(19, await azureSearchService.CountDocumentsAsync(EVENT_INDEX_NAME));
Assert.Equal(3, await azureSearchService.CountDocumentsAsync(FUNCTION_INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(EVENT_INDEX_NAME);
await azureSearchService.DeleteIndexAsync(FUNCTION_INDEX_NAME);
}
}
}
public async Task UsingTheIndividualComponents()
{
TransferMetadata.CurrentChainUrl = BlockchainUrl;
var web3 =
new Web3.Web3(BlockchainUrl);
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
try
{
using (var transferIndexer =
await azureSearchService.CreateEventIndexer <TransferEvent_ERC20>(AzureTransferIndexName))
{
using (var transferProcessor =
new EventIndexProcessor <TransferEvent_ERC20>(transferIndexer))
{
var logProcessor = new BlockRangeLogsProcessor(
web3.Eth.Filters.GetLogs,
new ILogProcessor[] { transferProcessor });
var progressRepository =
new JsonBlockProgressRepository(CreateJsonFileToHoldProgress());
var progressService = new StaticBlockRangeProgressService(
3146684, 3146694, progressRepository);
var batchProcessorService = new LogsProcessor(
logProcessor, progressService, maxNumberOfBlocksPerBatch: 2);
BlockRange?lastBlockRangeProcessed;
do
{
lastBlockRangeProcessed = await batchProcessorService.ProcessOnceAsync();
} while (lastBlockRangeProcessed != null);
Assert.Equal(19, transferIndexer.Indexed);
}
}
}
finally
{
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
}
}
}
public async Task OneEvent()
{
const string INDEX_NAME = "transfer-logs";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
var index = await azureSearchService.CreateIndexForEventLogAsync <TransferEvent_ERC20>(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20>(index.Name, documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>((transfer) => indexer.IndexAsync(transfer));
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(19, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task FilterLogsWithCriteria()
{
const string INDEX_NAME = "filter-logs-with-criteria";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
// create an index - if an existing index is required: azureSearchService.GetIndexAsync()
var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForLog(
index.Name,
documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor(
action: log => indexer.IndexAsync(log),
criteria: log => AddressUtil.Current.AreAddressesTheSame(log.Address, "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831"));
var cancellationTokenSource = new CancellationTokenSource();
await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684);
await Task.Delay(5000); // allow time to index
Assert.Equal(2, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task FilterLogs()
{
const string INDEX_NAME = "filter-logs";
//surround with "using" so that anything in a buffer is sent on dispose
//to clear the buffer manually - searchIndexProcessor.EventIndexer.SubmitPendingItemsAsync()
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
// create an index - if an existing index is required: azureSearchService.GetIndexAsync()
var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForLog(index.Name, documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor(log => indexer.IndexAsync(log));
var cancellationTokenSource = new CancellationTokenSource();
await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684);
await Task.Delay(5000); // allow time to index
Assert.Equal(25, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
static async Task Main(string[] args)
{
var searchServiceName = ConfigurationManager.AppSettings.Get("searchServiceName");
var searchServiceKey = ConfigurationManager.AppSettings.Get("searchServiceKey");
var searchServiceClient = new AzureSearchService(searchServiceName, searchServiceKey);
//first we delete existing search structures if exist in order to create them later from zero
await searchServiceClient.DeleteDataSourceAsync("jsonaudiosdatasource");
await searchServiceClient.DeleteIndexerAsync("netcoreconf2020-indexer");
await searchServiceClient.DeleteSkillsetAsync("jsonskillsets");
await searchServiceClient.DeleteIndexAsync("netcoreconf2020index");
//then we create our azure search structures
var json = ReadJsonResource("jsonaudiosdatasource");
await searchServiceClient.CreateorUpdateDataSource("jsonaudiosdatasource", json);
json = ReadJsonResource("netcoreconf2020index");
await searchServiceClient.CreateorUpdateAzureIndex("netcoreconf2020index", json);
json = ReadJsonResource("jsonskillsets");
await searchServiceClient.CreateorUpdateSkillset("jsonskillsets", json);
json = ReadJsonResource("netcoreconf2020indexer");
await searchServiceClient.CreateorUpdateIndexer("netcoreconf2020indexer", json);
}
public async Task OneEventWithMapping()
{
const string INDEX_NAME = "one-event-with-mapping";
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
//create a definition of an index -
//it describes the fields, key and data types
//at this stage - it is only an in-memory schema
var index = CreateAzureIndexDefinition(INDEX_NAME);
//create the index in azure
index = await azureSearchService.CreateIndexAsync(index);
// create a processor for a specific event and map to a custom DTO for the search
var indexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20, CustomTransferSearchDocumentDto>(
index.Name,
(e) =>
new CustomTransferSearchDocumentDto
{
From = e.Event.From,
To = e.Event.To,
Value = e.Event.Value.ToString(),
BlockNumber = e.Log.BlockNumber.Value.ToString(),
TxHash = e.Log.TransactionHash,
LogAddress = e.Log.Address,
LogIndex = (int)e.Log.LogIndex.Value,
DocumentKey = $"{e.Log.TransactionHash}_{e.Log.LogIndex.Value}"
},
documentsPerBatch: 1);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>(
transfer => indexer.IndexAsync(transfer));
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(19, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task CustomComposition()
{
TransferMetadata.CurrentChainUrl = BlockchainUrl;
var blockchainProxyService =
new BlockchainProxyService(BlockchainUrl);
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
using (var transferIndexer = await azureSearchService.GetOrCreateEventIndex <TransferEvent_ERC20>(AzureTransferIndexName))
{
using (var transferProcessor =
new EventIndexProcessor <TransferEvent_ERC20>(transferIndexer))
{
var logProcessor = new BlockchainLogProcessor(
blockchainProxyService,
new ILogProcessor[] { transferProcessor });
var progressRepository =
new JsonBlockProgressRepository(CreateJsonFileToHoldProgress());
var progressService = new StaticBlockRangeProgressService(
3146684, 3146694, progressRepository);
var batchProcessorService = new BlockchainBatchProcessorService(
logProcessor, progressService, maxNumberOfBlocksPerBatch: 2);
BlockRange?lastBlockRangeProcessed;
do
{
lastBlockRangeProcessed = await batchProcessorService.ProcessLatestBlocksAsync();
} while (lastBlockRangeProcessed != null);
Assert.Equal(19, transferIndexer.Indexed);
}
}
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
}
}
public async Task IndexingTransferFunctions()
{
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
try
{
using (var azureFunctionMessageIndexer =
await azureSearchService.CreateFunctionIndexer <TransferFunction>(
indexName: AzureTransferIndexName))
{
var transferHandler =
new FunctionIndexTransactionHandler <TransferFunction>(azureFunctionMessageIndexer);
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);
var blockchainProxy = new BlockchainProxyService(web3);
var handlers = new HandlerContainer {
TransactionHandler = transferHandler
};
var blockProcessor = BlockProcessorFactory.Create(blockchainProxy, handlers);
var processingStrategy = new ProcessingStrategy(blockProcessor);
var blockchainProcessor = new BlockchainProcessor(processingStrategy);
var blockRange = new BlockRange(3146684, 3146694);
await blockchainProcessor.ProcessAsync(blockRange);
await Task.Delay(TimeSpan.FromSeconds(5));
//ensure we have written the expected docs to the index
Assert.Equal(3, await azureFunctionMessageIndexer.DocumentCountAsync());
}
}
finally
{
await azureSearchService.DeleteIndexAsync(AzureTransferIndexName);
}
}
}
public async Task PendingItemsBuffer()
{
const string INDEX_NAME = "filter-logs-clearing-buffer";
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
// create an index - if an existing index is required: azureSearchService.GetIndexAsync()
var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForLog(INDEX_NAME, documentsPerBatch: 10);
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Logs.CreateProcessor(log => indexer.IndexAsync(log));
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684);
//as the indexer processes in batches and we've dictated a size of 10 items per batch
//we should have a buffer of items pending submission
//these are processed on disposal - but we can force this process manually
Assert.Equal(5, indexer.PendingDocumentCount);
Assert.Equal(20, indexer.Indexed);
//process the pending items
await indexer.IndexPendingDocumentsAsync();
//the buffer should be clear now
Assert.Equal(0, indexer.PendingDocumentCount);
Assert.Equal(25, indexer.Indexed);
await Task.Delay(5000); // allow time for Azure to index
Assert.Equal(25, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task WritingEventsToASearchIndex()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);
//Requires: Nethereum.BlockchainStore.Search
// Load config
// - this will contain the secrets and connection strings we don't want to hard code
var config = TestConfiguration.LoadConfig();
string ApiKeyName = "AzureSearchApiKey";
string AzureSearchServiceName = "blockchainsearch";
var apiKey = config[ApiKeyName];
//cancellation token to enable the listener to be stopped
//passing in a time limit as a safety valve for the unit test
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1));
//initialise the processor
//within "using" block so that the processor cleans up the search resources it creates
using (var processor = web3.Eth.LogsProcessor <TransferEventDto>()
.SetBlocksPerBatch(1) //optional: restrict batches to one block at a time
.SetMinimumBlockNumber(7540103) //optional: default is to start at current block on chain
.OnBatchProcessed((args) => cancellationTokenSource.Cancel())
.AddToSearchIndexAsync <TransferEventDto>(AzureSearchServiceName, apiKey, "sep-transfers")
.Result
.Build())
{
//run the processor for a while
var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token);
}
await Task.Delay(5000); //give azure time to update
using (var searchService = new AzureSearchService(AzureSearchServiceName, apiKey))
{
Assert.Equal((long)13, await searchService.CountDocumentsAsync("sep-transfers"));
await searchService.DeleteIndexAsync("sep-transfers");
}
}
public async Task IndexingTransferFunctions()
{
const string INDEX_NAME = "transfer-functions";
//surround with "using" so that anything in a buffer is sent on dispose
using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey))
{
try
{
//setup
var index = await azureSearchService.CreateIndexForFunctionMessageAsync <TransferFunction>(INDEX_NAME);
var indexer = azureSearchService.CreateIndexerForFunctionMessage <TransferFunction>(index.Name, documentsPerBatch: 1);
var processorHandler = indexer.CreateProcessorHandler();
var web3 = new Web3.Web3(BlockchainUrl);
var blockchainProcessor = web3.Processing.Blocks.CreateBlockProcessor((steps) => {
steps.TransactionStep.SetMatchCriteria(tx => tx.Transaction.IsFrom("0xf47a8bb5c9ff814d39509591281ae31c0c7c2f38"));
steps.TransactionReceiptStep.AddProcessorHandler(processorHandler);
});
var cancellationTokenSource = new CancellationTokenSource();
//execute
await blockchainProcessor.ExecuteAsync(3146689, cancellationTokenSource.Token, 3146689);
//assert
await Task.Delay(5000); // allow time to index
Assert.Equal(1, await azureSearchService.CountDocumentsAsync(INDEX_NAME));
}
finally
{
await azureSearchService.DeleteIndexAsync(INDEX_NAME);
}
}
}
public async Task CreateIndex_Upsert_Suggest_Search()
{
ConfigurationUtils.SetEnvironment("development");
var appConfig = ConfigurationUtils
.Build(Array.Empty <string>(), userSecretsId: "Nethereum.BlockchainStore.Search.Tests");
var apiKey = appConfig[ApiKeyName];
var eventSearchIndexDefinition = new EventIndexDefinition <TransferEvent>();
using (var searchService = new AzureSearchService(AzureSearchServiceName, apiKey))
{
try
{
await searchService.DeleteIndexAsync(eventSearchIndexDefinition);
using (var azureIndex = await searchService.GetOrCreateIndex(eventSearchIndexDefinition))
{
var transferEventLog = new EventLog <TransferEvent>(
new TransferEvent
{
From = "0x9209b29f2094457d3dba62d1953efea58176ba27",
To = "0x1209b29f2094457d3dba62d1953efea58176ba28",
Value = new HexBigInteger("2000000"),
Detail = new TransferDetail
{
Description = "A generic transfer",
Tags = new List <Tag>
{
new Tag {
Name = "Status", Value = "Good"
},
new Tag {
Name = "Year", Value = "2019"
}
}
}
},
new FilterLog
{
Address = "0x26bc47888b7bfdf77db41ec0a2fb4db00af1c92a",
TransactionHash = "0xcb00b69d2594a3583309f332ada97d0df48bae00170e36a4f7bbdad7783fc7e5",
BlockNumber = new HexBigInteger(7118507),
BlockHash = "0x337cd6feedafac6abba40eff40fb1957e08985180f5a03016924ef72fc7b04b9",
LogIndex = new HexBigInteger(0),
Removed = false,
TransactionIndex = new HexBigInteger(0)
});
await azureIndex.IndexAsync(transferEventLog);
await Task.Delay(TimeSpan.FromSeconds(5));
var suggestion = await azureIndex.SuggestAsync(transferEventLog.Log.BlockNumber.Value.ToString());
Assert.NotNull(suggestion);
Assert.Equal(1, suggestion.Results.Count);
var searchResult = await azureIndex.SearchAsync(
transferEventLog.Event.From);
Assert.NotNull(searchResult);
Assert.Equal(1, searchResult.Results.Count);
}
}
finally
{
await searchService.DeleteIndexAsync(eventSearchIndexDefinition);
}
}
}