public async Task IndexingTransferFunctions() { using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); using (var azureFunctionMessageIndexer = await azureSearchService.GetOrCreateFunctionIndex <TransferFunction>(indexName: AzureTransferIndexName)) { var transferHandler = new FunctionIndexTransactionHandler <TransferFunction>(azureFunctionMessageIndexer); var web3 = new Web3.Web3("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60"); var blockchainProxy = new BlockchainProxyService(web3); var handlers = new HandlerContainer { TransactionHandler = transferHandler }; var blockProcessor = BlockProcessorFactory.Create(blockchainProxy, handlers); var processingStrategy = new ProcessingStrategy(blockProcessor); var blockchainProcessor = new BlockchainProcessor(processingStrategy); var blockRange = new BlockRange(3146684, 3146694); await blockchainProcessor.ProcessAsync(blockRange); await Task.Delay(TimeSpan.FromSeconds(5)); //ensure we have written the expected docs to the index Assert.Equal(3, await azureFunctionMessageIndexer.DocumentCountAsync()); } await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); } }
public async Task AnEventAndItsTransaction() { const string EVENT_INDEX_NAME = "transfer-logs-related"; const string FUNCTION_INDEX_NAME = "related-transfer-functions"; //surround with "using" so that anything in a buffer is sent on dispose using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { try { //setup var transferEventIndex = await azureSearchService.CreateIndexForEventLogAsync <TransferEvent_ERC20>(EVENT_INDEX_NAME); var transferFunctionIndex = await azureSearchService.CreateIndexForFunctionMessageAsync <TransferFunction>(FUNCTION_INDEX_NAME); var transferIndexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20>(transferEventIndex.Name); var transferFunctionIndexer = azureSearchService.CreateIndexerForFunctionMessage <TransferFunction>(transferFunctionIndex.Name); //this handler ensures the transaction is a Transfer and invokes the indexer var transferFunctionProcessorHandler = transferFunctionIndexer.CreateProcessorHandler(); var web3 = new Web3.Web3(BlockchainUrl); var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>(async(log) => { await transferIndexer.IndexAsync(log); var transactionWithReceipt = await web3.Eth.GetTransactionReceiptVO(log.Log.BlockNumber, log.Log.TransactionHash).ConfigureAwait(false); await transferFunctionProcessorHandler.ExecuteAsync(transactionWithReceipt); }); var cancellationTokenSource = new CancellationTokenSource(); //execute await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684); //assert await Task.Delay(5000); // allow time to index Assert.Equal(19, await azureSearchService.CountDocumentsAsync(EVENT_INDEX_NAME)); Assert.Equal(3, await azureSearchService.CountDocumentsAsync(FUNCTION_INDEX_NAME)); } finally { await azureSearchService.DeleteIndexAsync(EVENT_INDEX_NAME); await azureSearchService.DeleteIndexAsync(FUNCTION_INDEX_NAME); } } }
public async Task UsingTheIndividualComponents() { TransferMetadata.CurrentChainUrl = BlockchainUrl; var web3 = new Web3.Web3(BlockchainUrl); using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); try { using (var transferIndexer = await azureSearchService.CreateEventIndexer <TransferEvent_ERC20>(AzureTransferIndexName)) { using (var transferProcessor = new EventIndexProcessor <TransferEvent_ERC20>(transferIndexer)) { var logProcessor = new BlockRangeLogsProcessor( web3.Eth.Filters.GetLogs, new ILogProcessor[] { transferProcessor }); var progressRepository = new JsonBlockProgressRepository(CreateJsonFileToHoldProgress()); var progressService = new StaticBlockRangeProgressService( 3146684, 3146694, progressRepository); var batchProcessorService = new LogsProcessor( logProcessor, progressService, maxNumberOfBlocksPerBatch: 2); BlockRange?lastBlockRangeProcessed; do { lastBlockRangeProcessed = await batchProcessorService.ProcessOnceAsync(); } while (lastBlockRangeProcessed != null); Assert.Equal(19, transferIndexer.Indexed); } } } finally { await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); } } }
public async Task OneEvent() { const string INDEX_NAME = "transfer-logs"; //surround with "using" so that anything in a buffer is sent on dispose using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { try { //setup var index = await azureSearchService.CreateIndexForEventLogAsync <TransferEvent_ERC20>(INDEX_NAME); var indexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20>(index.Name, documentsPerBatch: 1); var web3 = new Web3.Web3(BlockchainUrl); var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>((transfer) => indexer.IndexAsync(transfer)); var cancellationTokenSource = new CancellationTokenSource(); //execute await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684); //assert await Task.Delay(5000); // allow time to index Assert.Equal(19, await azureSearchService.CountDocumentsAsync(INDEX_NAME)); } finally { await azureSearchService.DeleteIndexAsync(INDEX_NAME); } } }
public async Task FilterLogsWithCriteria() { const string INDEX_NAME = "filter-logs-with-criteria"; //surround with "using" so that anything in a buffer is sent on dispose using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { try { // create an index - if an existing index is required: azureSearchService.GetIndexAsync() var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME); var indexer = azureSearchService.CreateIndexerForLog( index.Name, documentsPerBatch: 1); var web3 = new Web3.Web3(BlockchainUrl); var blockchainProcessor = web3.Processing.Logs.CreateProcessor( action: log => indexer.IndexAsync(log), criteria: log => AddressUtil.Current.AreAddressesTheSame(log.Address, "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831")); var cancellationTokenSource = new CancellationTokenSource(); await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684); await Task.Delay(5000); // allow time to index Assert.Equal(2, await azureSearchService.CountDocumentsAsync(INDEX_NAME)); } finally { await azureSearchService.DeleteIndexAsync(INDEX_NAME); } } }
public async Task FilterLogs() { const string INDEX_NAME = "filter-logs"; //surround with "using" so that anything in a buffer is sent on dispose //to clear the buffer manually - searchIndexProcessor.EventIndexer.SubmitPendingItemsAsync() using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { try { // create an index - if an existing index is required: azureSearchService.GetIndexAsync() var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME); var indexer = azureSearchService.CreateIndexerForLog(index.Name, documentsPerBatch: 1); var web3 = new Web3.Web3(BlockchainUrl); var blockchainProcessor = web3.Processing.Logs.CreateProcessor(log => indexer.IndexAsync(log)); var cancellationTokenSource = new CancellationTokenSource(); await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684); await Task.Delay(5000); // allow time to index Assert.Equal(25, await azureSearchService.CountDocumentsAsync(INDEX_NAME)); } finally { await azureSearchService.DeleteIndexAsync(INDEX_NAME); } } }
static async Task Main(string[] args) { var searchServiceName = ConfigurationManager.AppSettings.Get("searchServiceName"); var searchServiceKey = ConfigurationManager.AppSettings.Get("searchServiceKey"); var searchServiceClient = new AzureSearchService(searchServiceName, searchServiceKey); //first we delete existing search structures if exist in order to create them later from zero await searchServiceClient.DeleteDataSourceAsync("jsonaudiosdatasource"); await searchServiceClient.DeleteIndexerAsync("netcoreconf2020-indexer"); await searchServiceClient.DeleteSkillsetAsync("jsonskillsets"); await searchServiceClient.DeleteIndexAsync("netcoreconf2020index"); //then we create our azure search structures var json = ReadJsonResource("jsonaudiosdatasource"); await searchServiceClient.CreateorUpdateDataSource("jsonaudiosdatasource", json); json = ReadJsonResource("netcoreconf2020index"); await searchServiceClient.CreateorUpdateAzureIndex("netcoreconf2020index", json); json = ReadJsonResource("jsonskillsets"); await searchServiceClient.CreateorUpdateSkillset("jsonskillsets", json); json = ReadJsonResource("netcoreconf2020indexer"); await searchServiceClient.CreateorUpdateIndexer("netcoreconf2020indexer", json); }
public async Task OneEventWithMapping() { const string INDEX_NAME = "one-event-with-mapping"; using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { try { //setup //create a definition of an index - //it describes the fields, key and data types //at this stage - it is only an in-memory schema var index = CreateAzureIndexDefinition(INDEX_NAME); //create the index in azure index = await azureSearchService.CreateIndexAsync(index); // create a processor for a specific event and map to a custom DTO for the search var indexer = azureSearchService.CreateIndexerForEventLog <TransferEvent_ERC20, CustomTransferSearchDocumentDto>( index.Name, (e) => new CustomTransferSearchDocumentDto { From = e.Event.From, To = e.Event.To, Value = e.Event.Value.ToString(), BlockNumber = e.Log.BlockNumber.Value.ToString(), TxHash = e.Log.TransactionHash, LogAddress = e.Log.Address, LogIndex = (int)e.Log.LogIndex.Value, DocumentKey = $"{e.Log.TransactionHash}_{e.Log.LogIndex.Value}" }, documentsPerBatch: 1); var web3 = new Web3.Web3(BlockchainUrl); var blockchainProcessor = web3.Processing.Logs.CreateProcessor <TransferEvent_ERC20>( transfer => indexer.IndexAsync(transfer)); var cancellationTokenSource = new CancellationTokenSource(); //execute await blockchainProcessor.ExecuteAsync(3146694, cancellationTokenSource.Token, 3146684); //assert await Task.Delay(5000); // allow time to index Assert.Equal(19, await azureSearchService.CountDocumentsAsync(INDEX_NAME)); } finally { await azureSearchService.DeleteIndexAsync(INDEX_NAME); } } }
public async Task CustomComposition() { TransferMetadata.CurrentChainUrl = BlockchainUrl; var blockchainProxyService = new BlockchainProxyService(BlockchainUrl); using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); using (var transferIndexer = await azureSearchService.GetOrCreateEventIndex <TransferEvent_ERC20>(AzureTransferIndexName)) { using (var transferProcessor = new EventIndexProcessor <TransferEvent_ERC20>(transferIndexer)) { var logProcessor = new BlockchainLogProcessor( blockchainProxyService, new ILogProcessor[] { transferProcessor }); var progressRepository = new JsonBlockProgressRepository(CreateJsonFileToHoldProgress()); var progressService = new StaticBlockRangeProgressService( 3146684, 3146694, progressRepository); var batchProcessorService = new BlockchainBatchProcessorService( logProcessor, progressService, maxNumberOfBlocksPerBatch: 2); BlockRange?lastBlockRangeProcessed; do { lastBlockRangeProcessed = await batchProcessorService.ProcessLatestBlocksAsync(); } while (lastBlockRangeProcessed != null); Assert.Equal(19, transferIndexer.Indexed); } } await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); } }
public async Task IndexingTransferFunctions() { using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); try { using (var azureFunctionMessageIndexer = await azureSearchService.CreateFunctionIndexer <TransferFunction>( indexName: AzureTransferIndexName)) { var transferHandler = new FunctionIndexTransactionHandler <TransferFunction>(azureFunctionMessageIndexer); var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); var blockchainProxy = new BlockchainProxyService(web3); var handlers = new HandlerContainer { TransactionHandler = transferHandler }; var blockProcessor = BlockProcessorFactory.Create(blockchainProxy, handlers); var processingStrategy = new ProcessingStrategy(blockProcessor); var blockchainProcessor = new BlockchainProcessor(processingStrategy); var blockRange = new BlockRange(3146684, 3146694); await blockchainProcessor.ProcessAsync(blockRange); await Task.Delay(TimeSpan.FromSeconds(5)); //ensure we have written the expected docs to the index Assert.Equal(3, await azureFunctionMessageIndexer.DocumentCountAsync()); } } finally { await azureSearchService.DeleteIndexAsync(AzureTransferIndexName); } } }
public async Task PendingItemsBuffer() { const string INDEX_NAME = "filter-logs-clearing-buffer"; using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { try { // create an index - if an existing index is required: azureSearchService.GetIndexAsync() var index = await azureSearchService.CreateIndexForLogAsync(INDEX_NAME); var indexer = azureSearchService.CreateIndexerForLog(INDEX_NAME, documentsPerBatch: 10); var web3 = new Web3.Web3(BlockchainUrl); var blockchainProcessor = web3.Processing.Logs.CreateProcessor(log => indexer.IndexAsync(log)); var cancellationTokenSource = new CancellationTokenSource(); //execute await blockchainProcessor.ExecuteAsync(3146685, cancellationTokenSource.Token, 3146684); //as the indexer processes in batches and we've dictated a size of 10 items per batch //we should have a buffer of items pending submission //these are processed on disposal - but we can force this process manually Assert.Equal(5, indexer.PendingDocumentCount); Assert.Equal(20, indexer.Indexed); //process the pending items await indexer.IndexPendingDocumentsAsync(); //the buffer should be clear now Assert.Equal(0, indexer.PendingDocumentCount); Assert.Equal(25, indexer.Indexed); await Task.Delay(5000); // allow time for Azure to index Assert.Equal(25, await azureSearchService.CountDocumentsAsync(INDEX_NAME)); } finally { await azureSearchService.DeleteIndexAsync(INDEX_NAME); } } }
public async Task WritingEventsToASearchIndex() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); //Requires: Nethereum.BlockchainStore.Search // Load config // - this will contain the secrets and connection strings we don't want to hard code var config = TestConfiguration.LoadConfig(); string ApiKeyName = "AzureSearchApiKey"; string AzureSearchServiceName = "blockchainsearch"; var apiKey = config[ApiKeyName]; //cancellation token to enable the listener to be stopped //passing in a time limit as a safety valve for the unit test var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); //initialise the processor //within "using" block so that the processor cleans up the search resources it creates using (var processor = web3.Eth.LogsProcessor <TransferEventDto>() .SetBlocksPerBatch(1) //optional: restrict batches to one block at a time .SetMinimumBlockNumber(7540103) //optional: default is to start at current block on chain .OnBatchProcessed((args) => cancellationTokenSource.Cancel()) .AddToSearchIndexAsync <TransferEventDto>(AzureSearchServiceName, apiKey, "sep-transfers") .Result .Build()) { //run the processor for a while var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token); } await Task.Delay(5000); //give azure time to update using (var searchService = new AzureSearchService(AzureSearchServiceName, apiKey)) { Assert.Equal((long)13, await searchService.CountDocumentsAsync("sep-transfers")); await searchService.DeleteIndexAsync("sep-transfers"); } }
public async Task IndexingTransferFunctions() { const string INDEX_NAME = "transfer-functions"; //surround with "using" so that anything in a buffer is sent on dispose using (var azureSearchService = new AzureSearchService(AzureSearchServiceName, _azureSearchApiKey)) { try { //setup var index = await azureSearchService.CreateIndexForFunctionMessageAsync <TransferFunction>(INDEX_NAME); var indexer = azureSearchService.CreateIndexerForFunctionMessage <TransferFunction>(index.Name, documentsPerBatch: 1); var processorHandler = indexer.CreateProcessorHandler(); var web3 = new Web3.Web3(BlockchainUrl); var blockchainProcessor = web3.Processing.Blocks.CreateBlockProcessor((steps) => { steps.TransactionStep.SetMatchCriteria(tx => tx.Transaction.IsFrom("0xf47a8bb5c9ff814d39509591281ae31c0c7c2f38")); steps.TransactionReceiptStep.AddProcessorHandler(processorHandler); }); var cancellationTokenSource = new CancellationTokenSource(); //execute await blockchainProcessor.ExecuteAsync(3146689, cancellationTokenSource.Token, 3146689); //assert await Task.Delay(5000); // allow time to index Assert.Equal(1, await azureSearchService.CountDocumentsAsync(INDEX_NAME)); } finally { await azureSearchService.DeleteIndexAsync(INDEX_NAME); } } }
public async Task CreateIndex_Upsert_Suggest_Search() { ConfigurationUtils.SetEnvironment("development"); var appConfig = ConfigurationUtils .Build(Array.Empty <string>(), userSecretsId: "Nethereum.BlockchainStore.Search.Tests"); var apiKey = appConfig[ApiKeyName]; var eventSearchIndexDefinition = new EventIndexDefinition <TransferEvent>(); using (var searchService = new AzureSearchService(AzureSearchServiceName, apiKey)) { try { await searchService.DeleteIndexAsync(eventSearchIndexDefinition); using (var azureIndex = await searchService.GetOrCreateIndex(eventSearchIndexDefinition)) { var transferEventLog = new EventLog <TransferEvent>( new TransferEvent { From = "0x9209b29f2094457d3dba62d1953efea58176ba27", To = "0x1209b29f2094457d3dba62d1953efea58176ba28", Value = new HexBigInteger("2000000"), Detail = new TransferDetail { Description = "A generic transfer", Tags = new List <Tag> { new Tag { Name = "Status", Value = "Good" }, new Tag { Name = "Year", Value = "2019" } } } }, new FilterLog { Address = "0x26bc47888b7bfdf77db41ec0a2fb4db00af1c92a", TransactionHash = "0xcb00b69d2594a3583309f332ada97d0df48bae00170e36a4f7bbdad7783fc7e5", BlockNumber = new HexBigInteger(7118507), BlockHash = "0x337cd6feedafac6abba40eff40fb1957e08985180f5a03016924ef72fc7b04b9", LogIndex = new HexBigInteger(0), Removed = false, TransactionIndex = new HexBigInteger(0) }); await azureIndex.IndexAsync(transferEventLog); await Task.Delay(TimeSpan.FromSeconds(5)); var suggestion = await azureIndex.SuggestAsync(transferEventLog.Log.BlockNumber.Value.ToString()); Assert.NotNull(suggestion); Assert.Equal(1, suggestion.Results.Count); var searchResult = await azureIndex.SearchAsync( transferEventLog.Event.From); Assert.NotNull(searchResult); Assert.Equal(1, searchResult.Results.Count); } } finally { await searchService.DeleteIndexAsync(eventSearchIndexDefinition); } } }