public void CanRunIndexerAndGetIndexerStatus() { Run(() => { SearchServiceClient searchClient = Data.GetSearchServiceClient(); Indexer indexer = Data.CreateTestIndexer(); IndexerDefinitionResponse createResponse = searchClient.Indexers.Create(indexer); Assert.Equal(HttpStatusCode.Created, createResponse.StatusCode); IndexerGetStatusResponse statusResponse = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode); IndexerExecutionInfo info = statusResponse.ExecutionInfo; Assert.Equal(IndexerStatus.Running, info.Status); AzureOperationResponse runResponse = searchClient.Indexers.Run(indexer.Name); Assert.Equal(HttpStatusCode.Accepted, runResponse.StatusCode); // Set handler that injects mock_status query string, which results in service // returning a well-known mock response searchClient.AddHandlerToPipeline(new MockStatusDelegatingHandler()); statusResponse = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode); info = statusResponse.ExecutionInfo; Assert.Equal(IndexerStatus.Running, info.Status); Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status); Assert.Equal(3, info.ExecutionHistory.Count); IndexerExecutionResult newestResult = info.ExecutionHistory[0]; IndexerExecutionResult middleResult = info.ExecutionHistory[1]; IndexerExecutionResult oldestResult = info.ExecutionHistory[2]; Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status); Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage); AssertStartAndEndTimeValid(newestResult); Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status); AssertStartAndEndTimeValid(middleResult); Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status); Assert.Equal(124876, oldestResult.ItemCount); Assert.Equal(2, oldestResult.FailedItemCount); Assert.Equal("100", oldestResult.InitialTrackingState); Assert.Equal("200", oldestResult.FinalTrackingState); AssertStartAndEndTimeValid(oldestResult); Assert.Equal(2, oldestResult.Errors.Count); Assert.Equal("1", oldestResult.Errors[0].Key); Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage); Assert.Equal("121713", oldestResult.Errors[1].Key); Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage); }); }
static void CheckIndexerStatus(Indexer indexer, SearchServiceClient searchService) { try { IndexerExecutionInfo execInfo = searchService.Indexers.GetStatus(indexer.Name); Console.WriteLine("Indexer has run {0} times.", execInfo.ExecutionHistory.Count); Console.WriteLine("Indexer Status: " + execInfo.Status.ToString()); IndexerExecutionResult result = execInfo.LastResult; Console.WriteLine("Latest run"); Console.WriteLine(" Run Status: {0}", result.Status.ToString()); Console.WriteLine(" Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount); TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value; Console.WriteLine(" StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed); string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage; Console.WriteLine(" ErrorMessage: {0}", errorMsg); Console.WriteLine(" Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count); } catch (Exception e) { // Handle exception } }
private static void CheckIndexerOverallStatus(SearchServiceClient searchClient, Indexer indexer) { try { // Get the index status IndexerExecutionInfo indexerExecutionInfo = searchClient.Indexers.GetStatus(indexer.Name); switch (indexerExecutionInfo.Status) { case IndexerStatus.Error: Console.WriteLine("\nIndexer has error status. Check the Azure Portal to further understand the error."); break; case IndexerStatus.Running: Console.WriteLine("\nIndexer is running"); break; case IndexerStatus.Unknown: Console.WriteLine("\nIndexer status is unknown"); break; default: Console.WriteLine("\nNo indexer information"); break; } } catch (Exception e) { Console.WriteLine("\nFailed to get indexer overall status\n Exception message: {0}\n", e.Message); } }
/* * This function checks the status of the indexer every 5 seconds * and writes the current status to the console */ private static async Task <bool> CheckIndexerStatus() { Console.WriteLine("Waiting for indexing to complete..."); IndexerExecutionStatus requestStatus = IndexerExecutionStatus.InProgress; try { await _searchClient.Indexers.GetAsync(IndexerName); while (requestStatus.Equals(IndexerExecutionStatus.InProgress)) { Thread.Sleep(5000); IndexerExecutionInfo info = await _searchClient.Indexers.GetStatusAsync(IndexerName); requestStatus = info.LastResult.Status; if (DebugMode) { Console.WriteLine("Indexer Status: " + requestStatus.ToString()); } } } catch (Exception ex) { if (DebugMode) { Console.WriteLine("ERROR RETREIVING INDEXER STATUS: " + ex.Message + "\n\n" + ex.StackTrace); } return(false); } return(requestStatus.Equals(IndexerExecutionStatus.Success)); }
private static async Task <bool> CheckIndexerStatus() { Console.WriteLine("Waiting for indexing to complete..."); IndexerExecutionStatus requestStatus = IndexerExecutionStatus.InProgress; try { await _searchClient.Indexers.GetAsync(IndexerName); while (requestStatus.Equals(IndexerExecutionStatus.InProgress)) { Thread.Sleep(3000); IndexerExecutionInfo info = await _searchClient.Indexers.GetStatusAsync(IndexerName); requestStatus = info.LastResult.Status; if (DebugMode) { Console.WriteLine("Current indexer status: {0}", requestStatus.ToString()); } } } catch (Exception ex) { if (DebugMode) { Console.WriteLine("Error retrieving indexer status: {0}", ex.Message); } return(false); } return(requestStatus.Equals(IndexerExecutionStatus.Success)); }
private async Task CheckIndexerStatus(string[] fileNames, Guid anonymizationId, int referralId) { Console.WriteLine("Waiting for indexing to complete..."); IndexerExecutionStatus requestStatus = IndexerExecutionStatus.InProgress; try { await _searchClient.Indexers.GetAsync(_configuration["IndexerName"]); while (requestStatus.Equals(IndexerExecutionStatus.InProgress)) { Thread.Sleep(3000); IndexerExecutionInfo info = await _searchClient.Indexers.GetStatusAsync(_configuration["IndexerName"]); requestStatus = info.LastResult.Status; Console.WriteLine("Current indexer status: {0}", requestStatus.ToString()); } } catch (Exception ex) { Console.WriteLine("Error retrieving indexer status: {0}", ex.Message); } ISearchIndexClient indexClient = _searchClient.Indexes.GetClient(_configuration["IndexName"]); foreach (var f in fileNames) { var searchResult = await indexClient.Documents.SearchAsync("*", new SearchParameters { Filter = $"(search.in(fileName,'{f}', '|'))" }); foreach (var result in searchResult.Results) { if (result.Document["fileName"].ToString() == f) { var uploadBatch = new List <IndexSchema>(); var indexDoc = new IndexSchema { id = result.Document["id"].ToString(), anonymizationId = anonymizationId.ToString(), referralId = referralId.ToString() }; uploadBatch.Add(indexDoc); var batch = IndexBatch.MergeOrUpload(uploadBatch); await indexClient.Documents.IndexAsync(batch); uploadBatch.Clear(); } } } }
public void CanResetIndexerAndGetIndexerStatus() { Run(() => { SearchServiceClient searchClient = Data.GetSearchServiceClient(); Indexer indexer = Data.CreateTestIndexer(); searchClient.Indexers.Create(indexer); searchClient.Indexers.Reset(indexer.Name); IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(IndexerStatus.Running, info.Status); Assert.Equal(IndexerExecutionStatus.Reset, info.LastResult.Status); }); }
public static async Task RunAsync( [BlobTrigger("datum/{name}", Connection = "Search")] CloudBlockBlob myBlob, [CosmosDB("taskDatabase", "TaskCollection", ConnectionStringSetting = "CosmosDB")] IAsyncCollector <object> todos, string name, ILogger log) { SearchServiceClient serviceClient = new SearchServiceClient(searchServiceName: "jfk-search-service-qymjpzort5hho", credentials: new SearchCredentials("023DB82430FFA416AD39EEF8A6FDFF2A")); ISearchIndexClient indexClient = serviceClient.Indexes.GetClient("jfkindex"); await serviceClient.Indexers.RunAsync("jfkindexer"); try { IndexerExecutionInfo execInfo = serviceClient.Indexers.GetStatus("jfkindexer"); Console.WriteLine("{0} veces ejecutadas.", execInfo.ExecutionHistory.Count); Console.WriteLine("Indexer Status: " + execInfo.Status.ToString()); IndexerExecutionResult result = execInfo.LastResult; Console.WriteLine("Latest run"); Console.WriteLine(" Run Status: {0}", result.Status.ToString()); Console.WriteLine(" Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount); TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value; Console.WriteLine(" StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed); string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage; Console.WriteLine(" ErrorMessage: {0}", errorMsg); Console.WriteLine(" Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count); } catch (Exception e) { Console.WriteLine("Error: ", e); } var param = new SearchParameters { Select = new[] { "*" }, //IncludeTotalResultCount = true }; System.Threading.Thread.Sleep(19000); log.LogInformation($"espera terminada{name}"); var results = indexClient.Documents.Search <Book>($"{name}", param); log.LogInformation($"enviando a cosmos.... {myBlob.Name}"); await todos.AddAsync(results); }
private bool IndexBlobStorageStatus(SearchServiceClient client) { try { IndexerExecutionInfo demoIndexerExecutionInfo = client.Indexers.GetStatus(indexerName); switch (demoIndexerExecutionInfo.Status) { case IndexerStatus.Error: WriteLine("Indexer has error status"); return(false); case IndexerStatus.Running: WriteLine("Indexer is running"); if (demoIndexerExecutionInfo.LastResult.Status == IndexerExecutionStatus.Success) { return(true); } else { System.Threading.Thread.Sleep(5000); return(IndexBlobStorageStatus(client)); } case IndexerStatus.Unknown: WriteLine("Indexer status is unknown"); return(false); default: WriteLine("No indexer information. Assumign Completed"); return(true); } } catch (Exception ex) { WriteLine("Indexer Failed - " + ex.ToString()); return(false); } }
public void CanResetIndexerAndGetIndexerStatus() { Run(() => { SearchServiceClient searchClient = Data.GetSearchServiceClient(); Indexer indexer = Data.CreateTestIndexer(); IndexerDefinitionResponse createResponse = searchClient.Indexers.Create(indexer); Assert.Equal(HttpStatusCode.Created, createResponse.StatusCode); AzureOperationResponse resetResponse = searchClient.Indexers.Reset(indexer.Name); Assert.Equal(HttpStatusCode.NoContent, resetResponse.StatusCode); IndexerGetStatusResponse statusResponse = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode); IndexerExecutionInfo info = statusResponse.ExecutionInfo; Assert.Equal(IndexerStatus.Running, info.Status); Assert.Equal(IndexerExecutionStatus.Reset, info.LastResult.Status); }); }
private static void SyncDataFromBlobSorage() { // This will use the Azure Search Indexer to synchronize data from Blob Storage to Azure Search Console.WriteLine("{0}", "Creating Data Source...\n"); var dataSource = DataSource.AzureBlobStorage( name: InterpreterIntelligenceDataSource, storageConnectionString: containerName: ""); try { _searchClient.DataSources.Create(dataSource); } catch (Exception ex) { Console.WriteLine("Error creating data source: {0}", ex.Message); return; } Console.WriteLine("{0}", "Creating Indexer and syncing data...\n"); var indexer = new Indexer() { Name = InterpreterIntelligenceIndexer, Description = "Interpreter Intelligence indexer", DataSourceName = dataSource.Name, TargetIndexName = InterpreterIntelligenceIndex }; try { _searchClient.Indexers.Create(indexer); } catch (Exception ex) { Console.WriteLine("Error creating and running indexer: {0}", ex.Message); return; } bool running = true; Console.WriteLine("{0}", "Synchronization running...\n"); while (running) { IndexerExecutionInfo status = null; try { status = _searchClient.Indexers.GetStatus(indexer.Name); } catch (Exception ex) { Console.WriteLine("Error polling for indexer status: {0}", ex.Message); return; } IndexerExecutionResult lastResult = status.LastResult; if (lastResult != null) { switch (lastResult.Status) { case IndexerExecutionStatus.InProgress: Console.WriteLine("{0}", "Synchronization running...\n"); Thread.Sleep(1000); break; case IndexerExecutionStatus.Success: running = false; Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount); break; default: running = false; Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage); break; } } } }
private static void SyncDataFromAzureSQL() { // This will use the Azure Search Indexer to synchronize data from Azure Blob Storage to Azure Search Console.WriteLine("{0}", "Creating Data Source...\n"); var dataSource = DataSource.AzureBlobStorage( name: UsgsDataSource, storageConnectionString: "DefaultEndpointsProtocol=https;AccountName=ronanblobstorage;AccountKey=Z8nPhGaeduop96WgvZ2FP6QKGNjXeN/G8RmyoJtX97Cycusq5WOaIymucItSNnrv31ChYMICh04nmyHbCmySYw==;EndpointSuffix=core.windows.net;", containerName: "testmovies", description: "movies Json"); try { _searchClient.DataSources.Create(dataSource); } catch (Exception ex) { Console.WriteLine("Error creating data source: {0}", ex.Message); return; } Console.WriteLine("{0}", "Creating Indexer and syncing data...\n"); var indexer = new Indexer() { Name = UsgsIndexer, Description = "USGS data indexer", DataSourceName = dataSource.Name, TargetIndexName = GeoNamesIndex }; try { _searchClient.Indexers.Create(indexer); } catch (Exception ex) { Console.WriteLine("Error creating and running indexer: {0}", ex.Message); return; } bool running = true; Console.WriteLine("{0}", "Synchronization running...\n"); while (running) { IndexerExecutionInfo status = null; try { status = _searchClient.Indexers.GetStatus(indexer.Name); } catch (Exception ex) { Console.WriteLine("Error polling for indexer status: {0}", ex.Message); return; } IndexerExecutionResult lastResult = status.LastResult; if (lastResult != null) { switch (lastResult.Status) { case IndexerExecutionStatus.InProgress: Console.WriteLine("{0}", "Synchronization running...\n"); Thread.Sleep(1000); break; case IndexerExecutionStatus.Success: running = false; Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount); break; default: running = false; Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage); break; } } } }
private static void SyncProductDataFromAzureSQL() { // This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search Console.WriteLine("{0}", "Creating Product Data Source...\n"); var dataSource = DataSource.AzureSql( name: MowidoDataSource, sqlConnectionString: "Server=tcp:mowido.database.windows.net,1433;Initial Catalog=MowidoDB;Persist Security Info=False;User ID=mowido;Password=Billions123;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;", tableOrViewName: "Products", description: "Mowido Dataset"); try { _searchClient.DataSources.Create(dataSource); } catch (Exception ex) { Console.WriteLine("Error creating data source: {0}", ex.Message); return; } Console.WriteLine("{0}", "Creating Indexer and syncing data...\n"); var indexer = new Indexer() { Name = MowidoIndexer, Description = "Mowido start search product data indexer", DataSourceName = dataSource.Name, TargetIndexName = MowidoProductIndex }; try { _searchClient.Indexers.Create(indexer); } catch (Exception ex) { Console.WriteLine("Error creating and running indexer: {0}", ex.Message); return; } bool running = true; Console.WriteLine("{0}", "Synchronization running...\n"); while (running) { IndexerExecutionInfo status = null; try { status = _searchClient.Indexers.GetStatus(indexer.Name); } catch (Exception ex) { Console.WriteLine("Error polling for indexer status: {0}", ex.Message); return; } IndexerExecutionResult lastResult = status.LastResult; if (lastResult != null) { switch (lastResult.Status) { case IndexerExecutionStatus.InProgress: Console.WriteLine("{0}", "Synchronization running...\n"); Thread.Sleep(1000); break; case IndexerExecutionStatus.Success: running = false; Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount); break; default: running = false; Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage); break; } } } }
private static void SyncDataFromAzureSQL() { // This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search Console.WriteLine("{0}", "Creating Data Source...\n"); var dataSource = new DataSource() { Name = AssetDataSource, Description = "AssetManagement DataSet", Type = DataSourceType.AzureSql, Credentials = new DataSourceCredentials("Server=fckpvybjkl.database.windows.net,1433;Database=AssetManagement;User ID=brucewright18;Password=Mallard18;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;"), Container = new DataContainer("vAssetPwoCustomerSearch") }; try { _searchClient.DataSources.Create(dataSource); } catch (Exception ex) { Console.WriteLine("Error creating data source: {0}", ex.Message); return; } Console.WriteLine("{0}", "Creating Indexer and syncing data...\n"); var indexer = new Indexer() { Name = AssetIndexer, Description = "Asset Management data indexer", DataSourceName = dataSource.Name, TargetIndexName = AssetIndex }; try { _searchClient.Indexers.Create(indexer); } catch (Exception ex) { Console.WriteLine("Error creating and running indexer: {0}", ex.Message); return; } bool running = true; Console.WriteLine("{0}", "Synchronization running...\n"); while (running) { IndexerExecutionInfo status = null; try { status = _searchClient.Indexers.GetStatus(indexer.Name); } catch (Exception ex) { Console.WriteLine("Error polling for indexer status: {0}", ex.Message); return; } IndexerExecutionResult lastResult = status.LastResult; if (lastResult != null) { switch (lastResult.Status) { case IndexerExecutionStatus.InProgress: Console.WriteLine("{0}", "Synchronization running...\n"); Thread.Sleep(1000); break; case IndexerExecutionStatus.Success: running = false; Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount); break; default: running = false; Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage); break; } } } }
private static void SyncDataFromAzureSQL() { // This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search Console.WriteLine("{0}", "Creating Data Source...\n"); var dataSource = DataSource.AzureSql( name: UsgsDataSource, sqlConnectionString: "Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;", tableOrViewName: "GeoNamesRI", description: "USGS Dataset"); try { _searchClient.DataSources.Create(dataSource); } catch (Exception ex) { Console.WriteLine("Error creating data source: {0}", ex.Message); return; } Console.WriteLine("{0}", "Creating Indexer and syncing data...\n"); var indexer = new Indexer() { Name = UsgsIndexer, Description = "USGS data indexer", DataSourceName = dataSource.Name, TargetIndexName = GeoNamesIndex }; try { _searchClient.Indexers.Create(indexer); } catch (Exception ex) { Console.WriteLine("Error creating and running indexer: {0}", ex.Message); return; } bool running = true; Console.WriteLine("{0}", "Synchronization running...\n"); while (running) { IndexerExecutionInfo status = null; try { status = _searchClient.Indexers.GetStatus(indexer.Name); } catch (Exception ex) { Console.WriteLine("Error polling for indexer status: {0}", ex.Message); return; } IndexerExecutionResult lastResult = status.LastResult; if (lastResult != null) { switch (lastResult.Status) { case IndexerExecutionStatus.InProgress: Console.WriteLine("{0}", "Synchronization running...\n"); Thread.Sleep(1000); break; case IndexerExecutionStatus.Success: running = false; Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount); break; default: running = false; Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage); break; } } } }
private static void CreateAndSyncIndexer() { // Create a new indexer and sync it try { var creds = new DataSourceCredentials("Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30"); DataSource ds = new DataSource("usgs-datasource", DataSourceType.AzureSql, creds, new DataContainer("GeoNamesRI")); ds.Description = "USGS Dataset"; Indexer idx = new Indexer(); idx.Name = "usgs-indexer"; idx.Description = "USGS data indexer"; idx.DataSourceName = "usgs-datasource"; idx.TargetIndexName = "geonames"; idx.Parameters = new IndexingParameters(); idx.Parameters.MaxFailedItems = 10; idx.Parameters.MaxFailedItemsPerBatch = 5; idx.Parameters.Base64EncodeKeys = false; //Delete indexer and datasource if it existed _searchClient.DataSources.Delete("usgs-datasource"); _searchClient.Indexers.Delete("usgs-indexer"); //Create indexer and datasource _searchClient.DataSources.Create(ds); _searchClient.Indexers.Create(idx); //Launch the sync and then monitor progress until complete bool running = true; Console.WriteLine("{0}", "Synchronization running...\n"); while (running) { IndexerExecutionInfo status = null; try { status = _searchClient.Indexers.GetStatus(idx.Name); } catch (Exception ex) { Console.WriteLine("Error polling for indexer status: {0}", ex.Message); return; } IndexerExecutionResult lastResult = status.LastResult; if (lastResult != null) { switch (lastResult.Status) { case IndexerExecutionStatus.InProgress: Console.WriteLine("{0}", "Synchronization running...\n"); Thread.Sleep(1000); break; case IndexerExecutionStatus.Success: running = false; Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount); break; default: running = false; Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage); break; } } } } catch (Exception ex) { Console.WriteLine("Error creating indexer: {0}: \n", ex.Message.ToString()); } }
public void CanRunIndexerAndGetIndexerStatus() { Run(() => { // Set handler that injects mock_status query string, which results in service // returning a well-known mock response SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler()); Indexer indexer = Data.CreateTestIndexer(); searchClient.Indexers.Create(indexer); IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(IndexerStatus.Running, info.Status); AzureOperationResponse runResponse = searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result; Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode); info = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(IndexerStatus.Running, info.Status); Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status); Assert.Equal(3, info.ExecutionHistory.Count); Assert.NotNull(info.Limits); Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract); Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize); Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime); IndexerExecutionResult newestResult = info.ExecutionHistory[0]; IndexerExecutionResult middleResult = info.ExecutionHistory[1]; IndexerExecutionResult oldestResult = info.ExecutionHistory[2]; Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status); Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage); AssertStartAndEndTimeValid(newestResult); Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status); AssertStartAndEndTimeValid(middleResult); Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status); Assert.Equal(124876, oldestResult.ItemCount); Assert.Equal(2, oldestResult.FailedItemCount); Assert.Equal("100", oldestResult.InitialTrackingState); Assert.Equal("200", oldestResult.FinalTrackingState); AssertStartAndEndTimeValid(oldestResult); Assert.Equal(2, oldestResult.Errors.Count); Assert.Equal("1", oldestResult.Errors[0].Key); Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage); Assert.Equal("121713", oldestResult.Errors[1].Key); Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage); Assert.Equal(1, oldestResult.Warnings.Count); Assert.Equal("2", oldestResult.Warnings[0].Key); Assert.Equal("This is the first and last warning", oldestResult.Warnings[0].Message); }); }
public async Task CreateIndex(string searchAdminKey, string cosmosConnectionString, SearchConfig searchConfig) { Console.WriteLine("{0}", "Retrieving Search Service\n"); var azureSearchService = SearchHelper.GetSearchServiceClient(searchConfig.SearchServiceName, searchAdminKey); if (azureSearchService == null) { throw new WebException("Unable to find Search Service"); } Console.WriteLine("{0}", "Deleting Index...\n"); SearchHelper.DeleteIndexIfExists(searchConfig.CustomerSearchIndexName); Console.WriteLine("{0}", "Creating Index Model...\n"); var indexModelForCustomer = IndexModelHelper.CreateIndexModelForCustomer(searchConfig.CustomerSearchIndexName); if (indexModelForCustomer == null) { Console.WriteLine("Unable to create Index Model"); throw new Exception("Unable to create Index Model"); } Console.WriteLine("{0}", "Creating Index...\n"); SearchHelper.CreateIndex(indexModelForCustomer); Console.WriteLine("{0}", "Creating Data Source object...\n"); var dataSource = DataSourceHelper.CreateDataSource(searchConfig.CustomerSearchDataSourceQuery, searchConfig.CustomerCollectionId, searchConfig.CustomerSearchIndexName, searchConfig.CustomerSearchDataSourceName, cosmosConnectionString); try { Console.WriteLine("{0}", "Attempting to Create/Update Data Source...\n"); await azureSearchService.DataSources.CreateOrUpdateWithHttpMessagesAsync(dataSource); } catch (CloudException e) { Console.WriteLine(e.ToString()); throw; } Indexer indexer; try { Console.WriteLine("{0}", "Attempting to Create Indexer...\n"); indexer = IndexerHelper.CreateIndexer(azureSearchService, searchConfig.CustomerSearchIndexerName, indexModelForCustomer, searchConfig.CustomerSearchDataSourceName); } catch (CloudException e) { Console.WriteLine("{0}{1}", "Unable to Create Indexer...\n", e); throw; } Console.WriteLine("{0}", "Run Indexer...\n"); try { azureSearchService.Indexers.Run(indexer.Name); } catch (CloudException e) { Console.WriteLine("{0} {1}", "Unable to get data for the Indexer...\n", e); throw; } var running = true; Console.WriteLine("{0}", "Synchronization running...\n"); while (running) { IndexerExecutionInfo status = null; try { status = azureSearchService.Indexers.GetStatus(indexer.Name); } catch (Exception ex) { Console.WriteLine("Error polling for indexer status: {0}", ex.Message); throw; } var lastResult = status.LastResult; if (lastResult != null) { switch (lastResult.Status) { case IndexerExecutionStatus.Reset: case IndexerExecutionStatus.InProgress: Console.WriteLine("{0} Status: {1}, Item Count: {2}", "Synchronization running...\n", lastResult.Status, lastResult.ItemCount); Thread.Sleep(1000); break; case IndexerExecutionStatus.Success: running = false; Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount.ToString()); break; default: running = false; Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage); break; } } } Console.WriteLine("{0} Status: {1}", "Completed... ", HttpStatusCode.Created); }
public void CanRunIndexerAndGetIndexerStatus() { Run(() => { // Set handler that injects mock_status query string, which results in service // returning a well-known mock response SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler()); Indexer indexer = Data.CreateTestIndexer(); searchClient.Indexers.Create(indexer); IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(IndexerStatus.Running, info.Status); AzureOperationResponse runResponse = searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result; Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode); info = searchClient.Indexers.GetStatus(indexer.Name); Assert.Equal(IndexerStatus.Running, info.Status); Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status); Assert.Equal(3, info.ExecutionHistory.Count); Assert.NotNull(info.Limits); Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract); Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize); Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime); IndexerExecutionResult newestResult = info.ExecutionHistory[0]; IndexerExecutionResult middleResult = info.ExecutionHistory[1]; IndexerExecutionResult oldestResult = info.ExecutionHistory[2]; Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status); Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage); AssertStartAndEndTimeValid(newestResult); Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status); AssertStartAndEndTimeValid(middleResult); Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status); Assert.Equal(124876, oldestResult.ItemCount); Assert.Equal(2, oldestResult.FailedItemCount); Assert.Equal("100", oldestResult.InitialTrackingState); Assert.Equal("200", oldestResult.FinalTrackingState); AssertStartAndEndTimeValid(oldestResult); Assert.Equal(2, oldestResult.Errors.Count); Assert.Equal("1", oldestResult.Errors[0].Key); Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage); Assert.Equal("DocumentExtraction.AzureBlob.MyDataSource", oldestResult.Errors[0].Name); Assert.Equal("The file could not be parsed.", oldestResult.Errors[0].Details); Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[0].DocumentationLink); Assert.Equal("121713", oldestResult.Errors[1].Key); Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage); Assert.Equal("DocumentExtraction.AzureBlob.DataReader", oldestResult.Errors[1].Name); Assert.Equal("Blob size cannot exceed 256 MB.", oldestResult.Errors[1].Details); Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[1].DocumentationLink); Assert.Equal(1, oldestResult.Warnings.Count); Assert.Equal("2", oldestResult.Warnings[0].Key); Assert.Equal("Document was truncated to 50000 characters.", oldestResult.Warnings[0].Message); Assert.Equal("Enrichment.LanguageDetectionSkill.#4", oldestResult.Warnings[0].Name); Assert.Equal("Try to split the input into smaller chunks using Split skill.", oldestResult.Warnings[0].Details); Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2099692", oldestResult.Warnings[0].DocumentationLink); }); }