private async Task CreateAndRunBlobIndexerAsync(CancellationToken cancellationToken) { SearchIndexerDataSourceConnection blobDataSource = new SearchIndexerDataSourceConnection( name: _azureBlobOptions.Value.Name, type: SearchIndexerDataSourceType.AzureBlob, connectionString: _azureBlobOptions.Value.ConnectionString, container: _searchIndexerDataContainer); await _searchIndexerClient.CreateOrUpdateDataSourceConnectionAsync(blobDataSource, false, cancellationToken); try { await _searchIndexerClient.GetIndexerAsync(_azureBlobIndexer.Name); await _searchIndexerClient.ResetIndexerAsync(_azureBlobIndexer.Name); } catch (RequestFailedException ex) when(ex.Status == 404) { } await _searchIndexerClient.CreateOrUpdateIndexerAsync(_azureBlobIndexer); try { // Run the indexer. await _searchIndexerClient.RunIndexerAsync(_azureBlobIndexer.Name, cancellationToken); } catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429) { Console.WriteLine("Failed to run indexer: {0}", e.Response.Content); } }
private static async Task CreateAndRunCosmosDbIndexerAsync(string indexName, SearchIndexerClient indexerClient) { // Append the database name to the connection string string cosmosConnectString = configuration["CosmosDBConnectionString"] + ";Database=" + configuration["CosmosDBDatabaseName"]; SearchIndexerDataSourceConnection cosmosDbDataSource = new SearchIndexerDataSourceConnection( name: configuration["CosmosDBDatabaseName"], type: SearchIndexerDataSourceType.CosmosDb, connectionString: cosmosConnectString, container: new SearchIndexerDataContainer("hotels")); // The Cosmos DB data source does not need to be deleted if it already exists, // but the connection string might need to be updated if it has changed. await indexerClient.CreateOrUpdateDataSourceConnectionAsync(cosmosDbDataSource); Console.WriteLine("Creating Cosmos DB indexer...\n"); SearchIndexer cosmosDbIndexer = new SearchIndexer( name: "hotel-rooms-cosmos-indexer", dataSourceName: cosmosDbDataSource.Name, targetIndexName: indexName) { Schedule = new IndexingSchedule(TimeSpan.FromDays(1)) }; // Indexers keep metadata about how much they have already indexed. // If we already ran this sample, the indexer will remember that it already // indexed the sample data and not run again. // To avoid this, reset the indexer if it exists. try { await indexerClient.GetIndexerAsync(cosmosDbIndexer.Name); //Rest the indexer if it exsits. await indexerClient.ResetIndexerAsync(cosmosDbIndexer.Name); } catch (RequestFailedException ex) when(ex.Status == 404) { //if the specified indexer not exist, 404 will be thrown. } await indexerClient.CreateOrUpdateIndexerAsync(cosmosDbIndexer); Console.WriteLine("Running Cosmos DB indexer...\n"); try { await indexerClient.RunIndexerAsync(cosmosDbIndexer.Name); } catch (RequestFailedException ex) when(ex.Status == 429) { Console.WriteLine("Failed to run indexer: {0}", ex.Message); } }
private static async Task CreateAndRunSQLIndexerAsync(string indexName, SearchIndexerClient indexerClient) { SearchIndexerDataSourceConnection sqlDataSource = new SearchIndexerDataSourceConnection( name: configuration["SQLDatabaseName"], type: SearchIndexerDataSourceType.AzureSql, connectionString: configuration["SQLConnectSctring"], container: new SearchIndexerDataContainer("books")); // The data source does not need to be deleted if it already exists, // but the connection string might need to be updated if it has changed. await indexerClient.CreateOrUpdateDataSourceConnectionAsync(sqlDataSource); Console.WriteLine("Creating SQL indexer...\n"); SearchIndexer sqlIndexer = new SearchIndexer( name: "books-indexer", dataSourceName: sqlDataSource.Name, targetIndexName: indexName) { //here you can set the desired schedule for indexing repetitions Schedule = new IndexingSchedule(TimeSpan.FromDays(1)) }; // Indexers keep metadata about how much they have already indexed. // If we already ran this sample, the indexer will remember that it already // indexed the sample data and not run again. // To avoid this, reset the indexer if it exists. try { await indexerClient.GetIndexerAsync(sqlIndexer.Name); //Rest the indexer if it exsits. await indexerClient.ResetIndexerAsync(sqlIndexer.Name); } catch (RequestFailedException ex) when(ex.Status == 404) { //if the specified indexer not exist, 404 will be thrown. } await indexerClient.CreateOrUpdateIndexerAsync(sqlIndexer); Console.WriteLine("Running SQL indexer...\n"); try { await indexerClient.RunIndexerAsync(sqlIndexer.Name); } catch (RequestFailedException ex) when(ex.Status == 429) { Console.WriteLine("Failed to run sql indexer: {0}", ex.Message); } }
private static async Task CreateAndRunBlobIndexerAsync(string indexName, SearchIndexerClient indexerClient) { SearchIndexerDataSourceConnection blobDataSource = new SearchIndexerDataSourceConnection( name: configuration["BlobStorageAccountName"], type: SearchIndexerDataSourceType.AzureBlob, connectionString: configuration["BlobStorageConnectionString"], container: new SearchIndexerDataContainer("gapzap-pdf-docs")); // The blob data source does not need to be deleted if it already exists, // but the connection string might need to be updated if it has changed. await indexerClient.CreateOrUpdateDataSourceConnectionAsync(blobDataSource); Console.WriteLine("Creating Blob Storage indexer...\n"); // Add a field mapping to match the Id field in the documents to // the HotelId key field in the index List <FieldMapping> map = new List <FieldMapping> { new FieldMapping("Id") { TargetFieldName = "HotelId" } }; IndexingParameters parameters = new IndexingParameters(); parameters.Configuration.Add("parsingMode", "json"); SearchIndexer blobIndexer = new SearchIndexer( name: "hotel-rooms-blob-indexer", dataSourceName: blobDataSource.Name, targetIndexName: indexName) { Parameters = parameters, Schedule = new IndexingSchedule(TimeSpan.FromDays(1)) }; blobIndexer.FieldMappings.Add(new FieldMapping("Id") { TargetFieldName = "HotelId" }); // Reset the indexer if it already exists try { await indexerClient.GetIndexerAsync(blobIndexer.Name); //Rest the indexer if it exsits. await indexerClient.ResetIndexerAsync(blobIndexer.Name); } catch (RequestFailedException ex) when(ex.Status == 404) { } await indexerClient.CreateOrUpdateIndexerAsync(blobIndexer); Console.WriteLine("Running Blob Storage indexer...\n"); try { await indexerClient.RunIndexerAsync(blobIndexer.Name); } catch (RequestFailedException ex) when(ex.Status == 429) { Console.WriteLine("Failed to run indexer: {0}", ex.Message); } }
public static async Task Main(string[] args) { IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json"); IConfigurationRoot configuration = builder.Build(); if (configuration["SearchServiceEndPoint"] == "Put your search service endpoint here") { Console.Error.WriteLine("Specify SearchServiceEndPoint in appsettings.json"); Environment.Exit(-1); } if (configuration["SearchServiceAdminApiKey"] == "Put your search service admin API key here") { Console.Error.WriteLine("Specify SearchServiceAdminApiKey in appsettings.json"); Environment.Exit(-1); } if (configuration["AzureSQLConnectionString"] == "Put your Azure SQL database connection string here") { Console.Error.WriteLine("Specify AzureSQLConnectionString in appsettings.json"); Environment.Exit(-1); } SearchIndexClient indexClient = new SearchIndexClient(new Uri(configuration["SearchServiceEndPoint"]), new AzureKeyCredential(configuration["SearchServiceAdminApiKey"])); SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(configuration["SearchServiceEndPoint"]), new AzureKeyCredential(configuration["SearchServiceAdminApiKey"])); Console.WriteLine("Creating index..."); FieldBuilder fieldBuilder = new FieldBuilder(); var searchFields = fieldBuilder.Build(typeof(Hotel)); var searchIndex = new SearchIndex("hotels-sql-idx", searchFields); // If we have run the sample before, this index will be populated // We can clear the index by deleting it if it exists and creating // it again CleanupSearchIndexClientResources(indexClient, searchIndex); indexClient.CreateOrUpdateIndex(searchIndex); Console.WriteLine("Creating data source..."); // The sample data set has a table name of "hotels" // The sample data set table has a "soft delete" column named IsDeleted // When this column is set to true and the indexer sees it, it will remove the // corresponding document from the search service // See this link for more information // https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.search.models.softdeletecolumndeletiondetectionpolicy // The sample data set uses SQL integrated change tracking for change detection // This means that when the indexer runs, it will be able to detect which data has // changed since the last run using built in change tracking // See this link for more information // https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-tracking-sql-server var dataSource = new SearchIndexerDataSourceConnection( "hotels-sql-ds", SearchIndexerDataSourceType.AzureSql, configuration["AzureSQLConnectionString"], new SearchIndexerDataContainer("hotels")); // The data source does not need to be deleted if it was already created, // but the connection string may need to be updated if it was changed indexerClient.CreateOrUpdateDataSourceConnection(dataSource); Console.WriteLine("Creating Azure SQL indexer..."); var schedule = new IndexingSchedule(TimeSpan.FromDays(1)) { StartTime = DateTimeOffset.Now }; var parameters = new IndexingParameters() { BatchSize = 100, MaxFailedItems = 0, MaxFailedItemsPerBatch = 0 }; // Indexer declarations require a data source and search index. // Common optional properties include a schedule, parameters, and field mappings // The field mappings below are redundant due to how the Hotel class is defined, but // we included them anyway to show the syntax var indexer = new SearchIndexer("hotels-sql-idxr", dataSource.Name, searchIndex.Name) { Description = "Data indexer", Schedule = schedule, Parameters = parameters, FieldMappings = { new FieldMapping("_id") { TargetFieldName = "HotelId" }, new FieldMapping("Amenities") { TargetFieldName = "Tags" } } }; // Indexers contain metadata about how much they have already indexed // If we already ran the sample, the indexer will remember that it already // indexed the sample data and not run again // To avoid this, reset the indexer if it exists CleanupSearchIndexerClientResources(indexerClient, indexer); await indexerClient.CreateOrUpdateIndexerAsync(indexer); // We created the indexer with a schedule, but we also // want to run it immediately Console.WriteLine("Running Azure SQL indexer..."); try { await indexerClient.RunIndexerAsync(indexer.Name); } catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429) { Console.WriteLine("Failed to run indexer: {0}", e.Response.Content); } // Wait 5 seconds for indexing to complete before checking status Console.WriteLine("Waiting for indexing...\n"); System.Threading.Thread.Sleep(5000); // After an indexer run, you can retrieve status. CheckIndexerStatus(indexerClient, indexer); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); Environment.Exit(0); }
public void CreateCustomerIndex() { //creating index SearchIndexClient indexClient = new SearchIndexClient(Uri, keyCredential); SearchIndexerClient indexerClient = new SearchIndexerClient(Uri, keyCredential); Console.WriteLine("Creating index..."); FieldBuilder fieldBuilder = new FieldBuilder(); var searchFields = fieldBuilder.Build(typeof(ServiceOrder)); //var searchIndex = new SearchIndex("serviceorder-sql-idx", searchFields); var searchIndex = new SearchIndex("serviceorder-sql-idx", searchFields); // If we have run the sample before, this index will be populated // We can clear the index by deleting it if it exists and creating // it again CleanupSearchIndexClientResources(indexClient, searchIndex); indexClient.CreateOrUpdateIndex(searchIndex); //Creating data source Console.WriteLine("Creating data source..."); var dataSource = new SearchIndexerDataSourceConnection( "serviceorder-sql-ds", SearchIndexerDataSourceType.AzureSql, azureSQLConnectionStr, new SearchIndexerDataContainer("[ServiceOrder]")); indexerClient.CreateOrUpdateDataSourceConnection(dataSource); //Creating indexer Console.WriteLine("Creating Azure SQL indexer..."); //var schedule = new IndexingSchedule(TimeSpan.FromDays(1)) //{ // StartTime = DateTimeOffset.Now //}; var parameters = new IndexingParameters() { BatchSize = 100, MaxFailedItems = 0, MaxFailedItemsPerBatch = 0 }; // Indexer declarations require a data source and search index. // Common optional properties include a schedule, parameters, and field mappings // The field mappings below are redundant due to how the Hotel class is defined, but // we included them anyway to show the syntax var indexer = new SearchIndexer("serviceorder-sql-idxr", dataSource.Name, searchIndex.Name) { Description = "Service Order indexer", Schedule = new IndexingSchedule(TimeSpan.FromMinutes(5)), Parameters = parameters, }; indexerClient.CreateOrUpdateIndexerAsync(indexer); Console.WriteLine("Running Azure SQL indexer..."); try { indexerClient.RunIndexerAsync(indexer.Name); } catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429) { Console.WriteLine("Failed to run indexer: {0}", e.Response.Content); } // Wait 5 seconds for indexing to complete before checking status Console.WriteLine("Waiting for indexing...\n"); System.Threading.Thread.Sleep(5000); }
public void CreateIndex <T>() { string azureSQLConnectionStr = ConfigurationManager.AppSettings["AzureSqlConnectionString"].ToString(); Uri Uri = new Uri(ConfigurationManager.AppSettings["SearchServiceEndPoint"]); string searchServiceKey = ConfigurationManager.AppSettings["SearchServiceAdminApiKey"]; AzureKeyCredential keyCredential = new AzureKeyCredential(searchServiceKey); SearchIndexClient indexClient = new SearchIndexClient(Uri, keyCredential); SearchIndexerClient indexerClient = new SearchIndexerClient(Uri, keyCredential); Console.WriteLine("Creating index..."); _logger.LogInformation("Creating index for Orion Customer"); FieldBuilder fieldBuilder = new FieldBuilder(); var searchFields = fieldBuilder.Build(typeof(T)); var searchIndex = new SearchIndex(GetIndexName(HdsConstants.Orion, typeof(T).ToString()), searchFields); CleanupSearchIndexClientResources(indexClient, searchIndex); indexClient.CreateOrUpdateIndex(searchIndex); Console.WriteLine("Creating data source..."); _logger.LogInformation("Creating data source for Orion Customer"); var dataSource = new SearchIndexerDataSourceConnection( GetDataSourceName(HdsConstants.Orion, typeof(T).ToString()), SearchIndexerDataSourceType.AzureSql, azureSQLConnectionStr, new SearchIndexerDataContainer($"[{GetTableName(typeof(T).ToString())}]")); indexerClient.CreateOrUpdateDataSourceConnection(dataSource); //Creating indexer Console.WriteLine("Creating Azure SQL indexer..."); _logger.LogInformation("Creating Azure SQL indexer for Orion Customer"); var schedule = new IndexingSchedule(TimeSpan.FromDays(1)) { StartTime = DateTimeOffset.Now }; var parameters = new IndexingParameters() { BatchSize = 100, MaxFailedItems = 0, MaxFailedItemsPerBatch = 0 }; var indexer = new SearchIndexer(GetIndexerName(HdsConstants.Orion, typeof(T).ToString()), dataSource.Name, searchIndex.Name) { Description = "Data indexer", Schedule = schedule, Parameters = parameters, }; indexerClient.CreateOrUpdateIndexerAsync(indexer); Console.WriteLine("Running Azure SQL indexer..."); try { indexerClient.RunIndexerAsync(indexer.Name); } catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429) { Console.WriteLine("Failed to run indexer: {0}", e.Response.Content); _logger.LogError("Failed to run indexer: {0}", e.Response.Content); } }