private async Task CreateAndRunBlobIndexerAsync(CancellationToken cancellationToken)
{
SearchIndexerDataSourceConnection blobDataSource = new SearchIndexerDataSourceConnection(
name: _azureBlobOptions.Value.Name,
type: SearchIndexerDataSourceType.AzureBlob,
connectionString: _azureBlobOptions.Value.ConnectionString,
container: _searchIndexerDataContainer);
await _searchIndexerClient.CreateOrUpdateDataSourceConnectionAsync(blobDataSource, false, cancellationToken);
try
{
await _searchIndexerClient.GetIndexerAsync(_azureBlobIndexer.Name);
await _searchIndexerClient.ResetIndexerAsync(_azureBlobIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
}
await _searchIndexerClient.CreateOrUpdateIndexerAsync(_azureBlobIndexer);
try
{
// Run the indexer.
await _searchIndexerClient.RunIndexerAsync(_azureBlobIndexer.Name, cancellationToken);
}
catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429)
{
Console.WriteLine("Failed to run indexer: {0}", e.Response.Content);
}
}
private static async Task CreateAndRunCosmosDbIndexerAsync(string indexName, SearchIndexerClient indexerClient)
{
// Append the database name to the connection string
string cosmosConnectString =
configuration["CosmosDBConnectionString"]
+ ";Database="
+ configuration["CosmosDBDatabaseName"];
SearchIndexerDataSourceConnection cosmosDbDataSource = new SearchIndexerDataSourceConnection(
name: configuration["CosmosDBDatabaseName"],
type: SearchIndexerDataSourceType.CosmosDb,
connectionString: cosmosConnectString,
container: new SearchIndexerDataContainer("hotels"));
// The Cosmos DB data source does not need to be deleted if it already exists,
// but the connection string might need to be updated if it has changed.
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(cosmosDbDataSource);
Console.WriteLine("Creating Cosmos DB indexer...\n");
SearchIndexer cosmosDbIndexer = new SearchIndexer(
name: "hotel-rooms-cosmos-indexer",
dataSourceName: cosmosDbDataSource.Name,
targetIndexName: indexName)
{
Schedule = new IndexingSchedule(TimeSpan.FromDays(1))
};
// Indexers keep metadata about how much they have already indexed.
// If we already ran this sample, the indexer will remember that it already
// indexed the sample data and not run again.
// To avoid this, reset the indexer if it exists.
try
{
await indexerClient.GetIndexerAsync(cosmosDbIndexer.Name);
//Rest the indexer if it exsits.
await indexerClient.ResetIndexerAsync(cosmosDbIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
//if the specified indexer not exist, 404 will be thrown.
}
await indexerClient.CreateOrUpdateIndexerAsync(cosmosDbIndexer);
Console.WriteLine("Running Cosmos DB indexer...\n");
try
{
await indexerClient.RunIndexerAsync(cosmosDbIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 429)
{
Console.WriteLine("Failed to run indexer: {0}", ex.Message);
}
}
private static async Task CreateAndRunSQLIndexerAsync(string indexName, SearchIndexerClient indexerClient)
{
SearchIndexerDataSourceConnection sqlDataSource = new SearchIndexerDataSourceConnection(
name: configuration["SQLDatabaseName"],
type: SearchIndexerDataSourceType.AzureSql,
connectionString: configuration["SQLConnectSctring"],
container: new SearchIndexerDataContainer("books"));
// The data source does not need to be deleted if it already exists,
// but the connection string might need to be updated if it has changed.
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(sqlDataSource);
Console.WriteLine("Creating SQL indexer...\n");
SearchIndexer sqlIndexer = new SearchIndexer(
name: "books-indexer",
dataSourceName: sqlDataSource.Name,
targetIndexName: indexName)
{
//here you can set the desired schedule for indexing repetitions
Schedule = new IndexingSchedule(TimeSpan.FromDays(1))
};
// Indexers keep metadata about how much they have already indexed.
// If we already ran this sample, the indexer will remember that it already
// indexed the sample data and not run again.
// To avoid this, reset the indexer if it exists.
try
{
await indexerClient.GetIndexerAsync(sqlIndexer.Name);
//Rest the indexer if it exsits.
await indexerClient.ResetIndexerAsync(sqlIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
//if the specified indexer not exist, 404 will be thrown.
}
await indexerClient.CreateOrUpdateIndexerAsync(sqlIndexer);
Console.WriteLine("Running SQL indexer...\n");
try
{
await indexerClient.RunIndexerAsync(sqlIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 429)
{
Console.WriteLine("Failed to run sql indexer: {0}", ex.Message);
}
}
private static async Task CreateAndRunBlobIndexerAsync(string indexName, SearchIndexerClient indexerClient)
{
SearchIndexerDataSourceConnection blobDataSource = new SearchIndexerDataSourceConnection(
name: configuration["BlobStorageAccountName"],
type: SearchIndexerDataSourceType.AzureBlob,
connectionString: configuration["BlobStorageConnectionString"],
container: new SearchIndexerDataContainer("gapzap-pdf-docs"));
// The blob data source does not need to be deleted if it already exists,
// but the connection string might need to be updated if it has changed.
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(blobDataSource);
Console.WriteLine("Creating Blob Storage indexer...\n");
// Add a field mapping to match the Id field in the documents to
// the HotelId key field in the index
List <FieldMapping> map = new List <FieldMapping> {
new FieldMapping("Id")
{
TargetFieldName = "HotelId"
}
};
IndexingParameters parameters = new IndexingParameters();
parameters.Configuration.Add("parsingMode", "json");
SearchIndexer blobIndexer = new SearchIndexer(
name: "hotel-rooms-blob-indexer",
dataSourceName: blobDataSource.Name,
targetIndexName: indexName)
{
Parameters = parameters,
Schedule = new IndexingSchedule(TimeSpan.FromDays(1))
};
blobIndexer.FieldMappings.Add(new FieldMapping("Id")
{
TargetFieldName = "HotelId"
});
// Reset the indexer if it already exists
try
{
await indexerClient.GetIndexerAsync(blobIndexer.Name);
//Rest the indexer if it exsits.
await indexerClient.ResetIndexerAsync(blobIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
}
await indexerClient.CreateOrUpdateIndexerAsync(blobIndexer);
Console.WriteLine("Running Blob Storage indexer...\n");
try
{
await indexerClient.RunIndexerAsync(blobIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 429)
{
Console.WriteLine("Failed to run indexer: {0}", ex.Message);
}
}
public static async Task Main(string[] args)
{
IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json");
IConfigurationRoot configuration = builder.Build();
if (configuration["SearchServiceEndPoint"] == "Put your search service endpoint here")
{
Console.Error.WriteLine("Specify SearchServiceEndPoint in appsettings.json");
Environment.Exit(-1);
}
if (configuration["SearchServiceAdminApiKey"] == "Put your search service admin API key here")
{
Console.Error.WriteLine("Specify SearchServiceAdminApiKey in appsettings.json");
Environment.Exit(-1);
}
if (configuration["AzureSQLConnectionString"] == "Put your Azure SQL database connection string here")
{
Console.Error.WriteLine("Specify AzureSQLConnectionString in appsettings.json");
Environment.Exit(-1);
}
SearchIndexClient indexClient = new SearchIndexClient(new Uri(configuration["SearchServiceEndPoint"]), new AzureKeyCredential(configuration["SearchServiceAdminApiKey"]));
SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(configuration["SearchServiceEndPoint"]), new AzureKeyCredential(configuration["SearchServiceAdminApiKey"]));
Console.WriteLine("Creating index...");
FieldBuilder fieldBuilder = new FieldBuilder();
var searchFields = fieldBuilder.Build(typeof(Hotel));
var searchIndex = new SearchIndex("hotels-sql-idx", searchFields);
// If we have run the sample before, this index will be populated
// We can clear the index by deleting it if it exists and creating
// it again
CleanupSearchIndexClientResources(indexClient, searchIndex);
indexClient.CreateOrUpdateIndex(searchIndex);
Console.WriteLine("Creating data source...");
// The sample data set has a table name of "hotels"
// The sample data set table has a "soft delete" column named IsDeleted
// When this column is set to true and the indexer sees it, it will remove the
// corresponding document from the search service
// See this link for more information
// https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.search.models.softdeletecolumndeletiondetectionpolicy
// The sample data set uses SQL integrated change tracking for change detection
// This means that when the indexer runs, it will be able to detect which data has
// changed since the last run using built in change tracking
// See this link for more information
// https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-tracking-sql-server
var dataSource =
new SearchIndexerDataSourceConnection(
"hotels-sql-ds",
SearchIndexerDataSourceType.AzureSql,
configuration["AzureSQLConnectionString"],
new SearchIndexerDataContainer("hotels"));
// The data source does not need to be deleted if it was already created,
// but the connection string may need to be updated if it was changed
indexerClient.CreateOrUpdateDataSourceConnection(dataSource);
Console.WriteLine("Creating Azure SQL indexer...");
var schedule = new IndexingSchedule(TimeSpan.FromDays(1))
{
StartTime = DateTimeOffset.Now
};
var parameters = new IndexingParameters()
{
BatchSize = 100,
MaxFailedItems = 0,
MaxFailedItemsPerBatch = 0
};
// Indexer declarations require a data source and search index.
// Common optional properties include a schedule, parameters, and field mappings
// The field mappings below are redundant due to how the Hotel class is defined, but
// we included them anyway to show the syntax
var indexer = new SearchIndexer("hotels-sql-idxr", dataSource.Name, searchIndex.Name)
{
Description = "Data indexer",
Schedule = schedule,
Parameters = parameters,
FieldMappings =
{
new FieldMapping("_id")
{
TargetFieldName = "HotelId"
},
new FieldMapping("Amenities")
{
TargetFieldName = "Tags"
}
}
};
// Indexers contain metadata about how much they have already indexed
// If we already ran the sample, the indexer will remember that it already
// indexed the sample data and not run again
// To avoid this, reset the indexer if it exists
CleanupSearchIndexerClientResources(indexerClient, indexer);
await indexerClient.CreateOrUpdateIndexerAsync(indexer);
// We created the indexer with a schedule, but we also
// want to run it immediately
Console.WriteLine("Running Azure SQL indexer...");
try
{
await indexerClient.RunIndexerAsync(indexer.Name);
}
catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429)
{
Console.WriteLine("Failed to run indexer: {0}", e.Response.Content);
}
// Wait 5 seconds for indexing to complete before checking status
Console.WriteLine("Waiting for indexing...\n");
System.Threading.Thread.Sleep(5000);
// After an indexer run, you can retrieve status.
CheckIndexerStatus(indexerClient, indexer);
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
Environment.Exit(0);
}
public void CreateCustomerIndex()
{
//creating index
SearchIndexClient indexClient = new SearchIndexClient(Uri, keyCredential);
SearchIndexerClient indexerClient = new SearchIndexerClient(Uri, keyCredential);
Console.WriteLine("Creating index...");
FieldBuilder fieldBuilder = new FieldBuilder();
var searchFields = fieldBuilder.Build(typeof(ServiceOrder));
//var searchIndex = new SearchIndex("serviceorder-sql-idx", searchFields);
var searchIndex = new SearchIndex("serviceorder-sql-idx", searchFields);
// If we have run the sample before, this index will be populated
// We can clear the index by deleting it if it exists and creating
// it again
CleanupSearchIndexClientResources(indexClient, searchIndex);
indexClient.CreateOrUpdateIndex(searchIndex);
//Creating data source
Console.WriteLine("Creating data source...");
var dataSource =
new SearchIndexerDataSourceConnection(
"serviceorder-sql-ds",
SearchIndexerDataSourceType.AzureSql,
azureSQLConnectionStr,
new SearchIndexerDataContainer("[ServiceOrder]"));
indexerClient.CreateOrUpdateDataSourceConnection(dataSource);
//Creating indexer
Console.WriteLine("Creating Azure SQL indexer...");
//var schedule = new IndexingSchedule(TimeSpan.FromDays(1))
//{
// StartTime = DateTimeOffset.Now
//};
var parameters = new IndexingParameters()
{
BatchSize = 100,
MaxFailedItems = 0,
MaxFailedItemsPerBatch = 0
};
// Indexer declarations require a data source and search index.
// Common optional properties include a schedule, parameters, and field mappings
// The field mappings below are redundant due to how the Hotel class is defined, but
// we included them anyway to show the syntax
var indexer = new SearchIndexer("serviceorder-sql-idxr", dataSource.Name, searchIndex.Name)
{
Description = "Service Order indexer",
Schedule = new IndexingSchedule(TimeSpan.FromMinutes(5)),
Parameters = parameters,
};
indexerClient.CreateOrUpdateIndexerAsync(indexer);
Console.WriteLine("Running Azure SQL indexer...");
try
{
indexerClient.RunIndexerAsync(indexer.Name);
}
catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429)
{
Console.WriteLine("Failed to run indexer: {0}", e.Response.Content);
}
// Wait 5 seconds for indexing to complete before checking status
Console.WriteLine("Waiting for indexing...\n");
System.Threading.Thread.Sleep(5000);
}
public void CreateIndex <T>()
{
string azureSQLConnectionStr = ConfigurationManager.AppSettings["AzureSqlConnectionString"].ToString();
Uri Uri = new Uri(ConfigurationManager.AppSettings["SearchServiceEndPoint"]);
string searchServiceKey = ConfigurationManager.AppSettings["SearchServiceAdminApiKey"];
AzureKeyCredential keyCredential = new AzureKeyCredential(searchServiceKey);
SearchIndexClient indexClient = new SearchIndexClient(Uri, keyCredential);
SearchIndexerClient indexerClient = new SearchIndexerClient(Uri, keyCredential);
Console.WriteLine("Creating index...");
_logger.LogInformation("Creating index for Orion Customer");
FieldBuilder fieldBuilder = new FieldBuilder();
var searchFields = fieldBuilder.Build(typeof(T));
var searchIndex = new SearchIndex(GetIndexName(HdsConstants.Orion, typeof(T).ToString()), searchFields);
CleanupSearchIndexClientResources(indexClient, searchIndex);
indexClient.CreateOrUpdateIndex(searchIndex);
Console.WriteLine("Creating data source...");
_logger.LogInformation("Creating data source for Orion Customer");
var dataSource =
new SearchIndexerDataSourceConnection(
GetDataSourceName(HdsConstants.Orion, typeof(T).ToString()),
SearchIndexerDataSourceType.AzureSql,
azureSQLConnectionStr,
new SearchIndexerDataContainer($"[{GetTableName(typeof(T).ToString())}]"));
indexerClient.CreateOrUpdateDataSourceConnection(dataSource);
//Creating indexer
Console.WriteLine("Creating Azure SQL indexer...");
_logger.LogInformation("Creating Azure SQL indexer for Orion Customer");
var schedule = new IndexingSchedule(TimeSpan.FromDays(1))
{
StartTime = DateTimeOffset.Now
};
var parameters = new IndexingParameters()
{
BatchSize = 100,
MaxFailedItems = 0,
MaxFailedItemsPerBatch = 0
};
var indexer = new SearchIndexer(GetIndexerName(HdsConstants.Orion, typeof(T).ToString()), dataSource.Name, searchIndex.Name)
{
Description = "Data indexer",
Schedule = schedule,
Parameters = parameters,
};
indexerClient.CreateOrUpdateIndexerAsync(indexer);
Console.WriteLine("Running Azure SQL indexer...");
try
{
indexerClient.RunIndexerAsync(indexer.Name);
}
catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429)
{
Console.WriteLine("Failed to run indexer: {0}", e.Response.Content);
_logger.LogError("Failed to run indexer: {0}", e.Response.Content);
}
}