private async Task CreateAndRunBlobIndexerAsync(CancellationToken cancellationToken)
{
SearchIndexerDataSourceConnection blobDataSource = new SearchIndexerDataSourceConnection(
name: _azureBlobOptions.Value.Name,
type: SearchIndexerDataSourceType.AzureBlob,
connectionString: _azureBlobOptions.Value.ConnectionString,
container: _searchIndexerDataContainer);
await _searchIndexerClient.CreateOrUpdateDataSourceConnectionAsync(blobDataSource, false, cancellationToken);
try
{
await _searchIndexerClient.GetIndexerAsync(_azureBlobIndexer.Name);
await _searchIndexerClient.ResetIndexerAsync(_azureBlobIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
}
await _searchIndexerClient.CreateOrUpdateIndexerAsync(_azureBlobIndexer);
try
{
// Run the indexer.
await _searchIndexerClient.RunIndexerAsync(_azureBlobIndexer.Name, cancellationToken);
}
catch (CloudException e) when(e.Response.StatusCode == (HttpStatusCode)429)
{
Console.WriteLine("Failed to run indexer: {0}", e.Response.Content);
}
}
public static async Task <SearchIndexerDataSourceConnection> CreateOrUpdateAzureBlobDataSourceAsync(SearchIndexerClient indexerClient,
string blobConnectionString, string indexName, string blobContainerName)
{
SearchIndexerDataSourceConnection dataSource = new SearchIndexerDataSourceConnection(
name: indexName + "-azureblobdatasource",
type: SearchIndexerDataSourceType.AzureBlob,
connectionString: blobConnectionString,
container: new SearchIndexerDataContainer(blobContainerName))
{
Description = "Files to demonstrate cognitive search capabilities."
};
// The data source does not need to be deleted if it was already created
// since we are using the CreateOrUpdate method
try
{
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Failed to create or update the data source\n Exception message: {0}\n", ex.Message);
ExitProgram("Cannot continue without a data source");
}
return(dataSource);
}
private static async Task CreateAndRunCosmosDbIndexerAsync(string indexName, SearchIndexerClient indexerClient)
{
// Append the database name to the connection string
string cosmosConnectString =
configuration["CosmosDBConnectionString"]
+ ";Database="
+ configuration["CosmosDBDatabaseName"];
SearchIndexerDataSourceConnection cosmosDbDataSource = new SearchIndexerDataSourceConnection(
name: configuration["CosmosDBDatabaseName"],
type: SearchIndexerDataSourceType.CosmosDb,
connectionString: cosmosConnectString,
container: new SearchIndexerDataContainer("hotels"));
// The Cosmos DB data source does not need to be deleted if it already exists,
// but the connection string might need to be updated if it has changed.
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(cosmosDbDataSource);
Console.WriteLine("Creating Cosmos DB indexer...\n");
SearchIndexer cosmosDbIndexer = new SearchIndexer(
name: "hotel-rooms-cosmos-indexer",
dataSourceName: cosmosDbDataSource.Name,
targetIndexName: indexName)
{
Schedule = new IndexingSchedule(TimeSpan.FromDays(1))
};
// Indexers keep metadata about how much they have already indexed.
// If we already ran this sample, the indexer will remember that it already
// indexed the sample data and not run again.
// To avoid this, reset the indexer if it exists.
try
{
await indexerClient.GetIndexerAsync(cosmosDbIndexer.Name);
//Rest the indexer if it exsits.
await indexerClient.ResetIndexerAsync(cosmosDbIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
//if the specified indexer not exist, 404 will be thrown.
}
await indexerClient.CreateOrUpdateIndexerAsync(cosmosDbIndexer);
Console.WriteLine("Running Cosmos DB indexer...\n");
try
{
await indexerClient.RunIndexerAsync(cosmosDbIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 429)
{
Console.WriteLine("Failed to run indexer: {0}", ex.Message);
}
}
private static async Task CreateAndRunSQLIndexerAsync(string indexName, SearchIndexerClient indexerClient)
{
SearchIndexerDataSourceConnection sqlDataSource = new SearchIndexerDataSourceConnection(
name: configuration["SQLDatabaseName"],
type: SearchIndexerDataSourceType.AzureSql,
connectionString: configuration["SQLConnectSctring"],
container: new SearchIndexerDataContainer("books"));
// The data source does not need to be deleted if it already exists,
// but the connection string might need to be updated if it has changed.
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(sqlDataSource);
Console.WriteLine("Creating SQL indexer...\n");
SearchIndexer sqlIndexer = new SearchIndexer(
name: "books-indexer",
dataSourceName: sqlDataSource.Name,
targetIndexName: indexName)
{
//here you can set the desired schedule for indexing repetitions
Schedule = new IndexingSchedule(TimeSpan.FromDays(1))
};
// Indexers keep metadata about how much they have already indexed.
// If we already ran this sample, the indexer will remember that it already
// indexed the sample data and not run again.
// To avoid this, reset the indexer if it exists.
try
{
await indexerClient.GetIndexerAsync(sqlIndexer.Name);
//Rest the indexer if it exsits.
await indexerClient.ResetIndexerAsync(sqlIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
//if the specified indexer not exist, 404 will be thrown.
}
await indexerClient.CreateOrUpdateIndexerAsync(sqlIndexer);
Console.WriteLine("Running SQL indexer...\n");
try
{
await indexerClient.RunIndexerAsync(sqlIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 429)
{
Console.WriteLine("Failed to run sql indexer: {0}", ex.Message);
}
}
public async Task CreateDataSourceConnectionAsync(string dataSourceName, string containerName, string connectionString)
{
try
{
SearchIndexerDataContainer searchIndexerDataContainer = new SearchIndexerDataContainer(containerName);
SearchIndexerDataSourceConnection searchIndexerDataSourceConnection = new SearchIndexerDataSourceConnection(
dataSourceName,
SearchIndexerDataSourceType.AzureBlob,
connectionString,
searchIndexerDataContainer);
await _searchIndexerClient.CreateOrUpdateDataSourceConnectionAsync(searchIndexerDataSourceConnection);
}
catch (Exception)
{
throw new Exception("Cognitive Search APIs -> Failed To Create Data Source Connection!");
}
}
private static async Task CreateAndRunBlobIndexerAsync(string indexName, SearchIndexerClient indexerClient)
{
SearchIndexerDataSourceConnection blobDataSource = new SearchIndexerDataSourceConnection(
name: configuration["BlobStorageAccountName"],
type: SearchIndexerDataSourceType.AzureBlob,
connectionString: configuration["BlobStorageConnectionString"],
container: new SearchIndexerDataContainer("gapzap-pdf-docs"));
// The blob data source does not need to be deleted if it already exists,
// but the connection string might need to be updated if it has changed.
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(blobDataSource);
Console.WriteLine("Creating Blob Storage indexer...\n");
// Add a field mapping to match the Id field in the documents to
// the HotelId key field in the index
List <FieldMapping> map = new List <FieldMapping> {
new FieldMapping("Id")
{
TargetFieldName = "HotelId"
}
};
IndexingParameters parameters = new IndexingParameters();
parameters.Configuration.Add("parsingMode", "json");
SearchIndexer blobIndexer = new SearchIndexer(
name: "hotel-rooms-blob-indexer",
dataSourceName: blobDataSource.Name,
targetIndexName: indexName)
{
Parameters = parameters,
Schedule = new IndexingSchedule(TimeSpan.FromDays(1))
};
blobIndexer.FieldMappings.Add(new FieldMapping("Id")
{
TargetFieldName = "HotelId"
});
// Reset the indexer if it already exists
try
{
await indexerClient.GetIndexerAsync(blobIndexer.Name);
//Rest the indexer if it exsits.
await indexerClient.ResetIndexerAsync(blobIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
}
await indexerClient.CreateOrUpdateIndexerAsync(blobIndexer);
Console.WriteLine("Running Blob Storage indexer...\n");
try
{
await indexerClient.RunIndexerAsync(blobIndexer.Name);
}
catch (RequestFailedException ex) when(ex.Status == 429)
{
Console.WriteLine("Failed to run indexer: {0}", ex.Message);
}
}
static async Task CreateSearchResources(AppSettings settings)
{
SearchIndexClient indexClient = new SearchIndexClient(settings.SearchEndpointUri, settings.SearchKeyCredential);
Console.WriteLine("Deleting search index {0} if exists...", SEARCH_ACL_INDEX_NAME);
try
{
await indexClient.GetIndexAsync(SEARCH_ACL_INDEX_NAME);
await indexClient.DeleteIndexAsync(SEARCH_ACL_INDEX_NAME);
}
catch (RequestFailedException)
{
// Index didn't exist - continue
}
Console.WriteLine("Creating search index {0}...", SEARCH_ACL_INDEX_NAME);
await indexClient.CreateOrUpdateIndexAsync(
new SearchIndex(SEARCH_ACL_INDEX_NAME, fields : new[]
{
new SearchField("key", SearchFieldDataType.String)
{
IsKey = true
},
new SearchField("metadata_storage_path", SearchFieldDataType.String),
new SearchField("content", SearchFieldDataType.String)
}));
Console.WriteLine("Creating search data source {0}...", SEARCH_ACL_DATASOURCE_NAME);
SearchIndexerClient indexerClient = new SearchIndexerClient(settings.SearchEndpointUri, settings.SearchKeyCredential);
await indexerClient.CreateOrUpdateDataSourceConnectionAsync(
new SearchIndexerDataSourceConnection(
name : SEARCH_ACL_DATASOURCE_NAME,
type : SearchIndexerDataSourceType.AzureBlob,
connectionString : "ResourceId=" + settings.DataLakeResourceID,
container : new SearchIndexerDataContainer(name : DATA_LAKE_FILESYSTEM_NAME)));
Console.WriteLine("Deleting search indexer {0} if exists...", SEARCH_ACL_INDEXER_NAME);
try
{
await indexerClient.GetIndexerAsync(SEARCH_ACL_INDEXER_NAME);
await indexerClient.DeleteIndexerAsync(SEARCH_ACL_INDEXER_NAME);
}
catch (RequestFailedException)
{
// Indexer didn't exist - continue
}
Console.WriteLine("Creating search indexer {0}...", SEARCH_ACL_INDEXER_NAME);
await indexerClient.CreateIndexerAsync(
new SearchIndexer(
name : SEARCH_ACL_INDEXER_NAME,
dataSourceName : SEARCH_ACL_DATASOURCE_NAME,
targetIndexName : SEARCH_ACL_INDEX_NAME)
{
Parameters = new IndexingParameters
{
MaxFailedItems = -1,
IndexingParametersConfiguration = new IndexingParametersConfiguration
{
ParsingMode = BlobIndexerParsingMode.Text
}
}
});
}