internal HttpMessage CreateCreateOrUpdateRequest(string skillsetName, SearchIndexerSkillset skillset, string ifMatch, string ifNoneMatch) { var message = _pipeline.CreateMessage(); var request = message.Request; request.Method = RequestMethod.Put; var uri = new RawRequestUriBuilder(); uri.AppendRaw(endpoint, false); uri.AppendPath("/skillsets('", false); uri.AppendPath(skillsetName, true); uri.AppendPath("')", false); uri.AppendQuery("api-version", apiVersion, true); request.Uri = uri; if (ifMatch != null) { request.Headers.Add("If-Match", ifMatch); } if (ifNoneMatch != null) { request.Headers.Add("If-None-Match", ifNoneMatch); } request.Headers.Add("Prefer", "return=representation"); request.Headers.Add("Accept", "application/json; odata.metadata=minimal"); request.Headers.Add("Content-Type", "application/json"); var content = new Utf8JsonRequestContent(); content.JsonWriter.WriteObjectValue(skillset); request.Content = content; return(message); }
/// <summary> /// Creates a new skillset or updates an existing skillset. /// </summary> /// <param name="skillset">Required. The <see cref="SearchIndexerSkillset"/> to create or update.</param> /// <param name="onlyIfUnchanged"> /// True to throw a <see cref="RequestFailedException"/> if the <see cref="SearchIndexerSkillset.ETag"/> does not match the current service version; /// otherwise, the current service version will be overwritten. /// </param> /// <param name="ignoreCacheResetRequirements">Ignores cache reset requirements.</param> /// <param name="disableCacheReprocessingChangeDetection">Disables cache reprocessing change detection.</param> /// <param name="cancellationToken">Optional <see cref="CancellationToken"/> to propagate notifications that the operation should be canceled.</param> /// <returns> /// The <see cref="Response{T}"/> from the server containing the <see cref="SearchIndexerSkillset"/> that was created. /// This may differ slightly from what was passed in since the service may return back properties set to their default values. /// </returns> /// <exception cref="ArgumentNullException">Thrown when <paramref name="skillset"/> is null.</exception> /// <exception cref="RequestFailedException">Thrown when a failure is returned by the Search service.</exception> public virtual async Task <Response <SearchIndexerSkillset> > CreateOrUpdateSkillsetAsync( SearchIndexerSkillset skillset, bool onlyIfUnchanged = false, bool?ignoreCacheResetRequirements = null, bool?disableCacheReprocessingChangeDetection = null, CancellationToken cancellationToken = default) { // The REST client uses a different parameter name that would be confusing to reference. Argument.AssertNotNull(skillset, nameof(skillset)); using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(SearchIndexerClient)}.{nameof(CreateOrUpdateSkillset)}"); scope.Start(); try { return(await SkillsetsClient.CreateOrUpdateAsync( skillset?.Name, skillset, onlyIfUnchanged?skillset?.ETag?.ToString() : null, null, ignoreCacheResetRequirements, disableCacheReprocessingChangeDetection, cancellationToken) .ConfigureAwait(false)); } catch (Exception ex) { scope.Failed(ex); throw; } }
public async Task <Response <SearchIndexerSkillset> > CreateOrUpdateAsync(string skillsetName, SearchIndexerSkillset skillset, string ifMatch = null, string ifNoneMatch = null, CancellationToken cancellationToken = default) { if (skillsetName == null) { throw new ArgumentNullException(nameof(skillsetName)); } if (skillset == null) { throw new ArgumentNullException(nameof(skillset)); } using var message = CreateCreateOrUpdateRequest(skillsetName, skillset, ifMatch, ifNoneMatch); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); switch (message.Response.Status) { case 200: case 201: { SearchIndexerSkillset value = default; using var document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false); value = SearchIndexerSkillset.DeserializeSearchIndexerSkillset(document.RootElement); return(Response.FromValue(value, message.Response)); }
public async Task <string> CreateIndexAndIndexerAsync() { // Create or Update the data source SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(_indexerClient); // Create the skills OcrSkill ocrSkill = CreateOcrSkill(); MergeSkill mergeSkill = CreateMergeSkill(); EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill(); LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill(); SplitSkill splitSkill = CreateSplitSkill(); KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill(); // Create the skillset List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>(); skills.Add(ocrSkill); skills.Add(mergeSkill); skills.Add(languageDetectionSkill); skills.Add(splitSkill); skills.Add(entityRecognitionSkill); skills.Add(keyPhraseExtractionSkill); SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(_indexerClient, skills, _cognitiveServicesKey); // Create the index SearchIndex demoIndex = await CreateDemoIndexAsync(_indexClient); // Create the indexer, map fields, and execute transformations SearchIndexer demoIndexer = await CreateDemoIndexerAsync(_indexerClient, dataSource, skillset, demoIndex); // Check indexer overall status return(await CheckIndexerOverallStatusAsync(_indexerClient, demoIndexer.Name)); }
#pragma warning disable AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. public virtual Response <SearchIndexerSkillset> CreateOrUpdateSkillset( #pragma warning restore AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. SearchIndexerSkillset skillset, bool onlyIfUnchanged, CancellationToken cancellationToken) => CreateOrUpdateSkillset( skillset, onlyIfUnchanged, ignoreCacheResetRequirements: null, disableCacheReprocessingChangeDetection: null, cancellationToken);
#pragma warning disable AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. public virtual async Task <Response <SearchIndexerSkillset> > CreateOrUpdateSkillsetAsync( #pragma warning restore AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. SearchIndexerSkillset skillset, bool onlyIfUnchanged, bool disableCacheReprocessingChangeDetection, bool ignoreCacheResetRequirements, CancellationToken cancellationToken) => await CreateOrUpdateSkillsetAsync( skillset, onlyIfUnchanged, ignoreCacheResetRequirements, disableCacheReprocessingChangeDetection, cancellationToken).ConfigureAwait(false);
public static void Main(string[] args) { // Create service client IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json"); IConfigurationRoot configuration = builder.Build(); string searchServiceUri = configuration["SearchServiceUri"]; string adminApiKey = configuration["SearchServiceAdminApiKey"]; string cognitiveServicesKey = configuration["CognitiveServicesKey"]; SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); // Create or Update the data source Console.WriteLine("Creating or updating the data source..."); SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration); // Create the skills Console.WriteLine("Creating the skills..."); OcrSkill ocrSkill = CreateOcrSkill(); MergeSkill mergeSkill = CreateMergeSkill(); EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill(); LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill(); SplitSkill splitSkill = CreateSplitSkill(); KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill(); // Create the skillset Console.WriteLine("Creating or updating the skillset..."); List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>(); skills.Add(ocrSkill); skills.Add(mergeSkill); skills.Add(languageDetectionSkill); skills.Add(splitSkill); skills.Add(entityRecognitionSkill); skills.Add(keyPhraseExtractionSkill); SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey); // Create the index Console.WriteLine("Creating the index..."); SearchIndex demoIndex = CreateDemoIndex(indexClient); // Create the indexer, map fields, and execute transformations Console.WriteLine("Creating the indexer and executing the pipeline..."); SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex); // Check indexer overall status Console.WriteLine("Check the indexer overall status..."); CheckIndexerOverallStatus(indexerClient, demoIndexer); }
/// <summary> /// Deletes a skillset. /// </summary> /// <param name="skillset">The <see cref="SearchIndexerSkillset"/> to delete.</param> /// <param name="onlyIfUnchanged"> /// True to throw a <see cref="RequestFailedException"/> if the <see cref="SearchIndexerSkillset.ETag"/> does not match the current service version; /// otherwise, the current service version will be overwritten. /// </param> /// <param name="cancellationToken">Optional <see cref="CancellationToken"/> to propagate notifications that the operation should be canceled.</param> /// <returns>The <see cref="Response"/> from the server.</returns> /// <exception cref="ArgumentNullException">Thrown when <paramref name="skillset"/> is null.</exception> /// <exception cref="RequestFailedException">Thrown when a failure is returned by the Search service.</exception> public virtual Response DeleteSkillset( SearchIndexerSkillset skillset, bool onlyIfUnchanged = false, CancellationToken cancellationToken = default) { // The REST client uses a different parameter name that would be confusing to reference. Argument.AssertNotNull(skillset, nameof(skillset)); return(DeleteSkillset( skillset?.Name, skillset?.ETag, onlyIfUnchanged, cancellationToken)); }
public virtual Response <SearchIndexerSkillset> Create(SearchIndexerSkillset skillset, Guid?xMsClientRequestId = null, CancellationToken cancellationToken = default) { using var scope = _clientDiagnostics.CreateScope("SkillsetsClient.Create"); scope.Start(); try { return(RestClient.Create(skillset, xMsClientRequestId, cancellationToken)); } catch (Exception e) { scope.Failed(e); throw; } }
/// <summary> /// Creates a new skillset. /// </summary> /// <param name="skillset">Required. The <see cref="SearchIndexerSkillset"/> to create.</param> /// <param name="cancellationToken">Optional <see cref="CancellationToken"/> to propagate notifications that the operation should be canceled.</param> /// <returns> /// The <see cref="Response{T}"/> from the server containing the <see cref="SearchIndexerSkillset"/> that was created. /// This may differ slightly from what was passed in since the service may return back properties set to their default values. /// </returns> /// <exception cref="ArgumentNullException">Thrown when <paramref name="skillset"/> is null.</exception> /// <exception cref="RequestFailedException">Thrown when a failure is returned by the Search service.</exception> public virtual Response <SearchIndexerSkillset> CreateSkillset( SearchIndexerSkillset skillset, CancellationToken cancellationToken = default) { using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(SearchIndexerClient)}.{nameof(CreateSkillset)}"); scope.Start(); try { return(SkillsetsClient.Create( skillset, cancellationToken)); } catch (Exception ex) { scope.Failed(ex); throw; } }
private static async Task <bool> CreateSkillSet() { Console.WriteLine("Creating Skill Set..."); try { SearchIndexerSkillset skillset = SearchResources.GetSkillset(SkillsetName, BlobContainerNameForImageStore); await _searchIndexerClient.CreateSkillsetAsync(skillset); } catch (Exception ex) { if (DebugMode) { Console.WriteLine("Error creating skillset: {0}", ex.Message); } return(false); } return(true); }
/// <summary> /// Creates a new skillset. /// </summary> /// <param name="skillset">Required. The <see cref="SearchIndexerSkillset"/> to create.</param> /// <param name="cancellationToken">Optional <see cref="CancellationToken"/> to propagate notifications that the operation should be canceled.</param> /// <returns> /// The <see cref="Response{T}"/> from the server containing the <see cref="SearchIndexerSkillset"/> that was created. /// This may differ slightly from what was passed in since the service may return back properties set to their default values. /// </returns> /// <exception cref="ArgumentNullException">Thrown when <paramref name="skillset"/> is null.</exception> /// <exception cref="RequestFailedException">Thrown when a failure is returned by the Search service.</exception> public virtual async Task <Response <SearchIndexerSkillset> > CreateSkillsetAsync( SearchIndexerSkillset skillset, CancellationToken cancellationToken = default) { using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(SearchIndexerClient)}.{nameof(CreateSkillset)}"); scope.Start(); try { return(await SkillsetsClient.CreateAsync( skillset, cancellationToken) .ConfigureAwait(false)); } catch (Exception ex) { scope.Failed(ex); throw; } }
private SearchIndexerSkillset CreateOrUpdateDemoSkillSet(SearchIndexerClient indexerClient, IList <SearchIndexerSkill> skills, string cognitiveServicesKey) { SearchIndexerSkillset skillset = new SearchIndexerSkillset("demoskillset", skills) { Description = "Demo skillset", CognitiveServicesAccount = new CognitiveServicesAccountKey(cognitiveServicesKey) }; // Create the skillset in your search service. // The skillset does not need to be deleted if it was already created // since we are using the CreateOrUpdate method try { indexerClient.CreateOrUpdateSkillset(skillset); } catch (RequestFailedException ex) { throw new Exception("Failed to create the skillset", ex); } return(skillset); }
private static SearchIndexerSkillset CreateOrUpdateDemoSkillSet(SearchIndexerClient indexerClient, IList <SearchIndexerSkill> skills, string cognitiveServicesKey) { SearchIndexerSkillset skillset = new SearchIndexerSkillset("demoskillset", skills) { Description = "Demo skillset", CognitiveServicesAccount = new CognitiveServicesAccountKey(cognitiveServicesKey) }; // Create the skillset in your search service. // The skillset does not need to be deleted if it was already created // since we are using the CreateOrUpdate method try { indexerClient.CreateOrUpdateSkillset(skillset); } catch (RequestFailedException ex) { Console.WriteLine("Failed to create the skillset\n Exception message: {0}\n", ex.Message); ExitProgram("Cannot continue without a skillset"); } return(skillset); }
public async ValueTask <Response <SearchIndexerSkillset> > CreateOrUpdateAsync(string skillsetName, SearchIndexerSkillset skillset, Guid?xMsClientRequestId = null, string ifMatch = null, string ifNoneMatch = null, CancellationToken cancellationToken = default) { if (skillsetName == null) { throw new ArgumentNullException(nameof(skillsetName)); } if (skillset == null) { throw new ArgumentNullException(nameof(skillset)); } using var scope = _clientDiagnostics.CreateScope("SkillsetsClient.CreateOrUpdate"); scope.Start(); try { using var message = CreateCreateOrUpdateRequest(skillsetName, skillset, xMsClientRequestId, ifMatch, ifNoneMatch); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); switch (message.Response.Status) { case 200: case 201: { SearchIndexerSkillset value = default; using var document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false); if (document.RootElement.ValueKind == JsonValueKind.Null) { value = null; } else { value = SearchIndexerSkillset.DeserializeSearchIndexerSkillset(document.RootElement); } return(Response.FromValue(value, message.Response)); }
public async Task CreateIndexerAsync() { await using SearchResources resources = await SearchResources.CreateWithBlobStorageAsync(this, populate : true); Environment.SetEnvironmentVariable("SEARCH_ENDPOINT", resources.Endpoint.ToString()); Environment.SetEnvironmentVariable("SEARCH_API_KEY", resources.PrimaryApiKey); Environment.SetEnvironmentVariable("STORAGE_CONNECTION_STRING", resources.StorageAccountConnectionString); Environment.SetEnvironmentVariable("STORAGE_CONTAINER", resources.BlobContainerName); Environment.SetEnvironmentVariable("COGNITIVE_SERVICES_KEY", resources.CognitiveServicesKey); // Define clean up tasks to be invoked in reverse order added. Stack <Func <Task> > cleanUpTasks = new Stack <Func <Task> >(); try { #region Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateSynonymMap // Create a new SearchIndexClient Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT")); AzureKeyCredential credential = new AzureKeyCredential( Environment.GetEnvironmentVariable("SEARCH_API_KEY")); SearchIndexClient indexClient = new SearchIndexClient(endpoint, credential); #if !SNIPPET indexClient = resources.GetIndexClient(new SearchClientOptions()); #endif // Create a synonym map from a file containing country names and abbreviations // using the Solr format with entry on a new line using \n, for example: // United States of America,US,USA\n string synonymMapName = "countries"; #if !SNIPPET synonymMapName = Recording.Random.GetName(); #endif string synonymMapPath = "countries.txt"; #if !SNIPPET synonymMapPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Samples", "countries.txt"); #endif SynonymMap synonyms; #if SNIPPET using (StreamReader file = File.OpenText(synonymMapPath)) { synonyms = new SynonymMap(synonymMapName, file); } #else synonyms = new SynonymMap(synonymMapName, CountriesSolrSynonymMap); #endif await indexClient.CreateSynonymMapAsync(synonyms); #endregion Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateSynonymMap // Make sure our synonym map gets deleted, which is not deleted when our // index is deleted when our SearchResources goes out of scope. cleanUpTasks.Push(() => indexClient.DeleteSynonymMapAsync(synonymMapName)); #region Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateIndex // Create the index string indexName = "hotels"; #if !SNIPPET indexName = Recording.Random.GetName(); #endif SearchIndex index = new SearchIndex(indexName) { Fields = { new SimpleField("hotelId", SearchFieldDataType.String) { IsKey = true, IsFilterable = true, IsSortable = true }, new SearchableField("hotelName") { IsFilterable = true, IsSortable = true }, new SearchableField("description") { AnalyzerName = LexicalAnalyzerName.EnLucene }, new SearchableField("descriptionFr") { AnalyzerName = LexicalAnalyzerName.FrLucene }, new SearchableField("tags", collection: true) { IsFilterable = true, IsFacetable = true }, new ComplexField("address") { Fields = { new SearchableField("streetAddress"), new SearchableField("city") { IsFilterable = true, IsSortable = true, IsFacetable = true }, new SearchableField("stateProvince") { IsFilterable = true, IsSortable = true, IsFacetable = true }, new SearchableField("country") { SynonymMapNames = new[] { synonymMapName },IsFilterable = true, IsSortable = true,IsFacetable = true }, new SearchableField("postalCode") { IsFilterable = true, IsSortable = true, IsFacetable = true } } } } }; await indexClient.CreateIndexAsync(index); #endregion Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateIndex // Make sure our synonym map gets deleted, which is not deleted when our // index is deleted when our SearchResources goes out of scope. cleanUpTasks.Push(() => indexClient.DeleteIndexAsync(indexName)); #region Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateDataSourceConnection // Create a new SearchIndexerClient SearchIndexerClient indexerClient = new SearchIndexerClient(endpoint, credential); #if !SNIPPET indexerClient = resources.GetIndexerClient(); #endif string dataSourceConnectionName = "hotels"; #if !SNIPPET dataSourceConnectionName = Recording.Random.GetName(); #endif SearchIndexerDataSourceConnection dataSourceConnection = new SearchIndexerDataSourceConnection( dataSourceConnectionName, SearchIndexerDataSourceType.AzureBlob, Environment.GetEnvironmentVariable("STORAGE_CONNECTION_STRING"), new SearchIndexerDataContainer(Environment.GetEnvironmentVariable("STORAGE_CONTAINER"))); await indexerClient.CreateDataSourceConnectionAsync(dataSourceConnection); #endregion Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateDataSourceConnection // Make sure our data source gets deleted, which is not deleted when our // index is deleted when our SearchResources goes out of scope. cleanUpTasks.Push(() => indexerClient.DeleteDataSourceConnectionAsync(dataSourceConnectionName)); #region Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_Skillset // Translate English descriptions to French. // See https://docs.microsoft.com/azure/search/cognitive-search-skill-text-translation for details of the Text Translation skill. TextTranslationSkill translationSkill = new TextTranslationSkill( inputs: new[] { new InputFieldMappingEntry("text") { Source = "/document/description" } }, outputs: new[] { new OutputFieldMappingEntry("translatedText") { TargetName = "descriptionFrTranslated" } }, TextTranslationSkillLanguage.Fr) { Name = "descriptionFrTranslation", Context = "/document", DefaultFromLanguageCode = TextTranslationSkillLanguage.En }; // Use the human-translated French description if available; otherwise, use the translated description. // See https://docs.microsoft.com/azure/search/cognitive-search-skill-conditional for details of the Conditional skill. ConditionalSkill conditionalSkill = new ConditionalSkill( inputs: new[] { new InputFieldMappingEntry("condition") { Source = "= $(/document/descriptionFr) == null" }, new InputFieldMappingEntry("whenTrue") { Source = "/document/descriptionFrTranslated" }, new InputFieldMappingEntry("whenFalse") { Source = "/document/descriptionFr" } }, outputs: new[] { new OutputFieldMappingEntry("output") { TargetName = "descriptionFrFinal" } }) { Name = "descriptionFrConditional", Context = "/document", }; // Create a SearchIndexerSkillset that processes those skills in the order given below. string skillsetName = "translations"; #if !SNIPPET skillsetName = Recording.Random.GetName(); #endif SearchIndexerSkillset skillset = new SearchIndexerSkillset( skillsetName, new SearchIndexerSkill[] { translationSkill, conditionalSkill }) { CognitiveServicesAccount = new CognitiveServicesAccountKey( Environment.GetEnvironmentVariable("COGNITIVE_SERVICES_KEY")), KnowledgeStore = new SearchIndexerKnowledgeStore( Environment.GetEnvironmentVariable("STORAGE_CONNECTION_STRING"), new List <SearchIndexerKnowledgeStoreProjection>()), }; await indexerClient.CreateSkillsetAsync(skillset); #endregion Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_Skillset // Make sure our skillset gets deleted, which is not deleted when our // index is deleted when our SearchResources goes out of scope. cleanUpTasks.Push(() => indexerClient.DeleteSkillsetAsync(skillsetName)); #region Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateIndexer string indexerName = "hotels"; #if !SNIPPET indexerName = Recording.Random.GetName(); #endif SearchIndexer indexer = new SearchIndexer( indexerName, dataSourceConnectionName, indexName) { // We only want to index fields defined in our index, excluding descriptionFr if defined. FieldMappings = { new FieldMapping("hotelId"), new FieldMapping("hotelName"), new FieldMapping("description"), new FieldMapping("tags"), new FieldMapping("address") }, OutputFieldMappings = { new FieldMapping("/document/descriptionFrFinal") { TargetFieldName = "descriptionFr" } }, Parameters = new IndexingParameters { // Tell the indexer to parse each blob as a separate JSON document. IndexingParametersConfiguration = new IndexingParametersConfiguration { ParsingMode = BlobIndexerParsingMode.Json } }, SkillsetName = skillsetName }; // Create the indexer which, upon successful creation, also runs the indexer. await indexerClient.CreateIndexerAsync(indexer); #endregion Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateIndexer // Make sure our indexer gets deleted, which is not deleted when our // index is deleted when our SearchResources goes out of scope. cleanUpTasks.Push(() => indexerClient.DeleteIndexerAsync(indexerName)); // Wait till the indexer is done. await WaitForIndexingAsync(indexerClient, indexerName); #region Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_Query // Get a SearchClient from the SearchIndexClient to share its pipeline. SearchClient searchClient = indexClient.GetSearchClient(indexName); #if !SNIPPET searchClient = InstrumentClient(new SearchClient(endpoint, indexName, credential, GetSearchClientOptions())); #endif // Query for hotels with an ocean view. SearchResults <Hotel> results = await searchClient.SearchAsync <Hotel>("ocean view"); #if !SNIPPET bool found = false; #endif await foreach (SearchResult <Hotel> result in results.GetResultsAsync()) { Hotel hotel = result.Document; #if !SNIPPET if (hotel.HotelId == "6") { Assert.IsNotNull(hotel.DescriptionFr); found = true; } #endif Console.WriteLine($"{hotel.HotelName} ({hotel.HotelId})"); Console.WriteLine($" Description (English): {hotel.Description}"); Console.WriteLine($" Description (French): {hotel.DescriptionFr}"); } #endregion Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_Query Assert.IsTrue(found, "Expected hotel #6 not found in search results"); } finally { // We want to await these individual to create a deterministic order for playing back tests. foreach (Func <Task> cleanUpTask in cleanUpTasks) { await cleanUpTask(); } } }
public virtual async Task <Response <SearchIndexerSkillset> > CreateAsync(SearchIndexerSkillset skillset, Guid?xMsClientRequestId = null, CancellationToken cancellationToken = default) { return(await RestClient.CreateAsync(skillset, xMsClientRequestId, cancellationToken).ConfigureAwait(false)); }
public virtual Response <SearchIndexerSkillset> Create(SearchIndexerSkillset skillset, Guid?xMsClientRequestId = null, CancellationToken cancellationToken = default) { return(RestClient.Create(skillset, xMsClientRequestId, cancellationToken)); }
public virtual async Task <Response <SearchIndexerSkillset> > CreateOrUpdateAsync(string skillsetName, SearchIndexerSkillset skillset, Guid?xMsClientRequestId = null, string ifMatch = null, string ifNoneMatch = null, CancellationToken cancellationToken = default) { return(await RestClient.CreateOrUpdateAsync(skillsetName, skillset, xMsClientRequestId, ifMatch, ifNoneMatch, cancellationToken).ConfigureAwait(false)); }
public static async Task <SearchIndexer> CreateIndexerAsync(SearchIndexerClient indexerClient, SearchIndexerDataSourceConnection dataSource, SearchIndexerSkillset skillSet, SearchIndex index) { IndexingParameters indexingParameters = new IndexingParameters() { MaxFailedItems = -1, MaxFailedItemsPerBatch = -1, }; indexingParameters.IndexingParametersConfiguration = new IndexingParametersConfiguration(); indexingParameters.IndexingParametersConfiguration.DataToExtract = BlobIndexerDataToExtract.ContentAndMetadata; indexingParameters.IndexingParametersConfiguration.ParsingMode = BlobIndexerParsingMode.Text; string indexerName = index.Name + "-indexer"; SearchIndexer indexer = new SearchIndexer(indexerName, dataSource.Name, index.Name) { Description = index.Name + " Indexer", SkillsetName = skillSet.Name, Parameters = indexingParameters }; FieldMappingFunction mappingFunction = new FieldMappingFunction("base64Encode"); mappingFunction.Parameters.Add("useHttpServerUtilityUrlTokenEncode", true); indexer.FieldMappings.Add(new FieldMapping("metadata_storage_path") { TargetFieldName = "metadata_storage_path", MappingFunction = mappingFunction }); //indexer.FieldMappings.Add(new FieldMapping("metadata_storage_name") //{ // TargetFieldName = "FileName" //}); //indexer.FieldMappings.Add(new FieldMapping("content")5 //{ // TargetFieldName = "Content" //}); //indexer.OutputFieldMappings.Add(new FieldMapping("/document/pages/*/organizations/*") //{ // TargetFieldName = "organizations" //}); //indexer.OutputFieldMappings.Add(new FieldMapping("/document/pages/*/keyPhrases/*") //{ // TargetFieldName = "keyPhrases" //}); //indexer.OutputFieldMappings.Add(new FieldMapping("/document/languageCode") //{ // TargetFieldName = "languageCode" //}); try { await indexerClient.GetIndexerAsync(indexer.Name); await indexerClient.DeleteIndexerAsync(indexer.Name); } catch (RequestFailedException ex) when(ex.Status == 404) { //if the specified indexer not exist, 404 will be thrown. } try { await indexerClient.CreateIndexerAsync(indexer); } catch (RequestFailedException ex) { Console.WriteLine("Failed to create the indexer\n Exception message: {0}\n", ex.Message); ExitProgram("Cannot continue without creating an indexer"); } return(indexer); }
private static SearchIndexer CreateDemoIndexer(SearchIndexerClient indexerClient, SearchIndexerDataSourceConnection dataSource, SearchIndexerSkillset skillSet, SearchIndex index) { IndexingParameters indexingParameters = new IndexingParameters() { MaxFailedItems = -1, MaxFailedItemsPerBatch = -1, }; indexingParameters.Configuration.Add("dataToExtract", "contentAndMetadata"); indexingParameters.Configuration.Add("imageAction", "generateNormalizedImages"); SearchIndexer indexer = new SearchIndexer("demoindexer", dataSource.Name, index.Name) { Description = "Demo Indexer", SkillsetName = skillSet.Name, Parameters = indexingParameters }; FieldMappingFunction mappingFunction = new FieldMappingFunction("base64Encode"); mappingFunction.Parameters.Add("useHttpServerUtilityUrlTokenEncode", true); indexer.FieldMappings.Add(new FieldMapping("metadata_storage_path") { TargetFieldName = "id", MappingFunction = mappingFunction }); indexer.FieldMappings.Add(new FieldMapping("content") { TargetFieldName = "content" }); indexer.OutputFieldMappings.Add(new FieldMapping("/document/pages/*/organizations/*") { TargetFieldName = "organizations" }); indexer.OutputFieldMappings.Add(new FieldMapping("/document/pages/*/keyPhrases/*") { TargetFieldName = "keyPhrases" }); indexer.OutputFieldMappings.Add(new FieldMapping("/document/languageCode") { TargetFieldName = "languageCode" }); try { indexerClient.GetIndexer(indexer.Name); indexerClient.DeleteIndexer(indexer.Name); } catch (RequestFailedException ex) when(ex.Status == 404) { //if the specified indexer not exist, 404 will be thrown. } try { indexerClient.CreateIndexer(indexer); } catch (RequestFailedException ex) { Console.WriteLine("Failed to create the indexer\n Exception message: {0}\n", ex.Message); ExitProgram("Cannot continue without creating an indexer"); } return(indexer); }
public virtual Response <SearchIndexerSkillset> CreateOrUpdate(string skillsetName, SearchIndexerSkillset skillset, Guid?xMsClientRequestId = null, string ifMatch = null, string ifNoneMatch = null, CancellationToken cancellationToken = default) { return(RestClient.CreateOrUpdate(skillsetName, skillset, xMsClientRequestId, ifMatch, ifNoneMatch, cancellationToken)); }
public void ParsesETag(string value, string expected) { SearchIndexerSkillset sut = new SearchIndexerSkillset(null, null, null, null, value); Assert.AreEqual(expected, sut.ETag?.ToString()); }
static void Main(string[] args) { Console.BackgroundColor = ConsoleColor.Black; Console.ForegroundColor = ConsoleColor.Cyan; // Step 1 - Create datasource Console.WriteLine("\nStep 1 - Creating the data source..."); IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json"); IConfigurationRoot configuration = builder.Build(); string searchServiceUri = configuration["SearchServiceUri"]; string cognitiveServicesKey = configuration["CognitiveServicesKey"]; string adminApiKey = configuration["SearchServiceAdminApiKey"]; SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration); // Step 2 - Create the skillset Console.WriteLine("\nStep 2 - Creating the skillset..."); Console.WriteLine("\tStep 2.1 - Adding the skills..."); Console.WriteLine("\t\t OCR skill"); OcrSkill ocrSkill = CreateOcrSkill(); Console.WriteLine("\t\t Merge skill"); MergeSkill mergeSkill = CreateMergeSkill(); Console.WriteLine("\t\t Entity recognition skill"); EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill(); Console.WriteLine("\t\t Language detection skill"); LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill(); Console.WriteLine("\t\t Split skill"); SplitSkill splitSkill = CreateSplitSkill(); Console.WriteLine("\t\t Key phrase skill"); KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill(); List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>(); skills.Add(ocrSkill); skills.Add(mergeSkill); skills.Add(languageDetectionSkill); skills.Add(splitSkill); skills.Add(entityRecognitionSkill); skills.Add(keyPhraseExtractionSkill); Console.WriteLine("\tStep 2.2 - Building the skillset..."); SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey); // Step 3 - Create the index Console.WriteLine("\nStep 3 - Creating the index..."); SearchIndex demoIndex = CreateDemoIndex(indexClient); // Step 4 - Create the indexer, map fields, and execute transformations Console.WriteLine("\nStep 4 - Creating the indexer and executing the pipeline..."); SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex); // Step 5 - Monitor the indexing process Console.WriteLine("\nStep 5 - Check the indexer overall status..."); CheckIndexerOverallStatus(indexerClient, demoIndexer); }
public virtual async Task <Response <SearchIndexerSkillset> > CreateOrUpdateAsync(string skillsetName, SearchIndexerSkillset skillset, Guid?xMsClientRequestId = null, string ifMatch = null, string ifNoneMatch = null, CancellationToken cancellationToken = default) { using var scope = _clientDiagnostics.CreateScope("SkillsetsClient.CreateOrUpdate"); scope.Start(); try { return(await RestClient.CreateOrUpdateAsync(skillsetName, skillset, xMsClientRequestId, ifMatch, ifNoneMatch, cancellationToken).ConfigureAwait(false)); } catch (Exception e) { scope.Failed(e); throw; } }