private static List <Skill> CreateSkills() { OcrSkill ocrSkill = CreateOcrSkill(); MergeSkill mergeSkill = CreateMergeSkill(); LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill(); SentimentSkill sentimentSkill = CreateSentimentSkill(); KeyPhraseExtractionSkill keyPhraseSkill = CreateKeyPhraseExtractionSkill(); EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill(); ImageAnalysisSkill imageAnalysisSkill = CreateImageAnalysisSkill(); WebApiSkill webApiSkill = CreateWebApiSkill(); WebApiSkill topTenWordsSkill = CreateTopTenWordsSkill(); List <Skill> skills = new List <Skill>(); skills.Add(ocrSkill); skills.Add(imageAnalysisSkill); skills.Add(mergeSkill); skills.Add(sentimentSkill); skills.Add(keyPhraseSkill); skills.Add(entityRecognitionSkill); skills.Add(webApiSkill); skills.Add(topTenWordsSkill); return(skills); }
public async Task <string> CreateIndexAndIndexerAsync() { // Create or Update the data source SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(_indexerClient); // Create the skills OcrSkill ocrSkill = CreateOcrSkill(); MergeSkill mergeSkill = CreateMergeSkill(); EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill(); LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill(); SplitSkill splitSkill = CreateSplitSkill(); KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill(); // Create the skillset List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>(); skills.Add(ocrSkill); skills.Add(mergeSkill); skills.Add(languageDetectionSkill); skills.Add(splitSkill); skills.Add(entityRecognitionSkill); skills.Add(keyPhraseExtractionSkill); SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(_indexerClient, skills, _cognitiveServicesKey); // Create the index SearchIndex demoIndex = await CreateDemoIndexAsync(_indexClient); // Create the indexer, map fields, and execute transformations SearchIndexer demoIndexer = await CreateDemoIndexerAsync(_indexerClient, dataSource, skillset, demoIndex); // Check indexer overall status return(await CheckIndexerOverallStatusAsync(_indexerClient, demoIndexer.Name)); }
public static void Main(string[] args) { // Create service client IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json"); IConfigurationRoot configuration = builder.Build(); string searchServiceUri = configuration["SearchServiceUri"]; string adminApiKey = configuration["SearchServiceAdminApiKey"]; string cognitiveServicesKey = configuration["CognitiveServicesKey"]; SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); // Create or Update the data source Console.WriteLine("Creating or updating the data source..."); SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration); // Create the skills Console.WriteLine("Creating the skills..."); OcrSkill ocrSkill = CreateOcrSkill(); MergeSkill mergeSkill = CreateMergeSkill(); EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill(); LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill(); SplitSkill splitSkill = CreateSplitSkill(); KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill(); // Create the skillset Console.WriteLine("Creating or updating the skillset..."); List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>(); skills.Add(ocrSkill); skills.Add(mergeSkill); skills.Add(languageDetectionSkill); skills.Add(splitSkill); skills.Add(entityRecognitionSkill); skills.Add(keyPhraseExtractionSkill); SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey); // Create the index Console.WriteLine("Creating the index..."); SearchIndex demoIndex = CreateDemoIndex(indexClient); // Create the indexer, map fields, and execute transformations Console.WriteLine("Creating the indexer and executing the pipeline..."); SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex); // Check indexer overall status Console.WriteLine("Check the indexer overall status..."); CheckIndexerOverallStatus(indexerClient, demoIndexer); }
private static LanguageDetectionSkill CreateLanguageDetectionSkill() { List <InputFieldMappingEntry> inputMappings = new List <InputFieldMappingEntry>(); inputMappings.Add(new InputFieldMappingEntry( name: "text", source: "/document/file_name")); List <OutputFieldMappingEntry> outputMappings = new List <OutputFieldMappingEntry>(); outputMappings.Add(new OutputFieldMappingEntry( name: "languageCode", targetName: "languageCode")); LanguageDetectionSkill languageDetectionSkill = new LanguageDetectionSkill( description: "Detect the language used in the document", context: "/document", inputs: inputMappings, outputs: outputMappings); return(languageDetectionSkill); }
public static LanguageDetectionSkill CreateLanguageDetectionSkill() { List <InputFieldMappingEntry> inputMappings = new List <InputFieldMappingEntry>(); inputMappings.Add(new InputFieldMappingEntry("text") { Source = "/document/merged_text" }); List <OutputFieldMappingEntry> outputMappings = new List <OutputFieldMappingEntry>(); outputMappings.Add(new OutputFieldMappingEntry("languageCode") { TargetName = "languageCode" }); LanguageDetectionSkill languageDetectionSkill = new LanguageDetectionSkill(inputMappings, outputMappings) { Description = "Detect the language used in the document", Context = "/document" }; return(languageDetectionSkill); }
private static Skillset CreateBaseSkillset(string cognitiveServicesResourceId, string cognitiveServicesKey) { var skillset = new Skillset { Name = TweetSkillset, Description = "Cognitive skills collection" }; skillset.Skills = new List <Skill>(); var entityRecognitionSkill = new EntityRecognitionSkill { Description = "Entity recognition skill", Context = "/document", Categories = new List <string> { EntityCategory.Person, EntityCategory.Quantity, EntityCategory.Organization, EntityCategory.Location, EntityCategory.DateTime, EntityCategory.URL, EntityCategory.Email }, DefaultLanguageCode = "en", Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("text", "/document/text") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("persons", "people"), new OutputFieldMappingEntry("organizations", "organizations"), new OutputFieldMappingEntry("locations", "locations") } }; skillset.Skills.Add(entityRecognitionSkill); var keyPhraseExtractionSkill = new KeyPhraseExtractionSkill { Context = "/document", Description = "Key phrase extraction skill", DefaultLanguageCode = "en", Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("text", "/document/text") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("keyPhrases", "keyphrases") } }; skillset.Skills.Add(keyPhraseExtractionSkill); var languageDetectionSkill = new LanguageDetectionSkill { Context = "/document", Description = "Language detection skill", Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("text", "/document/text") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("languageCode", "language") } }; skillset.Skills.Add(languageDetectionSkill); skillset.CognitiveServices = new CognitiveServices(cognitiveServicesResourceId, cognitiveServicesKey); return(skillset); }
static void Main(string[] args) { Console.BackgroundColor = ConsoleColor.Black; Console.ForegroundColor = ConsoleColor.Cyan; // Step 1 - Create datasource Console.WriteLine("\nStep 1 - Creating the data source..."); IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json"); IConfigurationRoot configuration = builder.Build(); string searchServiceUri = configuration["SearchServiceUri"]; string cognitiveServicesKey = configuration["CognitiveServicesKey"]; string adminApiKey = configuration["SearchServiceAdminApiKey"]; SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey)); SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration); // Step 2 - Create the skillset Console.WriteLine("\nStep 2 - Creating the skillset..."); Console.WriteLine("\tStep 2.1 - Adding the skills..."); Console.WriteLine("\t\t OCR skill"); OcrSkill ocrSkill = CreateOcrSkill(); Console.WriteLine("\t\t Merge skill"); MergeSkill mergeSkill = CreateMergeSkill(); Console.WriteLine("\t\t Entity recognition skill"); EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill(); Console.WriteLine("\t\t Language detection skill"); LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill(); Console.WriteLine("\t\t Split skill"); SplitSkill splitSkill = CreateSplitSkill(); Console.WriteLine("\t\t Key phrase skill"); KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill(); List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>(); skills.Add(ocrSkill); skills.Add(mergeSkill); skills.Add(languageDetectionSkill); skills.Add(splitSkill); skills.Add(entityRecognitionSkill); skills.Add(keyPhraseExtractionSkill); Console.WriteLine("\tStep 2.2 - Building the skillset..."); SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey); // Step 3 - Create the index Console.WriteLine("\nStep 3 - Creating the index..."); SearchIndex demoIndex = CreateDemoIndex(indexClient); // Step 4 - Create the indexer, map fields, and execute transformations Console.WriteLine("\nStep 4 - Creating the indexer and executing the pipeline..."); SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex); // Step 5 - Monitor the indexing process Console.WriteLine("\nStep 5 - Check the indexer overall status..."); CheckIndexerOverallStatus(indexerClient, demoIndexer); }
private static void CreateSkillset(SearchServiceClient searchClient, string cognitiveServicesKey) { Console.WriteLine("\nDefining skills..."); // Create a list for the skills we need List <Skill> skills = new List <Skill>(); // Language detection skill Console.WriteLine(" - Language detection"); // inputs List <InputFieldMappingEntry> languageInputs = new List <InputFieldMappingEntry>(); languageInputs.Add(new InputFieldMappingEntry( name: "text", source: "/document/content")); // outputs List <OutputFieldMappingEntry> languageOutputs = new List <OutputFieldMappingEntry>(); languageOutputs.Add(new OutputFieldMappingEntry( name: "languageCode", targetName: "language")); // Create skill LanguageDetectionSkill languageDetectionSkill = new LanguageDetectionSkill( name: "get-language", description: "Detect the language used in the document", context: "/document", inputs: languageInputs, outputs: languageOutputs); // Add to list skills.Add(languageDetectionSkill); // Image Analysis skill Console.WriteLine(" - Image analysis"); // inputs List <InputFieldMappingEntry> imageAnalysisInputs = new List <InputFieldMappingEntry>(); imageAnalysisInputs.Add(new InputFieldMappingEntry( name: "image", source: "/document/normalized_images/*")); // Visual features to extract List <VisualFeature> features = new List <VisualFeature>(); features.Add(VisualFeature.Description); // outputs List <OutputFieldMappingEntry> imageAnalysisOutputs = new List <OutputFieldMappingEntry>(); imageAnalysisOutputs.Add(new OutputFieldMappingEntry( name: "description", targetName: "imageDescription")); // Create skill ImageAnalysisSkill imageAnalysisSkill = new ImageAnalysisSkill( name: "get-image-descriptions", description: "Generate descriptions of images.", context: "/document/normalized_images/*", visualFeatures: features, inputs: imageAnalysisInputs, outputs: imageAnalysisOutputs); //Add to list skills.Add(imageAnalysisSkill); // OCR skill Console.WriteLine(" - OCR"); // inputs List <InputFieldMappingEntry> ocrInputs = new List <InputFieldMappingEntry>(); ocrInputs.Add(new InputFieldMappingEntry( name: "image", source: "/document/normalized_images/*")); // outputs List <OutputFieldMappingEntry> ocrOutputs = new List <OutputFieldMappingEntry>(); ocrOutputs.Add(new OutputFieldMappingEntry( name: "text", targetName: "text")); // Create skill OcrSkill ocrSkill = new OcrSkill( name: "get-image-text", description: "Use OCR to extract text from images.", context: "/document/normalized_images/*", inputs: ocrInputs, outputs: ocrOutputs, shouldDetectOrientation: true); //Add to list skills.Add(ocrSkill); // Merge skill Console.WriteLine(" - Merge"); // inputs List <InputFieldMappingEntry> mergeInputs = new List <InputFieldMappingEntry>(); mergeInputs.Add(new InputFieldMappingEntry( name: "text", source: "/document/content")); mergeInputs.Add(new InputFieldMappingEntry( name: "itemsToInsert", source: "/document/normalized_images/*/text")); mergeInputs.Add(new InputFieldMappingEntry( name: "offsets", source: "/document/normalized_images/*/contentOffset")); // outputs List <OutputFieldMappingEntry> mergeOutputs = new List <OutputFieldMappingEntry>(); mergeOutputs.Add(new OutputFieldMappingEntry( name: "mergedText", targetName: "mergedText")); // Create skill MergeSkill mergeSkill = new MergeSkill( name: "merge-text", description: "Create merged_text which includes all the textual representation of each image inserted at the right location in the content field.", context: "/document", inputs: mergeInputs, outputs: mergeOutputs, insertPreTag: "[", insertPostTag: "]"); //Add to list skills.Add(mergeSkill); // Sentiment skill Console.WriteLine(" - Sentiment"); // inputs List <InputFieldMappingEntry> sentimentInputs = new List <InputFieldMappingEntry>(); sentimentInputs.Add(new InputFieldMappingEntry( name: "text", source: "/document/mergedText")); sentimentInputs.Add(new InputFieldMappingEntry( name: "languageCode", source: "/document/language")); // outputs List <OutputFieldMappingEntry> sentimentOutputs = new List <OutputFieldMappingEntry>(); sentimentOutputs.Add(new OutputFieldMappingEntry( name: "score", targetName: "sentimentScore")); // Create skill SentimentSkill sentimentSkill = new SentimentSkill( name: "get-sentiment", description: "Detect sentiment.", context: "/document", inputs: sentimentInputs, outputs: sentimentOutputs); //Add to list skills.Add(sentimentSkill); // Entity recognition skill Console.WriteLine(" - Text entities"); // inputs List <InputFieldMappingEntry> entityInputs = new List <InputFieldMappingEntry>(); entityInputs.Add(new InputFieldMappingEntry( name: "text", source: "/document/mergedText")); entityInputs.Add(new InputFieldMappingEntry( name: "languageCode", source: "/document/language")); // Categories to extract List <EntityCategory> entityCategories = new List <EntityCategory>(); entityCategories.Add(EntityCategory.Location); entityCategories.Add(EntityCategory.Url); // outputs List <OutputFieldMappingEntry> entityOutputs = new List <OutputFieldMappingEntry>(); entityOutputs.Add(new OutputFieldMappingEntry( name: "locations", targetName: "locations")); entityOutputs.Add(new OutputFieldMappingEntry( name: "urls", targetName: "urls")); // Create skill EntityRecognitionSkill entitySkill = new EntityRecognitionSkill( name: "get-text-entities", description: "Extract locations and URLs", context: "/document", categories: entityCategories, inputs: entityInputs, outputs: entityOutputs); //Add to list skills.Add(entitySkill); // Key phrases skill Console.WriteLine(" - Key phrases"); // inputs List <InputFieldMappingEntry> keyphraseInputs = new List <InputFieldMappingEntry>(); keyphraseInputs.Add(new InputFieldMappingEntry( name: "text", source: "/document/mergedText")); keyphraseInputs.Add(new InputFieldMappingEntry( name: "languageCode", source: "/document/language")); // outputs List <OutputFieldMappingEntry> keyphraseOutputs = new List <OutputFieldMappingEntry>(); keyphraseOutputs.Add(new OutputFieldMappingEntry( name: "keyPhrases", targetName: "keyPhrases")); // Create skill KeyPhraseExtractionSkill keyphraseSkill = new KeyPhraseExtractionSkill( name: "get-key-phrases", description: "Extract key phrases.", context: "/document", inputs: keyphraseInputs, outputs: keyphraseOutputs); //Add to list skills.Add(keyphraseSkill); // Uncomment below to add custom skill //skills.Add(CreateCustomSkill()); // Create skillset Console.WriteLine("\nCreating skillset...\n"); Skillset skillset = new Skillset( name: SkillsetName, description: "Extract and enrich data", skills: skills, cognitiveServices: new CognitiveServicesByKey( cognitiveServicesKey, "Cognitive Services account")); try { searchClient.Skillsets.CreateOrUpdate(skillset); } catch (Exception e) { Console.WriteLine("Failed to create the skillset\n Exception message: {0}\n", e.Message); } }