private static List <Skill> CreateSkills()
{
OcrSkill ocrSkill = CreateOcrSkill();
MergeSkill mergeSkill = CreateMergeSkill();
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
SentimentSkill sentimentSkill = CreateSentimentSkill();
KeyPhraseExtractionSkill keyPhraseSkill = CreateKeyPhraseExtractionSkill();
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
ImageAnalysisSkill imageAnalysisSkill = CreateImageAnalysisSkill();
WebApiSkill webApiSkill = CreateWebApiSkill();
WebApiSkill topTenWordsSkill = CreateTopTenWordsSkill();
List <Skill> skills = new List <Skill>();
skills.Add(ocrSkill);
skills.Add(imageAnalysisSkill);
skills.Add(mergeSkill);
skills.Add(sentimentSkill);
skills.Add(keyPhraseSkill);
skills.Add(entityRecognitionSkill);
skills.Add(webApiSkill);
skills.Add(topTenWordsSkill);
return(skills);
}
public async Task <string> CreateIndexAndIndexerAsync()
{
// Create or Update the data source
SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(_indexerClient);
// Create the skills
OcrSkill ocrSkill = CreateOcrSkill();
MergeSkill mergeSkill = CreateMergeSkill();
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
SplitSkill splitSkill = CreateSplitSkill();
KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill();
// Create the skillset
List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>();
skills.Add(ocrSkill);
skills.Add(mergeSkill);
skills.Add(languageDetectionSkill);
skills.Add(splitSkill);
skills.Add(entityRecognitionSkill);
skills.Add(keyPhraseExtractionSkill);
SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(_indexerClient, skills, _cognitiveServicesKey);
// Create the index
SearchIndex demoIndex = await CreateDemoIndexAsync(_indexClient);
// Create the indexer, map fields, and execute transformations
SearchIndexer demoIndexer = await CreateDemoIndexerAsync(_indexerClient, dataSource, skillset, demoIndex);
// Check indexer overall status
return(await CheckIndexerOverallStatusAsync(_indexerClient, demoIndexer.Name));
}
public static void Main(string[] args)
{
// Create service client
IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json");
IConfigurationRoot configuration = builder.Build();
string searchServiceUri = configuration["SearchServiceUri"];
string adminApiKey = configuration["SearchServiceAdminApiKey"];
string cognitiveServicesKey = configuration["CognitiveServicesKey"];
SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
// Create or Update the data source
Console.WriteLine("Creating or updating the data source...");
SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration);
// Create the skills
Console.WriteLine("Creating the skills...");
OcrSkill ocrSkill = CreateOcrSkill();
MergeSkill mergeSkill = CreateMergeSkill();
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
SplitSkill splitSkill = CreateSplitSkill();
KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill();
// Create the skillset
Console.WriteLine("Creating or updating the skillset...");
List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>();
skills.Add(ocrSkill);
skills.Add(mergeSkill);
skills.Add(languageDetectionSkill);
skills.Add(splitSkill);
skills.Add(entityRecognitionSkill);
skills.Add(keyPhraseExtractionSkill);
SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey);
// Create the index
Console.WriteLine("Creating the index...");
SearchIndex demoIndex = CreateDemoIndex(indexClient);
// Create the indexer, map fields, and execute transformations
Console.WriteLine("Creating the indexer and executing the pipeline...");
SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex);
// Check indexer overall status
Console.WriteLine("Check the indexer overall status...");
CheckIndexerOverallStatus(indexerClient, demoIndexer);
}
private static LanguageDetectionSkill CreateLanguageDetectionSkill()
{
List <InputFieldMappingEntry> inputMappings = new List <InputFieldMappingEntry>();
inputMappings.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/file_name"));
List <OutputFieldMappingEntry> outputMappings = new List <OutputFieldMappingEntry>();
outputMappings.Add(new OutputFieldMappingEntry(
name: "languageCode",
targetName: "languageCode"));
LanguageDetectionSkill languageDetectionSkill = new LanguageDetectionSkill(
description: "Detect the language used in the document",
context: "/document",
inputs: inputMappings,
outputs: outputMappings);
return(languageDetectionSkill);
}
public static LanguageDetectionSkill CreateLanguageDetectionSkill()
{
List <InputFieldMappingEntry> inputMappings = new List <InputFieldMappingEntry>();
inputMappings.Add(new InputFieldMappingEntry("text")
{
Source = "/document/merged_text"
});
List <OutputFieldMappingEntry> outputMappings = new List <OutputFieldMappingEntry>();
outputMappings.Add(new OutputFieldMappingEntry("languageCode")
{
TargetName = "languageCode"
});
LanguageDetectionSkill languageDetectionSkill = new LanguageDetectionSkill(inputMappings, outputMappings)
{
Description = "Detect the language used in the document",
Context = "/document"
};
return(languageDetectionSkill);
}
private static Skillset CreateBaseSkillset(string cognitiveServicesResourceId, string cognitiveServicesKey)
{
var skillset = new Skillset
{
Name = TweetSkillset,
Description = "Cognitive skills collection"
};
skillset.Skills = new List <Skill>();
var entityRecognitionSkill = new EntityRecognitionSkill
{
Description = "Entity recognition skill",
Context = "/document",
Categories = new List <string>
{
EntityCategory.Person,
EntityCategory.Quantity,
EntityCategory.Organization,
EntityCategory.Location,
EntityCategory.DateTime,
EntityCategory.URL,
EntityCategory.Email
},
DefaultLanguageCode = "en",
Inputs = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry("text", "/document/text")
},
Outputs = new List <OutputFieldMappingEntry> {
new OutputFieldMappingEntry("persons", "people"),
new OutputFieldMappingEntry("organizations", "organizations"),
new OutputFieldMappingEntry("locations", "locations")
}
};
skillset.Skills.Add(entityRecognitionSkill);
var keyPhraseExtractionSkill = new KeyPhraseExtractionSkill
{
Context = "/document",
Description = "Key phrase extraction skill",
DefaultLanguageCode = "en",
Inputs = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry("text", "/document/text")
},
Outputs = new List <OutputFieldMappingEntry> {
new OutputFieldMappingEntry("keyPhrases", "keyphrases")
}
};
skillset.Skills.Add(keyPhraseExtractionSkill);
var languageDetectionSkill = new LanguageDetectionSkill
{
Context = "/document",
Description = "Language detection skill",
Inputs = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry("text", "/document/text")
},
Outputs = new List <OutputFieldMappingEntry> {
new OutputFieldMappingEntry("languageCode", "language")
}
};
skillset.Skills.Add(languageDetectionSkill);
skillset.CognitiveServices = new CognitiveServices(cognitiveServicesResourceId, cognitiveServicesKey);
return(skillset);
}
static void Main(string[] args)
{
Console.BackgroundColor = ConsoleColor.Black;
Console.ForegroundColor = ConsoleColor.Cyan;
// Step 1 - Create datasource
Console.WriteLine("\nStep 1 - Creating the data source...");
IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json");
IConfigurationRoot configuration = builder.Build();
string searchServiceUri = configuration["SearchServiceUri"];
string cognitiveServicesKey = configuration["CognitiveServicesKey"];
string adminApiKey = configuration["SearchServiceAdminApiKey"];
SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration);
// Step 2 - Create the skillset
Console.WriteLine("\nStep 2 - Creating the skillset...");
Console.WriteLine("\tStep 2.1 - Adding the skills...");
Console.WriteLine("\t\t OCR skill");
OcrSkill ocrSkill = CreateOcrSkill();
Console.WriteLine("\t\t Merge skill");
MergeSkill mergeSkill = CreateMergeSkill();
Console.WriteLine("\t\t Entity recognition skill");
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
Console.WriteLine("\t\t Language detection skill");
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
Console.WriteLine("\t\t Split skill");
SplitSkill splitSkill = CreateSplitSkill();
Console.WriteLine("\t\t Key phrase skill");
KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill();
List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>();
skills.Add(ocrSkill);
skills.Add(mergeSkill);
skills.Add(languageDetectionSkill);
skills.Add(splitSkill);
skills.Add(entityRecognitionSkill);
skills.Add(keyPhraseExtractionSkill);
Console.WriteLine("\tStep 2.2 - Building the skillset...");
SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey);
// Step 3 - Create the index
Console.WriteLine("\nStep 3 - Creating the index...");
SearchIndex demoIndex = CreateDemoIndex(indexClient);
// Step 4 - Create the indexer, map fields, and execute transformations
Console.WriteLine("\nStep 4 - Creating the indexer and executing the pipeline...");
SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex);
// Step 5 - Monitor the indexing process
Console.WriteLine("\nStep 5 - Check the indexer overall status...");
CheckIndexerOverallStatus(indexerClient, demoIndexer);
}
private static void CreateSkillset(SearchServiceClient searchClient, string cognitiveServicesKey)
{
Console.WriteLine("\nDefining skills...");
// Create a list for the skills we need
List <Skill> skills = new List <Skill>();
// Language detection skill
Console.WriteLine(" - Language detection");
// inputs
List <InputFieldMappingEntry> languageInputs = new List <InputFieldMappingEntry>();
languageInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/content"));
// outputs
List <OutputFieldMappingEntry> languageOutputs = new List <OutputFieldMappingEntry>();
languageOutputs.Add(new OutputFieldMappingEntry(
name: "languageCode",
targetName: "language"));
// Create skill
LanguageDetectionSkill languageDetectionSkill = new LanguageDetectionSkill(
name: "get-language",
description: "Detect the language used in the document",
context: "/document",
inputs: languageInputs,
outputs: languageOutputs);
// Add to list
skills.Add(languageDetectionSkill);
// Image Analysis skill
Console.WriteLine(" - Image analysis");
// inputs
List <InputFieldMappingEntry> imageAnalysisInputs = new List <InputFieldMappingEntry>();
imageAnalysisInputs.Add(new InputFieldMappingEntry(
name: "image",
source: "/document/normalized_images/*"));
// Visual features to extract
List <VisualFeature> features = new List <VisualFeature>();
features.Add(VisualFeature.Description);
// outputs
List <OutputFieldMappingEntry> imageAnalysisOutputs = new List <OutputFieldMappingEntry>();
imageAnalysisOutputs.Add(new OutputFieldMappingEntry(
name: "description",
targetName: "imageDescription"));
// Create skill
ImageAnalysisSkill imageAnalysisSkill = new ImageAnalysisSkill(
name: "get-image-descriptions",
description: "Generate descriptions of images.",
context: "/document/normalized_images/*",
visualFeatures: features,
inputs: imageAnalysisInputs,
outputs: imageAnalysisOutputs);
//Add to list
skills.Add(imageAnalysisSkill);
// OCR skill
Console.WriteLine(" - OCR");
// inputs
List <InputFieldMappingEntry> ocrInputs = new List <InputFieldMappingEntry>();
ocrInputs.Add(new InputFieldMappingEntry(
name: "image",
source: "/document/normalized_images/*"));
// outputs
List <OutputFieldMappingEntry> ocrOutputs = new List <OutputFieldMappingEntry>();
ocrOutputs.Add(new OutputFieldMappingEntry(
name: "text",
targetName: "text"));
// Create skill
OcrSkill ocrSkill = new OcrSkill(
name: "get-image-text",
description: "Use OCR to extract text from images.",
context: "/document/normalized_images/*",
inputs: ocrInputs,
outputs: ocrOutputs,
shouldDetectOrientation: true);
//Add to list
skills.Add(ocrSkill);
// Merge skill
Console.WriteLine(" - Merge");
// inputs
List <InputFieldMappingEntry> mergeInputs = new List <InputFieldMappingEntry>();
mergeInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/content"));
mergeInputs.Add(new InputFieldMappingEntry(
name: "itemsToInsert",
source: "/document/normalized_images/*/text"));
mergeInputs.Add(new InputFieldMappingEntry(
name: "offsets",
source: "/document/normalized_images/*/contentOffset"));
// outputs
List <OutputFieldMappingEntry> mergeOutputs = new List <OutputFieldMappingEntry>();
mergeOutputs.Add(new OutputFieldMappingEntry(
name: "mergedText",
targetName: "mergedText"));
// Create skill
MergeSkill mergeSkill = new MergeSkill(
name: "merge-text",
description: "Create merged_text which includes all the textual representation of each image inserted at the right location in the content field.",
context: "/document",
inputs: mergeInputs,
outputs: mergeOutputs,
insertPreTag: "[",
insertPostTag: "]");
//Add to list
skills.Add(mergeSkill);
// Sentiment skill
Console.WriteLine(" - Sentiment");
// inputs
List <InputFieldMappingEntry> sentimentInputs = new List <InputFieldMappingEntry>();
sentimentInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/mergedText"));
sentimentInputs.Add(new InputFieldMappingEntry(
name: "languageCode",
source: "/document/language"));
// outputs
List <OutputFieldMappingEntry> sentimentOutputs = new List <OutputFieldMappingEntry>();
sentimentOutputs.Add(new OutputFieldMappingEntry(
name: "score",
targetName: "sentimentScore"));
// Create skill
SentimentSkill sentimentSkill = new SentimentSkill(
name: "get-sentiment",
description: "Detect sentiment.",
context: "/document",
inputs: sentimentInputs,
outputs: sentimentOutputs);
//Add to list
skills.Add(sentimentSkill);
// Entity recognition skill
Console.WriteLine(" - Text entities");
// inputs
List <InputFieldMappingEntry> entityInputs = new List <InputFieldMappingEntry>();
entityInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/mergedText"));
entityInputs.Add(new InputFieldMappingEntry(
name: "languageCode",
source: "/document/language"));
// Categories to extract
List <EntityCategory> entityCategories = new List <EntityCategory>();
entityCategories.Add(EntityCategory.Location);
entityCategories.Add(EntityCategory.Url);
// outputs
List <OutputFieldMappingEntry> entityOutputs = new List <OutputFieldMappingEntry>();
entityOutputs.Add(new OutputFieldMappingEntry(
name: "locations",
targetName: "locations"));
entityOutputs.Add(new OutputFieldMappingEntry(
name: "urls",
targetName: "urls"));
// Create skill
EntityRecognitionSkill entitySkill = new EntityRecognitionSkill(
name: "get-text-entities",
description: "Extract locations and URLs",
context: "/document",
categories: entityCategories,
inputs: entityInputs,
outputs: entityOutputs);
//Add to list
skills.Add(entitySkill);
// Key phrases skill
Console.WriteLine(" - Key phrases");
// inputs
List <InputFieldMappingEntry> keyphraseInputs = new List <InputFieldMappingEntry>();
keyphraseInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/mergedText"));
keyphraseInputs.Add(new InputFieldMappingEntry(
name: "languageCode",
source: "/document/language"));
// outputs
List <OutputFieldMappingEntry> keyphraseOutputs = new List <OutputFieldMappingEntry>();
keyphraseOutputs.Add(new OutputFieldMappingEntry(
name: "keyPhrases",
targetName: "keyPhrases"));
// Create skill
KeyPhraseExtractionSkill keyphraseSkill = new KeyPhraseExtractionSkill(
name: "get-key-phrases",
description: "Extract key phrases.",
context: "/document",
inputs: keyphraseInputs,
outputs: keyphraseOutputs);
//Add to list
skills.Add(keyphraseSkill);
// Uncomment below to add custom skill
//skills.Add(CreateCustomSkill());
// Create skillset
Console.WriteLine("\nCreating skillset...\n");
Skillset skillset = new Skillset(
name: SkillsetName,
description: "Extract and enrich data",
skills: skills,
cognitiveServices: new CognitiveServicesByKey(
cognitiveServicesKey,
"Cognitive Services account"));
try
{
searchClient.Skillsets.CreateOrUpdate(skillset);
}
catch (Exception e)
{
Console.WriteLine("Failed to create the skillset\n Exception message: {0}\n", e.Message);
}
}
