public static MergeSkill CreateMergeSkill()
{
List <InputFieldMappingEntry> inputMappings = new List <InputFieldMappingEntry>();
inputMappings.Add(new InputFieldMappingEntry("text")
{
Source = "/document/content"
});
inputMappings.Add(new InputFieldMappingEntry("itemsToInsert")
{
Source = "/document/normalized_images/*/text"
});
inputMappings.Add(new InputFieldMappingEntry("offsets")
{
Source = "/document/normalized_images/*/contentOffset"
});
List <OutputFieldMappingEntry> outputMappings = new List <OutputFieldMappingEntry>();
outputMappings.Add(new OutputFieldMappingEntry("mergedText")
{
TargetName = "merged_text"
});
MergeSkill mergeSkill = new MergeSkill(inputMappings, outputMappings)
{
Description = "Create merged_text which includes all the textual representation of each image inserted at the right location in the content field.",
Context = "/document",
InsertPreTag = " ",
InsertPostTag = " "
};
return(mergeSkill);
}
private static List <Skill> CreateSkills()
{
OcrSkill ocrSkill = CreateOcrSkill();
MergeSkill mergeSkill = CreateMergeSkill();
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
SentimentSkill sentimentSkill = CreateSentimentSkill();
KeyPhraseExtractionSkill keyPhraseSkill = CreateKeyPhraseExtractionSkill();
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
ImageAnalysisSkill imageAnalysisSkill = CreateImageAnalysisSkill();
WebApiSkill webApiSkill = CreateWebApiSkill();
WebApiSkill topTenWordsSkill = CreateTopTenWordsSkill();
List <Skill> skills = new List <Skill>();
skills.Add(ocrSkill);
skills.Add(imageAnalysisSkill);
skills.Add(mergeSkill);
skills.Add(sentimentSkill);
skills.Add(keyPhraseSkill);
skills.Add(entityRecognitionSkill);
skills.Add(webApiSkill);
skills.Add(topTenWordsSkill);
return(skills);
}
private static MergeSkill CreateMergeSkill()
{
List <InputFieldMappingEntry> inputMappings = new List <InputFieldMappingEntry>();
inputMappings.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/content"));
inputMappings.Add(new InputFieldMappingEntry(
name: "itemsToInsert",
source: "/document/normalized_images/*/text"));
inputMappings.Add(new InputFieldMappingEntry(
name: "offsets",
source: "/document/normalized_images/*/contentOffset"));
List <OutputFieldMappingEntry> outputMappings = new List <OutputFieldMappingEntry>();
outputMappings.Add(new OutputFieldMappingEntry(
name: "mergedText",
targetName: "merged_text"));
MergeSkill mergeSkill = new MergeSkill(
description: "Create merged_text which includes all the textual representation of each image inserted at the right location in the content field.",
context: "/document",
inputs: inputMappings,
outputs: outputMappings,
insertPreTag: " ",
insertPostTag: " ");
return(mergeSkill);
}
public async Task <string> CreateIndexAndIndexerAsync()
{
// Create or Update the data source
SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(_indexerClient);
// Create the skills
OcrSkill ocrSkill = CreateOcrSkill();
MergeSkill mergeSkill = CreateMergeSkill();
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
SplitSkill splitSkill = CreateSplitSkill();
KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill();
// Create the skillset
List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>();
skills.Add(ocrSkill);
skills.Add(mergeSkill);
skills.Add(languageDetectionSkill);
skills.Add(splitSkill);
skills.Add(entityRecognitionSkill);
skills.Add(keyPhraseExtractionSkill);
SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(_indexerClient, skills, _cognitiveServicesKey);
// Create the index
SearchIndex demoIndex = await CreateDemoIndexAsync(_indexClient);
// Create the indexer, map fields, and execute transformations
SearchIndexer demoIndexer = await CreateDemoIndexerAsync(_indexerClient, dataSource, skillset, demoIndex);
// Check indexer overall status
return(await CheckIndexerOverallStatusAsync(_indexerClient, demoIndexer.Name));
}
public static void Main(string[] args)
{
// Create service client
IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json");
IConfigurationRoot configuration = builder.Build();
string searchServiceUri = configuration["SearchServiceUri"];
string adminApiKey = configuration["SearchServiceAdminApiKey"];
string cognitiveServicesKey = configuration["CognitiveServicesKey"];
SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
// Create or Update the data source
Console.WriteLine("Creating or updating the data source...");
SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration);
// Create the skills
Console.WriteLine("Creating the skills...");
OcrSkill ocrSkill = CreateOcrSkill();
MergeSkill mergeSkill = CreateMergeSkill();
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
SplitSkill splitSkill = CreateSplitSkill();
KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill();
// Create the skillset
Console.WriteLine("Creating or updating the skillset...");
List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>();
skills.Add(ocrSkill);
skills.Add(mergeSkill);
skills.Add(languageDetectionSkill);
skills.Add(splitSkill);
skills.Add(entityRecognitionSkill);
skills.Add(keyPhraseExtractionSkill);
SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey);
// Create the index
Console.WriteLine("Creating the index...");
SearchIndex demoIndex = CreateDemoIndex(indexClient);
// Create the indexer, map fields, and execute transformations
Console.WriteLine("Creating the indexer and executing the pipeline...");
SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex);
// Check indexer overall status
Console.WriteLine("Check the indexer overall status...");
CheckIndexerOverallStatus(indexerClient, demoIndexer);
}
public void CreateSkillset()
{
IConfigurationRoot configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json").Build();
var inputMappings = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry(
name: "text",
source: "/document/Content")
};
var outputMappings = new List <OutputFieldMappingEntry>
{
new OutputFieldMappingEntry(name: "persons", "Persons"),
new OutputFieldMappingEntry(name: "locations", "Locations"),
new OutputFieldMappingEntry(name: "urls", "Urls"),
};
var entityCategory = new List <EntityCategory>()
{
EntityCategory.Location, EntityCategory.Person, EntityCategory.Url
};
var entityRecognitionSkill = new EntityRecognitionSkill(
description: "Recognize organizations",
context: "/document",
inputs: inputMappings,
outputs: outputMappings,
categories: entityCategory,
defaultLanguageCode: EntityRecognitionSkillLanguage.En);
var keyPhraseSkill = new KeyPhraseExtractionSkill(
name: "keyphraseextractionskill",
description: "Key Phrase Extraction Skill",
context: "/document",
inputs: new[] { new InputFieldMappingEntry("text", "/document/Content") },
outputs: new[] { new OutputFieldMappingEntry("keyPhrases", "KeyPhrases") }
);
var imageSkill = new ImageAnalysisSkill(
name: "imageanalysisskill",
context: "/document/normalized_images/*",
inputs: new[] { new InputFieldMappingEntry("image", "/document/normalized_images/*") },
outputs: new[] {
new OutputFieldMappingEntry("/document/normalized_images/*/tags/*", "Tags"),
new OutputFieldMappingEntry("/document/normalized_images/*/description", "Description")
}
);
var ocrSkill = new OcrSkill(
name: "ocrskill",
context: "/document/normalized_images/*",
inputs: new[] { new InputFieldMappingEntry("image", "/document/normalized_images/*") },
outputs: new[] { new OutputFieldMappingEntry("text", "OcrText") }
);
var mergeTextSkill = new MergeSkill(
name: "mergeTextSkill",
context: "/document",
inputs: new[] {
new InputFieldMappingEntry("text", "/document/Content"),
new InputFieldMappingEntry("itemsToInsert", "/document/normalized_images/*/OcrText")
},
outputs: new[] {
new OutputFieldMappingEntry("mergedText", "MergedText")
}
);
/*Microsoft.Rest.Azure.CloudException : One or more skills are invalid. Details: Skill 'mergeTextSkill' is not allowed to have recursively defined inputs
*/
//public SentimentSkill(IList<InputFieldMappingEntry> inputs, IList<OutputFieldMappingEntry> outputs, string name = null, string description = null, string context = null, SentimentSkillLanguage? defaultLanguageCode = null);
var sentimentskill = new SentimentSkill(
name: "sentimentskill",
description: "Our favorite Sentiment Skill",
context: "/document",
defaultLanguageCode: SentimentSkillLanguage.En,
inputs: new[] { new InputFieldMappingEntry("text", "/document/Content") },
outputs: new[] { new OutputFieldMappingEntry("score", "Sentiment") }
);
var commonWordsSkill = new WebApiSkill(
name: "commonWordsSkill",
description: "No description for you",
batchSize: 1,
httpMethod: "POST",
uri: configuration["CommonWordsApiUrl"],
context: "/document",
inputs: new[] { new InputFieldMappingEntry("text", "/document/MergedText") },
outputs: new[] { new OutputFieldMappingEntry("words", "CommonWords") }
);
var ss = new Skillset("fastracoontravelskillset", "self describing",
skills: new List <Skill>()
{
entityRecognitionSkill, keyPhraseSkill, sentimentskill, ocrSkill, mergeTextSkill, commonWordsSkill
},
cognitiveServices: new CognitiveServicesByKey(configuration["CogServicesKey"])
);
using (var serviceClient = CreateSearchServiceClient(configuration))
{
serviceClient.Skillsets.CreateOrUpdate(ss);
}
}
static void Main(string[] args)
{
Console.BackgroundColor = ConsoleColor.Black;
Console.ForegroundColor = ConsoleColor.Cyan;
// Step 1 - Create datasource
Console.WriteLine("\nStep 1 - Creating the data source...");
IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json");
IConfigurationRoot configuration = builder.Build();
string searchServiceUri = configuration["SearchServiceUri"];
string cognitiveServicesKey = configuration["CognitiveServicesKey"];
string adminApiKey = configuration["SearchServiceAdminApiKey"];
SearchIndexClient indexClient = new SearchIndexClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
SearchIndexerClient indexerClient = new SearchIndexerClient(new Uri(searchServiceUri), new AzureKeyCredential(adminApiKey));
SearchIndexerDataSourceConnection dataSource = CreateOrUpdateDataSource(indexerClient, configuration);
// Step 2 - Create the skillset
Console.WriteLine("\nStep 2 - Creating the skillset...");
Console.WriteLine("\tStep 2.1 - Adding the skills...");
Console.WriteLine("\t\t OCR skill");
OcrSkill ocrSkill = CreateOcrSkill();
Console.WriteLine("\t\t Merge skill");
MergeSkill mergeSkill = CreateMergeSkill();
Console.WriteLine("\t\t Entity recognition skill");
EntityRecognitionSkill entityRecognitionSkill = CreateEntityRecognitionSkill();
Console.WriteLine("\t\t Language detection skill");
LanguageDetectionSkill languageDetectionSkill = CreateLanguageDetectionSkill();
Console.WriteLine("\t\t Split skill");
SplitSkill splitSkill = CreateSplitSkill();
Console.WriteLine("\t\t Key phrase skill");
KeyPhraseExtractionSkill keyPhraseExtractionSkill = CreateKeyPhraseExtractionSkill();
List <SearchIndexerSkill> skills = new List <SearchIndexerSkill>();
skills.Add(ocrSkill);
skills.Add(mergeSkill);
skills.Add(languageDetectionSkill);
skills.Add(splitSkill);
skills.Add(entityRecognitionSkill);
skills.Add(keyPhraseExtractionSkill);
Console.WriteLine("\tStep 2.2 - Building the skillset...");
SearchIndexerSkillset skillset = CreateOrUpdateDemoSkillSet(indexerClient, skills, cognitiveServicesKey);
// Step 3 - Create the index
Console.WriteLine("\nStep 3 - Creating the index...");
SearchIndex demoIndex = CreateDemoIndex(indexClient);
// Step 4 - Create the indexer, map fields, and execute transformations
Console.WriteLine("\nStep 4 - Creating the indexer and executing the pipeline...");
SearchIndexer demoIndexer = CreateDemoIndexer(indexerClient, dataSource, skillset, demoIndex);
// Step 5 - Monitor the indexing process
Console.WriteLine("\nStep 5 - Check the indexer overall status...");
CheckIndexerOverallStatus(indexerClient, demoIndexer);
}
private static void CreateSkillset(SearchServiceClient searchClient, string cognitiveServicesKey)
{
Console.WriteLine("\nDefining skills...");
// Create a list for the skills we need
List <Skill> skills = new List <Skill>();
// Language detection skill
Console.WriteLine(" - Language detection");
// inputs
List <InputFieldMappingEntry> languageInputs = new List <InputFieldMappingEntry>();
languageInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/content"));
// outputs
List <OutputFieldMappingEntry> languageOutputs = new List <OutputFieldMappingEntry>();
languageOutputs.Add(new OutputFieldMappingEntry(
name: "languageCode",
targetName: "language"));
// Create skill
LanguageDetectionSkill languageDetectionSkill = new LanguageDetectionSkill(
name: "get-language",
description: "Detect the language used in the document",
context: "/document",
inputs: languageInputs,
outputs: languageOutputs);
// Add to list
skills.Add(languageDetectionSkill);
// Image Analysis skill
Console.WriteLine(" - Image analysis");
// inputs
List <InputFieldMappingEntry> imageAnalysisInputs = new List <InputFieldMappingEntry>();
imageAnalysisInputs.Add(new InputFieldMappingEntry(
name: "image",
source: "/document/normalized_images/*"));
// Visual features to extract
List <VisualFeature> features = new List <VisualFeature>();
features.Add(VisualFeature.Description);
// outputs
List <OutputFieldMappingEntry> imageAnalysisOutputs = new List <OutputFieldMappingEntry>();
imageAnalysisOutputs.Add(new OutputFieldMappingEntry(
name: "description",
targetName: "imageDescription"));
// Create skill
ImageAnalysisSkill imageAnalysisSkill = new ImageAnalysisSkill(
name: "get-image-descriptions",
description: "Generate descriptions of images.",
context: "/document/normalized_images/*",
visualFeatures: features,
inputs: imageAnalysisInputs,
outputs: imageAnalysisOutputs);
//Add to list
skills.Add(imageAnalysisSkill);
// OCR skill
Console.WriteLine(" - OCR");
// inputs
List <InputFieldMappingEntry> ocrInputs = new List <InputFieldMappingEntry>();
ocrInputs.Add(new InputFieldMappingEntry(
name: "image",
source: "/document/normalized_images/*"));
// outputs
List <OutputFieldMappingEntry> ocrOutputs = new List <OutputFieldMappingEntry>();
ocrOutputs.Add(new OutputFieldMappingEntry(
name: "text",
targetName: "text"));
// Create skill
OcrSkill ocrSkill = new OcrSkill(
name: "get-image-text",
description: "Use OCR to extract text from images.",
context: "/document/normalized_images/*",
inputs: ocrInputs,
outputs: ocrOutputs,
shouldDetectOrientation: true);
//Add to list
skills.Add(ocrSkill);
// Merge skill
Console.WriteLine(" - Merge");
// inputs
List <InputFieldMappingEntry> mergeInputs = new List <InputFieldMappingEntry>();
mergeInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/content"));
mergeInputs.Add(new InputFieldMappingEntry(
name: "itemsToInsert",
source: "/document/normalized_images/*/text"));
mergeInputs.Add(new InputFieldMappingEntry(
name: "offsets",
source: "/document/normalized_images/*/contentOffset"));
// outputs
List <OutputFieldMappingEntry> mergeOutputs = new List <OutputFieldMappingEntry>();
mergeOutputs.Add(new OutputFieldMappingEntry(
name: "mergedText",
targetName: "mergedText"));
// Create skill
MergeSkill mergeSkill = new MergeSkill(
name: "merge-text",
description: "Create merged_text which includes all the textual representation of each image inserted at the right location in the content field.",
context: "/document",
inputs: mergeInputs,
outputs: mergeOutputs,
insertPreTag: "[",
insertPostTag: "]");
//Add to list
skills.Add(mergeSkill);
// Sentiment skill
Console.WriteLine(" - Sentiment");
// inputs
List <InputFieldMappingEntry> sentimentInputs = new List <InputFieldMappingEntry>();
sentimentInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/mergedText"));
sentimentInputs.Add(new InputFieldMappingEntry(
name: "languageCode",
source: "/document/language"));
// outputs
List <OutputFieldMappingEntry> sentimentOutputs = new List <OutputFieldMappingEntry>();
sentimentOutputs.Add(new OutputFieldMappingEntry(
name: "score",
targetName: "sentimentScore"));
// Create skill
SentimentSkill sentimentSkill = new SentimentSkill(
name: "get-sentiment",
description: "Detect sentiment.",
context: "/document",
inputs: sentimentInputs,
outputs: sentimentOutputs);
//Add to list
skills.Add(sentimentSkill);
// Entity recognition skill
Console.WriteLine(" - Text entities");
// inputs
List <InputFieldMappingEntry> entityInputs = new List <InputFieldMappingEntry>();
entityInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/mergedText"));
entityInputs.Add(new InputFieldMappingEntry(
name: "languageCode",
source: "/document/language"));
// Categories to extract
List <EntityCategory> entityCategories = new List <EntityCategory>();
entityCategories.Add(EntityCategory.Location);
entityCategories.Add(EntityCategory.Url);
// outputs
List <OutputFieldMappingEntry> entityOutputs = new List <OutputFieldMappingEntry>();
entityOutputs.Add(new OutputFieldMappingEntry(
name: "locations",
targetName: "locations"));
entityOutputs.Add(new OutputFieldMappingEntry(
name: "urls",
targetName: "urls"));
// Create skill
EntityRecognitionSkill entitySkill = new EntityRecognitionSkill(
name: "get-text-entities",
description: "Extract locations and URLs",
context: "/document",
categories: entityCategories,
inputs: entityInputs,
outputs: entityOutputs);
//Add to list
skills.Add(entitySkill);
// Key phrases skill
Console.WriteLine(" - Key phrases");
// inputs
List <InputFieldMappingEntry> keyphraseInputs = new List <InputFieldMappingEntry>();
keyphraseInputs.Add(new InputFieldMappingEntry(
name: "text",
source: "/document/mergedText"));
keyphraseInputs.Add(new InputFieldMappingEntry(
name: "languageCode",
source: "/document/language"));
// outputs
List <OutputFieldMappingEntry> keyphraseOutputs = new List <OutputFieldMappingEntry>();
keyphraseOutputs.Add(new OutputFieldMappingEntry(
name: "keyPhrases",
targetName: "keyPhrases"));
// Create skill
KeyPhraseExtractionSkill keyphraseSkill = new KeyPhraseExtractionSkill(
name: "get-key-phrases",
description: "Extract key phrases.",
context: "/document",
inputs: keyphraseInputs,
outputs: keyphraseOutputs);
//Add to list
skills.Add(keyphraseSkill);
// Uncomment below to add custom skill
//skills.Add(CreateCustomSkill());
// Create skillset
Console.WriteLine("\nCreating skillset...\n");
Skillset skillset = new Skillset(
name: SkillsetName,
description: "Extract and enrich data",
skills: skills,
cognitiveServices: new CognitiveServicesByKey(
cognitiveServicesKey,
"Cognitive Services account"));
try
{
searchClient.Skillsets.CreateOrUpdate(skillset);
}
catch (Exception e)
{
Console.WriteLine("Failed to create the skillset\n Exception message: {0}\n", e.Message);
}
}