private static async Task CreateFormsRecognitionPipeline(SearchServiceClient searchClient, AppConfig appConfig, string modelId)
{
var formsSearchConfig = new SearchConfig
{
DataSourceName = "forms-datasource",
IndexName = "forms-index",
IndexerName = "forms-indexer",
SkillsetName = "forms-skillset"
};
var formsDataSource = await CognitiveSearchHelper.GetOrCreateBlobDataSource(searchClient, formsSearchConfig.DataSourceName, DataSourceType.AzureBlob, appConfig.BlobStorage);
Console.WriteLine($"Successfully created data source {formsSearchConfig.DataSourceName}");
var formsIndex = await CognitiveSearchHelper.GetIndexFromFile(formsSearchConfig.IndexName);
var formsIndexer = await CognitiveSearchHelper.GetIndexerFromFile(formsSearchConfig);
var formsSkillset = await CognitiveSearchHelper.GetSkillsetFromFile(formsSearchConfig.SkillsetName, appConfig.CognitiveServices);
Console.WriteLine("Adding Custom Form Recognizer skill to pipeline");
AddCustomFormRecognizerSkill(ref formsIndex, ref formsIndexer, ref formsSkillset, appConfig, modelId);
await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, formsIndex, formsIndexer, formsSkillset)
.ContinueWith(t =>
{
Console.WriteLine(t.IsFaulted
? t.Exception.Message
: "Your forms recognizer pipeline was successfully created.");
});
}
private static void AddSentimentAnalysisSkill(ref Index index, ref Indexer indexer, ref Skillset skillset)
{
if (index.Fields.Any(f => f.Name == "sentiment"))
{
return;
}
var sentimentField = new Field("sentiment", DataType.Double)
{
IsSortable = true,
IsFilterable = true
};
index.Fields.Add(sentimentField);
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping("document/sentiment", "sentiment").GetAwaiter().GetResult());
skillset.Skills.Add(new SentimentSkill
{
Context = "/document",
Description = "Sentiment analysis skill",
DefaultLanguageCode = SentimentSkillLanguage.En,
Inputs = new List <InputFieldMappingEntry> {
new InputFieldMappingEntry("text", "/document/text")
},
Outputs = new List <OutputFieldMappingEntry> {
new OutputFieldMappingEntry("score", "sentiment")
}
});
}
private static void AddCustomSummarizerSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, FunctionAppConfig config)
{
var targetField = "summary";
var headers = new Dictionary <string, string>
{
{ "Content-Type", "application/json" }
};
index.Fields.Add(new Field(targetField, AnalyzerName.StandardLucene));
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/{targetField}", targetField).GetAwaiter().GetResult());
// Create the custom translate skill
skillset.Skills.Add(new WebApiSkill
{
Description = "Custom summarization skill",
Context = "/document",
Uri = $"{config.Url}/api/Summarize?code={config.DefaultHostKey}",
HttpMethod = "POST",
//HttpHeaders = new WebApiHttpHeaders(headers),
BatchSize = 1,
Inputs = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry("text", "/document/textTranslated")
},
Outputs = new List <OutputFieldMappingEntry>
{
new OutputFieldMappingEntry("summaryText", targetField)
}
});
}
private static void AddUserInfoToIndex(ref Index index, ref Indexer indexer)
{
var analyzer = AnalyzerName.StandardLucene;
// Create a new index fields for userName and userLocation
index.Fields.Add(new Field("userName", analyzer));
index.Fields.Add(new Field("userLocation", analyzer));
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping("/document/user/name", "userName").GetAwaiter().GetResult());
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping("/document/user/location", "userLocation").GetAwaiter().GetResult());
}
private static void AddCustomTranslateSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, FunctionAppConfig config)
{
var targetField = "textTranslated";
var headers = new Dictionary <string, string>
{
{ "Content-Type", "application/json" }
};
index.Fields.Add(new Field(targetField, AnalyzerName.StandardLucene));
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/{targetField}", targetField).GetAwaiter().GetResult());
// Create the custom translate skill
skillset.Skills.Add(new WebApiSkill
{
Description = "Custom translator skill",
Context = "/document",
Uri = $"{config.Url}/api/Translate?code={config.DefaultHostKey}",
HttpMethod = "POST",
//HttpHeaders = new WebApiHttpHeaders(headers),
BatchSize = 1,
Inputs = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry("text", "/document/text"),
new InputFieldMappingEntry("language", "/document/Language")
},
Outputs = new List <OutputFieldMappingEntry>
{
new OutputFieldMappingEntry("text", targetField)
}
});
// Update all the other skills, except for the LanguageDetectionSkill, to use the new textTranslated field.
foreach (var skill in skillset.Skills)
{
var type = skill.GetType();
var typeName = type.Name;
if (typeName != "WebApiSkill" && typeName != "LanguageDetectionSkill")
{
foreach (var input in skill.Inputs)
{
if (input.Source == "/document/text")
{
input.Source = $"/document/{targetField}";
}
}
}
}
}
private static void AddCustomAnomalyDetectorSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, AppConfig config)
{
var headers = new Dictionary <string, string>
{
{ "Content-Type", "application/json" }
};
var anomalyFields = new List <Field>
{
new Field($"isAnomaly", DataType.Boolean),
new Field($"isPositiveAnomaly", DataType.Boolean),
new Field($"isNegativeAnomaly", DataType.Boolean),
new Field($"expectedValue", DataType.Double),
new Field($"upperMargin", DataType.Double),
new Field($"lowerMargin", DataType.Double)
};
index.Fields.Add(new Field("engineTemperatureAnalysis", DataType.Complex, anomalyFields));
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/engineTemperatureAnalysis", "engineTemperatureAnalysis").GetAwaiter().GetResult());
// Create the custom translate skill
skillset.Skills.Add(new WebApiSkill
{
Description = "Custom Anomaly Detector skill",
Context = "/document",
Uri = $"{config.FunctionApp.Url}/api/DetectAnomalies?code={config.FunctionApp.DefaultHostKey}",
HttpMethod = "POST",
//HttpHeaders = new WebApiHttpHeaders(), // This is broken in the SDK, so handle by sending JSON directly to Rest API.
BatchSize = 1,
Inputs = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry("timestamp", "/document/timestamp"),
new InputFieldMappingEntry("engineTemperature", "/document/engineTemperature")
},
Outputs = new List <OutputFieldMappingEntry>
{
new OutputFieldMappingEntry("anomalyResult", "engineTemperatureAnalysis")
}
});
}
private static void AddKnowledgeStore(SearchServiceClient searchClient, AppConfig appConfig, ref Index index, ref Indexer indexer, ref Skillset skillset)
{
// Add the skills from previous steps
AddSentimentAnalysisSkill(ref index, ref indexer, ref skillset);
// Convert the Skillset into a JSON string.
var skillsetJson = CognitiveSearchHelper.GetSkillsetJson(skillset).GetAwaiter().GetResult();
// Insert ShaperSkill into the Skillset as a JSON string
Console.WriteLine("Adding Shaper Skill to the pipeline to set up the table projections.");
var shaperSkillJson = GetJsonFromFile("shaper-skill").GetAwaiter().GetResult();
var skillsetJsonWithShaperSkill = CognitiveSearchHelper.InsertSkillAsJson(skillsetJson, shaperSkillJson);
// Insert knowledge store JSON string into the Skillset JSON
Console.WriteLine("Inserting the knowledge store with table projections.");
var updatedSkillset = InsertKnowledgeStoreJson(skillsetJsonWithShaperSkill, appConfig.BlobStorage);
// Create the search pipeline using the updated skillset JSON. No SDK exists yet for doing this using the SDK objects, so must use the REST API to accomplish adding a knowledge store via code.
Console.WriteLine("Rebuilding cognitive search pipeline...");
CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset.Name, updatedSkillset).GetAwaiter().GetResult();
}
private static void AddCustomFormRecognizerSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, AppConfig config, string modelId)
{
var headers = new Dictionary <string, string>
{
{ "Content-Type", "application/json" }
};
index.Fields.Add(new Field($"formHeight", DataType.Int32));
index.Fields.Add(new Field($"formWidth", DataType.Int32));
index.Fields.Add(new Field($"formKeyValuePairs", DataType.Collection(DataType.String)));
index.Fields.Add(new Field($"formColumns", DataType.Collection(DataType.String)));
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formHeight", "formHeight").GetAwaiter().GetResult());
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formWidth", "formWidth").GetAwaiter().GetResult());
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formKeyValuePairs", "formKeyValuePairs").GetAwaiter().GetResult());
indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formColumns", "formColumns").GetAwaiter().GetResult());
// Create the custom translate skill
skillset.Skills.Add(new WebApiSkill
{
Description = "Custom Form Recognizer skill",
Context = "/document",
Uri = $"{config.FunctionApp.Url}/api/AnalyzeForm?code={config.FunctionApp.DefaultHostKey}&modelId={modelId}",
HttpMethod = "POST",
//HttpHeaders = new WebApiHttpHeaders(), // This is broken in the SDK, so handle by sending JSON directly to Rest API.
BatchSize = 1,
Inputs = new List <InputFieldMappingEntry>
{
new InputFieldMappingEntry("contentType", "/document/fileContentType"),
new InputFieldMappingEntry("storageUri", "/document/storageUri"),
new InputFieldMappingEntry("storageSasToken", "/document/sasToken")
},
Outputs = new List <OutputFieldMappingEntry>
{
new OutputFieldMappingEntry("formHeight", "formHeight"),
new OutputFieldMappingEntry("formWidth", "formWidth"),
new OutputFieldMappingEntry("formKeyValuePairs", "formKeyValuePairs"),
new OutputFieldMappingEntry("formColumns", "formColumns"),
}
});
}
private static async Task CreateAnomalyDetectionPipeline(SearchServiceClient searchClient, AppConfig appConfig)
{
var searchConfig = new SearchConfig
{
DataSourceName = "telemetry-datasource",
IndexName = "telemetry-index",
IndexerName = "telemetry-indexer",
SkillsetName = "telemetry-skillset"
};
var dataSource = await CognitiveSearchHelper.GetOrCreateCosmosDataSource(searchClient, searchConfig.DataSourceName, DataSourceType.CosmosDb, appConfig.CosmosDb);
Console.WriteLine($"Successfully created data source {searchConfig.DataSourceName}");
var index = await CognitiveSearchHelper.GetIndexFromFile(searchConfig.IndexName);
var indexer = await CognitiveSearchHelper.GetIndexerFromFile(searchConfig);
var skillset = new Skillset
{
Name = searchConfig.SkillsetName,
Description = "Anomaly detection skills",
CognitiveServices = new CognitiveServicesByKey(appConfig.CognitiveServices.Key, appConfig.CognitiveServices.ResourceId),
Skills = new List <Skill>()
};
Console.WriteLine("Adding Custom Anomaly Detector skill to pipeline");
AddCustomAnomalyDetectorSkill(ref index, ref indexer, ref skillset, appConfig);
await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset)
.ContinueWith(t =>
{
Console.WriteLine(t.IsFaulted
? t.Exception.Message
: "Your anomaly detection pipeline was successfully created.");
});
Console.WriteLine("");
Console.WriteLine("");
}
static async Task Main(string[] args)
{
// Setup configuration to read from the appsettings.json file.
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
_configuration = builder.Build();
var appConfig = new AppConfig();
_configuration.Bind(appConfig);
using (var searchClient = new SearchServiceClient(appConfig.Search.ServiceName, new SearchCredentials(appConfig.Search.Key)))
{
while (true)
{
Console.ResetColor();
Console.WriteLine("=============");
Console.WriteLine("Choose an option below to run the PipelineEnhancer.");
Console.WriteLine("** Enter 1 to add a Sentiment Analysis cognitive skill to the Tweets search index.");
Console.WriteLine("** Enter 2 to add a knowledge store.");
Console.WriteLine("** Enter 3 to create a new search pipeline for searching and recognizing forms in Blob Storage.");
Console.WriteLine("** Enter 4 to create a new search pipeline for indexing vehicle telemetry and inspecting for engine temperature anomalies.");
Console.WriteLine("** Enter X to exit the console application.");
Console.WriteLine("=============");
Console.WriteLine("");
var userInput = "";
while (true)
{
Console.Write("Enter the number of the operation you would like to perform > ");
var input = Console.ReadLine().Trim();
if (input.Equals("1", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("2", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("3", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("4", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("X", StringComparison.InvariantCultureIgnoreCase))
{
userInput = input;
break;
}
else
{
Console.WriteLine("Invalid input entered. Please enter a number between 1 and 6, or X.");
}
}
if (userInput.Equals("x", StringComparison.InvariantCultureIgnoreCase))
{
break;
}
try
{
// Set the components back to their base objects, from JSON files
var index = await CognitiveSearchHelper.GetIndexFromFile(appConfig.Search.IndexName);
var indexer = await CognitiveSearchHelper.GetIndexerFromFile(appConfig.Search);
var skillset = await CognitiveSearchHelper.GetSkillsetFromFile(appConfig.Search.SkillsetName, appConfig.CognitiveServices);
var message = "";
Console.WriteLine("");
switch (userInput)
{
case "4":
await CreateAnomalyDetectionPipeline(searchClient, appConfig);
break;
case "3":
var modelId = await TrainFormRecognizerModel(appConfig.FormRecognizer, appConfig.BlobStorage);
await CreateFormsRecognitionPipeline(searchClient, appConfig, modelId);
break;
case "2":
AddKnowledgeStore(searchClient, appConfig, ref index, ref indexer, ref skillset);
Console.WriteLine("Successfully added the knowledge store to the cognitive search pipeline.");
break;
case "1":
AddSentimentAnalysisSkill(ref index, ref indexer, ref skillset);
message = "The sentiment analysis skill was successfully added to the search pipeline.";
await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset)
.ContinueWith(t =>
{
Console.WriteLine(t.IsFaulted
? t.Exception.Message
: message);
});
break;
default:
message = "Resetting the search pipeline to its initial state...";
await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset)
.ContinueWith(t =>
{
Console.WriteLine(t.IsFaulted
? t.Exception.Message
: message);
});
break;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.WriteLine("");
Console.WriteLine("");
}
}
}
static async Task Main(string[] args)
{
// Setup configuration to read from the appsettings.json file.
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
_configuration = builder.Build();
var appConfig = new AppConfig();
_configuration.Bind(appConfig);
using (var searchClient = new SearchServiceClient(appConfig.Search.ServiceName, new SearchCredentials(appConfig.Search.Key)))
{
while (true)
{
Console.WriteLine("PipelineEnhancer creates and updates Cognitive Search pipelines.");
Console.WriteLine("=============");
Console.WriteLine("** Enter 1 to add a Sentiment Analysis cognitive skill to the Tweets search index.");
//Console.WriteLine("** Enter 2 to include Personalized ranking information in the Tweets search index.");
Console.WriteLine("** Enter 2 to integrate a custom text translator skill to the Tweets search index.");
Console.WriteLine("** Enter 3 to create a new search pipeline for searching and recognizing forms in Blob Storage.");
Console.WriteLine("** Enter 4 to create a new search pipeline for indexing vehicle telemetry and inspecting for engine temperature anomalies.");
Console.WriteLine("=============");
Console.WriteLine("** Enter X to exit the console application.");
Console.WriteLine("=============");
Console.WriteLine("");
var userInput = "";
while (true)
{
Console.Write("Enter the number of the operation you would like to perform > ");
var input = Console.ReadLine();
if (input.Equals("1", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("2", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("3", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("4", StringComparison.InvariantCultureIgnoreCase) ||
//input.Equals("5", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("X", StringComparison.InvariantCultureIgnoreCase))
{
userInput = input.Trim();
break;
}
else
{
Console.WriteLine("Invalid input entered. Please enter a number between 1 and 4, or X.");
}
}
if (userInput.Equals("x", StringComparison.InvariantCultureIgnoreCase))
{
break;
}
try
{
// Retrieve the components from the API, or create base objects if they don't exist.
var index = await CognitiveSearchHelper.GetIndex(searchClient, appConfig.Search.IndexName);
var indexer = await CognitiveSearchHelper.GetIndexer(searchClient, appConfig.Search);
var skillset = await CognitiveSearchHelper.GetSkillset(searchClient, appConfig.Search.SkillsetName, appConfig.CognitiveServices);
var message = "";
Console.WriteLine("");
switch (userInput)
{
case "4":
await CreateAnomalyDetectionPipeline(searchClient, appConfig);
continue;
case "3":
var modelId = await TrainFormRecognizerModel(appConfig.FormRecognizer, appConfig.BlobStorage);
await CreateFormsRecognitionPipeline(searchClient, appConfig, modelId);
continue;
case "2":
AddCustomTranslateSkill(ref index, ref indexer, ref skillset, appConfig.FunctionApp);
Console.WriteLine("Your custom translator skill was successfully integrated to the search pipeline.");
goto case "1";
case "1":
AddSentimentAnalysisSkill(ref index, ref indexer, ref skillset);
message = "The sentiment analysis skill was successfully added to the search pipeline.";
await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset)
.ContinueWith(t =>
{
Console.WriteLine(t.IsFaulted
? t.Exception.Message
: message);
});
break;
default:
index = await CognitiveSearchHelper.GetIndexFromFile(appConfig.Search.IndexName);
indexer = await CognitiveSearchHelper.GetIndexerFromFile(appConfig.Search);
skillset = await CognitiveSearchHelper.GetSkillsetFromFile(appConfig.Search.SkillsetName, appConfig.CognitiveServices);
message = "The search pipeline has been restored to its initial state";
await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset)
.ContinueWith(t =>
{
Console.WriteLine(t.IsFaulted
? t.Exception.Message
: message);
});
break;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.WriteLine("");
Console.WriteLine("");
}
}
}