private static async Task CreateFormsRecognitionPipeline(SearchServiceClient searchClient, AppConfig appConfig, string modelId) { var formsSearchConfig = new SearchConfig { DataSourceName = "forms-datasource", IndexName = "forms-index", IndexerName = "forms-indexer", SkillsetName = "forms-skillset" }; var formsDataSource = await CognitiveSearchHelper.GetOrCreateBlobDataSource(searchClient, formsSearchConfig.DataSourceName, DataSourceType.AzureBlob, appConfig.BlobStorage); Console.WriteLine($"Successfully created data source {formsSearchConfig.DataSourceName}"); var formsIndex = await CognitiveSearchHelper.GetIndexFromFile(formsSearchConfig.IndexName); var formsIndexer = await CognitiveSearchHelper.GetIndexerFromFile(formsSearchConfig); var formsSkillset = await CognitiveSearchHelper.GetSkillsetFromFile(formsSearchConfig.SkillsetName, appConfig.CognitiveServices); Console.WriteLine("Adding Custom Form Recognizer skill to pipeline"); AddCustomFormRecognizerSkill(ref formsIndex, ref formsIndexer, ref formsSkillset, appConfig, modelId); await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, formsIndex, formsIndexer, formsSkillset) .ContinueWith(t => { Console.WriteLine(t.IsFaulted ? t.Exception.Message : "Your forms recognizer pipeline was successfully created."); }); }
private static void AddSentimentAnalysisSkill(ref Index index, ref Indexer indexer, ref Skillset skillset) { if (index.Fields.Any(f => f.Name == "sentiment")) { return; } var sentimentField = new Field("sentiment", DataType.Double) { IsSortable = true, IsFilterable = true }; index.Fields.Add(sentimentField); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping("document/sentiment", "sentiment").GetAwaiter().GetResult()); skillset.Skills.Add(new SentimentSkill { Context = "/document", Description = "Sentiment analysis skill", DefaultLanguageCode = SentimentSkillLanguage.En, Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("text", "/document/text") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("score", "sentiment") } }); }
private static void AddCustomSummarizerSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, FunctionAppConfig config) { var targetField = "summary"; var headers = new Dictionary <string, string> { { "Content-Type", "application/json" } }; index.Fields.Add(new Field(targetField, AnalyzerName.StandardLucene)); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/{targetField}", targetField).GetAwaiter().GetResult()); // Create the custom translate skill skillset.Skills.Add(new WebApiSkill { Description = "Custom summarization skill", Context = "/document", Uri = $"{config.Url}/api/Summarize?code={config.DefaultHostKey}", HttpMethod = "POST", //HttpHeaders = new WebApiHttpHeaders(headers), BatchSize = 1, Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("text", "/document/textTranslated") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("summaryText", targetField) } }); }
private static void AddUserInfoToIndex(ref Index index, ref Indexer indexer) { var analyzer = AnalyzerName.StandardLucene; // Create a new index fields for userName and userLocation index.Fields.Add(new Field("userName", analyzer)); index.Fields.Add(new Field("userLocation", analyzer)); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping("/document/user/name", "userName").GetAwaiter().GetResult()); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping("/document/user/location", "userLocation").GetAwaiter().GetResult()); }
private static void AddCustomTranslateSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, FunctionAppConfig config) { var targetField = "textTranslated"; var headers = new Dictionary <string, string> { { "Content-Type", "application/json" } }; index.Fields.Add(new Field(targetField, AnalyzerName.StandardLucene)); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/{targetField}", targetField).GetAwaiter().GetResult()); // Create the custom translate skill skillset.Skills.Add(new WebApiSkill { Description = "Custom translator skill", Context = "/document", Uri = $"{config.Url}/api/Translate?code={config.DefaultHostKey}", HttpMethod = "POST", //HttpHeaders = new WebApiHttpHeaders(headers), BatchSize = 1, Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("text", "/document/text"), new InputFieldMappingEntry("language", "/document/Language") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("text", targetField) } }); // Update all the other skills, except for the LanguageDetectionSkill, to use the new textTranslated field. foreach (var skill in skillset.Skills) { var type = skill.GetType(); var typeName = type.Name; if (typeName != "WebApiSkill" && typeName != "LanguageDetectionSkill") { foreach (var input in skill.Inputs) { if (input.Source == "/document/text") { input.Source = $"/document/{targetField}"; } } } } }
private static void AddCustomAnomalyDetectorSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, AppConfig config) { var headers = new Dictionary <string, string> { { "Content-Type", "application/json" } }; var anomalyFields = new List <Field> { new Field($"isAnomaly", DataType.Boolean), new Field($"isPositiveAnomaly", DataType.Boolean), new Field($"isNegativeAnomaly", DataType.Boolean), new Field($"expectedValue", DataType.Double), new Field($"upperMargin", DataType.Double), new Field($"lowerMargin", DataType.Double) }; index.Fields.Add(new Field("engineTemperatureAnalysis", DataType.Complex, anomalyFields)); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/engineTemperatureAnalysis", "engineTemperatureAnalysis").GetAwaiter().GetResult()); // Create the custom translate skill skillset.Skills.Add(new WebApiSkill { Description = "Custom Anomaly Detector skill", Context = "/document", Uri = $"{config.FunctionApp.Url}/api/DetectAnomalies?code={config.FunctionApp.DefaultHostKey}", HttpMethod = "POST", //HttpHeaders = new WebApiHttpHeaders(), // This is broken in the SDK, so handle by sending JSON directly to Rest API. BatchSize = 1, Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("timestamp", "/document/timestamp"), new InputFieldMappingEntry("engineTemperature", "/document/engineTemperature") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("anomalyResult", "engineTemperatureAnalysis") } }); }
private static void AddKnowledgeStore(SearchServiceClient searchClient, AppConfig appConfig, ref Index index, ref Indexer indexer, ref Skillset skillset) { // Add the skills from previous steps AddSentimentAnalysisSkill(ref index, ref indexer, ref skillset); // Convert the Skillset into a JSON string. var skillsetJson = CognitiveSearchHelper.GetSkillsetJson(skillset).GetAwaiter().GetResult(); // Insert ShaperSkill into the Skillset as a JSON string Console.WriteLine("Adding Shaper Skill to the pipeline to set up the table projections."); var shaperSkillJson = GetJsonFromFile("shaper-skill").GetAwaiter().GetResult(); var skillsetJsonWithShaperSkill = CognitiveSearchHelper.InsertSkillAsJson(skillsetJson, shaperSkillJson); // Insert knowledge store JSON string into the Skillset JSON Console.WriteLine("Inserting the knowledge store with table projections."); var updatedSkillset = InsertKnowledgeStoreJson(skillsetJsonWithShaperSkill, appConfig.BlobStorage); // Create the search pipeline using the updated skillset JSON. No SDK exists yet for doing this using the SDK objects, so must use the REST API to accomplish adding a knowledge store via code. Console.WriteLine("Rebuilding cognitive search pipeline..."); CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset.Name, updatedSkillset).GetAwaiter().GetResult(); }
private static void AddCustomFormRecognizerSkill(ref Index index, ref Indexer indexer, ref Skillset skillset, AppConfig config, string modelId) { var headers = new Dictionary <string, string> { { "Content-Type", "application/json" } }; index.Fields.Add(new Field($"formHeight", DataType.Int32)); index.Fields.Add(new Field($"formWidth", DataType.Int32)); index.Fields.Add(new Field($"formKeyValuePairs", DataType.Collection(DataType.String))); index.Fields.Add(new Field($"formColumns", DataType.Collection(DataType.String))); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formHeight", "formHeight").GetAwaiter().GetResult()); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formWidth", "formWidth").GetAwaiter().GetResult()); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formKeyValuePairs", "formKeyValuePairs").GetAwaiter().GetResult()); indexer.OutputFieldMappings.Add(CognitiveSearchHelper.CreateFieldMapping($"/document/formColumns", "formColumns").GetAwaiter().GetResult()); // Create the custom translate skill skillset.Skills.Add(new WebApiSkill { Description = "Custom Form Recognizer skill", Context = "/document", Uri = $"{config.FunctionApp.Url}/api/AnalyzeForm?code={config.FunctionApp.DefaultHostKey}&modelId={modelId}", HttpMethod = "POST", //HttpHeaders = new WebApiHttpHeaders(), // This is broken in the SDK, so handle by sending JSON directly to Rest API. BatchSize = 1, Inputs = new List <InputFieldMappingEntry> { new InputFieldMappingEntry("contentType", "/document/fileContentType"), new InputFieldMappingEntry("storageUri", "/document/storageUri"), new InputFieldMappingEntry("storageSasToken", "/document/sasToken") }, Outputs = new List <OutputFieldMappingEntry> { new OutputFieldMappingEntry("formHeight", "formHeight"), new OutputFieldMappingEntry("formWidth", "formWidth"), new OutputFieldMappingEntry("formKeyValuePairs", "formKeyValuePairs"), new OutputFieldMappingEntry("formColumns", "formColumns"), } }); }
private static async Task CreateAnomalyDetectionPipeline(SearchServiceClient searchClient, AppConfig appConfig) { var searchConfig = new SearchConfig { DataSourceName = "telemetry-datasource", IndexName = "telemetry-index", IndexerName = "telemetry-indexer", SkillsetName = "telemetry-skillset" }; var dataSource = await CognitiveSearchHelper.GetOrCreateCosmosDataSource(searchClient, searchConfig.DataSourceName, DataSourceType.CosmosDb, appConfig.CosmosDb); Console.WriteLine($"Successfully created data source {searchConfig.DataSourceName}"); var index = await CognitiveSearchHelper.GetIndexFromFile(searchConfig.IndexName); var indexer = await CognitiveSearchHelper.GetIndexerFromFile(searchConfig); var skillset = new Skillset { Name = searchConfig.SkillsetName, Description = "Anomaly detection skills", CognitiveServices = new CognitiveServicesByKey(appConfig.CognitiveServices.Key, appConfig.CognitiveServices.ResourceId), Skills = new List <Skill>() }; Console.WriteLine("Adding Custom Anomaly Detector skill to pipeline"); AddCustomAnomalyDetectorSkill(ref index, ref indexer, ref skillset, appConfig); await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset) .ContinueWith(t => { Console.WriteLine(t.IsFaulted ? t.Exception.Message : "Your anomaly detection pipeline was successfully created."); }); Console.WriteLine(""); Console.WriteLine(""); }
static async Task Main(string[] args) { // Setup configuration to read from the appsettings.json file. var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true); _configuration = builder.Build(); var appConfig = new AppConfig(); _configuration.Bind(appConfig); using (var searchClient = new SearchServiceClient(appConfig.Search.ServiceName, new SearchCredentials(appConfig.Search.Key))) { while (true) { Console.ResetColor(); Console.WriteLine("============="); Console.WriteLine("Choose an option below to run the PipelineEnhancer."); Console.WriteLine("** Enter 1 to add a Sentiment Analysis cognitive skill to the Tweets search index."); Console.WriteLine("** Enter 2 to add a knowledge store."); Console.WriteLine("** Enter 3 to create a new search pipeline for searching and recognizing forms in Blob Storage."); Console.WriteLine("** Enter 4 to create a new search pipeline for indexing vehicle telemetry and inspecting for engine temperature anomalies."); Console.WriteLine("** Enter X to exit the console application."); Console.WriteLine("============="); Console.WriteLine(""); var userInput = ""; while (true) { Console.Write("Enter the number of the operation you would like to perform > "); var input = Console.ReadLine().Trim(); if (input.Equals("1", StringComparison.InvariantCultureIgnoreCase) || input.Equals("2", StringComparison.InvariantCultureIgnoreCase) || input.Equals("3", StringComparison.InvariantCultureIgnoreCase) || input.Equals("4", StringComparison.InvariantCultureIgnoreCase) || input.Equals("X", StringComparison.InvariantCultureIgnoreCase)) { userInput = input; break; } else { Console.WriteLine("Invalid input entered. Please enter a number between 1 and 6, or X."); } } if (userInput.Equals("x", StringComparison.InvariantCultureIgnoreCase)) { break; } try { // Set the components back to their base objects, from JSON files var index = await CognitiveSearchHelper.GetIndexFromFile(appConfig.Search.IndexName); var indexer = await CognitiveSearchHelper.GetIndexerFromFile(appConfig.Search); var skillset = await CognitiveSearchHelper.GetSkillsetFromFile(appConfig.Search.SkillsetName, appConfig.CognitiveServices); var message = ""; Console.WriteLine(""); switch (userInput) { case "4": await CreateAnomalyDetectionPipeline(searchClient, appConfig); break; case "3": var modelId = await TrainFormRecognizerModel(appConfig.FormRecognizer, appConfig.BlobStorage); await CreateFormsRecognitionPipeline(searchClient, appConfig, modelId); break; case "2": AddKnowledgeStore(searchClient, appConfig, ref index, ref indexer, ref skillset); Console.WriteLine("Successfully added the knowledge store to the cognitive search pipeline."); break; case "1": AddSentimentAnalysisSkill(ref index, ref indexer, ref skillset); message = "The sentiment analysis skill was successfully added to the search pipeline."; await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset) .ContinueWith(t => { Console.WriteLine(t.IsFaulted ? t.Exception.Message : message); }); break; default: message = "Resetting the search pipeline to its initial state..."; await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset) .ContinueWith(t => { Console.WriteLine(t.IsFaulted ? t.Exception.Message : message); }); break; } } catch (Exception ex) { Console.WriteLine(ex.Message); } Console.WriteLine(""); Console.WriteLine(""); } } }
static async Task Main(string[] args) { // Setup configuration to read from the appsettings.json file. var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true); _configuration = builder.Build(); var appConfig = new AppConfig(); _configuration.Bind(appConfig); using (var searchClient = new SearchServiceClient(appConfig.Search.ServiceName, new SearchCredentials(appConfig.Search.Key))) { while (true) { Console.WriteLine("PipelineEnhancer creates and updates Cognitive Search pipelines."); Console.WriteLine("============="); Console.WriteLine("** Enter 1 to add a Sentiment Analysis cognitive skill to the Tweets search index."); //Console.WriteLine("** Enter 2 to include Personalized ranking information in the Tweets search index."); Console.WriteLine("** Enter 2 to integrate a custom text translator skill to the Tweets search index."); Console.WriteLine("** Enter 3 to create a new search pipeline for searching and recognizing forms in Blob Storage."); Console.WriteLine("** Enter 4 to create a new search pipeline for indexing vehicle telemetry and inspecting for engine temperature anomalies."); Console.WriteLine("============="); Console.WriteLine("** Enter X to exit the console application."); Console.WriteLine("============="); Console.WriteLine(""); var userInput = ""; while (true) { Console.Write("Enter the number of the operation you would like to perform > "); var input = Console.ReadLine(); if (input.Equals("1", StringComparison.InvariantCultureIgnoreCase) || input.Equals("2", StringComparison.InvariantCultureIgnoreCase) || input.Equals("3", StringComparison.InvariantCultureIgnoreCase) || input.Equals("4", StringComparison.InvariantCultureIgnoreCase) || //input.Equals("5", StringComparison.InvariantCultureIgnoreCase) || input.Equals("X", StringComparison.InvariantCultureIgnoreCase)) { userInput = input.Trim(); break; } else { Console.WriteLine("Invalid input entered. Please enter a number between 1 and 4, or X."); } } if (userInput.Equals("x", StringComparison.InvariantCultureIgnoreCase)) { break; } try { // Retrieve the components from the API, or create base objects if they don't exist. var index = await CognitiveSearchHelper.GetIndex(searchClient, appConfig.Search.IndexName); var indexer = await CognitiveSearchHelper.GetIndexer(searchClient, appConfig.Search); var skillset = await CognitiveSearchHelper.GetSkillset(searchClient, appConfig.Search.SkillsetName, appConfig.CognitiveServices); var message = ""; Console.WriteLine(""); switch (userInput) { case "4": await CreateAnomalyDetectionPipeline(searchClient, appConfig); continue; case "3": var modelId = await TrainFormRecognizerModel(appConfig.FormRecognizer, appConfig.BlobStorage); await CreateFormsRecognitionPipeline(searchClient, appConfig, modelId); continue; case "2": AddCustomTranslateSkill(ref index, ref indexer, ref skillset, appConfig.FunctionApp); Console.WriteLine("Your custom translator skill was successfully integrated to the search pipeline."); goto case "1"; case "1": AddSentimentAnalysisSkill(ref index, ref indexer, ref skillset); message = "The sentiment analysis skill was successfully added to the search pipeline."; await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset) .ContinueWith(t => { Console.WriteLine(t.IsFaulted ? t.Exception.Message : message); }); break; default: index = await CognitiveSearchHelper.GetIndexFromFile(appConfig.Search.IndexName); indexer = await CognitiveSearchHelper.GetIndexerFromFile(appConfig.Search); skillset = await CognitiveSearchHelper.GetSkillsetFromFile(appConfig.Search.SkillsetName, appConfig.CognitiveServices); message = "The search pipeline has been restored to its initial state"; await CognitiveSearchHelper.CreateCognitiveSearchPipeline(searchClient, appConfig.Search, index, indexer, skillset) .ContinueWith(t => { Console.WriteLine(t.IsFaulted ? t.Exception.Message : message); }); break; } } catch (Exception ex) { Console.WriteLine(ex.Message); } Console.WriteLine(""); Console.WriteLine(""); } } }