// Return a List of pair {URI, Question} from a dialog file and a question file private static List <String[]> MapQuestionUri(FileInfo sourceOrDialogFileInfo, FileInfo annotatedQuestionsFileInfo) { // Load new dialog file string sourceOrDialogFileName; Dialog dialog = LoadDialogFile(sourceOrDialogFileInfo, out sourceOrDialogFileName); List <String[]> resultList = new List <String[]>(); using (StreamReader sr = new StreamReader(annotatedQuestionsFileInfo.FullName, Encoding.GetEncoding("UTF-8"))) { string line = null; while ((line = sr.ReadLine()) != null) { string[] columns = line.Split(';'); string questionId = columns[0]; string questionText = columns[1]; string intentName = columns[2]; // Simulate new dialog execution var result = DialogInterpreter.AnalyzeInitialQuestion(dialog, questionId, questionText, intentName); var type = result.ExecutionResult != null?result.ExecutionResult.DialogNode.Type.ToString() : "[no result]"; if (type == "FatHeadAnswers") { var node = result.ExecutionResult as FatHeadAnswerNodeExecution; if (node != null) { var newURI = node.MappingURI; List <String> listUriWithFederation = MappingUriGenerator.AddFederationsToUri(newURI); foreach (string uriTemp in listUriWithFederation) { string[] temp = { uriTemp, questionText }; resultList.Add(temp); } } } } } return(resultList); }
private static void InternalTest_DisplayMatchResults(FileInfo sourceOrDialogFileInfo) { Console.WriteLine("Internal test - Check entity values matches :"); Console.WriteLine(); // Load dialog file string sourceOrDialogFileName; Dialog dialog = LoadDialogFile(sourceOrDialogFileInfo, out sourceOrDialogFileName, true); Console.WriteLine(dialog.Errors.Count + " inconsistencies found :"); Console.WriteLine("- " + dialog.Errors.Where(error => error.Contains(MessageType.DuplicateKey.ToString())).Count() + " duplicate keys"); Console.WriteLine("- " + dialog.Errors.Where(error => error.Contains(MessageType.IncorrectPattern.ToString())).Count() + " incorrect patterns"); Console.WriteLine("- " + dialog.Errors.Where(error => error.Contains(MessageType.InvalidReference.ToString())).Count() + " invalid references"); Console.WriteLine("- " + dialog.Errors.Where(error => error.Contains(MessageType.NeverUsed.ToString())).Count() + " elements never used"); Console.WriteLine("- " + dialog.Errors.Where(error => error.Contains(MessageType.Info.ToString())).Count() + " infos"); Console.WriteLine(""); // Write list of inconsistencies var errorsFilePath = @"result\" + sourceOrDialogFileName + ".errors.csv"; Console.Write("Writing " + errorsFilePath + " ... "); using (StreamWriter sw = new StreamWriter(errorsFilePath, false, Encoding.GetEncoding("iso8859-1"))) { foreach (var error in dialog.Errors.OrderBy(error => error)) { sw.WriteLine(error); } } Console.WriteLine("OK"); Console.WriteLine(""); // Write list of entity values matches var testsIntentName = "TESTS"; var simulationFilePath = @"result\" + sourceOrDialogFileName + ".internaltest.csv"; Console.Write("Writing " + simulationFilePath + " ... "); using (StreamWriter sw = new StreamWriter(simulationFilePath, false, Encoding.GetEncoding("iso8859-1"))) { sw.WriteLine("#num;question;Test_Var;Test_Var_2;Test2_Var;Test2_Var_2"); var testIntent = dialog.Intents[testsIntentName]; int i = 0; foreach (var question in testIntent.Questions) { i++; var result = DialogInterpreter.AnalyzeInitialQuestion(dialog, i.ToString(), question, testsIntentName); string var11 = null; result.VariablesValues.TryGetValue("Test_Var", out var11); string var12 = null; result.VariablesValues.TryGetValue("Test_Var_2", out var12); string var21 = null; result.VariablesValues.TryGetValue("Test2_Var", out var21); string var22 = null; result.VariablesValues.TryGetValue("Test2_Var_2", out var22); sw.WriteLine(i.ToString() + ";" + question + ";" + var11 + ";" + var12 + ";" + var21 + ";" + var22); //sw.WriteLine(i.ToString() + ";" + question + ";" + "{{\"TestMatch1\": \"{0}\", \"TestMatch2\": \"{1}\", \"Test2Match1\": \"{2}\", \"Test2Match2\": \"{3}\"}}", var11, var12, var21, var22); } } Console.WriteLine("OK"); Console.WriteLine(""); }
private static void CompareResultsAcrossDialogVersions(FileInfo newSourceOrDialogFileInfo, FileInfo oldSourceOrDialogFileInfo, FileInfo annotatedQuestionsFileInfo) { Console.WriteLine("Compare results for two dialog files :"); Console.WriteLine(); // Load new dialog file string newSourceOrDialogFileName; Dialog newDialog = LoadDialogFile(newSourceOrDialogFileInfo, out newSourceOrDialogFileName); // Load old dialog file string oldSourceOrDialogFileName; Dialog oldDialog = LoadDialogFile(oldSourceOrDialogFileInfo, out oldSourceOrDialogFileName); // Load all questions Console.WriteLine("Reading " + annotatedQuestionsFileInfo.Name + " ... "); int questionCount = 0; IList <string> impacts = new List <string>(); impacts.Add("id;question;intent;diff;new result;old result;new trace;old trace"); using (StreamReader sr = new StreamReader(annotatedQuestionsFileInfo.FullName, Encoding.GetEncoding("iso8859-1"))) { string line = null; while ((line = sr.ReadLine()) != null) { string[] columns = line.Split(';'); string questionId = columns[0]; string questionText = columns[1]; string intentName = columns[2]; // Simulate new dialog execution var newResult = DialogInterpreter.AnalyzeInitialQuestion(newDialog, questionId, questionText, intentName); // Simulate old dialog execution var oldResult = DialogInterpreter.AnalyzeInitialQuestion(oldDialog, questionId, questionText, intentName); // Compare results if (!newResult.ReturnsSameResultAs(oldResult)) { var newType = newResult.ExecutionResult != null?newResult.ExecutionResult.DialogNode.Type.ToString() : "[no result]"; var oldType = oldResult.ExecutionResult != null?oldResult.ExecutionResult.DialogNode.Type.ToString() : "[no result]"; string diff = null; if (newType != oldType) { diff = oldType + " => " + newType; } else { if (newType == "FatHeadAnswers") { var newURI = ((FatHeadAnswerNodeExecution)newResult.ExecutionResult).MappingURI; var oldURI = ((FatHeadAnswerNodeExecution)oldResult.ExecutionResult).MappingURI; diff = "FH: " + FatHeadAnswerNodeExecution.CompareMappingURIs(newURI, oldURI); } else { diff = oldType + " changed"; } } impacts.Add(newResult.QuestionId + ";" + newResult.QuestionText + ";" + newResult.IntentName + ";" + diff + ";" + newResult.ResultString + ";" + oldResult.ResultString + ";" + newResult.ToString() + ";" + oldResult.ToString()); } questionCount++; if (questionCount % 500 == 0) { Console.WriteLine(questionCount + " test set questions analyzed"); } } // TO DO : do the same thing with all DisambiguationQuestion nodes found in the tree } Console.WriteLine("OK"); Console.WriteLine("Analysis completed :"); Console.WriteLine("- impacts found on " + impacts.Count + " questions of the test set"); Console.WriteLine(""); // Write comparison results file var comparisonFilePath = @"result\" + newSourceOrDialogFileName + "." + oldSourceOrDialogFileName + "." + annotatedQuestionsFileInfo.Name + ".compare.csv"; Console.Write("Writing " + comparisonFilePath + " ... "); HashSet <string> mappingURISet = new HashSet <string>(); using (StreamWriter sw = new StreamWriter(comparisonFilePath, false, Encoding.GetEncoding("iso8859-1"))) { foreach (var impact in impacts) { sw.WriteLine(impact); } } Console.WriteLine("OK"); Console.WriteLine(""); }