public ConfusionMatrixAnalysis CreateConfusionMatrix(List <TestResultModel[]> foldResults) { var confusionMatrix = new ConfusionMatrixAnalysis(this._applicationVersion); foreach (var testResults in foldResults) { foreach (var utterance in testResults.Where(x => !x.IsCorrect)) { var confusionModel = confusionMatrix.MatrixItems.FirstOrDefault(x => x.ExpectedIntentName == utterance.IntentLabel); if (confusionModel == null) { confusionModel = new MatrixItem(confusionMatrix, utterance.IntentLabel); confusionMatrix.MatrixItems.Add(confusionModel); } var confusion = confusionModel.Confusions.FirstOrDefault(x => x.FoundIntent == utterance.FirstIntent.Name); if (confusion == null) { confusion = new Confusion(confusionModel, utterance.FirstIntent.Name); confusionModel.Confusions.Add(confusion); } var predictions = utterance.IntentPredictions.OrderByDescending(x => x.Score).Take(10).Select(x => new UtteranceIntents(x.Name, x.Score)).ToList(); confusion.Utterances.Add(new Utterance(confusion, utterance.Text, utterance.TokenizedText, utterance.FirstIntent.Score, predictions)); } } return(confusionMatrix); }
private static void WriteConfusionHtmlFile(ConfusionMatrixAnalysis confusionMatrix) { var templateFilePath = Path.Combine(AppContext.BaseDirectory, "templates", "results.html"); string templateFile = File.ReadAllText(templateFilePath, Encoding.UTF8); var maxConfusions = confusionMatrix.MatrixItems.Max(x => x.Confusions.Count); var confusionColumns = new StringBuilder(); var confusionHeaders = new StringBuilder(); for (var i = 0; i < maxConfusions; i++) { confusionHeaders.AppendLine($"<th style='width:150px !important;'>intent_{i + 1}</th>"); confusionColumns.AppendLine(JsonConvert.SerializeObject(new { data = $"intent_{i + 1}" }) + ","); } templateFile = templateFile.Replace("#CONFUSION_COLUMNS#", confusionColumns.ToString()); templateFile = templateFile.Replace("#CONFUSION_HEADERS#", confusionHeaders.ToString()); Func <Utterance, string> FormatUtterance = (item) => { var words = item.Text.Split(' '); var tokenAnalysis = item.TokenizedAnalysis; var falsePositiveIcon = item.FalsePosiive ? "<span class='badge badge-warning'><i class='fa fa-ban'></i></span>" : ""; var phrase = String.Join(" ", words.Select(word => { var apperance = tokenAnalysis.FirstOrDefault(x => x.Token == word); if (apperance != null) { var color = apperance.ApperancesInExpectedIntent <= 1 ? "style='color: orange;'" : ""; return($"<span data-toggle='tooltip' {color} data-placement='bottom' title='{apperance.ApperancesInExpectedIntent}({apperance.PercentageInExpectedIntent.ToString("F2")}%) Occurences in Expected Intent | {apperance.ApperancesInFoundIntent}({apperance.PercentageInFoundIntent.ToString("F2")}%) Occurences in Found Intent'>{word}</span>"); } else { return(word); } })); phrase = falsePositiveIcon + phrase + $"({item.Score})"; return(phrase); }; var data = confusionMatrix.MatrixItems.Select(x => { JObject model = new JObject(); model["id"] = x.ExpectedIntentName; model["confusions"] = x.Confusions.Sum(y => y.Utterances.Count); model["expected_intent"] = x.ExpectedIntentName; var orderedConfusions = x.Confusions.OrderByDescending(c => c.Utterances.Count).ToList(); for (var i = 0; i < maxConfusions; i++) { var examples = new JArray(orderedConfusions.Count > i ? orderedConfusions[i].Utterances.Select(FormatUtterance).ToArray() : new string[] { }); var examplesData = JArray.FromObject(orderedConfusions.Count > i ? orderedConfusions[i].Utterances.ToArray() : new object[] { }); model.Add("examples_intent_" + (i + 1), examples); model.Add("examples_data_intent_" + (i + 1), examplesData); model["intent_" + (i + 1)] = orderedConfusions.Count > i ? $"{orderedConfusions[i].FoundIntent} ({orderedConfusions[i].Utterances.Count})" : "-"; } return(model); }); templateFile = templateFile.Replace("#DATA#", JsonConvert.SerializeObject(data)); var intents = new JObject(); confusionMatrix.ApplicationVersion.Utterances.GroupBy(x => x.Intent).ToList().ForEach(item => { intents.Add(item.Key, new JArray(item.Select(x => x.Text).ToArray())); }); templateFile = templateFile.Replace("#INTENTS#", JsonConvert.SerializeObject(intents)); using (var writer = new StreamWriter($"{directory}/ConfusionMatrix-{DateTime.Now.ToString().CleanFileName()}.html", false, Encoding.UTF8)) { writer.Write(templateFile); } }