public ConfusionMatrixAnalysis CreateConfusionMatrix(List <TestResultModel[]> foldResults)
{
var confusionMatrix = new ConfusionMatrixAnalysis(this._applicationVersion);
foreach (var testResults in foldResults)
{
foreach (var utterance in testResults.Where(x => !x.IsCorrect))
{
var confusionModel = confusionMatrix.MatrixItems.FirstOrDefault(x => x.ExpectedIntentName == utterance.IntentLabel);
if (confusionModel == null)
{
confusionModel = new MatrixItem(confusionMatrix, utterance.IntentLabel);
confusionMatrix.MatrixItems.Add(confusionModel);
}
var confusion = confusionModel.Confusions.FirstOrDefault(x => x.FoundIntent == utterance.FirstIntent.Name);
if (confusion == null)
{
confusion = new Confusion(confusionModel, utterance.FirstIntent.Name);
confusionModel.Confusions.Add(confusion);
}
var predictions = utterance.IntentPredictions.OrderByDescending(x => x.Score).Take(10).Select(x => new UtteranceIntents(x.Name, x.Score)).ToList();
confusion.Utterances.Add(new Utterance(confusion, utterance.Text, utterance.TokenizedText, utterance.FirstIntent.Score, predictions));
}
}
return(confusionMatrix);
}
private static void WriteConfusionHtmlFile(ConfusionMatrixAnalysis confusionMatrix)
{
var templateFilePath = Path.Combine(AppContext.BaseDirectory, "templates", "results.html");
string templateFile = File.ReadAllText(templateFilePath, Encoding.UTF8);
var maxConfusions = confusionMatrix.MatrixItems.Max(x => x.Confusions.Count);
var confusionColumns = new StringBuilder();
var confusionHeaders = new StringBuilder();
for (var i = 0; i < maxConfusions; i++)
{
confusionHeaders.AppendLine($"<th style='width:150px !important;'>intent_{i + 1}</th>");
confusionColumns.AppendLine(JsonConvert.SerializeObject(new { data = $"intent_{i + 1}" }) + ",");
}
templateFile = templateFile.Replace("#CONFUSION_COLUMNS#", confusionColumns.ToString());
templateFile = templateFile.Replace("#CONFUSION_HEADERS#", confusionHeaders.ToString());
Func <Utterance, string> FormatUtterance = (item) =>
{
var words = item.Text.Split(' ');
var tokenAnalysis = item.TokenizedAnalysis;
var falsePositiveIcon = item.FalsePosiive ? "<span class='badge badge-warning'><i class='fa fa-ban'></i></span>" : "";
var phrase = String.Join(" ", words.Select(word =>
{
var apperance = tokenAnalysis.FirstOrDefault(x => x.Token == word);
if (apperance != null)
{
var color = apperance.ApperancesInExpectedIntent <= 1 ? "style='color: orange;'" : "";
return($"<span data-toggle='tooltip' {color} data-placement='bottom' title='{apperance.ApperancesInExpectedIntent}({apperance.PercentageInExpectedIntent.ToString("F2")}%) Occurences in Expected Intent | {apperance.ApperancesInFoundIntent}({apperance.PercentageInFoundIntent.ToString("F2")}%) Occurences in Found Intent'>{word}</span>");
}
else
{
return(word);
}
}));
phrase = falsePositiveIcon + phrase + $"({item.Score})";
return(phrase);
};
var data = confusionMatrix.MatrixItems.Select(x =>
{
JObject model = new JObject();
model["id"] = x.ExpectedIntentName;
model["confusions"] = x.Confusions.Sum(y => y.Utterances.Count);
model["expected_intent"] = x.ExpectedIntentName;
var orderedConfusions = x.Confusions.OrderByDescending(c => c.Utterances.Count).ToList();
for (var i = 0; i < maxConfusions; i++)
{
var examples = new JArray(orderedConfusions.Count > i ? orderedConfusions[i].Utterances.Select(FormatUtterance).ToArray() : new string[] { });
var examplesData = JArray.FromObject(orderedConfusions.Count > i ? orderedConfusions[i].Utterances.ToArray() : new object[] { });
model.Add("examples_intent_" + (i + 1), examples);
model.Add("examples_data_intent_" + (i + 1), examplesData);
model["intent_" + (i + 1)] = orderedConfusions.Count > i ? $"{orderedConfusions[i].FoundIntent} ({orderedConfusions[i].Utterances.Count})" : "-";
}
return(model);
});
templateFile = templateFile.Replace("#DATA#", JsonConvert.SerializeObject(data));
var intents = new JObject();
confusionMatrix.ApplicationVersion.Utterances.GroupBy(x => x.Intent).ToList().ForEach(item =>
{
intents.Add(item.Key, new JArray(item.Select(x => x.Text).ToArray()));
});
templateFile = templateFile.Replace("#INTENTS#", JsonConvert.SerializeObject(intents));
using (var writer = new StreamWriter($"{directory}/ConfusionMatrix-{DateTime.Now.ToString().CleanFileName()}.html", false, Encoding.UTF8))
{
writer.Write(templateFile);
}
}
