private EntityInfo[] getQuestionEntities(string question)
{
var linkedQuestion = Linker.LinkUtterance(question);
var questionEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray();
return(questionEntities);
}
internal static double evaluateLinker(QuestionDialogDatasetReader dataset, ILinker linker)
{
var totalDialogCount = 0;
var correctLinkCount = 0;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
totalDialogCount += 1;
var linkedQuestion = linker.LinkUtterance(dialog.Question);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities.Take(1)).ToArray();
var answerPhrase = getAnswerPhrase(dialog);
var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities);
var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities;
var isLinkingCorrect = answerPhraseEntities.Select(e => e.Mid).Contains(dialog.AnswerMid);
if (isLinkingCorrect)
{
++correctLinkCount;
}
}
return(1.0 * correctLinkCount / totalDialogCount);
}
private static void exportAugmentedLinkedAnswerHints(ILinker linker, string filePath, IEnumerable <QuestionDialog> dialogs)
{
var file = new StreamWriter(filePath);
foreach (var dialog in dialogs)
{
var answerEntity = new EntityInfo(dialog.AnswerMid, null, 0, 0);
var linkedQuestion = linker.LinkUtterance(dialog.Question);
//inject answer entities to question - forcing correct linking
var questionEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).Concat(new[] { answerEntity }).ToArray();
var answerTurn = dialog.AnswerTurns.Last();
var answerHint = answerTurn.InputChat;
var linkedAnswerHint = linker.LinkUtterance(answerHint, questionEntities);
if (linkedAnswerHint == null)
{
continue;
}
var featureText = linkedQuestion.GetEntityBasedRepresentation() + " ## " + linkedAnswerHint.GetEntityBasedRepresentation();
if (featureText.Contains("|"))
{
throw new NotImplementedException("escape feature text");
}
file.WriteLine("{0}|{1}|{2}", dialog.Id, featureText, dialog.AnswerMid);
}
file.Close();
}
private static void extractAnswer(string utterance, ILinker linker, LinkBasedExtractor extractor, QuestionDialogDatasetReader dataset)
{
var utteranceWords = getCleanWords(utterance).Distinct();
QuestionDialog dialogToAnswer = null;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
var questionWords = getCleanWords(dialog.Question);
if (utteranceWords.Except(questionWords).Count() == 0)
{
dialogToAnswer = dialog;
}
foreach (var turn in dialog.AnswerTurns)
{
var words = getCleanWords(turn.InputChat);
if (utteranceWords.Except(words).Count() == 0)
{
dialogToAnswer = dialog;
break;
}
}
if (dialogToAnswer != null)
{
break;
}
}
Console.WriteLine(dialogToAnswer.Question);
var answerPhrase = getAnswerPhrase(dialogToAnswer);
var linkedQuestion = linker.LinkUtterance(dialogToAnswer.Question);
Console.WriteLine(linkedQuestion);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray();
var result = linker.LinkUtterance(answerPhrase, contextEntities);
Console.WriteLine(result);
var answers = extractor.ExtractAnswerEntity(dialogToAnswer);
foreach (var answer in answers)
{
Console.WriteLine(answer);
}
}
internal static AnswerExtractionResult evaluateExtractor(QuestionDialogDatasetReader dataset, ILinker linker, Func <QuestionDialog, IEnumerable <EntityInfo> > extractor, int n = 1)
{
var totalDialogCount = 0;
var correctLinkCount = 0;
var correctExtractionCount = 0;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
totalDialogCount += 1;
var linkedQuestion = linker.LinkUtterance(dialog.Question);
//Console.WriteLine(linkedQuestion);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray();
var answerPhrase = getAnswerPhrase(dialog);
var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities);
var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities;
var utteranceEntities = linkedAnswer.Parts.SelectMany(p => p.Entities).ToArray();
var isLinkingCorrect = utteranceEntities.Select(e => e.Mid).Contains(dialog.AnswerMid);
if (!isLinkingCorrect)
{
continue;
}
++correctLinkCount;
var denotations = extractor(dialog).Take(n).ToArray();
var isExtractionCorrect = denotations.Any(d => d.Mid == dialog.AnswerMid);
if (isExtractionCorrect)
{
++correctExtractionCount;
}
}
return(new AnswerExtractionResult(1.0 * correctExtractionCount / totalDialogCount, 1.0 * correctExtractionCount / correctLinkCount));
}