internal static double evaluateLinker(QuestionDialogDatasetReader dataset, ILinker linker)
{
var totalDialogCount = 0;
var correctLinkCount = 0;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
totalDialogCount += 1;
var linkedQuestion = linker.LinkUtterance(dialog.Question);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities.Take(1)).ToArray();
var answerPhrase = getAnswerPhrase(dialog);
var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities);
var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities;
var isLinkingCorrect = answerPhraseEntities.Select(e => e.Mid).Contains(dialog.AnswerMid);
if (isLinkingCorrect)
{
++correctLinkCount;
}
}
return(1.0 * correctLinkCount / totalDialogCount);
}
private static void fillWithHintedQuestions(QuestionDialogDatasetReader qdd, List <string> questions)
{
foreach (var dialog in qdd.Dialogs)
{
if (dialog.HasCorrectAnswer)
{
questions.Add(dialog.Question);
}
}
}
private static int countDialogsWithCorrectAnswerHint(QuestionDialogDatasetReader reader)
{
var count = 0;
foreach (var dialog in reader.Dialogs)
{
if (dialog.HasCorrectAnswer)
{
count += 1;
}
}
return(count);
}
internal static AnswerExtractionResult evaluateExtractor(QuestionDialogDatasetReader dataset, ILinker linker, Func <QuestionDialog, IEnumerable <EntityInfo> > extractor, int n = 1)
{
var totalDialogCount = 0;
var correctLinkCount = 0;
var correctExtractionCount = 0;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
totalDialogCount += 1;
var linkedQuestion = linker.LinkUtterance(dialog.Question);
//Console.WriteLine(linkedQuestion);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray();
var answerPhrase = getAnswerPhrase(dialog);
var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities);
var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities;
var utteranceEntities = linkedAnswer.Parts.SelectMany(p => p.Entities).ToArray();
var isLinkingCorrect = utteranceEntities.Select(e => e.Mid).Contains(dialog.AnswerMid);
if (!isLinkingCorrect)
{
continue;
}
++correctLinkCount;
var denotations = extractor(dialog).Take(n).ToArray();
var isExtractionCorrect = denotations.Any(d => d.Mid == dialog.AnswerMid);
if (isExtractionCorrect)
{
++correctExtractionCount;
}
}
return(new AnswerExtractionResult(1.0 * correctExtractionCount / totalDialogCount, 1.0 * correctExtractionCount / correctLinkCount));
}
internal static IEnumerable <string> LoadPhrases(QuestionDialogDatasetReader seedDialogs, FreebaseDbProvider db)
{
var entities = seedDialogs.Dialogs.Select(d => db.GetEntryFromMid(d.AnswerMid)).Where(e => e != null).ToArray();
var phrases = new List <string>();
foreach (var entity in entities)
{
if (entity.Aliases.Count() < 2)
{
continue;
}
var alias = entity.Aliases.First();
if (!meetsPhraseRequirements(alias))
{
continue;
}
phrases.Add(entity.Aliases.First());
}
return(phrases.Distinct().ToArray());
}
public GraphNavigationExperiment(string experimentsRoot, string experimentId, int taskCount, QuestionDialogDatasetReader seedDialogs)
: base(experimentsRoot, experimentId)
{
_db = Configuration.Db;
var phrases = LoadPhrases(seedDialogs, _db);
_phrases = phrases.ToArray();
var navigationDataPath = Path.Combine(ExperimentRootPath, "navigation_data.nvd");
_data = new NavigationData(navigationDataPath);
_linker = createLinker(_phrases);
var writer = new CrowdFlowerCodeWriter(ExperimentRootPath, experimentId);
//generate all tasks
for (var taskIndex = 0; taskIndex < taskCount; ++taskIndex)
{
add(taskIndex, writer);
}
writer.Close();
}
private static void linkOnAnswerHint(string utterance, GraphDisambiguatedLinker linker, QuestionDialogDatasetReader dataset)
{
var utteranceWords = utterance.ToLowerInvariant().Split(' ').Distinct();
QuestionDialog dialogToLink = null;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
var questionWords = dialog.Question.ToLowerInvariant().Split(' ');
if (utteranceWords.Except(questionWords).Count() == 0)
{
dialogToLink = dialog;
}
foreach (var turn in dialog.AnswerTurns)
{
var words = turn.InputChat.ToLowerInvariant().Split(' ').Distinct();
if (utteranceWords.Except(words).Count() == 0)
{
dialogToLink = dialog;
break;
}
}
if (dialogToLink != null)
{
break;
}
}
Console.WriteLine(dialogToLink.Question);
var answerPhrase = getAnswerPhrase(dialogToLink);
var linkedQuestion = linker.LinkUtterance(dialogToLink.Question);
Console.WriteLine(linkedQuestion);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray();
var result = linker.LinkUtterance(answerPhrase, contextEntities);
Console.WriteLine(result);
}
private static void extractAnswer(string utterance, ILinker linker, LinkBasedExtractor extractor, QuestionDialogDatasetReader dataset)
{
var utteranceWords = getCleanWords(utterance).Distinct();
QuestionDialog dialogToAnswer = null;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
var questionWords = getCleanWords(dialog.Question);
if (utteranceWords.Except(questionWords).Count() == 0)
{
dialogToAnswer = dialog;
}
foreach (var turn in dialog.AnswerTurns)
{
var words = getCleanWords(turn.InputChat);
if (utteranceWords.Except(words).Count() == 0)
{
dialogToAnswer = dialog;
break;
}
}
if (dialogToAnswer != null)
{
break;
}
}
Console.WriteLine(dialogToAnswer.Question);
var answerPhrase = getAnswerPhrase(dialogToAnswer);
var linkedQuestion = linker.LinkUtterance(dialogToAnswer.Question);
Console.WriteLine(linkedQuestion);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray();
var result = linker.LinkUtterance(answerPhrase, contextEntities);
Console.WriteLine(result);
var answers = extractor.ExtractAnswerEntity(dialogToAnswer);
foreach (var answer in answers)
{
Console.WriteLine(answer);
}
}
