internal static void BasicCancelation_WithEnumAndNgrams() { var testData = Configuration.GetQuestionDialogsTest(); var trainDataset = Configuration.GetQuestionDialogsTrain(); var linker = Configuration.Linker; var linkedExtractor = new LinkBasedExtractor(linker, Configuration.Db); //enumeration detection is not disabled here linkedExtractor.UseNgramOrdering = true; //train the extractor foreach (var dialog in trainDataset.Dialogs) { if (dialog.HasCorrectAnswer) { Console.WriteLine(dialog.Question); linkedExtractor.Train(dialog); } } var result = evaluateExtractor(testData, linker, (d) => linkedExtractor.ExtractAnswerEntity(d)); Console.WriteLine(result); }
private static void extractAnswer(string utterance, ILinker linker, LinkBasedExtractor extractor, QuestionDialogDatasetReader dataset) { var utteranceWords = getCleanWords(utterance).Distinct(); QuestionDialog dialogToAnswer = null; foreach (var dialog in dataset.Dialogs) { if (!dialog.HasCorrectAnswer) { continue; } var questionWords = getCleanWords(dialog.Question); if (utteranceWords.Except(questionWords).Count() == 0) { dialogToAnswer = dialog; } foreach (var turn in dialog.AnswerTurns) { var words = getCleanWords(turn.InputChat); if (utteranceWords.Except(words).Count() == 0) { dialogToAnswer = dialog; break; } } if (dialogToAnswer != null) { break; } } Console.WriteLine(dialogToAnswer.Question); var answerPhrase = getAnswerPhrase(dialogToAnswer); var linkedQuestion = linker.LinkUtterance(dialogToAnswer.Question); Console.WriteLine(linkedQuestion); var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray(); var result = linker.LinkUtterance(answerPhrase, contextEntities); Console.WriteLine(result); var answers = extractor.ExtractAnswerEntity(dialogToAnswer); foreach (var answer in answers) { Console.WriteLine(answer); } }
internal static void BasicCancelation_WithEnumDetection() { var testData = Configuration.GetQuestionDialogsTest(); var linker = Configuration.Linker; var linkedExtractor = new LinkBasedExtractor(linker, Configuration.Db); //enumeration detection is not disabled here var result = evaluateExtractor(testData, linker, (d) => linkedExtractor.ExtractAnswerEntity(d), 2); Console.WriteLine(result); }
internal QuestionReport(QuestionInfo info, string answerId, LinkBasedExtractor extractor) { var linker = extractor.Linker; Question = linker.LinkUtterance(info.Utterance.OriginalSentence); AnswerLabel = extractor.Db.GetEntryFromId(answerId); var denotations = new List <Tuple <LinkedUtterance, EntityInfo, bool> >(); foreach (var answerHint in info.AnswerHints) { var linkedHint = linker.LinkUtterance(answerHint.OriginalSentence, Question.Entities); var denotation = extractor.ExtractAnswerEntity(info.Utterance.OriginalSentence, answerHint.OriginalSentence).FirstOrDefault(); var item = Tuple.Create(linkedHint, denotation, answerId == FreebaseDbProvider.GetId(denotation.Mid)); denotations.Add(item); } CollectedDenotations = denotations; var denotationCounts = from denotation in denotations group denotation by FreebaseDbProvider.GetId(denotation.Item2.Mid) into grouped select Tuple.Create(grouped.Key, grouped.Count()); var maxDenotation = denotationCounts.OrderByDescending(t => t.Item2).FirstOrDefault(); if (maxDenotation != null && AnswerLabel != null) { HasCorrectDenotation = maxDenotation.Item1 == AnswerLabel.Id; } if (maxDenotation != null) { TopDenotationEvidence = maxDenotation.Item2; } }
internal static void RunLinkedAnswerExtractionExperiment() { var trainDataset = Configuration.GetQuestionDialogsTrain(); var devDataset = Configuration.GetQuestionDialogsDev(); var db = Configuration.Db; var linker = getFullDataLinker(db); var linkedExtractor = new LinkBasedExtractor(linker, db); foreach (var dialog in trainDataset.Dialogs) { if (dialog.HasCorrectAnswer) { Console.WriteLine(dialog.Question); linkedExtractor.Train(dialog); } } linkedExtractor.PrintInfo(); var desiredEntityInfoPrintingEnabled = false; //extractAnswer("what is the gender of paul boutilier", linker, linkedExtractor, devDataset); var evaluationN = 1; var correctLinkCount = 0; var correctAnswerCount = 0; var totalCount = 0; foreach (var dialog in devDataset.Dialogs) { if (!dialog.HasCorrectAnswer) { continue; } var linkedQuestion = linker.LinkUtterance(dialog.Question); var contextEntities = linkedQuestion.Entities; Console.WriteLine(linkedQuestion); var answerPhrase = getAnswerPhrase(dialog); var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities); var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities; Console.WriteLine(linkedAnswer); if (desiredEntityInfoPrintingEnabled) { var correctAnswer = db.GetLabel(dialog.AnswerMid); var answerInBounds = db.GetInBounds(dialog.AnswerMid); var answerOutBounds = db.GetOutBounds(dialog.AnswerMid); Console.WriteLine("\tdesired: {0}({1})[{2}/{3}]", correctAnswer, db.GetFreebaseId(dialog.AnswerMid), answerInBounds, answerOutBounds); } else { Console.WriteLine("\tdesired: {0}", db.GetFreebaseId(dialog.AnswerMid)); } var answerEntities = linkedExtractor.ExtractAnswerEntity(dialog); var isCorrect = answerEntities.Select(e => e.Mid).Take(evaluationN).Contains(dialog.AnswerMid); if (isCorrect) { Console.WriteLine("\tOK"); ++correctAnswerCount; ++correctLinkCount; } else { if (answerPhraseEntities.Select(e => e.Mid).Contains(dialog.AnswerMid)) { ++correctLinkCount; Console.WriteLine("\tLINK ONLY"); } else { Console.WriteLine("\tNO"); } foreach (var entity in answerEntities) { Console.WriteLine("\t\t{0}({1})[{2}/{3}] {4:0.00}", entity.Label, db.GetFreebaseId(entity.Mid), entity.InBounds, entity.OutBounds, entity.Score); } } ++totalCount; Console.WriteLine("\tprecision answer {0:00.00}%, link {1:00.00}%, extraction {2:00.00}%", 100.0 * correctAnswerCount / totalCount, 100.0 * correctLinkCount / totalCount, 100.0 * correctAnswerCount / correctLinkCount); Console.WriteLine(); } Console.WriteLine(linkedExtractor.TotalEntityCount + " answer entities"); Console.WriteLine("END"); Console.ReadKey(); }