internal static void BasicCancelation_WithEnumAndNgrams() { var testData = Configuration.GetQuestionDialogsTest(); var trainDataset = Configuration.GetQuestionDialogsTrain(); var linker = Configuration.Linker; var linkedExtractor = new LinkBasedExtractor(linker, Configuration.Db); //enumeration detection is not disabled here linkedExtractor.UseNgramOrdering = true; //train the extractor foreach (var dialog in trainDataset.Dialogs) { if (dialog.HasCorrectAnswer) { Console.WriteLine(dialog.Question); linkedExtractor.Train(dialog); } } var result = evaluateExtractor(testData, linker, (d) => linkedExtractor.ExtractAnswerEntity(d)); Console.WriteLine(result); }
internal static void RunLinkedAnswerExtractionExperiment() { var trainDataset = Configuration.GetQuestionDialogsTrain(); var devDataset = Configuration.GetQuestionDialogsDev(); var db = Configuration.Db; var linker = getFullDataLinker(db); var linkedExtractor = new LinkBasedExtractor(linker, db); foreach (var dialog in trainDataset.Dialogs) { if (dialog.HasCorrectAnswer) { Console.WriteLine(dialog.Question); linkedExtractor.Train(dialog); } } linkedExtractor.PrintInfo(); var desiredEntityInfoPrintingEnabled = false; //extractAnswer("what is the gender of paul boutilier", linker, linkedExtractor, devDataset); var evaluationN = 1; var correctLinkCount = 0; var correctAnswerCount = 0; var totalCount = 0; foreach (var dialog in devDataset.Dialogs) { if (!dialog.HasCorrectAnswer) { continue; } var linkedQuestion = linker.LinkUtterance(dialog.Question); var contextEntities = linkedQuestion.Entities; Console.WriteLine(linkedQuestion); var answerPhrase = getAnswerPhrase(dialog); var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities); var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities; Console.WriteLine(linkedAnswer); if (desiredEntityInfoPrintingEnabled) { var correctAnswer = db.GetLabel(dialog.AnswerMid); var answerInBounds = db.GetInBounds(dialog.AnswerMid); var answerOutBounds = db.GetOutBounds(dialog.AnswerMid); Console.WriteLine("\tdesired: {0}({1})[{2}/{3}]", correctAnswer, db.GetFreebaseId(dialog.AnswerMid), answerInBounds, answerOutBounds); } else { Console.WriteLine("\tdesired: {0}", db.GetFreebaseId(dialog.AnswerMid)); } var answerEntities = linkedExtractor.ExtractAnswerEntity(dialog); var isCorrect = answerEntities.Select(e => e.Mid).Take(evaluationN).Contains(dialog.AnswerMid); if (isCorrect) { Console.WriteLine("\tOK"); ++correctAnswerCount; ++correctLinkCount; } else { if (answerPhraseEntities.Select(e => e.Mid).Contains(dialog.AnswerMid)) { ++correctLinkCount; Console.WriteLine("\tLINK ONLY"); } else { Console.WriteLine("\tNO"); } foreach (var entity in answerEntities) { Console.WriteLine("\t\t{0}({1})[{2}/{3}] {4:0.00}", entity.Label, db.GetFreebaseId(entity.Mid), entity.InBounds, entity.OutBounds, entity.Score); } } ++totalCount; Console.WriteLine("\tprecision answer {0:00.00}%, link {1:00.00}%, extraction {2:00.00}%", 100.0 * correctAnswerCount / totalCount, 100.0 * correctLinkCount / totalCount, 100.0 * correctAnswerCount / correctLinkCount); Console.WriteLine(); } Console.WriteLine(linkedExtractor.TotalEntityCount + " answer entities"); Console.WriteLine("END"); Console.ReadKey(); }