internal static void BasicCancelation_WithEnumAndNgrams()
{
var testData = Configuration.GetQuestionDialogsTest();
var trainDataset = Configuration.GetQuestionDialogsTrain();
var linker = Configuration.Linker;
var linkedExtractor = new LinkBasedExtractor(linker, Configuration.Db);
//enumeration detection is not disabled here
linkedExtractor.UseNgramOrdering = true;
//train the extractor
foreach (var dialog in trainDataset.Dialogs)
{
if (dialog.HasCorrectAnswer)
{
Console.WriteLine(dialog.Question);
linkedExtractor.Train(dialog);
}
}
var result = evaluateExtractor(testData, linker, (d) => linkedExtractor.ExtractAnswerEntity(d));
Console.WriteLine(result);
}
internal KnowledgeReport(ExtractionKnowledge knowledge, LinkBasedExtractor extractor, QuestionCollection questions, bool fruitOnly)
{
StoragePath = knowledge.StoragePath;
QuestionCount = knowledge.Questions.Count();
var reports = new List <QuestionReport>();
foreach (var question in knowledge.Questions)
{
if (!question.AnswerHints.Any())
{
continue;
}
var answerId = FreebaseDbProvider.GetId(questions.GetAnswerMid(question.Utterance.OriginalSentence));
var report = new QuestionReport(question, answerId, extractor);
if (fruitOnly)
{
if (report.TopDenotationEvidence < 2 || report.CollectedDenotations.Count() - report.TopDenotationEvidence * 2 >= 0)
{
//not enough evidence
continue;
}
}
reports.Add(report);
}
Questions = reports.OrderByDescending(r => r.CollectedDenotations.Count());
}
private static void extractAnswer(string utterance, ILinker linker, LinkBasedExtractor extractor, QuestionDialogDatasetReader dataset)
{
var utteranceWords = getCleanWords(utterance).Distinct();
QuestionDialog dialogToAnswer = null;
foreach (var dialog in dataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
var questionWords = getCleanWords(dialog.Question);
if (utteranceWords.Except(questionWords).Count() == 0)
{
dialogToAnswer = dialog;
}
foreach (var turn in dialog.AnswerTurns)
{
var words = getCleanWords(turn.InputChat);
if (utteranceWords.Except(words).Count() == 0)
{
dialogToAnswer = dialog;
break;
}
}
if (dialogToAnswer != null)
{
break;
}
}
Console.WriteLine(dialogToAnswer.Question);
var answerPhrase = getAnswerPhrase(dialogToAnswer);
var linkedQuestion = linker.LinkUtterance(dialogToAnswer.Question);
Console.WriteLine(linkedQuestion);
var contextEntities = linkedQuestion.Parts.SelectMany(p => p.Entities).ToArray();
var result = linker.LinkUtterance(answerPhrase, contextEntities);
Console.WriteLine(result);
var answers = extractor.ExtractAnswerEntity(dialogToAnswer);
foreach (var answer in answers)
{
Console.WriteLine(answer);
}
}
internal static void BasicCancelation_WithEnumDetection()
{
var testData = Configuration.GetQuestionDialogsTest();
var linker = Configuration.Linker;
var linkedExtractor = new LinkBasedExtractor(linker, Configuration.Db);
//enumeration detection is not disabled here
var result = evaluateExtractor(testData, linker, (d) => linkedExtractor.ExtractAnswerEntity(d), 2);
Console.WriteLine(result);
}
internal QuestionReport(QuestionInfo info, string answerId, LinkBasedExtractor extractor)
{
var linker = extractor.Linker;
Question = linker.LinkUtterance(info.Utterance.OriginalSentence);
AnswerLabel = extractor.Db.GetEntryFromId(answerId);
var denotations = new List <Tuple <LinkedUtterance, EntityInfo, bool> >();
foreach (var answerHint in info.AnswerHints)
{
var linkedHint = linker.LinkUtterance(answerHint.OriginalSentence, Question.Entities);
var denotation = extractor.ExtractAnswerEntity(info.Utterance.OriginalSentence, answerHint.OriginalSentence).FirstOrDefault();
var item = Tuple.Create(linkedHint, denotation, answerId == FreebaseDbProvider.GetId(denotation.Mid));
denotations.Add(item);
}
CollectedDenotations = denotations;
var denotationCounts = from denotation in denotations
group denotation by FreebaseDbProvider.GetId(denotation.Item2.Mid)
into grouped
select Tuple.Create(grouped.Key, grouped.Count());
var maxDenotation = denotationCounts.OrderByDescending(t => t.Item2).FirstOrDefault();
if (maxDenotation != null && AnswerLabel != null)
{
HasCorrectDenotation = maxDenotation.Item1 == AnswerLabel.Id;
}
if (maxDenotation != null)
{
TopDenotationEvidence = maxDenotation.Item2;
}
}
internal AnswerExtractionManager(QuestionCollection questions, ExtractionKnowledge knowledge, LinkBasedExtractor extractor)
{
_questions = questions;
_knowledge = knowledge;
_extractor = extractor;
}
internal static void RunLinkedAnswerExtractionExperiment()
{
var trainDataset = Configuration.GetQuestionDialogsTrain();
var devDataset = Configuration.GetQuestionDialogsDev();
var db = Configuration.Db;
var linker = getFullDataLinker(db);
var linkedExtractor = new LinkBasedExtractor(linker, db);
foreach (var dialog in trainDataset.Dialogs)
{
if (dialog.HasCorrectAnswer)
{
Console.WriteLine(dialog.Question);
linkedExtractor.Train(dialog);
}
}
linkedExtractor.PrintInfo();
var desiredEntityInfoPrintingEnabled = false;
//extractAnswer("what is the gender of paul boutilier", linker, linkedExtractor, devDataset);
var evaluationN = 1;
var correctLinkCount = 0;
var correctAnswerCount = 0;
var totalCount = 0;
foreach (var dialog in devDataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
var linkedQuestion = linker.LinkUtterance(dialog.Question);
var contextEntities = linkedQuestion.Entities;
Console.WriteLine(linkedQuestion);
var answerPhrase = getAnswerPhrase(dialog);
var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities);
var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities;
Console.WriteLine(linkedAnswer);
if (desiredEntityInfoPrintingEnabled)
{
var correctAnswer = db.GetLabel(dialog.AnswerMid);
var answerInBounds = db.GetInBounds(dialog.AnswerMid);
var answerOutBounds = db.GetOutBounds(dialog.AnswerMid);
Console.WriteLine("\tdesired: {0}({1})[{2}/{3}]", correctAnswer, db.GetFreebaseId(dialog.AnswerMid), answerInBounds, answerOutBounds);
}
else
{
Console.WriteLine("\tdesired: {0}", db.GetFreebaseId(dialog.AnswerMid));
}
var answerEntities = linkedExtractor.ExtractAnswerEntity(dialog);
var isCorrect = answerEntities.Select(e => e.Mid).Take(evaluationN).Contains(dialog.AnswerMid);
if (isCorrect)
{
Console.WriteLine("\tOK");
++correctAnswerCount;
++correctLinkCount;
}
else
{
if (answerPhraseEntities.Select(e => e.Mid).Contains(dialog.AnswerMid))
{
++correctLinkCount;
Console.WriteLine("\tLINK ONLY");
}
else
{
Console.WriteLine("\tNO");
}
foreach (var entity in answerEntities)
{
Console.WriteLine("\t\t{0}({1})[{2}/{3}] {4:0.00}", entity.Label, db.GetFreebaseId(entity.Mid), entity.InBounds, entity.OutBounds, entity.Score);
}
}
++totalCount;
Console.WriteLine("\tprecision answer {0:00.00}%, link {1:00.00}%, extraction {2:00.00}%", 100.0 * correctAnswerCount / totalCount, 100.0 * correctLinkCount / totalCount, 100.0 * correctAnswerCount / correctLinkCount);
Console.WriteLine();
}
Console.WriteLine(linkedExtractor.TotalEntityCount + " answer entities");
Console.WriteLine("END");
Console.ReadKey();
}