internal static void BasicCancelation_WithEnumAndNgrams()
{
var testData = Configuration.GetQuestionDialogsTest();
var trainDataset = Configuration.GetQuestionDialogsTrain();
var linker = Configuration.Linker;
var linkedExtractor = new LinkBasedExtractor(linker, Configuration.Db);
//enumeration detection is not disabled here
linkedExtractor.UseNgramOrdering = true;
//train the extractor
foreach (var dialog in trainDataset.Dialogs)
{
if (dialog.HasCorrectAnswer)
{
Console.WriteLine(dialog.Question);
linkedExtractor.Train(dialog);
}
}
var result = evaluateExtractor(testData, linker, (d) => linkedExtractor.ExtractAnswerEntity(d));
Console.WriteLine(result);
}
internal static void RunLinkedAnswerExtractionExperiment()
{
var trainDataset = Configuration.GetQuestionDialogsTrain();
var devDataset = Configuration.GetQuestionDialogsDev();
var db = Configuration.Db;
var linker = getFullDataLinker(db);
var linkedExtractor = new LinkBasedExtractor(linker, db);
foreach (var dialog in trainDataset.Dialogs)
{
if (dialog.HasCorrectAnswer)
{
Console.WriteLine(dialog.Question);
linkedExtractor.Train(dialog);
}
}
linkedExtractor.PrintInfo();
var desiredEntityInfoPrintingEnabled = false;
//extractAnswer("what is the gender of paul boutilier", linker, linkedExtractor, devDataset);
var evaluationN = 1;
var correctLinkCount = 0;
var correctAnswerCount = 0;
var totalCount = 0;
foreach (var dialog in devDataset.Dialogs)
{
if (!dialog.HasCorrectAnswer)
{
continue;
}
var linkedQuestion = linker.LinkUtterance(dialog.Question);
var contextEntities = linkedQuestion.Entities;
Console.WriteLine(linkedQuestion);
var answerPhrase = getAnswerPhrase(dialog);
var linkedAnswer = linker.LinkUtterance(answerPhrase, contextEntities);
var answerPhraseEntities = linkedAnswer == null ? new EntityInfo[0] : linkedAnswer.Entities;
Console.WriteLine(linkedAnswer);
if (desiredEntityInfoPrintingEnabled)
{
var correctAnswer = db.GetLabel(dialog.AnswerMid);
var answerInBounds = db.GetInBounds(dialog.AnswerMid);
var answerOutBounds = db.GetOutBounds(dialog.AnswerMid);
Console.WriteLine("\tdesired: {0}({1})[{2}/{3}]", correctAnswer, db.GetFreebaseId(dialog.AnswerMid), answerInBounds, answerOutBounds);
}
else
{
Console.WriteLine("\tdesired: {0}", db.GetFreebaseId(dialog.AnswerMid));
}
var answerEntities = linkedExtractor.ExtractAnswerEntity(dialog);
var isCorrect = answerEntities.Select(e => e.Mid).Take(evaluationN).Contains(dialog.AnswerMid);
if (isCorrect)
{
Console.WriteLine("\tOK");
++correctAnswerCount;
++correctLinkCount;
}
else
{
if (answerPhraseEntities.Select(e => e.Mid).Contains(dialog.AnswerMid))
{
++correctLinkCount;
Console.WriteLine("\tLINK ONLY");
}
else
{
Console.WriteLine("\tNO");
}
foreach (var entity in answerEntities)
{
Console.WriteLine("\t\t{0}({1})[{2}/{3}] {4:0.00}", entity.Label, db.GetFreebaseId(entity.Mid), entity.InBounds, entity.OutBounds, entity.Score);
}
}
++totalCount;
Console.WriteLine("\tprecision answer {0:00.00}%, link {1:00.00}%, extraction {2:00.00}%", 100.0 * correctAnswerCount / totalCount, 100.0 * correctLinkCount / totalCount, 100.0 * correctAnswerCount / correctLinkCount);
Console.WriteLine();
}
Console.WriteLine(linkedExtractor.TotalEntityCount + " answer entities");
Console.WriteLine("END");
Console.ReadKey();
}