internal QuestionAnsweringModuleBase(ComposedGraph graph, CallStorage storage)
{
Storage = storage;
Pool = new ContextPool(graph);
_adviceAnswer = storage.RegisterCall("AdviceAnswer", c =>
{
_AdviceAnswer(c.String("question"), c.Bool("isBasedOnContext"), c.Node("correctAnswerNode", Graph), c.Nodes("context", Graph));
});
_repairAnswer = storage.RegisterCall("RepairAnswer", c =>
{
_RepairAnswer(c.String("question"), c.Node("suggestedAnswer", Graph), c.Nodes("context", Graph));
});
_setEquivalencies = storage.RegisterCall("SetEquivalence", c =>
{
SetEquivalence(c.String("patternQuestion"), c.String("queriedQuestion"), c.Bool("isEquivalent"));
});
_negate = storage.RegisterCall("Negate", c =>
{
Negate(c.String("question"));
});
}
internal Interpretation GeneralizeBy(FeatureCover cover, ComposedGraph graph)
{
var mapping = cover.CreateNodeMapping(graph);
var isContractable = _rules.Any(r => r is InsertPoolRule);
if (mapping.IsEmpty)
{
//there is nothing to generalize
if (isContractable)
{
//contractable interpretation without generalization is trivial
return(null);
}
else
{
return(this);
}
}
mapping.IsGeneralizeMapping = true;
var newRules = mapRules(mapping);
if (!mapping.WasUsed && isContractable)
{
//mapping wasnt used, so the contractable interpretation is trivial
return(null);
}
return(new Interpretation(newRules));
}
private static void ProbabilisticMappingQATest(string dbPath)
{
var loader = loadDB(dbPath);
var graph = new ComposedGraph(loader.DataLayer);
var qa = new PoolComputation.ProbabilisticQA.ProbabilisticQAModule(graph, new CallStorage(null));
var denotation1 = graph.GetNode("Barack Obama");
var q1 = "Who is United States of America president?";
var denotation2 = graph.GetNode("Vladimir Putin");
var q2 = "Who is Russia president?";
var q3 = "Who is Czech republic president?";
var denotation3 = graph.GetNode("Miloš Zeman");
qa.AdviceAnswer(q1, false, denotation1);
qa.AdviceAnswer(q2, false, denotation2);
qa.Optimize(100);
var pool = new ContextPool(graph);
// var a1 = qa.GetAnswer(q1, pool).ToArray();
var a3 = qa.GetAnswer(q3, pool).ToArray();
var repl_a1 = qa.GetAnswer(q1, pool).ToArray();
throw new NotImplementedException();
}
private static Database.TripletLoader.Loader loadDB(string dbPath)
{
var loader = new Database.TripletLoader.Loader(dbPath);
var graph = new ComposedGraph(loader.DataLayer);
WikidataHelper.PreprocessData(loader, graph);
return(loader);
}
internal SurroundingPatternPart(ComposedGraph graph, string defaultValue)
{
SingleNodeClassifier = new KnowledgeClassifier <string>(graph);
MultipleNodesClassifier = new KnowledgeClassifier <string>(graph);
_singleDefaultValue = defaultValue;
_multiDefaultValue = defaultValue;
}
/// <inheritdoc/>
protected override IEnumerable <PoolRuleBase> extend(NodeReference node, ComposedGraph graph)
{
foreach (var path in graph.GetPaths(_targetNode, node, 3, 100))
{
yield return(extendWith(path));
}
yield break;
}
private static void MappingQATest(string dbPath)
{
var loader = loadDB(dbPath);
var graph = new ComposedGraph(loader.DataLayer);
var qa = new PoolComputation.MappedQA.MappedQAModule(graph, new CallStorage(null));
qa.AdviceAnswer("Who is United States of America president?", false, graph.GetNode("Barack Obama"));
qa.Optimize();
}
internal static void RunAnswerGeneralizationDev()
{
var trainDataset = Configuration.GetQuestionDialogsTrain();
var devDataset = Configuration.GetQuestionDialogsDev();
var simpleQuestions = Configuration.GetSimpleQuestionsDump();
var db = Configuration.Db;
var trainDialogs = trainDataset.Dialogs.ToArray();
var linkedUtterancesTrain = cachedLinkedUtterancesTrain(simpleQuestions, db, trainDialogs);
//var graph = cachedEntityGraph(simpleQuestions, trainDialogs, linkedUtterancesTrain);
var graph = new ComposedGraph(new FreebaseGraphLayer(db));
var linker = new GraphDisambiguatedLinker(db, "./verbs.lex");
var cachedLinker = new CachedLinker(trainDialogs.Select(d => d.Question).ToArray(), linkedUtterancesTrain, linker);
var generalizer = new PatternGeneralizer(graph, cachedLinker.LinkUtterance);
var testDialogs = 0;
//train
for (var i = 0; i < trainDialogs.Length - testDialogs; ++i)
{
var trainDialog = trainDialogs[i];
var question = trainDialog.Question;
var answerNodeId = FreebaseDbProvider.GetId(trainDialog.AnswerMid);
var answerNode = graph.GetNode(answerNodeId);
generalizer.AddExample(question, answerNode);
}
/*/
* //evaluation on dev set
* foreach (var devDialog in trainDialogs)
* {
* writeLine(devDialog.Question);
* writeLine("\t" + cachedLinker.LinkUtterance(devDialog.Question));
* var desiredAnswerLabel = db.GetLabel(devDialog.AnswerMid);
* writeLine("\tDesired answer: {0} ({1})", desiredAnswerLabel, devDialog.AnswerMid);
* var answer = generalizer.GetAnswer(devDialog.Question);
* if (answer == null)
* {
* writeLine("\tNo answer.");
* }
* else
* {
* var answerLabel = db.GetLabel(FreebaseLoader.GetMid(answer.Value.Data));
* writeLine("\tGeneralizer output: {0} {1}", answerLabel, answer);
* }
* writeLine();
* }
* /**/
var result = generalizer.GetAnswer("What county is ovens auditorium in");
//var result = generalizer.GetAnswer("What is Obama gender?");
//var result = generalizer.GetAnswer("is mir khasim ali of the male or female gender");
}
internal static void RunToyGeneralization()
{
var data = new ExplicitLayer();
var A = data.CreateReference("A");
var B = data.CreateReference("B");
var C = data.CreateReference("C");
var D = data.CreateReference("D");
var H1 = data.CreateReference("H1");
var H2 = data.CreateReference("H2");
var H3 = data.CreateReference("H3");
var fH1 = data.CreateReference("fH1");
var fH2 = data.CreateReference("fH2");
var fH3 = data.CreateReference("fH3");
var H = data.CreateReference("H");
var X = data.CreateReference("X");
var edge1 = "e1";
var edgeZ1 = "z1";
var edgeZ2 = "z2";
var edgeIs = "is";
data.AddEdge(A, edge1, H1);
data.AddEdge(B, edge1, H2);
data.AddEdge(C, edge1, H3);
data.AddEdge(A, edge1, fH1);
data.AddEdge(B, edge1, fH2);
data.AddEdge(C, edge1, fH3);
data.AddEdge(H1, edgeIs, H);
data.AddEdge(H2, edgeIs, H);
data.AddEdge(H3, edgeIs, H);
data.AddEdge(D, edgeZ1, H1);
data.AddEdge(D, edgeZ2, H2);
var graph = new ComposedGraph(data);
var group = new KnowledgeDialog.Knowledge.Group(graph);
group.AddNode(A);
group.AddNode(B);
var linker = new SingleWordLinker();
linker.Add(A, B, C, D, H1, H2, H3);
var generalizer = new PatternGeneralizer(graph, linker.LinkUtterance);
generalizer.AddExample("Where A lives?", H1);
generalizer.AddExample("Where B lives?", H2);
var answer = generalizer.GetAnswer("Where does C live?");
}
internal NodeMapping CreateNodeMapping(ComposedGraph graph)
{
var mapping = new NodeMapping(graph);
foreach (var instance in FeatureInstances)
{
instance.SetMapping(mapping);
}
return(mapping);
}
private static NodeReference getPropertyNode(NodeReference positionRoot, ComposedGraph graph)
{
var isTemporary = IsTemporaryNode(positionRoot);
if (isTemporary)
{
positionRoot = graph.OutcommingTargets(positionRoot, TemporaryMainEdge).FirstOrDefault();
}
return(positionRoot);
}
internal ConstraintSelector(ComposedGraph graph, NodeReference targetNode, IEnumerable <NodeReference> selectedNodes)
{
_factory = new PathFactory(targetNode, graph, false, InterpretationsFactory.MaxSearchWidth, InterpretationsFactory.MaxConstraintLength);
if (!selectedNodes.Contains(targetNode))
{
throw new NotSupportedException("Cannot constraint nodes without target");
}
_originalRemainingNodes = new HashSet <NodeReference>(selectedNodes.Except(new[] { targetNode }));
}
internal ActionBlock(ComposedGraph graph, IEnumerable <IPoolAction> actions)
{
var semantics = actions.FirstOrDefault();
if (semantics != null && semantics.SemanticOrigin != null)
{
OriginalSentence = Dialog.UtteranceParser.Parse(semantics.SemanticOrigin.Utterance).OriginalSentence;
}
OutputFilter = new KnowledgeClassifier <bool>(graph);
_actions.AddRange(actions);
RequiredSubstitutions = findRequiredSubstitutions(actions);
}
private NodesEnumeration findeQuestionNodes(ComposedGraph graph)
{
var nodes = new List <NodeReference>();
foreach (var word in ParsedQuestion.Words)
{
if (graph.HasEvidence(word))
{
nodes.Add(graph.GetNode(word));
}
}
return(new NodesEnumeration(nodes));
}
public IEnumerable <NodeReference> Nodes(string p, ComposedGraph graph)
{
var result = new List <NodeReference>();
var data = _storage[p] as IEnumerable <object>;
foreach (var nodeData in data)
{
//TODO not only strings can be present as node data
var stringData = nodeData.ToString();
result.Add(graph.GetNode(stringData));
}
return(result);
}
internal Interpretation InstantiateBy(FeatureCover cover, ComposedGraph graph)
{
var mapping = cover.CreateNodeMapping(graph);
if (mapping.IsEmpty)
{
return(this);
}
mapping.IsGeneralizeMapping = false;
var newRules = mapRules(mapping);
return(new Interpretation(newRules));
}
private static void printInfo(ComposedGraph graph, FreebaseDbProvider db, params string[] ids)
{
foreach (var id in ids)
{
var mid = FreebaseDbProvider.GetMid(id);
var label = db.GetLabel(mid);
var description = db.GetDescription(mid);
Console.WriteLine(id + " " + label);
Console.WriteLine("\t" + description);
Console.WriteLine();
}
}
private static ComposedGraph cachedEntityGraph(SimpleQuestionDumpProcessor simpleQuestions, QuestionDialog[] trainDialogs, LinkedUtterance[] linkedUtterances)
{
return(ComputationCache.Load("knowledge_all_train", 1, () =>
{
var trainEntities = getQAEntities(trainDialogs, linkedUtterances);
//var layer = simpleQuestions.GetLayerFromIds(trainEntities);
foreach (var entityId in trainEntities)
{
simpleQuestions.AddTargetMid(FreebaseDbProvider.GetMid(entityId));
}
simpleQuestions.RunIteration();
var layer = simpleQuestions.GetLayerFromIds(simpleQuestions.AllIds);
var graph = new ComposedGraph(layer);
return graph;
}));
}
private static double getDistance(NodeReference node1, NodeReference node2, ComposedGraph graph)
{
var paths = graph.GetPaths(node1, node2, MaximumGraphDepth, MaximumGraphWidth).Take(1000);
double minDistance = double.MaxValue;
foreach (var path in paths)
{
ConsoleServices.Print(path);
var distance = getDistance(path);
if (minDistance > distance)
{
minDistance = distance;
}
}
return(minDistance);
}
private static void RuleQuestionTest(string dbPath)
{
var loader = loadDB(dbPath);
var graph = new ComposedGraph(loader.DataLayer);
var generator = new StructuredInterpretationGenerator(graph);
var denotation1 = graph.GetNode("Barack Obama");
var q1 = "Who is United States of America president?";
var denotation2 = graph.GetNode("Vladimir Putin");
var q2 = "Who is Russia president?";
var q3 = "Who is Czech republic president?";
var denotation3 = graph.GetNode("Miloš Zeman");
generator.AdviceAnswer(q1, denotation1);
generator.AdviceAnswer(q2, denotation2);
generator.Optimize(5000);
var interpretations = new List <Ranked <StructuredInterpretation> >();
foreach (var evidence in generator.GetEvidences(q3))
{
foreach (var interpretation in evidence.AvailableRankedInterpretations)
{
interpretations.Add(interpretation);
}
}
interpretations.Sort((a, b) => a.Rank.CompareTo(b.Rank));
foreach (var interpretation in interpretations)
{
var answer = generator.Evaluate(q3, interpretation.Value);
ConsoleServices.Print(interpretation);
ConsoleServices.Print(answer);
ConsoleServices.PrintEmptyLine();
}
var qGenerator = new QuestionGenerator(generator);
//var questions = Generator.FindDistinguishingNodeQuestions();
throw new NotImplementedException();
}
private static bool IsPresidentPosition(NodeReference positionRoot, ComposedGraph graph)
{
positionRoot = getPropertyNode(positionRoot, graph);
if (positionRoot == null)
{
return(false);
}
var president = graph.GetNode(PresidentNode);
if (president.Equals(positionRoot))
{
return(true);
}
var paths = graph.GetPaths(president, positionRoot, 1, 100).ToArray();
return(paths.Length > 0);
}
public static void PreprocessData(Loader loader, ComposedGraph graph)
{
foreach (var node in loader.Nodes)
{
var positions = graph.OutcommingTargets(node, HeldPositionEdge).ToArray();
foreach (var position in positions)
{
if (IsPresidentPosition(position, graph))
{
//repair form of president position (fill end=null, if there is no end specified)
RepairPresidentPosition(node, position, loader.DataLayer, graph);
}
else
{
//keep only president positions
RemovePosition(node, position, loader.DataLayer);
}
}
}
}
private static void KnowledgeClassifier()
{
var dataLayer = new PresidentLayer();
var graph = new ComposedGraph(dataLayer);
var node1 = graph.GetNode("Barack_Obama");
var node2 = graph.GetNode("Miloš_Zeman");
var node3 = graph.GetNode("Michelle_Obama");
var log = new MultiTraceLog(new[] { node1, node2, node3 }, graph);
var classifier = new KnowledgeClassifier <string>(graph);
classifier.Advice(node1, "president");
classifier.Advice(node2, "president");
classifier.Advice(node3, "wife");
var node4 = graph.GetNode("Ivana_Zemanová");
var node5 = graph.GetNode("Andrej_Kiska");
var test1 = classifier.Classify(node4);
var test2 = classifier.Classify(node5);
}
private static void InconsistencyDBTesting(string dbPath)
{
var loader = new Database.TripletLoader.Loader(dbPath);
var graph = new ComposedGraph(loader.DataLayer);
WikidataHelper.PreprocessData(loader, graph);
var manager = new PoolComputation.StateDialogManager(null, loader.DataLayer);
var provider = new DialogConsole(manager);
provider.SimulateInput(
"name of Czech Republic president",
"it is Miloš Zeman",
"name of Russia president",
"it is Vladimir Putin",
"name of United States of America president",
"it is Barack Obama",
"name of United States of America president",
"name of Czech Republic president"
);
provider.Run();
}
protected override IEnumerable <FeatureInstance> generateFeatures(ParsedUtterance expression, ComposedGraph graph)
{
return(new[] { CreateSimpleFeatureInstance(expression) });
}
internal HashSet <NodeReference> FindSet(NodeReference constraintNode, ComposedGraph graph)
{
return(new HashSet <NodeReference>(graph.GetForwardTargets(new[] { constraintNode }, Path)));
}
internal bool IsSatisfiedBy(NodeReference featureNode, NodeReference answer, ComposedGraph graph)
{
return(FindSet(featureNode, graph).Contains(answer));
}
public StructuredTopicGenerator(IEnumerable <KnowledgeConstraintOptions> constraintOptions, ComposedGraph graph)
{
Graph = graph;
_constraintOptions = constraintOptions.ToArray();
}
internal NodeReference GetFeatureNode(NodeReference generalFeatureNode, ComposedGraph graph)
{
return(Cover.GetInstanceNode(generalFeatureNode, graph));
}
private static void RepairPresidentPosition(NodeReference node, NodeReference positionRoot, ExplicitLayer layer, ComposedGraph graph)
{
if (!IsTemporaryNode(positionRoot))
{
//only temporary nodes will be repaired
return;
}
var endNode = graph.OutcommingTargets(positionRoot, EndEdge).FirstOrDefault();
if (endNode != null)
{
//nothing to repair
return;
}
layer.AddEdge(positionRoot, EndEdge, graph.GetNode(NullNode));
}