public virtual double DeletionProbability(SemanticGraphEdge edge, IEnumerable <SemanticGraphEdge> neighbors)
{
string edgeRel = edge.GetRelation().ToString();
if (edgeRel.Contains("prep"))
{
return(PpDeletionProbability(edge, neighbors));
}
else
{
if (edgeRel.Contains("obj"))
{
return(ObjDeletionProbability(edge, neighbors));
}
else
{
if (edgeRel.Contains("subj"))
{
return(SubjDeletionProbability(edge, neighbors));
}
else
{
if (edgeRel.Equals("amod"))
{
string word = (edge.GetDependent().Lemma() != null ? edge.GetDependent().Lemma() : edge.GetDependent().Word()).ToLower();
if (Edu.Stanford.Nlp.Naturalli.Util.PrivativeAdjectives.Contains(word))
{
return(0.0);
}
else
{
return(1.0);
}
}
else
{
return(DeletionProbability(edgeRel));
}
}
}
}
}
public virtual double ObjDeletionProbability(SemanticGraphEdge edge, IEnumerable <SemanticGraphEdge> neighbors)
{
// Get information about the neighbors
// (in a totally not-creepy-stalker sort of way)
Optional <string> subj = Optional.Empty();
Optional <string> pp = Optional.Empty();
foreach (SemanticGraphEdge neighbor in neighbors)
{
if (neighbor != edge)
{
string neighborRel = neighbor.GetRelation().ToString();
if (neighborRel.Contains("subj"))
{
subj = Optional.Of(neighbor.GetDependent().OriginalText().ToLower());
}
if (neighborRel.Contains("prep"))
{
pp = Optional.Of(neighborRel);
}
if (neighborRel.Contains("obj"))
{
return(1.0);
}
}
}
// allow deleting second object
string obj = edge.GetDependent().OriginalText().ToLower();
string verb = edge.GetGovernor().OriginalText().ToLower();
// Compute the most informative drop probability we can
double rawScore = null;
if (subj.IsPresent())
{
if (pp.IsPresent())
{
// Case: subj+obj
rawScore = verbSubjPPObjAffinity[Quadruple.MakeQuadruple(verb, subj.Get(), pp.Get(), obj)];
}
}
if (rawScore == null)
{
rawScore = verbObjAffinity[verb];
}
if (rawScore == null)
{
return(DeletionProbability(edge.GetRelation().ToString()));
}
else
{
return(1.0 - Math.Min(1.0, rawScore / upperProbabilityCap));
}
}
/// <summary>The search algorithm, starting with a full sentence and iteratively shortening it to its entailed sentences.</summary>
/// <returns>A list of search results, corresponding to shortenings of the sentence.</returns>
private IList <ForwardEntailerSearchProblem.SearchResult> SearchImplementation()
{
// Pre-process the tree
SemanticGraph parseTree = new SemanticGraph(this.parseTree);
System.Diagnostics.Debug.Assert(Edu.Stanford.Nlp.Naturalli.Util.IsTree(parseTree));
// (remove common determiners)
IList <string> determinerRemovals = new List <string>();
parseTree.GetLeafVertices().Stream().Filter(null).ForEach(null);
// (cut conj_and nodes)
ICollection <SemanticGraphEdge> andsToAdd = new HashSet <SemanticGraphEdge>();
foreach (IndexedWord vertex in parseTree.VertexSet())
{
if (parseTree.InDegree(vertex) > 1)
{
SemanticGraphEdge conjAnd = null;
foreach (SemanticGraphEdge edge in parseTree.IncomingEdgeIterable(vertex))
{
if ("conj:and".Equals(edge.GetRelation().ToString()))
{
conjAnd = edge;
}
}
if (conjAnd != null)
{
parseTree.RemoveEdge(conjAnd);
System.Diagnostics.Debug.Assert(Edu.Stanford.Nlp.Naturalli.Util.IsTree(parseTree));
andsToAdd.Add(conjAnd);
}
}
}
// Clean the tree
Edu.Stanford.Nlp.Naturalli.Util.CleanTree(parseTree);
System.Diagnostics.Debug.Assert(Edu.Stanford.Nlp.Naturalli.Util.IsTree(parseTree));
// Find the subject / object split
// This takes max O(n^2) time, expected O(n*log(n)) time.
// Optimal is O(n), but I'm too lazy to implement it.
BitSet isSubject = new BitSet(256);
foreach (IndexedWord vertex_1 in parseTree.VertexSet())
{
// Search up the tree for a subj node; if found, mark that vertex as a subject.
IEnumerator <SemanticGraphEdge> incomingEdges = parseTree.IncomingEdgeIterator(vertex_1);
SemanticGraphEdge edge = null;
if (incomingEdges.MoveNext())
{
edge = incomingEdges.Current;
}
int numIters = 0;
while (edge != null)
{
if (edge.GetRelation().ToString().EndsWith("subj"))
{
System.Diagnostics.Debug.Assert(vertex_1.Index() > 0);
isSubject.Set(vertex_1.Index() - 1);
break;
}
incomingEdges = parseTree.IncomingEdgeIterator(edge.GetGovernor());
if (incomingEdges.MoveNext())
{
edge = incomingEdges.Current;
}
else
{
edge = null;
}
numIters += 1;
if (numIters > 100)
{
// log.error("tree has apparent depth > 100");
return(Java.Util.Collections.EmptyList);
}
}
}
// Outputs
IList <ForwardEntailerSearchProblem.SearchResult> results = new List <ForwardEntailerSearchProblem.SearchResult>();
if (!determinerRemovals.IsEmpty())
{
if (andsToAdd.IsEmpty())
{
double score = Math.Pow(weights.DeletionProbability("det"), (double)determinerRemovals.Count);
System.Diagnostics.Debug.Assert(!double.IsNaN(score));
System.Diagnostics.Debug.Assert(!double.IsInfinite(score));
results.Add(new ForwardEntailerSearchProblem.SearchResult(parseTree, determinerRemovals, score));
}
else
{
SemanticGraph treeWithAnds = new SemanticGraph(parseTree);
System.Diagnostics.Debug.Assert(Edu.Stanford.Nlp.Naturalli.Util.IsTree(treeWithAnds));
foreach (SemanticGraphEdge and in andsToAdd)
{
treeWithAnds.AddEdge(and.GetGovernor(), and.GetDependent(), and.GetRelation(), double.NegativeInfinity, false);
}
System.Diagnostics.Debug.Assert(Edu.Stanford.Nlp.Naturalli.Util.IsTree(treeWithAnds));
results.Add(new ForwardEntailerSearchProblem.SearchResult(treeWithAnds, determinerRemovals, Math.Pow(weights.DeletionProbability("det"), (double)determinerRemovals.Count)));
}
}
// Initialize the search
System.Diagnostics.Debug.Assert(Edu.Stanford.Nlp.Naturalli.Util.IsTree(parseTree));
IList <IndexedWord> topologicalVertices;
try
{
topologicalVertices = parseTree.TopologicalSort();
}
catch (InvalidOperationException)
{
// log.info("Could not topologically sort the vertices! Using left-to-right traversal.");
topologicalVertices = parseTree.VertexListSorted();
}
if (topologicalVertices.IsEmpty())
{
return(results);
}
Stack <ForwardEntailerSearchProblem.SearchState> fringe = new Stack <ForwardEntailerSearchProblem.SearchState>();
fringe.Push(new ForwardEntailerSearchProblem.SearchState(new BitSet(256), 0, parseTree, null, null, 1.0));
// Start the search
int numTicks = 0;
while (!fringe.IsEmpty())
{
// Overhead with popping a node.
if (numTicks >= maxTicks)
{
return(results);
}
numTicks += 1;
if (results.Count >= maxResults)
{
return(results);
}
ForwardEntailerSearchProblem.SearchState state = fringe.Pop();
System.Diagnostics.Debug.Assert(state.score > 0.0);
IndexedWord currentWord = topologicalVertices[state.currentIndex];
// Push the case where we don't delete
int nextIndex = state.currentIndex + 1;
int numIters = 0;
while (nextIndex < topologicalVertices.Count)
{
IndexedWord nextWord = topologicalVertices[nextIndex];
System.Diagnostics.Debug.Assert(nextWord.Index() > 0);
if (!state.deletionMask.Get(nextWord.Index() - 1))
{
fringe.Push(new ForwardEntailerSearchProblem.SearchState(state.deletionMask, nextIndex, state.tree, null, state, state.score));
break;
}
else
{
nextIndex += 1;
}
numIters += 1;
if (numIters > 10000)
{
// log.error("logic error (apparent infinite loop); returning");
return(results);
}
}
// Check if we can delete this subtree
bool canDelete = !state.tree.GetFirstRoot().Equals(currentWord);
foreach (SemanticGraphEdge edge in state.tree.IncomingEdgeIterable(currentWord))
{
if ("CD".Equals(edge.GetGovernor().Tag()))
{
canDelete = false;
}
else
{
// Get token information
CoreLabel token = edge.GetDependent().BackingLabel();
OperatorSpec @operator;
NaturalLogicRelation lexicalRelation;
Polarity tokenPolarity = token.Get(typeof(NaturalLogicAnnotations.PolarityAnnotation));
if (tokenPolarity == null)
{
tokenPolarity = Polarity.Default;
}
// Get the relation for this deletion
if ((@operator = token.Get(typeof(NaturalLogicAnnotations.OperatorAnnotation))) != null)
{
lexicalRelation = @operator.instance.deleteRelation;
}
else
{
System.Diagnostics.Debug.Assert(edge.GetDependent().Index() > 0);
lexicalRelation = NaturalLogicRelation.ForDependencyDeletion(edge.GetRelation().ToString(), isSubject.Get(edge.GetDependent().Index() - 1));
}
NaturalLogicRelation projectedRelation = tokenPolarity.ProjectLexicalRelation(lexicalRelation);
// Make sure this is a valid entailment
if (!projectedRelation.ApplyToTruthValue(truthOfPremise).IsTrue())
{
canDelete = false;
}
}
}
if (canDelete)
{
// Register the deletion
Lazy <Pair <SemanticGraph, BitSet> > treeWithDeletionsAndNewMask = Lazy.Of(null);
// Compute the score of the sentence
double newScore = state.score;
foreach (SemanticGraphEdge edge_1 in state.tree.IncomingEdgeIterable(currentWord))
{
double multiplier = weights.DeletionProbability(edge_1, state.tree.OutgoingEdgeIterable(edge_1.GetGovernor()));
System.Diagnostics.Debug.Assert(!double.IsNaN(multiplier));
System.Diagnostics.Debug.Assert(!double.IsInfinite(multiplier));
newScore *= multiplier;
}
// Register the result
if (newScore > 0.0)
{
SemanticGraph resultTree = new SemanticGraph(treeWithDeletionsAndNewMask.Get().first);
andsToAdd.Stream().Filter(null).ForEach(null);
results.Add(new ForwardEntailerSearchProblem.SearchResult(resultTree, AggregateDeletedEdges(state, state.tree.IncomingEdgeIterable(currentWord), determinerRemovals), newScore));
// Push the state with this subtree deleted
nextIndex = state.currentIndex + 1;
numIters = 0;
while (nextIndex < topologicalVertices.Count)
{
IndexedWord nextWord = topologicalVertices[nextIndex];
BitSet newMask = treeWithDeletionsAndNewMask.Get().second;
SemanticGraph treeWithDeletions = treeWithDeletionsAndNewMask.Get().first;
if (!newMask.Get(nextWord.Index() - 1))
{
System.Diagnostics.Debug.Assert(treeWithDeletions.ContainsVertex(topologicalVertices[nextIndex]));
fringe.Push(new ForwardEntailerSearchProblem.SearchState(newMask, nextIndex, treeWithDeletions, null, state, newScore));
break;
}
else
{
nextIndex += 1;
}
numIters += 1;
if (numIters > 10000)
{
// log.error("logic error (apparent infinite loop); returning");
return(results);
}
}
}
}
}
// Return
return(results);
}
/// <summary>Fix some bizarre peculiarities with certain trees.</summary>
/// <remarks>
/// Fix some bizarre peculiarities with certain trees.
/// So far, these include:
/// <ul>
/// <li>Sometimes there's a node from a word to itself. This seems wrong.</li>
/// </ul>
/// </remarks>
/// <param name="tree">The tree to clean (in place!).</param>
/// <returns>A list of extra edges, which are valid but were removed.</returns>
public static IList <SemanticGraphEdge> CleanTree(SemanticGraph tree)
{
// assert !isCyclic(tree);
// Clean nodes
IList <IndexedWord> toDelete = new List <IndexedWord>();
foreach (IndexedWord vertex in tree.VertexSet())
{
// Clean punctuation
if (vertex.Tag() == null)
{
continue;
}
char tag = vertex.BackingLabel().Tag()[0];
if (tag == '.' || tag == ',' || tag == '(' || tag == ')' || tag == ':')
{
if (!tree.OutgoingEdgeIterator(vertex).MoveNext())
{
// This should really never happen, but it does.
toDelete.Add(vertex);
}
}
}
toDelete.ForEach(null);
// Clean edges
IEnumerator <SemanticGraphEdge> iter = tree.EdgeIterable().GetEnumerator();
IList <Triple <IndexedWord, IndexedWord, SemanticGraphEdge> > toAdd = new List <Triple <IndexedWord, IndexedWord, SemanticGraphEdge> >();
toDelete.Clear();
while (iter.MoveNext())
{
SemanticGraphEdge edge = iter.Current;
if (edge.GetDependent().Index() == edge.GetGovernor().Index())
{
// Clean up copy-edges
if (edge.GetDependent().IsCopy(edge.GetGovernor()))
{
foreach (SemanticGraphEdge toCopy in tree.OutgoingEdgeIterable(edge.GetDependent()))
{
toAdd.Add(Triple.MakeTriple(edge.GetGovernor(), toCopy.GetDependent(), toCopy));
}
toDelete.Add(edge.GetDependent());
}
if (edge.GetGovernor().IsCopy(edge.GetDependent()))
{
foreach (SemanticGraphEdge toCopy in tree.OutgoingEdgeIterable(edge.GetGovernor()))
{
toAdd.Add(Triple.MakeTriple(edge.GetDependent(), toCopy.GetDependent(), toCopy));
}
toDelete.Add(edge.GetGovernor());
}
// Clean self-edges
iter.Remove();
}
else
{
if (edge.GetRelation().ToString().Equals("punct"))
{
// Clean punctuation (again)
if (!tree.OutgoingEdgeIterator(edge.GetDependent()).MoveNext())
{
// This should really never happen, but it does.
iter.Remove();
}
}
}
}
// (add edges we wanted to add)
toDelete.ForEach(null);
foreach (Triple <IndexedWord, IndexedWord, SemanticGraphEdge> edge_1 in toAdd)
{
tree.AddEdge(edge_1.first, edge_1.second, edge_1.third.GetRelation(), edge_1.third.GetWeight(), edge_1.third.IsExtra());
}
// Handle extra edges.
// Two cases:
// (1) the extra edge is a subj/obj edge and the main edge is a conj:.*
// in this case, keep the extra
// (2) otherwise, delete the extra
IList <SemanticGraphEdge> extraEdges = new List <SemanticGraphEdge>();
foreach (SemanticGraphEdge edge_2 in tree.EdgeIterable())
{
if (edge_2.IsExtra())
{
IList <SemanticGraphEdge> incomingEdges = tree.IncomingEdgeList(edge_2.GetDependent());
SemanticGraphEdge toKeep = null;
foreach (SemanticGraphEdge candidate in incomingEdges)
{
if (toKeep == null)
{
toKeep = candidate;
}
else
{
if (toKeep.GetRelation().ToString().StartsWith("conj") && candidate.GetRelation().ToString().Matches(".subj.*|.obj.*"))
{
toKeep = candidate;
}
else
{
if (!candidate.IsExtra() && !(candidate.GetRelation().ToString().StartsWith("conj") && toKeep.GetRelation().ToString().Matches(".subj.*|.obj.*")))
{
toKeep = candidate;
}
}
}
}
foreach (SemanticGraphEdge candidate_1 in incomingEdges)
{
if (candidate_1 != toKeep)
{
extraEdges.Add(candidate_1);
}
}
}
}
extraEdges.ForEach(null);
// Add apposition edges (simple coref)
foreach (SemanticGraphEdge extraEdge in new List <SemanticGraphEdge>(extraEdges))
{
// note[gabor] prevent concurrent modification exception
foreach (SemanticGraphEdge candidateAppos in tree.IncomingEdgeIterable(extraEdge.GetDependent()))
{
if (candidateAppos.GetRelation().ToString().Equals("appos"))
{
extraEdges.Add(new SemanticGraphEdge(extraEdge.GetGovernor(), candidateAppos.GetGovernor(), extraEdge.GetRelation(), extraEdge.GetWeight(), extraEdge.IsExtra()));
}
}
foreach (SemanticGraphEdge candidateAppos_1 in tree.OutgoingEdgeIterable(extraEdge.GetDependent()))
{
if (candidateAppos_1.GetRelation().ToString().Equals("appos"))
{
extraEdges.Add(new SemanticGraphEdge(extraEdge.GetGovernor(), candidateAppos_1.GetDependent(), extraEdge.GetRelation(), extraEdge.GetWeight(), extraEdge.IsExtra()));
}
}
}
// Brute force ensure tree
// Remove incoming edges from roots
IList <SemanticGraphEdge> rootIncomingEdges = new List <SemanticGraphEdge>();
foreach (IndexedWord root in tree.GetRoots())
{
foreach (SemanticGraphEdge incomingEdge in tree.IncomingEdgeIterable(root))
{
rootIncomingEdges.Add(incomingEdge);
}
}
rootIncomingEdges.ForEach(null);
// Loop until it becomes a tree.
bool changed = true;
while (changed)
{
// I just want trees to be trees; is that so much to ask!?
changed = false;
IList <IndexedWord> danglingNodes = new List <IndexedWord>();
IList <SemanticGraphEdge> invalidEdges = new List <SemanticGraphEdge>();
foreach (IndexedWord vertex_1 in tree.VertexSet())
{
// Collect statistics
IEnumerator <SemanticGraphEdge> incomingIter = tree.IncomingEdgeIterator(vertex_1);
bool hasIncoming = incomingIter.MoveNext();
bool hasMultipleIncoming = false;
if (hasIncoming)
{
incomingIter.Current;
hasMultipleIncoming = incomingIter.MoveNext();
}
// Register actions
if (!hasIncoming && !tree.GetRoots().Contains(vertex_1))
{
danglingNodes.Add(vertex_1);
}
else
{
if (hasMultipleIncoming)
{
foreach (SemanticGraphEdge edge in new IterableIterator <SemanticGraphEdge>(incomingIter))
{
invalidEdges.Add(edge_2);
}
}
}
}
// Perform actions
foreach (IndexedWord vertex_2 in danglingNodes)
{
tree.RemoveVertex(vertex_2);
changed = true;
}
foreach (SemanticGraphEdge edge_3 in invalidEdges)
{
tree.RemoveEdge(edge_3);
changed = true;
}
}
// Edge case: remove duplicate dobj to "that."
// This is a common parse error.
foreach (IndexedWord vertex_3 in tree.VertexSet())
{
SemanticGraphEdge thatEdge = null;
int dobjCount = 0;
foreach (SemanticGraphEdge edge in tree.OutgoingEdgeIterable(vertex_3))
{
if (Sharpen.Runtime.EqualsIgnoreCase("that", edge_2.GetDependent().Word()))
{
thatEdge = edge_2;
}
if ("dobj".Equals(edge_2.GetRelation().ToString()))
{
dobjCount += 1;
}
}
if (dobjCount > 1 && thatEdge != null)
{
// Case: there are two dobj edges, one of which goes to the word "that"
// Action: rewrite the dobj edge to "that" to be a "mark" edge.
tree.RemoveEdge(thatEdge);
tree.AddEdge(thatEdge.GetGovernor(), thatEdge.GetDependent(), GrammaticalRelation.ValueOf(thatEdge.GetRelation().GetLanguage(), "mark"), thatEdge.GetWeight(), thatEdge.IsExtra());
}
}
// Return
System.Diagnostics.Debug.Assert(IsTree(tree));
return(extraEdges);
}