void ISppfNodeVisitor.VisitAlternatives(SppfNode alternatives) { var token = alternatives.GetTokenId(grammar); var symbol = grammar.Symbols[token]; string label = symbol.Name + " pos: " + alternatives.Location.Position #if DEBUG + " t: " + currentNode.timestamp #endif ; object parentIdentity = currentNodeIdentity; graph.AddNode(parentIdentity, label: label, shape: Shape.Rect); var familyNode = alternatives; var altIdentity = alternatives.Children ?? new object(); int altIndex = 0; do { graph.AddEdge( parentIdentity, altIdentity, style: Style.Dotted, label: "alt#" + ++altIndex); this.currentNodeIdentity = altIdentity; this.currentNode = familyNode; visited.Add(familyNode); familyNode.Accept(this, ignoreAlternatives: true); familyNode = familyNode.NextAlternative; altIdentity = familyNode; } while (familyNode != null); }
private static HashSet <SppfNode> GetAllNodes(SppfNode sppf) { var nodes = new HashSet <SppfNode>(); var stack = new Stack <SppfNode>(); stack.Push(sppf); while (stack.Count > 0) { var node = stack.Pop(); if (nodes.Contains(node)) { continue; } nodes.Add(node); foreach (var family in node.Families) { foreach (var child in family.Members) { stack.Push(child); } } } return(nodes); }
private double StepProbability(SppfNode node, Dictionary <SppfNode, int> nodeToIndex, double[] previousEstimates) { if (node.Families.Count == 0) { return(1.0); } var l = node.Families; var familyProbs = new double[l.Count]; for (int i = 0; i < l.Count; i++) { var alternative = l[i]; double prob = GetChildProb(node, i); var childrenProbs = l[i].Members.Select((child) => previousEstimates[nodeToIndex[child]]).ToList(); var childrenProb = childrenProbs.Aggregate(1.0, (p1, p2) => p1 * p2); familyProbs[i] = prob * childrenProb; } var familyProb = familyProbs.Sum(); if (familyProb > 1) { familyProb = 1.0; } var result = familyProb; return(result); }
public GraphBuilder(SppfNode root) { _root = root; var graphNode = new SppfNodeNode(root, 0, GetOrSetId(root)); _g = new Graph(graphNode); GetGraphHelper(root, graphNode); }
public Pipe <TContext> Compile(string input) { SppfNode root = BuildTree(input); var cachedMethod = new CachedMethod <Pipe <TContext> >("temp", (emit, args) => Build(emit, args, root)); return(cachedMethod.Delegate); }
private int GetOrSetId(SppfNode node) { if (!_nodeIdDict.TryGetValue(node, out var id)) { id = _nextId; _nextId++; _nodeIdDict[node] = id; } return(id); }
public void FillEpsilonSuffix(int ruleId, int prefixSize, SppfNode[] dest, int destIndex, IStackLookback<SppfNode> stackLookback) { int i = ruleOffsetInCache[ruleId] + prefixSize; int end = ruleEndOffsetInCache[ruleId]; while (i != end) { dest[destIndex++] = ruleCache[i++]; } }
//TODO this is so horribly terrible. There's got to be a better way of thinking about this structure private void AnnotateWithProductions(SppfNode node, HashSet <SppfNode> seen = null, InteriorNode parent = null, int place = 0) { if (seen == null) { seen = new HashSet <SppfNode>(); } if (node is IntermediateNode) { var intermediateNode = (IntermediateNode)node; var production = intermediateNode.Item.Production; if (intermediateNode.Item.CurrentPosition == production.Rhs.Count - 1) { parent.AddChild(place, production); } } if (seen.Contains(node)) { return; } seen.Add(node); var l = node.Families; for (int i = 0; i < l.Count; i++) { var alternative = l[i]; if (!(node is InteriorNode)) { throw new Exception(); } var members = l[i].Members; if (members.Count == 1) { var child = members[0]; AnnotateWithProductionsChildren((InteriorNode)node, seen, child, i); } else if (members.Count == 2) { var left = members[0]; var right = members[1]; AnnotateWithProductionsChildren((InteriorNode)node, seen, left, right, i); } else { throw new Exception("Should only be 0--2 children"); } } }
private void BuildCache() { int tokenCount = grammar.SymbolCount; tokenCache = new SppfNode[tokenCount]; for (int token = 0; token != tokenCount; ++token) { if (grammar.IsNullable(token)) { tokenCache[token] = new SppfNode(token, null, Loc.Unknown, HLoc.Unknown); } } ruleOffsetInCache = new int[grammar.Productions.Count]; ruleEndOffsetInCache = new int[grammar.Productions.Count]; int nullableCount = 0; foreach (var rule in grammar.Productions) { int i = rule.PatternTokens.Length; while (i != 0) { int token = rule.PatternTokens[--i]; if (tokenCache[token] == null) { break; } ++nullableCount; } ruleEndOffsetInCache[rule.Index] = nullableCount; ruleOffsetInCache[rule.Index] = nullableCount - rule.PatternTokens.Length; } this.ruleCache = new SppfNode[nullableCount]; foreach (var rule in grammar.Productions) { int endOffset = ruleOffsetInCache[rule.Index] + rule.PatternTokens.Length; int i = rule.PatternTokens.Length; while (i != 0) { int token = rule.PatternTokens[--i]; if (tokenCache[token] == null) { break; } ruleCache[--endOffset] = tokenCache[token]; } } }
private double GetChildProb(SppfNode node, int i) { var production = node.Families[i].Production; var prob = 1.0; if (production != null) { prob = _grammar.GetProbability(production); } return(prob); }
private void PrintForest(SppfNode node, Dictionary <SppfNode, double> nodeProbs = null, string padding = "", HashSet <SppfNode> seen = null) { if (seen == null) { seen = new HashSet <SppfNode>(); } var nodeProb = ""; if (nodeProbs != null) { nodeProb = " p=" + nodeProbs[node]; } Console.WriteLine("{0}{1}{2}", padding, node, nodeProb); if (node.Families.Count > 0 && seen.Contains(node)) { Console.WriteLine("{0}Already seen this node!", padding); return; } seen.Add(node); //if (node is IntermediateNode) { // foreach (var family in node.Families) { // if (family.Production != null) { // // throw new Exception(); // } // } // if (node.Families.Count > 1) { // } //} var l = node.Families; for (int i = 0; i < l.Count; i++) { var alternative = l[i]; if (l.Count > 1) { Console.WriteLine("{0}Alternative {1}", padding, i); } foreach (var member in l[i].Members) { PrintForest(member, nodeProbs, padding + " ", seen); } } }
public void Test2Sppf() { var filePath = DataSamples.CompileSample2FilePath; using (var source = new StreamReader(filePath)) using (var interpreter = new Interpreter <ILLanguage>()) { SppfNode sppf = interpreter.BuildTree(source, filePath); using (var graph = new GvGraphView("Cil_Sample2_sppf.gv")) { var lang = Language.Get(typeof(ILLanguage)); sppf.WriteGraph(graph, lang.Grammar, true); } } }
public EarleyItem(DecoratedProduction decoratedProduction, int startPosition, SppfNode sppfNode) { if (decoratedProduction == null) { throw new ArgumentNullException(); } DecoratedProduction = decoratedProduction; StartPosition = startPosition; SppfNode = sppfNode; unchecked { int hash = 17; hash = hash * 23 + this.DecoratedProduction.GetHashCode(); hash = hash * 23 + this.StartPosition.GetHashCode(); _cachedHash = hash; } }
private static double StepProbability(BaseGrammar _grammar, SppfNode node, Dictionary <SppfNode, int> nodeToIndex, double[] previousEstimates) { var l = node.Families; var familyCount = l.Count(); if (familyCount == 0) { return(1.0); } var familyProbs = new double[familyCount]; var i = 0; foreach (var alternative in l) { // for (int i = 0; i < familyCount; i++) { // var alternative = l[i]; double prob = GetChildProb(_grammar, alternative.Production); //var childrenProbs = l[i].Members.Select((child) => previousEstimates[nodeToIndex[child]]); //var childrenProb = childrenProbs.Aggregate(1.0, (p1, p2) => p1 * p2); var childrenProb = 1.0; foreach (var child in alternative.Members) { var index = nodeToIndex[child]; var estimate = previousEstimates[index]; childrenProb *= estimate; } familyProbs[i] = prob * childrenProb; i++; } var familyProb = familyProbs.Sum(); if (familyProb > 1) { familyProb = 1.0; } var result = familyProb; return(result); }
internal void GetGraphHelper(SppfNode node, SppfNodeNode myNode) { if (_visited.Contains(node)) { return; } _visited.Add(node); var i = 0; foreach (var family in node.Families) { // for (int i = 0; i < Families.Count; i++) { // var family = Families[i]; Production lowerProduction = null; IGraphNode prevNode; if (node.Families.Count() == 1) { prevNode = myNode; lowerProduction = node.Families.Single().Production; } else { var id = GetOrSetId(node); prevNode = new FamilyNode(family, id + "-" + i, myNode.Rank + 1); _g.AddEdge(myNode, prevNode, family.Production); //g.AddEdge(myNode, prevNode); } prevNode.TheFamily = family; foreach (var child in family.Members) { var id = GetOrSetId(child); var childNode = new SppfNodeNode(child, prevNode.Rank + 1, id); // var childNode = g.GetNode(child, prevNode.Rank + 1); // g.AddEdge(prevNode, childNode, singletonProduction); // g.AddEdge(prevNode, childNode, singletonProduction); _g.AddEdge(prevNode, childNode, lowerProduction); GetGraphHelper(child, childNode); } i++; } }
public static Sentence RemoveLayout(Sentence input, out SppfNode sppf) { var layoutGrammar = Ebnf.GrammarLayout(); var earley = new EarleyParser2(layoutGrammar); sppf = earley.ParseGetForest(input); if (sppf == null) { throw new Exception(); } var traversal = new Traversal(sppf, layoutGrammar); var result = traversal.Traverse(); if (result.Count() != 1) { throw new Exception(); } var inputNoLayout = new Sentence((List <Terminal>)result.First().Payload); return(inputNoLayout); }
void ISppfNodeVisitor.VisitBranch(int ruleIndex, SppfNode[] children, Loc location) { var rule = grammar.Productions[ruleIndex]; var tokenName = grammar.Symbols[rule.OutcomeToken].Name; string label; if (showRules) { label = string.Format( "{0} -> {1}" #if DEBUG + " t: " + currentNode.timestamp #endif , tokenName, string.Join(" ", from s in rule.Pattern select s.Name)); } else { label = tokenName #if DEBUG + " t: " + currentNode.timestamp #endif ; } graph.AddNode(currentNodeIdentity, label: label, shape: Shape.Ellipse); if (children != null && children.Length != 0) { int i = 0; foreach (var child in children) { graph.AddEdge(currentNodeIdentity, child, label: "child" + ++i); EnqueueNode(child); } } }
private EmitSyntax Build(EmitSyntax emit, Ref <Args>[] args, SppfNode root) { int index = 0; return(BuildNodeInvocation(ref index, emit, args, root).Ret()); }
// MAKE_NODE(B ::= αx · β, j, i, w, v, V) { private SppfNode MakeNodeOriginal(DecoratedProduction decoratedProduction, int j, int i, SppfNode w, SppfNode v) { // var α = decoratedProduction.Prefix; // if α = ϵ and β ̸= ϵ { let y = v } if (decoratedProduction.CurrentPosition == 1 && !decoratedProduction.AtEnd) { return(v); } Production production = null; SppfNode y; // if β = ϵ { let s = B } else { let s = (B ::= αx · β) } // if there is no node y ∈ V labelled (s,j,i) create one and add it to V if (decoratedProduction.AtEnd) { production = decoratedProduction.Production; var s = production.Lhs; y = _V.GetOrSet(s, j, i); } else { var s = decoratedProduction; y = _V.GetOrSet(s, j, i); } // if w = null and y does not have a family of children (v) add one if (w == null) { y.AddFamily(production, v); } // if w ̸= null and y does not have a family of children(w, v) add one else { y.AddFamily(production, w, v); } return(y); }
public SppfNodeNode(SppfNode node, int rank, int id) { Node = node; Rank = rank; _id = id; }
private EmitSyntax BuildNodeInvocation(ref int startArgIndex, EmitSyntax emit, Ref <Args>[] args, SppfNode node) { if (node.IsTerminal) { return(emit.Ldarg(args[startArgIndex++])); } int count = node.Children.Length; for (int i = 0; i != count; ++i) { emit = BuildNodeInvocation(ref startArgIndex, emit, args, node.Children[i]); } return(emit); }
// MAKE_NODE(B ::= αx · β, j, i, w, v, V) { private SppfNode MakeNodeSimplified(DecoratedProduction decoratedProduction, int j, int i, SppfNode w, SppfNode v) { SppfNode y; Production production = null; // if β = ϵ { let s = B } else { let s = (B ::= αx · β) } if (decoratedProduction.AtEnd) { // s = ValueTuple.Create<Word, DecoratedProduction>(decoratedProduction.Production.Lhs, null); y = _V.GetOrSet(decoratedProduction.Production.Lhs, j, i); production = decoratedProduction.Production; } else { // s = ValueTuple.Create<Word, DecoratedProduction>(null, decoratedProduction); y = _V.GetOrSet(decoratedProduction, j, i); } // if w = null and y does not have a family of children (v) add one if (w == null) { y.AddFamily(production, v); } // if w ̸= null and y does not have a family of children(w, v) add one else { y.AddFamily(production, w, v); } return(y); }
internal TraverseResult(object payload, SppfNode node, Production production) { Payload = payload; Node = node; Production = production; }
public static double GetProbFromSppf(BaseGrammar _grammar, SppfNode node) { var prob = CalculateProbability(_grammar, node); return(prob); }
private string GetLabel(SppfNode node) { if (node.Id > 0) { return grammar.Symbols[node.Id].Name + " pos: " + node.Location.Position; } if (node.NextAlternative != null) { return grammar.Symbols[node.GetTokenId(grammar)].Name; } var production = grammar.Productions[-node.Id]; if (showRules) { return string.Format( "{0} -> {1}", production.Outcome.Name, string.Join(" ", from s in production.Pattern select s.Name)); } return production.Outcome.Name; }
private void AnnotateWithProductionsChildren(InteriorNode parent, HashSet <SppfNode> seen, SppfNode left, SppfNode right, int place) { if (!(left is IntermediateNode)) { throw new Exception(); } //if (!(right is SymbolNode)) { // throw new Exception(); //} AnnotateWithProductions(left, seen, parent, place); AnnotateWithProductions(right, seen, parent, place); }
private void AnnotateWithProductionsChildren(InteriorNode parent, HashSet <SppfNode> seen, SppfNode child, int place) { Word parentSymbol = null; if (parent is SymbolNode) { var symbolParent = (SymbolNode)parent; parentSymbol = symbolParent.Symbol; } else { var intermediateParent = (IntermediateNode)parent; if (intermediateParent.Item.CurrentPosition != 1) { throw new Exception("Expected to be at beginning of item"); } parentSymbol = intermediateParent.Item.Production.Rhs[0]; } if (child is SymbolNode) { var symbolChild = (SymbolNode)child; if (parent is SymbolNode) { var symbolParent = (SymbolNode)parent; var production = _grammar.FindProduction((Nonterminal)parentSymbol, new Sentence { symbolChild.Symbol }); symbolParent.AddChild(place, production); } AnnotateWithProductions(symbolChild, seen, parent, place); return; } else if (child is IntermediateNode) { throw new Exception("Don't handle intermediate"); } else if (child is LeafNode) { if (parentSymbol is Nonterminal) { var leafChild = (LeafNode)child; var childSentence = leafChild.GetSentence(); var production = _grammar.FindProduction((Nonterminal)parentSymbol, childSentence); parent.AddChild(place, production); } return; } throw new Exception(); }
private void BuildAndSetResult(SppfNode node) { if (_cache.ContainsKey(node) || _branchCache.ContainsKey(node)) { return; } if (node.Families.Count() == 0) { if (node is SppfEpsilon) { _cache[node] = new TraverseResultCollection(new List <TraverseResult> { new TraverseResult("", node, null) }); } else if (node is SppfWord sppfWord) { if (!(sppfWord.Word is Terminal terminal)) { throw new Exception(); } _cache[node] = new TraverseResultCollection(new List <TraverseResult> { new TraverseResult(terminal.Name, node, null) }); } return; } var resultList = new List <TraverseResult>(); foreach (var family in node.Families) { // var argList = new List<TraverseResultCollection>(); if (family.Members.Count == 0) { throw new NotImplementedException(); } else if (family.Members.Count == 1) { var child = family.Members[0]; var childValue = _cache[child]; if (node is SppfBranch) // this only happens in old (not contracted) sppf { if (family.Production != null) { throw new Exception(); } var newValues = new TraverseResultCollection[] { childValue }; _branchCache[node] = newValues; } else { if (family.Production == null) { throw new Exception(); } foreach (var tr in childValue) { var args = new TraverseResult[1] { tr }; var payload = GetPayload(args, family.Production); var oneResult = new TraverseResult(payload, node, family.Production); resultList.Add(oneResult); } } } else if (family.Members.Count == 2) { var left = family.Members[0]; var right = family.Members[1]; if (family.Production == null) { if (left is SppfBranch) // middle of long production { var leftValues = _branchCache[left]; var rightValue = _cache[right]; var newValues = AppendToArray(leftValues, rightValue); _branchCache[node] = newValues; } else // bottom left of a long production { var leftValue = _cache[left]; var rightValue = _cache[right]; var newValues = new TraverseResultCollection[] { leftValue, rightValue }; _branchCache[node] = newValues; } } else // top of production { var rightValue = _cache[right]; TraverseResultCollection[] args; if (left is SppfBranch) { var leftValues = _branchCache[left]; args = AppendToArray(leftValues, rightValue); } else { var leftValue = _cache[left]; args = new TraverseResultCollection[] { leftValue, rightValue }; } var someResults = BuildResultList(args, node, family.Production); resultList.AddRange(someResults); // throw new NotImplementedException(); } } else { throw new Exception(); } //foreach (var child in family.Members) { // if (!_cache.TryGetValue(child, out var childValue)) { // throw new Exception(); // } // argList.Add(childValue); //} // argSet.Add(argList.ToArray()); } var collection = new TraverseResultCollection(resultList); _cache[node] = collection; // var value = new TraverseResultCollection // var payload = GetPayload(_emptyArgs, ) }
private List <TraverseResult> BuildResultList(TraverseResultCollection[] args, SppfNode node, Production production) { var resultList = new List <TraverseResult>(); foreach (var oneSet in OneOfEach(args)) { object payload = null; var action = production.Annotations.Action; if (action == null) { if (oneSet.Length > 0) { payload = oneSet[0].Payload; // default action } else { payload = null; } } else { payload = action.Act(oneSet); } var oneResult = new TraverseResult(payload, node, production); resultList.Add(oneResult); } return(resultList); }
private void PrintDebugForest(SppfNode node, Sentence s, Dictionary <SppfNode, double> nodeProbs = null, string padding = "", HashSet <SppfNode> seen = null) { if (seen == null) { seen = new HashSet <SppfNode>(); } double?nodeProb = null; if (nodeProbs != null) { nodeProb = nodeProbs[node]; } string lhs = ""; if (node is SymbolNode) { var symbol = (SymbolNode)node; lhs = symbol.Symbol.ToString(); } else if (node is IntermediateNode) { var inter = (IntermediateNode)node; lhs = inter.Item.ProductionToString(); } else if (node is LeafNode) { lhs = ((LeafNode)node).GetSentence().ToString(); } else { throw new Exception(); } string rhs = ""; if (node is InteriorNode) { var interior = (InteriorNode)node; rhs = s.GetRange(interior.StartPosition, interior.EndPosition - interior.StartPosition).ToString(); } Console.WriteLine("{0}{1} --> {2} [{4}]\t{3}", padding, lhs, rhs, nodeProb, node.ProductionsToString()); if (node.Families.Count > 0 && seen.Contains(node)) { Console.WriteLine("{0}Already seen this node!", padding); return; } seen.Add(node); if (node.Families.Count == 0) { return; } var l = node.Families; for (int i = 0; i < l.Count; i++) { var alternative = l[i]; if (l.Count > 1) { Console.WriteLine("{0}Alternative {1}", padding, i); } foreach (var member in l[i].Members) { PrintDebugForest(member, s, nodeProbs, padding + " ", seen); } } }
public void WriteGraph(SppfNode rootNode) { front.Push(rootNode); graph.BeginDigraph("SPPF"); graph.SetGraphProperties(ordering: Ordering.Out); while (front.Count != 0) { var node = front.Pop(); WriteNode(node); } graph.EndDigraph(); }
private static double CalculateProbability(BaseGrammar _grammar, SppfNode sppf) { if (sppf == null) { return(0.0); } var nodeProbs = new Dictionary <SppfNode, double>(); var nodes = GetAllNodes(sppf); var indexToNode = nodes.ToArray(); var nodeToIndex = new Dictionary <SppfNode, int>(nodes.Count); for (int i = 0; i < indexToNode.Length; i++) { var node = indexToNode[i]; nodeToIndex[node] = i; } var previousEstimates = Enumerable.Repeat(1.0, indexToNode.Length).ToArray(); var currentEstimates = new double[indexToNode.Length]; //for (var i = 0; i < indexToNode.Length; i++) { // Console.WriteLine("{0,-40}: {1}", indexToNode[i], previousEstimates[i]); //} bool changed = true; while (changed == true) { changed = false; Array.Clear(currentEstimates, 0, currentEstimates.Length); for (var i = 0; i < indexToNode.Length; i++) { var node = indexToNode[i]; var estimate = StepProbability(_grammar, node, nodeToIndex, previousEstimates); currentEstimates[i] = estimate; if (currentEstimates[i] > previousEstimates[i]) { throw new Exception("Didn't expect estimates to increase"); } else if (currentEstimates[i] < previousEstimates[i]) { var diff = previousEstimates[i] - currentEstimates[i]; var tolerance = _probabilityChangePercentage * previousEstimates[i]; if (diff > _probabilityChangePercentage) { changed = true; } } } //Console.WriteLine("--------------------------"); //for (var i = 0; i < indexToNode.Length; i++) { // Console.WriteLine("{0,-40}: {1}", indexToNode[i], currentEstimates[i]); //} Helpers.Swap(ref previousEstimates, ref currentEstimates); } for (var i = 0; i < indexToNode.Length; i++) { nodeProbs[indexToNode[i]] = currentEstimates[i]; } return(currentEstimates[nodeToIndex[sppf]]); }
private bool EnqueueNode(SppfNode node) { bool result = !visited.Contains(node); if (result) { visited.Add(node); front.Push(node); } return result; }
public double ProbabilityOfSppf(SppfNode sppf) { return(ProbabilityCalculator.GetProbFromSppf(Grammar, sppf)); }
private void WriteNode(SppfNode node) { this.currentNodeIdentity = node; this.currentNode = node; node.Accept(this, false); }
public TraversalLoopException(SppfNode node) { Node = node; }
public Traversal(SppfNode root, BaseGrammar grammar) { _root = root; _grammar = grammar; }
public SppfNodeNode(SppfNode node, int rank) { Node = node; Rank = rank; }