private void Assign(NonTerminal head, Terminal term, Production production) { if (predictiveParseTable[head][term].HasValue) throw new InvalidGrammarException("Grammar is not LL(1)"); predictiveParseTable[head][term] = production; }
public void ConstructANonTerminal() { string name = "foo"; NonTerminal n = new NonTerminal(name); Assert.AreEqual(name, n.Name); Assert.AreEqual(name, n.ToString()); }
public void ConcatenationOfStringAndNonTerminal() { NonTerminal b = new NonTerminal("b"); var c = "a" + b; Assert.IsNotNull(c); Assert.AreEqual(2, c.Count); Assert.IsInstanceOfType(c, typeof(ConcatenationRule)); }
public void ConcatenationOfNonTerminalAndString() { NonTerminal a = new NonTerminal("a"); var c = a | "b"; Assert.IsNotNull(c); Assert.AreEqual(2, c.Count); Assert.IsInstanceOfType(c, typeof(RuleAlternation)); }
public NonTerminal Optional() { NonTerminal alt = new NonTerminal("alt"); alt.Rules = this; NonTerminal opt = new NonTerminal("optional"); opt.Rules = alt + "".AsTerminal(); return(opt); }
private static NonTerminal ConstructGrammar(string openBracket, string closeBracket) { Terminal open = new Terminal(Regex.Escape(openBracket)); Terminal close = new Terminal(Regex.Escape(closeBracket)); NonTerminal s = new NonTerminal("S"); NonTerminal sPrime = new NonTerminal("S'"); s.Rules = open + sPrime + close; sPrime.Rules = s | "a"; return s; }
public Grammar(NonTerminal root, params Terminal[] whitespace) { NonTerminal augmentedRoot = new NonTerminal("Root'"); augmentedRoot.Rules = root; this.Root = augmentedRoot; this.whitespace = whitespace; ConstructGrammar(Root); CalculateNullable(); CalculateFirstSymbols(); CalculateFollowSets(); }
private static NonTerminal ConstructGrammar() { NonTerminal R = new NonTerminal("R"); NonTerminal S = new NonTerminal("S"); NonTerminal V = new NonTerminal("V"); Terminal id = new Terminal("id", @"([A-Za-z]+)([0-9A-Za-z]*)"); R.Rules = S; S.Rules = V + "=" + V; V.Rules = id; return R; }
public static Grammar ConstructTestGrammar(out NonTerminal S, out NonTerminal T, out NonTerminal F) { T = new NonTerminal("T"); F = new NonTerminal("F"); T.Rules = T + "*" + F | F; F.Rules = "(" + T + ")" | new Terminal("ID", "([A-Z]|[a-z])+"); var g = new Grammar(T, new Terminal(" ", isIgnored:true)); S = g.Root; return g; }
public NonTerminal Star() { NonTerminal alt = new NonTerminal("alt"); alt.Rules = this; NonTerminal listTail = new NonTerminal("listTail"); listTail.Rules = alt | ""; NonTerminal list = new NonTerminal("list"); list.Rules = alt + listTail | "".AsTerminal(); return(list); }
protected override ParseTreeNode Parse(Lexer lexer, NonTerminal root) { IEnumerator<Token> tokens = lexer.GetEnumerator(); var eof = !tokens.MoveNext(); if (eof) { if (root.Rules.Any(a => a.IsNullable())) return new ParseTreeNode(null, new Token(new Terminal(""), "", 0, 0)); else throw new ParseException("Empty string not matched by grammar"); } return Parse(tokens, root, null); }
public void GetFirstSetWithEmptyStrings() { NonTerminal a = new NonTerminal("A"); NonTerminal b = new NonTerminal("B"); a.Rules = b + "s"; b.Rules = "".AsTerminal() | "x"; Grammar g = new Grammar(a); var set = g.GetFirstSet(a); Assert.IsNotNull(set); Assert.AreEqual(2, set.Count); Assert.IsTrue(set.Contains("x")); Assert.IsTrue(set.Contains("s")); }
private IEnumerable<Item> CalculateInitialState(out Grammar g, out NonTerminal S, out NonTerminal T, out NonTerminal F) { S = new NonTerminal("S"); var sCopy = S; T = new NonTerminal("T"); F = new NonTerminal("F"); S.Rules = T + "$"; T.Rules = T + "*" + F | F; F.Rules = "(" + T + ")" | new Terminal("ID", "([A-Z]|[a-z])+"); g = new Grammar(S, new Terminal(" ")); Item start = new Item(g.Productions.First(a => a.Head.Equals(sCopy)), 0); return new[] { start }.Closure(g); }
public void GetFollowSet() { NonTerminal a = new NonTerminal("A"); NonTerminal b = new NonTerminal("B"); a.Rules = "a" + b; b.Rules = b + "a" | "c"; Grammar g = new Grammar(a); var followA = g.GetFollowSet(a); Assert.AreEqual(0, followA.Count); var followB = g.GetFollowSet(b); Assert.AreEqual(1, followB.Count); Assert.IsTrue(followB.Contains("a")); }
private ParseTreeNode Parse(IEnumerator<Token> lexer, NonTerminal current, ParseTreeNode parent) { Production? prod = predictiveParseTable[current][lexer.Current.Terminal]; if (!prod.HasValue) { if (current.IsNullable) { var n = new ParseTreeNode(parent, current); var c = new ParseTreeNode(n, new Token(new Terminal(""), "", lexer.Current.Line, lexer.Current.Column)); return n; } else throw new ParseException("Unable to match " + lexer.Current + " at line " + lexer.Current.Line + " column " + lexer.Current.Column); } ParseTreeNode node = new ParseTreeNode(parent, current); List<ParseTreeNode> nodes = new List<ParseTreeNode>(); for (int i = 0; i < prod.Value.Body.Length; i++) { var term = prod.Value.Body[i]; if (term is Terminal) { string match; (term as Terminal).Match(lexer.Current.Value, 0, out match); nodes.Add(new ParseTreeNode(node, lexer.Current)); if (!lexer.MoveNext()) for (int j = i + 1; j < prod.Value.Body.Length; j++) if (!prod.Value.Body[j].IsNullable) throw new ParseException("unexpected EOF"); } else if (term is NonTerminal) { nodes.Add(Parse(lexer, term as NonTerminal, node)); } else throw new Exception("wtf?"); } return node; }
private static NonTerminal ConstructGrammar(string openBracket, string closeBracket) { Terminal open = new Terminal(Regex.Escape(openBracket)); Terminal close = new Terminal(Regex.Escape(closeBracket)); NonTerminal start = new NonTerminal("start"); NonTerminal wbbList = new NonTerminal("wbb List"); NonTerminal openBracketPlusStuff = new NonTerminal("( + L|)"); NonTerminal stuffPlusClose = new NonTerminal("L + ) | )"); start.Rules = openBracketPlusStuff + wbbList; wbbList.Rules = openBracketPlusStuff + wbbList | ""; openBracketPlusStuff.Rules = open + stuffPlusClose; stuffPlusClose.Rules = wbbList + close | close; return start; }
private static NonTerminal ConstructGrammar() { NonTerminal expression = new NonTerminal("expression"); NonTerminal term = new NonTerminal("term"); NonTerminal factor = new NonTerminal("factor"); Terminal number = new Terminal("number", @"[0-9]+(\.[0-9]+)?"); Terminal multiply = new Terminal("*", Regex.Escape("*")); Terminal divide = new Terminal("/", Regex.Escape("/")); Terminal add = new Terminal("+", Regex.Escape("+")); Terminal subtract = new Terminal("-", Regex.Escape("-")); Terminal openBracket = new Terminal("(", Regex.Escape("(")); Terminal closeBracket = new Terminal(")", Regex.Escape(")")); expression.Rules = expression + add + term | expression + subtract + term | term; term.Rules = term + multiply + factor | term + divide + factor | factor; factor.Rules = openBracket + expression + closeBracket | number; return expression; }
public void GetFirstSet() { NonTerminal a = new NonTerminal("A"); NonTerminal b = new NonTerminal("B"); NonTerminal c = new NonTerminal("C"); NonTerminal d = new NonTerminal("D"); a.Rules = b + d | c + d; b.Rules = "b"; c.Rules = "c"; d.Rules = "d"; Grammar g = new Grammar(a); var set = g.GetFirstSet(a); Assert.IsNotNull(set); Assert.AreEqual(2, set.Count); Assert.IsFalse(set.Contains("d")); Assert.IsTrue(set.Contains("b")); Assert.IsTrue(set.Contains("c")); }
public void ConstructAGrammar() { NonTerminal op = new NonTerminal("Operator"); NonTerminal expr = new NonTerminal("Expression"); Terminal number = new Terminal("Number", @"\b\d+\b"); Terminal plus = new Terminal("Plus", @"\+"); Terminal multiply = @"\*"; op.Rules = plus | multiply; expr.Rules = number + op + expr; Terminal whitespace = new Terminal("Whitespace", " |\n|\r", true); Grammar g = new Grammar(expr, whitespace); Assert.AreEqual(3, g.Terminals.Count()); Assert.AreEqual(3, g.NonTerminals.Count()); //Although the grammar only specifies 2 Nonterminals, grammar creates 1 internally for the root Lexer lexer = g.CreateLexer("1 + 2 * 3"); Assert.AreEqual(5, lexer.Count()); }
private static NonTerminal ConstructGrammar() { NonTerminal chunk = new NonTerminal("chunk"); NonTerminal block = new NonTerminal("block"); NonTerminal stat = new NonTerminal("stat"); NonTerminal laststat = new NonTerminal("laststat"); NonTerminal funcname = new NonTerminal("funcname"); NonTerminal varlist = new NonTerminal("varlist"); NonTerminal var = new NonTerminal("var"); NonTerminal namelist = new NonTerminal("namelist"); NonTerminal explist = new NonTerminal("explist"); NonTerminal exp = new NonTerminal("exp"); NonTerminal prefixexp = new NonTerminal("prefixexp"); NonTerminal functioncall = new NonTerminal("functioncall"); NonTerminal args = new NonTerminal("args"); NonTerminal function = new NonTerminal("function"); NonTerminal funcbody = new NonTerminal("funcbody"); NonTerminal parlist = new NonTerminal("parlist"); NonTerminal tableConstructor = new NonTerminal("tableConstructor"); NonTerminal fieldList = new NonTerminal("fieldList"); NonTerminal field = new NonTerminal("field"); NonTerminal fieldsep = new NonTerminal("fieldsep"); NonTerminal binop = new NonTerminal("binop"); NonTerminal unop = new NonTerminal("unop"); chunk.Rules = stat.Star() + laststat.Optional() + ";".Optional(); //block.Rules = chunk; //stat.Rules = varlist + "=" + explist // | functioncall // | "do" + block + "end" // | "while" + exp + "do" + block + "end" // | "repeat" + block + "until" + exp // | "if" + exp + "then" + block + ("elseif" + exp + "then" + block).Star() + ("else" + block).Optional() + "end"; return chunk; }
public ISet<Terminal> GetFollowSet(NonTerminal a) { return follow[a]; }
public ParseTreeNode(ParseTreeNode parent, NonTerminal nonTerminal) { this.Parent = parent; this.nonTerminal = nonTerminal; }
public Production(NonTerminal head, ConcatenationRule body) { Body = body.ToArray(); Head = head; }
protected override ParseTreeNode Parse(Lexer lexer, NonTerminal root) { IEnumerator<Token> tokens = lexer.GetEnumerator(); var eof = !tokens.MoveNext(); if (eof) { if (root.Rules.Any(r => r.IsNullable())) return new ParseTreeNode(null, new Token(new Terminal(""), "", 0, 0)); else throw new ParseException("Empty string not matched by grammar"); } Stack<ParseState> states = new Stack<ParseState>(); states.Push(MakeStateState(root, Grammar)); ParserAction action = ParserAction.Start; Stack<ParseTreeNode> nodes = new Stack<ParseTreeNode>(); do { Production reduceProduction; ParseState shiftState; action = Action(states.Peek(), eof ? null : tokens.Current, out reduceProduction, out shiftState); switch (action) { case ParserAction.Accept: break; case ParserAction.Error: throw new ParseException(tokens.Current.ToString()); case ParserAction.Start: default: throw new NotImplementedException(); case ParserAction.Shift: var newLeafNode = new ParseTreeNode(tokens.Current); nodes.Push(newLeafNode); states.Push(shiftState); eof = !tokens.MoveNext(); break; case ParserAction.Reduce: var newNode = new ParseTreeNode(reduceProduction.Head); var children = nodes.Take(reduceProduction.Body.Length).Reverse().Select((n, i) => new KeyValuePair<int, ParseTreeNode>(i, n)); foreach (var child in children) { child.Value.Parent = newNode; nodes.Pop(); var bnfTerm = reduceProduction.Body[child.Key]; var terminal = bnfTerm as Terminal; string match; if (terminal != null) { if (!terminal.Match(child.Value.Token.Value, 0, out match)) throw new ParseException("Unexpected crap"); } else if (!reduceProduction.Body[child.Key].Equals(child.Value.NonTerminal)) throw new ParseException("Unexpected crap"); } nodes.Push(newNode); for (int i = 0; i < reduceProduction.Body.Length; i++) states.Pop(); states.Push(Goto(states.Peek(), reduceProduction.Head)); break; } } while (action != ParserAction.Accept); if (nodes.Count != 1) throw new ParseException("o_0"); return nodes.Pop(); }
protected abstract ParseState Goto(ParseState state, NonTerminal nonTerminal);
private ParseState MakeStateState(NonTerminal root, Grammar g) { return new HashSet<Item>(root.Rules.Select(a => new Item(new Production(root, a), 0))).Closure(g); }
public ParseTreeNode(NonTerminal nonTerminal) { this.nonTerminal = nonTerminal; }
public void ConcatenationOfTerminalAndNonTerminal() { Terminal a = new Terminal("a"); NonTerminal b = new NonTerminal("b"); var c = a | b; Assert.IsNotNull(c); Assert.AreEqual(2, c.Count); Assert.IsInstanceOfType(c, typeof(RuleAlternation)); }
private void ConstructGrammar(NonTerminal root) { nonTerminals.Add(root); foreach (ConcatenationRule option in root.Rules) { foreach (var term in option) { if (term is NonTerminal) { if (nonTerminals.Add(term as NonTerminal)) ConstructGrammar(term as NonTerminal); } else terminals.Add(term as Terminal); } } }
public void IsNullable() { NonTerminal a = new NonTerminal("A"); NonTerminal b = new NonTerminal("B"); NonTerminal c = new NonTerminal("C"); a.Rules = b + "s"; b.Rules = "".AsTerminal() | c; c.Rules = "c"; Grammar g = new Grammar(a); Assert.IsTrue(b.IsNullable); Assert.IsFalse(a.IsNullable); Assert.IsFalse(c.IsNullable); }
protected override ParseState Goto(ParseState state, NonTerminal nonTerminal) { return automaton[state, nonTerminal]; }
protected abstract ParseTreeNode Parse(Lexer lexer, NonTerminal root);