private void Assign(NonTerminal head, Terminal term, Production production)
{
if (predictiveParseTable[head][term].HasValue)
throw new InvalidGrammarException("Grammar is not LL(1)");
predictiveParseTable[head][term] = production;
}
public void ConstructANonTerminal()
{
string name = "foo";
NonTerminal n = new NonTerminal(name);
Assert.AreEqual(name, n.Name);
Assert.AreEqual(name, n.ToString());
}
public void ConcatenationOfStringAndNonTerminal()
{
NonTerminal b = new NonTerminal("b");
var c = "a" + b;
Assert.IsNotNull(c);
Assert.AreEqual(2, c.Count);
Assert.IsInstanceOfType(c, typeof(ConcatenationRule));
}
public void ConcatenationOfNonTerminalAndString()
{
NonTerminal a = new NonTerminal("a");
var c = a | "b";
Assert.IsNotNull(c);
Assert.AreEqual(2, c.Count);
Assert.IsInstanceOfType(c, typeof(RuleAlternation));
}
public NonTerminal Optional()
{
NonTerminal alt = new NonTerminal("alt");
alt.Rules = this;
NonTerminal opt = new NonTerminal("optional");
opt.Rules = alt + "".AsTerminal();
return(opt);
}
private static NonTerminal ConstructGrammar(string openBracket, string closeBracket)
{
Terminal open = new Terminal(Regex.Escape(openBracket));
Terminal close = new Terminal(Regex.Escape(closeBracket));
NonTerminal s = new NonTerminal("S");
NonTerminal sPrime = new NonTerminal("S'");
s.Rules = open + sPrime + close;
sPrime.Rules = s | "a";
return s;
}
public Grammar(NonTerminal root, params Terminal[] whitespace)
{
NonTerminal augmentedRoot = new NonTerminal("Root'");
augmentedRoot.Rules = root;
this.Root = augmentedRoot;
this.whitespace = whitespace;
ConstructGrammar(Root);
CalculateNullable();
CalculateFirstSymbols();
CalculateFollowSets();
}
private static NonTerminal ConstructGrammar()
{
NonTerminal R = new NonTerminal("R");
NonTerminal S = new NonTerminal("S");
NonTerminal V = new NonTerminal("V");
Terminal id = new Terminal("id", @"([A-Za-z]+)([0-9A-Za-z]*)");
R.Rules = S;
S.Rules = V + "=" + V;
V.Rules = id;
return R;
}
public static Grammar ConstructTestGrammar(out NonTerminal S, out NonTerminal T, out NonTerminal F)
{
T = new NonTerminal("T");
F = new NonTerminal("F");
T.Rules = T + "*" + F | F;
F.Rules = "(" + T + ")" | new Terminal("ID", "([A-Z]|[a-z])+");
var g = new Grammar(T, new Terminal(" ", isIgnored:true));
S = g.Root;
return g;
}
public NonTerminal Star()
{
NonTerminal alt = new NonTerminal("alt");
alt.Rules = this;
NonTerminal listTail = new NonTerminal("listTail");
listTail.Rules = alt | "";
NonTerminal list = new NonTerminal("list");
list.Rules = alt + listTail | "".AsTerminal();
return(list);
}
protected override ParseTreeNode Parse(Lexer lexer, NonTerminal root)
{
IEnumerator<Token> tokens = lexer.GetEnumerator();
var eof = !tokens.MoveNext();
if (eof)
{
if (root.Rules.Any(a => a.IsNullable()))
return new ParseTreeNode(null, new Token(new Terminal(""), "", 0, 0));
else
throw new ParseException("Empty string not matched by grammar");
}
return Parse(tokens, root, null);
}
public void GetFirstSetWithEmptyStrings()
{
NonTerminal a = new NonTerminal("A");
NonTerminal b = new NonTerminal("B");
a.Rules = b + "s";
b.Rules = "".AsTerminal() | "x";
Grammar g = new Grammar(a);
var set = g.GetFirstSet(a);
Assert.IsNotNull(set);
Assert.AreEqual(2, set.Count);
Assert.IsTrue(set.Contains("x"));
Assert.IsTrue(set.Contains("s"));
}
private IEnumerable<Item> CalculateInitialState(out Grammar g, out NonTerminal S, out NonTerminal T, out NonTerminal F)
{
S = new NonTerminal("S");
var sCopy = S;
T = new NonTerminal("T");
F = new NonTerminal("F");
S.Rules = T + "$";
T.Rules = T + "*" + F | F;
F.Rules = "(" + T + ")" | new Terminal("ID", "([A-Z]|[a-z])+");
g = new Grammar(S, new Terminal(" "));
Item start = new Item(g.Productions.First(a => a.Head.Equals(sCopy)), 0);
return new[] { start }.Closure(g);
}
public void GetFollowSet()
{
NonTerminal a = new NonTerminal("A");
NonTerminal b = new NonTerminal("B");
a.Rules = "a" + b;
b.Rules = b + "a" | "c";
Grammar g = new Grammar(a);
var followA = g.GetFollowSet(a);
Assert.AreEqual(0, followA.Count);
var followB = g.GetFollowSet(b);
Assert.AreEqual(1, followB.Count);
Assert.IsTrue(followB.Contains("a"));
}
private ParseTreeNode Parse(IEnumerator<Token> lexer, NonTerminal current, ParseTreeNode parent)
{
Production? prod = predictiveParseTable[current][lexer.Current.Terminal];
if (!prod.HasValue)
{
if (current.IsNullable)
{
var n = new ParseTreeNode(parent, current);
var c = new ParseTreeNode(n, new Token(new Terminal(""), "", lexer.Current.Line, lexer.Current.Column));
return n;
}
else
throw new ParseException("Unable to match " + lexer.Current + " at line " + lexer.Current.Line + " column " + lexer.Current.Column);
}
ParseTreeNode node = new ParseTreeNode(parent, current);
List<ParseTreeNode> nodes = new List<ParseTreeNode>();
for (int i = 0; i < prod.Value.Body.Length; i++)
{
var term = prod.Value.Body[i];
if (term is Terminal)
{
string match;
(term as Terminal).Match(lexer.Current.Value, 0, out match);
nodes.Add(new ParseTreeNode(node, lexer.Current));
if (!lexer.MoveNext())
for (int j = i + 1; j < prod.Value.Body.Length; j++)
if (!prod.Value.Body[j].IsNullable)
throw new ParseException("unexpected EOF");
}
else if (term is NonTerminal)
{
nodes.Add(Parse(lexer, term as NonTerminal, node));
}
else
throw new Exception("wtf?");
}
return node;
}
private static NonTerminal ConstructGrammar(string openBracket, string closeBracket)
{
Terminal open = new Terminal(Regex.Escape(openBracket));
Terminal close = new Terminal(Regex.Escape(closeBracket));
NonTerminal start = new NonTerminal("start");
NonTerminal wbbList = new NonTerminal("wbb List");
NonTerminal openBracketPlusStuff = new NonTerminal("( + L|)");
NonTerminal stuffPlusClose = new NonTerminal("L + ) | )");
start.Rules = openBracketPlusStuff + wbbList;
wbbList.Rules = openBracketPlusStuff + wbbList | "";
openBracketPlusStuff.Rules = open + stuffPlusClose;
stuffPlusClose.Rules = wbbList + close | close;
return start;
}
private static NonTerminal ConstructGrammar()
{
NonTerminal expression = new NonTerminal("expression");
NonTerminal term = new NonTerminal("term");
NonTerminal factor = new NonTerminal("factor");
Terminal number = new Terminal("number", @"[0-9]+(\.[0-9]+)?");
Terminal multiply = new Terminal("*", Regex.Escape("*"));
Terminal divide = new Terminal("/", Regex.Escape("/"));
Terminal add = new Terminal("+", Regex.Escape("+"));
Terminal subtract = new Terminal("-", Regex.Escape("-"));
Terminal openBracket = new Terminal("(", Regex.Escape("("));
Terminal closeBracket = new Terminal(")", Regex.Escape(")"));
expression.Rules = expression + add + term | expression + subtract + term | term;
term.Rules = term + multiply + factor | term + divide + factor | factor;
factor.Rules = openBracket + expression + closeBracket | number;
return expression;
}
public void GetFirstSet()
{
NonTerminal a = new NonTerminal("A");
NonTerminal b = new NonTerminal("B");
NonTerminal c = new NonTerminal("C");
NonTerminal d = new NonTerminal("D");
a.Rules = b + d | c + d;
b.Rules = "b";
c.Rules = "c";
d.Rules = "d";
Grammar g = new Grammar(a);
var set = g.GetFirstSet(a);
Assert.IsNotNull(set);
Assert.AreEqual(2, set.Count);
Assert.IsFalse(set.Contains("d"));
Assert.IsTrue(set.Contains("b"));
Assert.IsTrue(set.Contains("c"));
}
public void ConstructAGrammar()
{
NonTerminal op = new NonTerminal("Operator");
NonTerminal expr = new NonTerminal("Expression");
Terminal number = new Terminal("Number", @"\b\d+\b");
Terminal plus = new Terminal("Plus", @"\+");
Terminal multiply = @"\*";
op.Rules = plus | multiply;
expr.Rules = number + op + expr;
Terminal whitespace = new Terminal("Whitespace", " |\n|\r", true);
Grammar g = new Grammar(expr, whitespace);
Assert.AreEqual(3, g.Terminals.Count());
Assert.AreEqual(3, g.NonTerminals.Count()); //Although the grammar only specifies 2 Nonterminals, grammar creates 1 internally for the root
Lexer lexer = g.CreateLexer("1 + 2 * 3");
Assert.AreEqual(5, lexer.Count());
}
private static NonTerminal ConstructGrammar()
{
NonTerminal chunk = new NonTerminal("chunk");
NonTerminal block = new NonTerminal("block");
NonTerminal stat = new NonTerminal("stat");
NonTerminal laststat = new NonTerminal("laststat");
NonTerminal funcname = new NonTerminal("funcname");
NonTerminal varlist = new NonTerminal("varlist");
NonTerminal var = new NonTerminal("var");
NonTerminal namelist = new NonTerminal("namelist");
NonTerminal explist = new NonTerminal("explist");
NonTerminal exp = new NonTerminal("exp");
NonTerminal prefixexp = new NonTerminal("prefixexp");
NonTerminal functioncall = new NonTerminal("functioncall");
NonTerminal args = new NonTerminal("args");
NonTerminal function = new NonTerminal("function");
NonTerminal funcbody = new NonTerminal("funcbody");
NonTerminal parlist = new NonTerminal("parlist");
NonTerminal tableConstructor = new NonTerminal("tableConstructor");
NonTerminal fieldList = new NonTerminal("fieldList");
NonTerminal field = new NonTerminal("field");
NonTerminal fieldsep = new NonTerminal("fieldsep");
NonTerminal binop = new NonTerminal("binop");
NonTerminal unop = new NonTerminal("unop");
chunk.Rules = stat.Star() + laststat.Optional() + ";".Optional();
//block.Rules = chunk;
//stat.Rules = varlist + "=" + explist
// | functioncall
// | "do" + block + "end"
// | "while" + exp + "do" + block + "end"
// | "repeat" + block + "until" + exp
// | "if" + exp + "then" + block + ("elseif" + exp + "then" + block).Star() + ("else" + block).Optional() + "end";
return chunk;
}
public ISet<Terminal> GetFollowSet(NonTerminal a)
{
return follow[a];
}
public ParseTreeNode(ParseTreeNode parent, NonTerminal nonTerminal)
{
this.Parent = parent;
this.nonTerminal = nonTerminal;
}
public Production(NonTerminal head, ConcatenationRule body)
{
Body = body.ToArray();
Head = head;
}
protected override ParseTreeNode Parse(Lexer lexer, NonTerminal root)
{
IEnumerator<Token> tokens = lexer.GetEnumerator();
var eof = !tokens.MoveNext();
if (eof)
{
if (root.Rules.Any(r => r.IsNullable()))
return new ParseTreeNode(null, new Token(new Terminal(""), "", 0, 0));
else
throw new ParseException("Empty string not matched by grammar");
}
Stack<ParseState> states = new Stack<ParseState>();
states.Push(MakeStateState(root, Grammar));
ParserAction action = ParserAction.Start;
Stack<ParseTreeNode> nodes = new Stack<ParseTreeNode>();
do
{
Production reduceProduction;
ParseState shiftState;
action = Action(states.Peek(), eof ? null : tokens.Current, out reduceProduction, out shiftState);
switch (action)
{
case ParserAction.Accept:
break;
case ParserAction.Error:
throw new ParseException(tokens.Current.ToString());
case ParserAction.Start:
default:
throw new NotImplementedException();
case ParserAction.Shift:
var newLeafNode = new ParseTreeNode(tokens.Current);
nodes.Push(newLeafNode);
states.Push(shiftState);
eof = !tokens.MoveNext();
break;
case ParserAction.Reduce:
var newNode = new ParseTreeNode(reduceProduction.Head);
var children = nodes.Take(reduceProduction.Body.Length).Reverse().Select((n, i) => new KeyValuePair<int, ParseTreeNode>(i, n));
foreach (var child in children)
{
child.Value.Parent = newNode;
nodes.Pop();
var bnfTerm = reduceProduction.Body[child.Key];
var terminal = bnfTerm as Terminal;
string match;
if (terminal != null)
{
if (!terminal.Match(child.Value.Token.Value, 0, out match))
throw new ParseException("Unexpected crap");
}
else if (!reduceProduction.Body[child.Key].Equals(child.Value.NonTerminal))
throw new ParseException("Unexpected crap");
}
nodes.Push(newNode);
for (int i = 0; i < reduceProduction.Body.Length; i++)
states.Pop();
states.Push(Goto(states.Peek(), reduceProduction.Head));
break;
}
} while (action != ParserAction.Accept);
if (nodes.Count != 1)
throw new ParseException("o_0");
return nodes.Pop();
}
protected abstract ParseState Goto(ParseState state, NonTerminal nonTerminal);
private ParseState MakeStateState(NonTerminal root, Grammar g)
{
return new HashSet<Item>(root.Rules.Select(a => new Item(new Production(root, a), 0))).Closure(g);
}
public ParseTreeNode(NonTerminal nonTerminal)
{
this.nonTerminal = nonTerminal;
}
public void ConcatenationOfTerminalAndNonTerminal()
{
Terminal a = new Terminal("a");
NonTerminal b = new NonTerminal("b");
var c = a | b;
Assert.IsNotNull(c);
Assert.AreEqual(2, c.Count);
Assert.IsInstanceOfType(c, typeof(RuleAlternation));
}
private void ConstructGrammar(NonTerminal root)
{
nonTerminals.Add(root);
foreach (ConcatenationRule option in root.Rules)
{
foreach (var term in option)
{
if (term is NonTerminal)
{
if (nonTerminals.Add(term as NonTerminal))
ConstructGrammar(term as NonTerminal);
}
else
terminals.Add(term as Terminal);
}
}
}
public void IsNullable()
{
NonTerminal a = new NonTerminal("A");
NonTerminal b = new NonTerminal("B");
NonTerminal c = new NonTerminal("C");
a.Rules = b + "s";
b.Rules = "".AsTerminal() | c;
c.Rules = "c";
Grammar g = new Grammar(a);
Assert.IsTrue(b.IsNullable);
Assert.IsFalse(a.IsNullable);
Assert.IsFalse(c.IsNullable);
}
protected override ParseState Goto(ParseState state, NonTerminal nonTerminal)
{
return automaton[state, nonTerminal];
}
protected abstract ParseTreeNode Parse(Lexer lexer, NonTerminal root);
