public void TestParseErrorGetsReported()
{
// Create an instance of our language grammar and set handlers for its rules.
LanguageGrammar<Expression> g = new LanguageGrammar<Expression>();
g.DivideRule.Action = (e1, e2) => Expression.Divide(e1, e2);
g.DoubleLiteralRule.Action = (d) => Expression.Constant(d);
g.IntLiteralRule.Action = (i) => Expression.Constant(i);
g.StringLiteralRule.Action = (s) => Expression.Constant(s);
g.MinusRule.Action = (e1, e2) => Expression.Subtract(e1, e2);
g.ModRule.Action = (e1, e2) => Expression.Modulo(e1, e2);
g.MultiplyRule.Action = (e1, e2) => Expression.Multiply(e1, e2);
g.NegateRule.Action = (e) => Expression.Negate(e);
g.PlusRule.Action = (e1, e2) => Expression.Add(e1, e2);
g.VariableRefRule.SetStatefulAction<ParserState>((s, name) => s.GetVariable(name));
// Spin up a parser for our language.
// TODO: Package this up and simplify it.
var expressionHelper = new ExpressionHelper();
var classifierSession = new TerminalClassifierSession<char, ParserState, int>()
.AddSkipTerminal(new Terminal<char> { Name = "Whitespace", InitialState = RegexCharNFABuilder.RegexCompiler(@"\s+") })
.CurrentCharExprIs(x => x.CurrentChar())
.GetFromMarkExprIs(x => x.GetFromMarkedPos())
.HasCurrentCharExprIs(x => x.HasCurrentChar())
.MarkPosExprIs(x => x.MarkPos())
.MoveNextCharExprIs(x => x.MoveNextChar())
.GetLocationIs(x=>new ParseLocation {Line = 1, Column = x.GetPos()})
.ErrorCollectionIs(x=>x.Errors)
.UnmarkPosExprIs(x => x.UnmarkPos());
var parserGen = new ParserGenerator<char>(expressionHelper);
var parseTableBuilder = new LRParseTableBuilder();
var parseTable = parseTableBuilder.BuildParseTable(g);
var session = parserGen.NewSession<ParserState>()
.NonTerminalValueExprIs<object>(x => x.NonTerminalValue)
.TerminalValueExprIs<string>(x => x.TerminalStringValue)
.TerminalValueExprIs<int>(x => x.TerminalIntValue)
.TerminalValueExprIs<double>(x => x.TerminalDoubleValue)
.TerminalIs(x => x.Terminal)
.NonTerminalIs(x => x.NonTerminal)
.IncludeSymbols(true)
.DebugOutputIs(x => Debug.WriteLine(x))
.Generate("LanguageParser", parseTable, classifierSession);
// At this point, session is an Expression<ParserState,int> representing our parser.
// We can compile it into a delegate or a MethodBuilder. For the examples, we'll use a delegate.
var compiler = session.Compile();
// Create a parser state object and initialize it.
ParserState ps = new ParserState("x*y +* 2.0");
ps.SetParameters(
Expression.Parameter(typeof(double), "x"),
Expression.Parameter(typeof(double), "y"));
Assert.AreNotEqual(0, compiler(ps));
Assert.AreNotEqual(0, ps.Errors.Count);
Assert.Less(ps.Errors.First().ExpectedTerminalNames.Count, g.Terminals.Count);
}
public void TestCreateClassifier()
{
Expression<Func<TextReader, TerminalMatch>> foundOpenTag = r => TerminalMatch.OpenTag;
Expression<Func<TextReader, TerminalMatch>> foundCloseTag = r => TerminalMatch.CloseTag;
Expression<Func<TextReader, TerminalMatch>> foundLeafTag = r => TerminalMatch.LeafTag;
Expression<Func<TextReader, TerminalMatch>> foundNoTag = r => TerminalMatch.None;
//var classifier = _parserGenerator.Classifier<TextReader, TerminalMatch>()
var classifier = new TerminalClassifierSession<char, TextReader, TerminalMatch>()
.HasCurrentCharExprIs(r => r.Peek() != -1)
.CurrentCharExprIs(r => (char)r.Peek())
.MoveNextCharExprIs(r => r.Read())
.AddTerminalHandler(_openTag, r => TerminalMatch.OpenTag)
.AddTerminalHandler(_closeTag, r => TerminalMatch.CloseTag)
.AddTerminalHandler(_leafTag, r => TerminalMatch.LeafTag)
.RejectHandlerIs(r => TerminalMatch.None)
.Generate();
var open1 = new StringReader("<Open1>");
var open2 = new StringReader("<x>");
var close1 = new StringReader("</Close1>");
var close2 = new StringReader("</x>");
var leaf1 = new StringReader("<Leaf/>");
var leaf2 = new StringReader("<x/>");
var reject1 = new StringReader("<>");
var reject2 = new StringReader("</>");
var reject3 = new StringReader("x/>");
var reject4 = new StringReader("<reject4");
var reject5 = new StringReader("<reject5/");
var reject6 = new StringReader("</reject6");
var reject7 = new StringReader("");
var f = classifier.Compile();
Assert.AreEqual(TerminalMatch.OpenTag, f(open1));
Assert.AreEqual(TerminalMatch.OpenTag, f(open2));
Assert.AreEqual(TerminalMatch.CloseTag, f(close1));
Assert.AreEqual(TerminalMatch.CloseTag, f(close2));
Assert.AreEqual(TerminalMatch.LeafTag, f(leaf1));
Assert.AreEqual(TerminalMatch.LeafTag, f(leaf2));
Assert.AreEqual(TerminalMatch.None, f(reject1));
Assert.AreEqual(TerminalMatch.None, f(reject2));
Assert.AreEqual(TerminalMatch.None, f(reject3));
Assert.AreEqual(TerminalMatch.None, f(reject4));
Assert.AreEqual(TerminalMatch.None, f(reject5));
Assert.AreEqual(TerminalMatch.None, f(reject6));
Assert.AreEqual(TerminalMatch.None, f(reject7));
}
public void TestClassifyTokensAndCaptureIdentifiersUntilEof()
{
Expression<Func<TestStringInput, string, TerminalMatch>> capture = ((t, m) => t.Capture(m, TerminalMatch.Identifier));
var classifier = new TerminalClassifierSession<char, TestStringInput, TerminalMatch>()
.HasCurrentCharExprIs(r => r.HasCurrentChar())
.CurrentCharExprIs(r => r.CurrentChar())
.MoveNextCharExprIs(r => r.MoveNextChar())
.MarkPosExprIs(r => r.MarkPos())
.UnmarkPosExprIs(r => r.UnmarkPos())
.GetFromMarkExprIs(r => r.GetFromMarkedPos())
.AddTerminalHandler(_if, r => TerminalMatch.If)
.AddTerminalHandler(_while, r => TerminalMatch.While)
.AddTerminalHandler(_identifier, capture)
.AddSkipTerminal(_whitespace)
.EofHandlerIs(r => TerminalMatch.Eof)
.RejectHandlerIs(r => TerminalMatch.None)
.Generate();
var f = classifier.Compile();
var strIds = new TestStringInput("if while bob ida\nwhere iffy wh wherefore i if8 d7");
Assert.AreEqual(TerminalMatch.If, f(strIds));
Assert.AreEqual(TerminalMatch.While, f(strIds));
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("bob", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("ida", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("where", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("iffy", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("wh", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("wherefore", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("i", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("if8", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("d7", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Eof, f(strIds));
}
public void TestClassifyKeywordsAndIdentifiers()
{
Expression<Func<TextReader, TerminalMatch>> foundIdentifer = (r) => TerminalMatch.Identifier;
Expression<Func<TextReader, TerminalMatch>> foundIf = (r) => TerminalMatch.If;
Expression<Func<TextReader, TerminalMatch>> foundWhile = (r) => TerminalMatch.While;
Expression<Func<TextReader, TerminalMatch>> foundNone = (r) => TerminalMatch.None;
var classifier = new TerminalClassifierSession<char, TextReader, TerminalMatch>()
.HasCurrentCharExprIs(r => r.Peek() != -1)
.CurrentCharExprIs(r => (char)r.Peek())
.MoveNextCharExprIs(r => r.Read())
.AddTerminalHandler(_if, r => TerminalMatch.If)
.AddTerminalHandler(_while, r => TerminalMatch.While)
.AddTerminalHandler(_identifier, r => TerminalMatch.Identifier)
.RejectHandlerIs(r => TerminalMatch.None)
.Generate();
var f = classifier.Compile();
var strIf = new StringReader("if");
var strWhile = new StringReader("while");
var id1 = new StringReader("bob");
var id2 = new StringReader("ida");
var id3 = new StringReader("where");
var id4 = new StringReader("iffy");
var id5 = new StringReader("wh");
var id6 = new StringReader("wherefore");
var id7 = new StringReader("i");
var id8 = new StringReader("if8");
var id9 = new StringReader("d7 = 123");
var reject1 = new StringReader("99");
var reject2 = new StringReader("(@*#");
var reject3 = new StringReader("");
Assert.AreEqual(TerminalMatch.If, f(strIf));
Assert.AreEqual(TerminalMatch.While, f(strWhile));
Assert.AreEqual(TerminalMatch.Identifier, f(id1));
Assert.AreEqual(TerminalMatch.Identifier, f(id2));
Assert.AreEqual(TerminalMatch.Identifier, f(id3));
Assert.AreEqual(TerminalMatch.Identifier, f(id4));
Assert.AreEqual(TerminalMatch.Identifier, f(id5));
Assert.AreEqual(TerminalMatch.Identifier, f(id6));
Assert.AreEqual(TerminalMatch.Identifier, f(id7));
Assert.AreEqual(TerminalMatch.Identifier, f(id8));
Assert.AreEqual(TerminalMatch.Identifier, f(id9));
Assert.AreEqual(TerminalMatch.None, f(reject1));
Assert.AreEqual(TerminalMatch.None, f(reject2));
Assert.AreEqual(TerminalMatch.None, f(reject3));
}
public void TestQuotedStringValue()
{
LanguageGrammar<string> g = new LanguageGrammar<string>();
g.StringLiteralRule.Action = (s) => s;
// Spin up a parser for our language.
// TODO: Package this up and simplify it.
var expressionHelper = new ExpressionHelper();
//var classifierGen = new TerminalClassifier<char>();
var classifierSession = new TerminalClassifierSession<char, ParserState, int>()
.AddSkipTerminal(new Terminal<char> { Name = "Whitespace", InitialState = RegexCharNFABuilder.RegexCompiler(@"\s+") })
.CurrentCharExprIs(x => x.CurrentChar())
.GetFromMarkExprIs(x => x.GetFromMarkedPos())
.HasCurrentCharExprIs(x => x.HasCurrentChar())
.MarkPosExprIs(x => x.MarkPos())
.MoveNextCharExprIs(x => x.MoveNextChar())
.UnmarkPosExprIs(x => x.UnmarkPos());
var parserGen = new ParserGenerator<char>(expressionHelper);
var parseTableBuilder = new LRParseTableBuilder();
var parseTable = parseTableBuilder.BuildParseTable(g);
var session = parserGen.NewSession<ParserState>()
.NonTerminalValueExprIs<object>(x => x.NonTerminalValue)
.TerminalValueExprIs<string>(x => x.TerminalStringValue)
.TerminalValueExprIs<int>(x => x.TerminalIntValue)
.TerminalValueExprIs<double>(x => x.TerminalDoubleValue)
.TerminalIs(x => x.Terminal)
.NonTerminalIs(x => x.NonTerminal)
.IncludeSymbols(true)
.UseDefaultValue(true)
.DebugOutputIs(x => Debug.WriteLine(x))
.Generate("LanguageParser", parseTable, classifierSession);
// At this point, session is an Expression<ParserState,int> representing our parser.
// We can compile it into a delegate or a MethodBuilder. For the examples, we'll use a delegate.
var compiler = session.Compile();
ParserState ps = new ParserState("\"This is a quoted string\"");
Assert.AreEqual(0, compiler(ps));
Assert.AreEqual("This is a quoted string", ps.NonTerminalValue);
ps = new ParserState("\"Here are some \\t escape characters\\r\\netc...\"");
Assert.AreEqual(0, compiler(ps));
Assert.AreEqual("Here are some \t escape characters\r\netc...", ps.NonTerminalValue);
ps = new ParserState("\"And more \\\\ escape \\\" characters\"");
Assert.AreEqual(0, compiler(ps));
Assert.AreEqual("And more \\ escape \" characters", ps.NonTerminalValue);
}
public void TestParserSucceedsWithoutAValue()
{
LanguageGrammar<Expression> g = new LanguageGrammar<Expression>();
// Spin up a parser for our language.
// TODO: Package this up and simplify it.
var expressionHelper = new ExpressionHelper();
//var classifierGen = new TerminalClassifier<char>();
var classifierSession = new TerminalClassifierSession<char, ParserState, int>()
.CurrentCharExprIs(x => x.CurrentChar())
.GetFromMarkExprIs(x => x.GetFromMarkedPos())
.HasCurrentCharExprIs(x => x.HasCurrentChar())
.MarkPosExprIs(x => x.MarkPos())
.MoveNextCharExprIs(x => x.MoveNextChar())
.UnmarkPosExprIs(x => x.UnmarkPos());
var parserGen = new ParserGenerator<char>(expressionHelper);
var parseTableBuilder = new LRParseTableBuilder();
var parseTable = parseTableBuilder.BuildParseTable(g);
var session = parserGen.NewSession<ParserState>()
.NonTerminalValueExprIs<object>(x => x.NonTerminalValue)
.TerminalValueExprIs<string>(x => x.TerminalStringValue)
.TerminalValueExprIs<int>(x => x.TerminalIntValue)
.TerminalValueExprIs<double>(x => x.TerminalDoubleValue)
.TerminalIs(x => x.Terminal)
.NonTerminalIs(x => x.NonTerminal)
.IncludeSymbols(true)
.UseDefaultValue(true)
.DebugOutputIs(x => Debug.WriteLine(x))
.Generate("LanguageParser", parseTable, classifierSession);
// At this point, session is an Expression<ParserState,int> representing our parser.
// We can compile it into a delegate or a MethodBuilder. For the examples, we'll use a delegate.
var compiler = session.Compile();
// Create a parser state object and initialize it.
ParserState ps = new ParserState("x*y+2.0");
Assert.AreEqual(0, compiler(ps));
}
