public void TestClassifyTokensAndCaptureIdentifiersUntilEof()
{
Expression<Func<TestStringInput, string, TerminalMatch>> capture = ((t, m) => t.Capture(m, TerminalMatch.Identifier));
var classifier = new TerminalClassifierSession<char, TestStringInput, TerminalMatch>()
.HasCurrentCharExprIs(r => r.HasCurrentChar())
.CurrentCharExprIs(r => r.CurrentChar())
.MoveNextCharExprIs(r => r.MoveNextChar())
.MarkPosExprIs(r => r.MarkPos())
.UnmarkPosExprIs(r => r.UnmarkPos())
.GetFromMarkExprIs(r => r.GetFromMarkedPos())
.AddTerminalHandler(_if, r => TerminalMatch.If)
.AddTerminalHandler(_while, r => TerminalMatch.While)
.AddTerminalHandler(_identifier, capture)
.AddSkipTerminal(_whitespace)
.EofHandlerIs(r => TerminalMatch.Eof)
.RejectHandlerIs(r => TerminalMatch.None)
.Generate();
var f = classifier.Compile();
var strIds = new TestStringInput("if while bob ida\nwhere iffy wh wherefore i if8 d7");
Assert.AreEqual(TerminalMatch.If, f(strIds));
Assert.AreEqual(TerminalMatch.While, f(strIds));
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("bob", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("ida", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("where", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("iffy", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("wh", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("wherefore", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("i", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("if8", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Identifier, f(strIds));
Assert.AreEqual("d7", strIds.LastCapture);
Assert.AreEqual(TerminalMatch.Eof, f(strIds));
}
public void TestCreateClassifier()
{
Expression<Func<TextReader, TerminalMatch>> foundOpenTag = r => TerminalMatch.OpenTag;
Expression<Func<TextReader, TerminalMatch>> foundCloseTag = r => TerminalMatch.CloseTag;
Expression<Func<TextReader, TerminalMatch>> foundLeafTag = r => TerminalMatch.LeafTag;
Expression<Func<TextReader, TerminalMatch>> foundNoTag = r => TerminalMatch.None;
//var classifier = _parserGenerator.Classifier<TextReader, TerminalMatch>()
var classifier = new TerminalClassifierSession<char, TextReader, TerminalMatch>()
.HasCurrentCharExprIs(r => r.Peek() != -1)
.CurrentCharExprIs(r => (char)r.Peek())
.MoveNextCharExprIs(r => r.Read())
.AddTerminalHandler(_openTag, r => TerminalMatch.OpenTag)
.AddTerminalHandler(_closeTag, r => TerminalMatch.CloseTag)
.AddTerminalHandler(_leafTag, r => TerminalMatch.LeafTag)
.RejectHandlerIs(r => TerminalMatch.None)
.Generate();
var open1 = new StringReader("<Open1>");
var open2 = new StringReader("<x>");
var close1 = new StringReader("</Close1>");
var close2 = new StringReader("</x>");
var leaf1 = new StringReader("<Leaf/>");
var leaf2 = new StringReader("<x/>");
var reject1 = new StringReader("<>");
var reject2 = new StringReader("</>");
var reject3 = new StringReader("x/>");
var reject4 = new StringReader("<reject4");
var reject5 = new StringReader("<reject5/");
var reject6 = new StringReader("</reject6");
var reject7 = new StringReader("");
var f = classifier.Compile();
Assert.AreEqual(TerminalMatch.OpenTag, f(open1));
Assert.AreEqual(TerminalMatch.OpenTag, f(open2));
Assert.AreEqual(TerminalMatch.CloseTag, f(close1));
Assert.AreEqual(TerminalMatch.CloseTag, f(close2));
Assert.AreEqual(TerminalMatch.LeafTag, f(leaf1));
Assert.AreEqual(TerminalMatch.LeafTag, f(leaf2));
Assert.AreEqual(TerminalMatch.None, f(reject1));
Assert.AreEqual(TerminalMatch.None, f(reject2));
Assert.AreEqual(TerminalMatch.None, f(reject3));
Assert.AreEqual(TerminalMatch.None, f(reject4));
Assert.AreEqual(TerminalMatch.None, f(reject5));
Assert.AreEqual(TerminalMatch.None, f(reject6));
Assert.AreEqual(TerminalMatch.None, f(reject7));
}
public void TestClassifyKeywordsAndIdentifiers()
{
Expression<Func<TextReader, TerminalMatch>> foundIdentifer = (r) => TerminalMatch.Identifier;
Expression<Func<TextReader, TerminalMatch>> foundIf = (r) => TerminalMatch.If;
Expression<Func<TextReader, TerminalMatch>> foundWhile = (r) => TerminalMatch.While;
Expression<Func<TextReader, TerminalMatch>> foundNone = (r) => TerminalMatch.None;
var classifier = new TerminalClassifierSession<char, TextReader, TerminalMatch>()
.HasCurrentCharExprIs(r => r.Peek() != -1)
.CurrentCharExprIs(r => (char)r.Peek())
.MoveNextCharExprIs(r => r.Read())
.AddTerminalHandler(_if, r => TerminalMatch.If)
.AddTerminalHandler(_while, r => TerminalMatch.While)
.AddTerminalHandler(_identifier, r => TerminalMatch.Identifier)
.RejectHandlerIs(r => TerminalMatch.None)
.Generate();
var f = classifier.Compile();
var strIf = new StringReader("if");
var strWhile = new StringReader("while");
var id1 = new StringReader("bob");
var id2 = new StringReader("ida");
var id3 = new StringReader("where");
var id4 = new StringReader("iffy");
var id5 = new StringReader("wh");
var id6 = new StringReader("wherefore");
var id7 = new StringReader("i");
var id8 = new StringReader("if8");
var id9 = new StringReader("d7 = 123");
var reject1 = new StringReader("99");
var reject2 = new StringReader("(@*#");
var reject3 = new StringReader("");
Assert.AreEqual(TerminalMatch.If, f(strIf));
Assert.AreEqual(TerminalMatch.While, f(strWhile));
Assert.AreEqual(TerminalMatch.Identifier, f(id1));
Assert.AreEqual(TerminalMatch.Identifier, f(id2));
Assert.AreEqual(TerminalMatch.Identifier, f(id3));
Assert.AreEqual(TerminalMatch.Identifier, f(id4));
Assert.AreEqual(TerminalMatch.Identifier, f(id5));
Assert.AreEqual(TerminalMatch.Identifier, f(id6));
Assert.AreEqual(TerminalMatch.Identifier, f(id7));
Assert.AreEqual(TerminalMatch.Identifier, f(id8));
Assert.AreEqual(TerminalMatch.Identifier, f(id9));
Assert.AreEqual(TerminalMatch.None, f(reject1));
Assert.AreEqual(TerminalMatch.None, f(reject2));
Assert.AreEqual(TerminalMatch.None, f(reject3));
}