public void LexerWhiteSpaceTest() { //var lexer = new Nicodem.Lexer.Lexer(RegExFactory.Concat(MakeCharRegex(' '), RegExFactory.Star(MakeCharRegex(' ')))); var lexer = new Nicodem.Lexer.Lexer(RegExParser.Parse("[:space:]+")); Assert.IsEmpty(lexer.Process("a ")); Assert.IsEmpty(lexer.Process("a")); Assert.IsTrue(lexer.Process(" a").Tokens().SequenceEqual(new[] { " " })); Assert.IsTrue(lexer.Process("\n\t").Tokens().SequenceEqual(new[] { "\n\t" })); Assert.IsTrue(lexer.Process("\n").Tokens().SequenceEqual(new[] { "\n" })); Assert.IsTrue(lexer.Process("\t").Tokens().SequenceEqual(new[] { "\t" })); Assert.IsTrue(lexer.Process("\n ").Tokens().SequenceEqual(new[] { "\n " })); Assert.IsTrue(lexer.Process("\t\r\n a b c").Tokens().SequenceEqual(new[] { "\t\r\n " })); }
public void LexerEmpty() { foreach (var arr in new[] { new RegEx <char> [0], new[] { RegExFactory.Empty <char>() } }) { var lexer = new Nicodem.Lexer.Lexer(arr); Assert.IsEmpty(lexer.Process("")); Assert.IsEmpty(lexer.Process(" ")); Assert.IsEmpty(lexer.Process("a")); Assert.IsEmpty(lexer.Process("asdf")); Assert.IsEmpty(lexer.Process(" _\n")); Assert.IsEmpty(lexer.Process("*^& GY?'\t\n\t\tlikjd")); Assert.IsEmpty(lexer.Process("\\")); Assert.IsEmpty(lexer.Process("\\n")); } }
public void LexerComplexLanguages() { // 0: a+ // 1: a*b // 2: a*b+ var regex0 = RegExFactory.Concat(MakeCharRegex('a'), RegExFactory.Star(MakeCharRegex('a'))); var regex1 = RegExFactory.Concat(RegExFactory.Star(MakeCharRegex('a')), MakeCharRegex('b')); var regex2 = RegExFactory.Concat(RegExFactory.Star(MakeCharRegex('a')), RegExFactory.Concat(MakeCharRegex('b'), RegExFactory.Star(MakeCharRegex('b')))); var lexex = new Nicodem.Lexer.Lexer(regex0, regex1, regex2); Assert.IsTrue(lexex.Process("b").TokenizedTo(T("b", 1, 2))); Assert.IsEmpty(lexex.Process("")); Assert.IsTrue(lexex.Process("a").TokenizedTo(T("a", 0))); Assert.IsTrue(lexex.Process("aaa").TokenizedTo(T("aaa", 0))); Assert.IsTrue(lexex.Process("ab").TokenizedTo(T("ab", 1, 2))); Assert.IsTrue(lexex.Process("abb").TokenizedTo(T("abb", 2))); Assert.IsTrue(lexex.Process("bb").TokenizedTo(T("bb", 2))); Assert.IsTrue(lexex.Process("baabab").TokenizedTo(T("b", 1, 2), T("aab", 1, 2), T("ab", 1, 2))); Assert.IsTrue(lexex.Process("bbab").TokenizedTo(T("bb", 2), T("ab", 1, 2))); Assert.IsTrue(lexex.Process("bbacb").TokenizedTo(T("bb", 2), T("a", 0))); }
public void LexerMainTest() { var lexer = new Nicodem.Lexer.Lexer(RegExParser.Parse("[:space:]*main[:space:]*\\(\\)[:space:]*")); Assert.IsEmpty(lexer.Process("a")); Assert.IsEmpty(lexer.Process("\tain()")); Assert.IsTrue(lexer.Process("\rmain()").Tokens().SequenceEqual(new[] { "\rmain()" })); Assert.IsTrue(lexer.Process("\tmain()\n\n\n\n\n").Tokens().SequenceEqual(new[] { "\tmain()\n\n\n\n\n" })); Assert.IsTrue(lexer.Process(" main \n \r \t ()tttt").Tokens().SequenceEqual(new[] { " main \n \r \t ()" })); Assert.IsTrue(lexer.Process("main()").Tokens().SequenceEqual(new[] { "main()" })); }
public void LexerSimpleRanges() { var lexer = new Nicodem.Lexer.Lexer(MakeCharRangeRegex('b', 'd'), MakeCharRangeRegex('d', 'g'), MakeCharRangeRegex('g', 'i')); Assert.IsEmpty(lexer.Process("a")); Assert.IsEmpty(lexer.Process("ab")); Assert.IsEmpty(lexer.Process("j")); Assert.IsTrue(lexer.Process("b").TokenizedTo(T("b", 0))); Assert.IsTrue(lexer.Process("c").TokenizedTo(T("c", 0))); Assert.IsTrue(lexer.Process("d").TokenizedTo(T("d", 0, 1))); Assert.IsTrue(lexer.Process("e").TokenizedTo(T("e", 1))); Assert.IsTrue(lexer.Process("f").TokenizedTo(T("f", 1))); Assert.IsTrue(lexer.Process("g").TokenizedTo(T("g", 1, 2))); Assert.IsTrue(lexer.Process("h").TokenizedTo(T("h", 2))); Assert.IsTrue(lexer.Process("i").TokenizedTo(T("i", 2))); Assert.IsTrue(lexer.Process("bbdgheijb").TokenizedTo( T("b", 0), T("b", 0), T("d", 0, 1), T("g", 1, 2), T("h", 2), T("e", 1), T("i", 2))); }
public void LexerSingleRange() { var lexer = new Nicodem.Lexer.Lexer(MakeCharRangeRegex('b', 'd')); Assert.IsEmpty(lexer.Process("")); Assert.IsEmpty(lexer.Process(" ")); Assert.IsEmpty(lexer.Process("a")); Assert.IsEmpty(lexer.Process("asdf")); Assert.IsEmpty(lexer.Process(" _\n")); Assert.IsEmpty(lexer.Process("*^& GY?'\t\n\t\tlikjd")); Assert.IsEmpty(lexer.Process("\\")); Assert.IsEmpty(lexer.Process("\\n")); Assert.IsTrue(lexer.Process("ba").Tokens().SequenceEqual(new[] { "b" })); Assert.IsTrue(lexer.Process("bb").Tokens().SequenceEqual(new[] { "b", "b" })); Assert.IsEmpty(lexer.Process("ab")); Assert.IsTrue(lexer.Process("db").Tokens().SequenceEqual(new[] { "d", "b" })); Assert.IsEmpty(lexer.Process("eb")); }