public void LexerSimpleRanges()
{
var lexer =
new Nicodem.Lexer.Lexer(MakeCharRangeRegex('b', 'd'), MakeCharRangeRegex('d', 'g'),
MakeCharRangeRegex('g', 'i'));
Assert.IsEmpty(lexer.Process("a"));
Assert.IsEmpty(lexer.Process("ab"));
Assert.IsEmpty(lexer.Process("j"));
Assert.IsTrue(lexer.Process("b").TokenizedTo(T("b", 0)));
Assert.IsTrue(lexer.Process("c").TokenizedTo(T("c", 0)));
Assert.IsTrue(lexer.Process("d").TokenizedTo(T("d", 0, 1)));
Assert.IsTrue(lexer.Process("e").TokenizedTo(T("e", 1)));
Assert.IsTrue(lexer.Process("f").TokenizedTo(T("f", 1)));
Assert.IsTrue(lexer.Process("g").TokenizedTo(T("g", 1, 2)));
Assert.IsTrue(lexer.Process("h").TokenizedTo(T("h", 2)));
Assert.IsTrue(lexer.Process("i").TokenizedTo(T("i", 2)));
Assert.IsTrue(lexer.Process("bbdgheijb").TokenizedTo(
T("b", 0),
T("b", 0),
T("d", 0, 1),
T("g", 1, 2),
T("h", 2),
T("e", 1),
T("i", 2)));
}
internal static IEnumerable <Tuple <string, IEnumerable <int> > > Process(this Nicodem.Lexer.Lexer lexer, string s)
{
var origin = new StringOrigin(s);
return(lexer
.ProcessBare(origin)
.Select(t => new Tuple <string, IEnumerable <int> >(origin.GetText(t.Item1), t.Item2)));
}
public void LexerMainTest()
{
var lexer = new Nicodem.Lexer.Lexer(RegExParser.Parse("[:space:]*main[:space:]*\\(\\)[:space:]*"));
Assert.IsEmpty(lexer.Process("a"));
Assert.IsEmpty(lexer.Process("\tain()"));
Assert.IsTrue(lexer.Process("\rmain()").Tokens().SequenceEqual(new[] { "\rmain()" }));
Assert.IsTrue(lexer.Process("\tmain()\n\n\n\n\n").Tokens().SequenceEqual(new[] { "\tmain()\n\n\n\n\n" }));
Assert.IsTrue(lexer.Process(" main \n \r \t ()tttt").Tokens().SequenceEqual(new[] { " main \n \r \t ()" }));
Assert.IsTrue(lexer.Process("main()").Tokens().SequenceEqual(new[] { "main()" }));
}
public void LexerWhiteSpaceTest()
{
//var lexer = new Nicodem.Lexer.Lexer(RegExFactory.Concat(MakeCharRegex(' '), RegExFactory.Star(MakeCharRegex(' '))));
var lexer = new Nicodem.Lexer.Lexer(RegExParser.Parse("[:space:]+"));
Assert.IsEmpty(lexer.Process("a "));
Assert.IsEmpty(lexer.Process("a"));
Assert.IsTrue(lexer.Process(" a").Tokens().SequenceEqual(new[] { " " }));
Assert.IsTrue(lexer.Process("\n\t").Tokens().SequenceEqual(new[] { "\n\t" }));
Assert.IsTrue(lexer.Process("\n").Tokens().SequenceEqual(new[] { "\n" }));
Assert.IsTrue(lexer.Process("\t").Tokens().SequenceEqual(new[] { "\t" }));
Assert.IsTrue(lexer.Process("\n ").Tokens().SequenceEqual(new[] { "\n " }));
Assert.IsTrue(lexer.Process("\t\r\n a b c").Tokens().SequenceEqual(new[] { "\t\r\n " }));
}
public void LexerEmpty()
{
foreach (var arr in new[] { new RegEx <char> [0], new[] { RegExFactory.Empty <char>() } })
{
var lexer = new Nicodem.Lexer.Lexer(arr);
Assert.IsEmpty(lexer.Process(""));
Assert.IsEmpty(lexer.Process(" "));
Assert.IsEmpty(lexer.Process("a"));
Assert.IsEmpty(lexer.Process("asdf"));
Assert.IsEmpty(lexer.Process(" _\n"));
Assert.IsEmpty(lexer.Process("*^& GY?'\t\n\t\tlikjd"));
Assert.IsEmpty(lexer.Process("\\"));
Assert.IsEmpty(lexer.Process("\\n"));
}
}
public void LexerSingleRange()
{
var lexer = new Nicodem.Lexer.Lexer(MakeCharRangeRegex('b', 'd'));
Assert.IsEmpty(lexer.Process(""));
Assert.IsEmpty(lexer.Process(" "));
Assert.IsEmpty(lexer.Process("a"));
Assert.IsEmpty(lexer.Process("asdf"));
Assert.IsEmpty(lexer.Process(" _\n"));
Assert.IsEmpty(lexer.Process("*^& GY?'\t\n\t\tlikjd"));
Assert.IsEmpty(lexer.Process("\\"));
Assert.IsEmpty(lexer.Process("\\n"));
Assert.IsTrue(lexer.Process("ba").Tokens().SequenceEqual(new[] { "b" }));
Assert.IsTrue(lexer.Process("bb").Tokens().SequenceEqual(new[] { "b", "b" }));
Assert.IsEmpty(lexer.Process("ab"));
Assert.IsTrue(lexer.Process("db").Tokens().SequenceEqual(new[] { "d", "b" }));
Assert.IsEmpty(lexer.Process("eb"));
}
public void LexerComplexLanguages()
{
// 0: a+
// 1: a*b
// 2: a*b+
var regex0 = RegExFactory.Concat(MakeCharRegex('a'), RegExFactory.Star(MakeCharRegex('a')));
var regex1 = RegExFactory.Concat(RegExFactory.Star(MakeCharRegex('a')), MakeCharRegex('b'));
var regex2 = RegExFactory.Concat(RegExFactory.Star(MakeCharRegex('a')),
RegExFactory.Concat(MakeCharRegex('b'), RegExFactory.Star(MakeCharRegex('b'))));
var lexex = new Nicodem.Lexer.Lexer(regex0, regex1, regex2);
Assert.IsTrue(lexex.Process("b").TokenizedTo(T("b", 1, 2)));
Assert.IsEmpty(lexex.Process(""));
Assert.IsTrue(lexex.Process("a").TokenizedTo(T("a", 0)));
Assert.IsTrue(lexex.Process("aaa").TokenizedTo(T("aaa", 0)));
Assert.IsTrue(lexex.Process("ab").TokenizedTo(T("ab", 1, 2)));
Assert.IsTrue(lexex.Process("abb").TokenizedTo(T("abb", 2)));
Assert.IsTrue(lexex.Process("bb").TokenizedTo(T("bb", 2)));
Assert.IsTrue(lexex.Process("baabab").TokenizedTo(T("b", 1, 2), T("aab", 1, 2), T("ab", 1, 2)));
Assert.IsTrue(lexex.Process("bbab").TokenizedTo(T("bb", 2), T("ab", 1, 2)));
Assert.IsTrue(lexex.Process("bbacb").TokenizedTo(T("bb", 2), T("a", 0)));
}
