Пример #1
0
        public static void TestManyString()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <ManyString> >());

            Assert.False(lexerRes.IsError);
            var lexer         = lexerRes.Result;
            var string1       = "\"hello \\\"world \"";
            var expectString1 = "\"hello \"world \"";
            var string2       = "'that''s it'";
            var expectString2 = "'that's it'";
            var source1       = $"{string1} {string2}";
            var r             = lexer.Tokenize(source1);

            Assert.True(r.IsOk);
            Assert.Equal(3, r.Tokens.Count);
            var tok1 = r.Tokens[0];

            Assert.Equal(ManyString.STRING, tok1.TokenID);
            Assert.Equal(expectString1, tok1.Value);
            Assert.Equal('"', tok1.StringDelimiter);

            var tok2 = r.Tokens[1];

            Assert.Equal(ManyString.STRING, tok2.TokenID);
            Assert.Equal(expectString2, tok2.Value);
            Assert.Equal('\'', tok2.StringDelimiter);
        }
Пример #2
0
        public static void Issue218()
        {
            var lexerResult = LexerBuilder.BuildLexer <Token218>();

            Assert.True(lexerResult.IsOk);
            var lexer  = lexerResult.Result;
            var result = lexer.Tokenize("a = 0.0;");

            Assert.True(result.IsOk);
            var tokens = result.Tokens;
            var dump   = string.Join(" ", tokens.Select(x => x.ToString()));
            int n      = 0;
            int count  = 0;

            while ((n = dump.IndexOf("<<EOS>>", n)) != -1)
            {
                n++;
                count++;
            }
            Console.WriteLine(dump);
            Assert.Equal(1, count);
            var eoss = tokens.Where(x => x.IsEOS);

            Assert.Single(eoss);
        }
Пример #3
0
        private static void Main(string[] args)
        {
            if (args.Length < 1 || args.Any(arg => !File.Exists(arg)))
            {
                Console.WriteLine(
                    $"Usage {Path.GetFileNameWithoutExtension(Assembly.GetExecutingAssembly().Location)} codefile.llama [, codefile2.llama, ...]"
                    );
            }

            var time           = Stopwatch.StartNew();
            var sourceFilePath = args[0];
            var source         = string.Join("\n", args.Select(File.ReadAllText));
            var lexer          = LexerBuilder.BuildLlamaLexer();
            var parser         = new ParseContext(LlamaParseStore.Instance, lexer, source);
            var document       = parser.ReadNode <LlamaDocument>();

            Console.WriteLine($"Parsing time: {time.Elapsed.TotalMilliseconds:F2} ms");
            time.Restart();

            var functionDeclarations = document.Functions.Select(fun => fun.Declaration).ToArray();
            var compiler             = new Compiler(new LlamaCompilerStore(), new LinkerFactory(), document.Imports, functionDeclarations);

            foreach (var functionImplementation in document.Functions)
            {
                compiler.AddFunction(functionImplementation);
            }

            var codeBlob = compiler.Code;
            var linker   = (Linker)compiler.LinkingInfo;

            Console.WriteLine($"Compilation time: {time.Elapsed.TotalMilliseconds:F2} ms");
            time.Restart();

            var pe32PlusExeBuilder = new ExecutableBuilder
            {
                Subsystem = Subsystem.WindowsCui // Change to WindowsGui to make a Window application (without console)
            };

            linker.LinkPreBuild(pe32PlusExeBuilder);

            var mainOffset = linker.GetCodeOffsetOfKnownFunction("Main");

            if (mainOffset < 0)
            {
                Console.WriteLine("No main function");
                return;
            }

            pe32PlusExeBuilder.AddAdditionalSection("A\nB\t\0", 200, SectionCharacteristics.MemWrite | SectionCharacteristics.MemRead);
            var peResult = pe32PlusExeBuilder.Build((uint)codeBlob.Length, (uint)mainOffset);

            codeBlob.CopyTo(peResult.GetCodeSectionBuffer());
            linker.LinkPostBuild(peResult);

            Console.WriteLine($"Linking time: {time.Elapsed.TotalMilliseconds:F2} ms");
            time.Restart();

            using var fileStream = File.Open(Path.ChangeExtension(sourceFilePath, "exe"), FileMode.Create);
            peResult.Finish(fileStream);
        }
Пример #4
0
        /// <summary>
        /// Handles construction of the elements yielded by the
        /// <see cref="ParserBuilder"/> with the <paramref name="input"/>
        /// provided.
        /// </summary>
        /// <param name="input">The <see cref="Tuple{T1, T2, T3}"/> which provides the
        /// <see cref="ParserCompiler"/>, <see cref="TokenSymbolBuilder"/> and
        /// <see cref="IIntermediateAssembly"/> necessary to perform the operation.</param>
        /// <returns>A <see cref="Tuple{T1, T2, T3}"/> which yields the
        /// <see cref="IIntermediateInterfaceType"/>, <see cref="IIntermediateClassType"/>
        /// for the result parser of the language, and
        /// <see cref="LexerBuilder"/> of the language which denotes the class
        /// and interface of the tokenizer.</returns>
        public Tuple <IIntermediateInterfaceType, IIntermediateClassType, LexerBuilder> Build(Tuple <ParserCompiler, TokenSymbolBuilder, IIntermediateAssembly, RegularLanguageDFAState> input)
        {
            this.compiler            = input.Item1;
            this._tokenSymbolBuilder = input.Item2;
            this._assembly           = input.Item3;
            this._identityManager    = (IIntermediateCliManager)this._assembly.IdentityManager;
            this._parserInterface    = _assembly.DefaultNamespace.Parts.Add().Interfaces.Add("I{0}", compiler.Source.Options.ParserName);
            this._parserClass        = _assembly.DefaultNamespace.Parts.Add().Classes.Add(compiler.Source.Options.ParserName);
            this._parserClass.Assembly.ScopeCoercions.Add("System.Linq");

            this._genericSymbolStreamBuilder = new GenericSymbolStreamBuilder();
            this._symbolStreamBuilder        = new SymbolStreamBuilder();
            this._lexerBuilder                = new LexerBuilder();
            this._parserClass.AccessLevel     = AccessLevelModifiers.Internal;
            this._parserInterface.AccessLevel = AccessLevelModifiers.Public;
            this._lexerCore                                 = input.Item4;
            this._parserState                               = this._parserClass.Fields.Add(new TypedName("_state", _assembly.IdentityManager.ObtainTypeReference(RuntimeCoreType.Int32)));
            this._parserState.AccessLevel                   = AccessLevelModifiers.Private;
            this._parserStateProp                           = this._parserInterface.Properties.Add(new TypedName("State", this._parserState.FieldType), true, false);
            this._parserStatePropImpl                       = this._parserClass.Properties.Add(new TypedName("State", this._parserState.FieldType), true, true);
            this._parserStatePropImpl.AccessLevel           = AccessLevelModifiers.Public;
            this._parserStatePropImpl.SetMethod.AccessLevel = AccessLevelModifiers.Private;
            this._parserStatePropImpl.GetMethod.Return(this._parserState.GetReference());
            this._parserStatePropImpl.SetMethod.Assign(this._parserState.GetReference(), this._parserStatePropImpl.SetMethod.ValueParameter.GetReference());
            this._parserClass.ImplementedInterfaces.ImplementInterfaceQuick(this._parserInterface);
            return(Tuple.Create(this.ParserInterface, this.ParserClass, this._lexerBuilder));
        }
Пример #5
0
        public override bool TryMatchCurrent(LexerBuilder lexer)
        {
            var prev  = lexer.Previous;
            var first = lexer.Current;

            // check for keywords
            if (lexer.IsDelimiter(prev, true) && char.IsLetter(first))
            {
                // can be a keyword...

                var sb = new StringBuilder();
                sb.Append(lexer.Current);
                while (!lexer.EndOfInput && char.IsLetter(lexer.PeekNext()))
                {
                    sb.Append(lexer.Current);
                }

                if (keywords.Contains(sb.ToString()))
                {
                    if (!lexer.EndOfInput)
                    {
                        lexer.RollbackBy(1);
                    }
                    lexer.Commit();
                    return(true);
                }

                return(false);
            }
            else
            {
                return(false);
            }
        }
Пример #6
0
        public void TestExtensions()
        {
            var lexerRes =
                LexerBuilder.BuildLexer(new BuildResult <ILexer <Extensions> >(), ExtendedGenericLexer.AddExtension);

            Assert.False(lexerRes.IsError);
            var lexer = lexerRes.Result as GenericLexer <Extensions>;

            Assert.NotNull(lexer);

            var tokens = lexer.Tokenize("20.02.2018 3.14").ToList();

            Assert.Equal(3, tokens.Count);
            Assert.Equal(Extensions.DATE, tokens[0].TokenID);
            Assert.Equal("20.02.2018", tokens[0].Value);
            Assert.Equal(Extensions.DOUBLE, tokens[1].TokenID);
            Assert.Equal("3.14", tokens[1].Value);

            tokens = lexer.Tokenize("'that''s it'").ToList();
            Assert.Equal(2, tokens.Count);
            var tok = tokens[0];

            Assert.Equal(Extensions.CHAINE, tok.TokenID);
            Assert.Equal("'that's it'", tokens[0].Value);

            tokens = lexer.Tokenize("'et voilà'").ToList();
            Assert.Equal(2, tokens.Count);
            tok = tokens[0];
            Assert.Equal(Extensions.CHAINE, tok.TokenID);
            Assert.Equal("'et voilà'", tokens[0].Value);
        }
Пример #7
0
        public override bool TryMatchCurrent(LexerBuilder lexer)
        {
            if (lexer.Current == '/')
            {
                lexer.PeekNext();
                if (lexer.Current == '/')
                {
                    // line comment. read to end of line or file.
                    do
                    {
                        lexer.Commit();
                        lexer.PeekNext();
                    }while (!lexer.EndOrNewLine);

                    return(true);
                }
                else if (lexer.Current == '*')
                {
                    // block comment, read until end of file or closing '*/'
                    lexer.PeekNext();
                    do
                    {
                        lexer.PeekNext();
                        lexer.Commit();
                    }while (!lexer.EndOfInput && !(lexer.Current == '/' && lexer.Previous == '*'));

                    return(true);
                }
            }

            return(false);
        }
Пример #8
0
        public void Test__Numeric()
        {
            // Arrange
            var lexer = LexerBuilder.New <BaseNode>()
                        .Init()
                        .WithAmbiguityResolverEnum(AmbiguityResolverEnum.FirstMatch)
                        .NilNode(() => new NilNode())
                        .WithHandler(new Regex("[\\d]+"), opt => opt.Yield(opt.Buffer, new NumericNode {
                Value = decimal.Parse(opt.Buffer)
            }))
                        .WithHandler(new Regex("[+-\\/*]"), opt => opt.Yield(opt.Buffer, new OpNode {
                Operator = opt.Buffer[0]
            }))
                        .WithHandler(new Regex("[\\s]+"), opt => opt.Dispose(opt.Buffer))
                        .WithHandler(new Regex("[\\w]+"), opt => opt.SetBuffer(string.Empty))
                        .WithHandler(new Regex("$"), opt => opt.Stop())
                        .FinalizeHandlers()
                        .Build();

            // Act
            var nodes = lexer.Process(new MemoryStream(Encoding.Default.GetBytes("12 * 23 + 34 / 45 - 56 Error 12 * 23 + 34 / 45 - 56")));

            // Assert
            Assert.Equal(18, nodes.Count());
        }
        public void TestCharTokenDelimiterConflict()
        {
            var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CharTokensConflicts> >());

            Assert.True(res.IsError);
            Assert.Equal(2, res.Errors.Count);
        }
        public void TestIssue114()
        {
            var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <Issue114> >());

            Assert.False(res.IsError);
            var lexer = res.Result as GenericLexer <Issue114>;

            var r = lexer?.Tokenize("// /&");

            Assert.True(r.IsError);

            Assert.Equal('&', r.Error.UnexpectedChar);

            r = lexer?.Tokenize("/&");


            Assert.Equal('&', r.Error.UnexpectedChar);

            r = lexer?.Tokenize("&/");
            Assert.True(r.IsError);

            Assert.Equal('&', r.Error.UnexpectedChar);

            r = lexer?.Tokenize("// &");
            Assert.True(r.IsError);
            Assert.Equal('&', r.Error.UnexpectedChar);
        }
Пример #11
0
        private static void TestChars()
        {
            var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CharTokens> >());

            if (res.IsOk)
            {
                var lexer = res.Result as GenericLexer <CharTokens>;

                var dump  = lexer.ToString();
                var graph = lexer.ToGraphViz();
                Console.WriteLine(graph);
                var source = "'\\''";
                Console.WriteLine(source);
                var res2 = lexer.Tokenize(source);
                Console.WriteLine($"{res2.IsOk} - {res2.Tokens[0].Value}");
                var sourceU = "'\\u0066'";
                var res3    = lexer.Tokenize(sourceU);
                Console.WriteLine($"{res3.IsOk} - {res3.Tokens[0].Value}");
            }
            else
            {
                var errors = string.Join('\n', res.Errors.Select(e => e.Level + " - " + e.Message).ToList());
                Console.WriteLine("error building lexer : ");
                Console.WriteLine(errors);
            }
        }
        public void TestCustomIdWithOther()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <CustomId> >());

            Assert.False(lexerRes.IsError);
            var lexer = lexerRes.Result;
            var r     = lexer.Tokenize("_b_ -C-");

            Assert.True(r.IsOk);
            Assert.Equal(5, r.Tokens.Count);
            var tok1 = r.Tokens[0];

            Assert.Equal(CustomId.OTHER, tok1.TokenID);
            Assert.Equal("_", tok1.Value);
            var tok2 = r.Tokens[1];

            Assert.Equal(CustomId.ID, tok2.TokenID);
            Assert.Equal("b_", tok2.Value);
            var tok3 = r.Tokens[2];

            Assert.Equal(CustomId.OTHER, tok3.TokenID);
            Assert.Equal("-", tok3.Value);
            var tok4 = r.Tokens[3];

            Assert.Equal(CustomId.ID, tok4.TokenID);
            Assert.Equal("C-", tok4.Value);
        }
Пример #13
0
        public void TestIssue210()
        {
            var lexResult = LexerBuilder.BuildLexer <Issue210Token>(Issue210Extensions.AddExtensions);

            Assert.True(lexResult.IsOk);
            var lexer = lexResult.Result;

            Assert.NotNull(lexer);
            var r = lexer.Tokenize("?");

            Assert.True(r.IsOk);
            var l = ToTokens2(r);

            Assert.Equal("QMARK[?]EOF[]", l);
            r = lexer.Tokenize("?special?");
            Assert.True(r.IsOk);
            l = ToTokens2(r);
            Assert.Equal("SPECIAL[special]EOF[]", l);
            r = lexer.Tokenize("x?x");
            Assert.True(r.IsOk);
            l = ToTokens2(r);
            Assert.Equal("X[x]QMARK[?]X[x]EOF[]", l);
            r = lexer.Tokenize("??");
            Assert.True(r.IsOk);
            l = ToTokens2(r);
            Assert.Equal("SPECIAL[]EOF[]", l);
        }
Пример #14
0
        public void TestIssue114()
        {
            var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <Issue114> >());

            Assert.False(res.IsError);
            var lexer = res.Result as GenericLexer <Issue114>;
            var error = Assert.Throws <LexerException>(() =>
            {
                lexer?.Tokenize("// /&").ToList();
            });

            Assert.Equal('&', error.Error.UnexpectedChar);

            error = Assert.Throws <LexerException>(() =>
            {
                lexer?.Tokenize("/&").ToList();
            });

            Assert.Equal('&', error.Error.UnexpectedChar);

            error = Assert.Throws <LexerException>(() =>
            {
                lexer?.Tokenize("&/").ToList();
            });
            Assert.Equal('&', error.Error.UnexpectedChar);

            error = Assert.Throws <LexerException>(() =>
            {
                lexer?.Tokenize("// &").ToList();
            });
            Assert.Equal('&', error.Error.UnexpectedChar);
        }
        public void TestSelfEscapedString()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <SelfEscapedString> >());

            Assert.False(lexerRes.IsError);
            var lexer = lexerRes.Result as GenericLexer <SelfEscapedString>;

            Assert.NotNull(lexer);
            var r = lexer.Tokenize("'that''s it'");

            Assert.True(r.IsOk);
            var tokens = r.Tokens;

            Assert.Equal(2, tokens.Count);
            var tok = tokens[0];

            Assert.Equal(SelfEscapedString.STRING, tok.TokenID);
            Assert.Equal("'that's it'", tokens[0].Value);

            r = lexer.Tokenize("'et voilà'");
            Assert.True(r.IsOk);
            tokens = r.Tokens;
            Assert.Equal(2, tokens.Count);
            tok = tokens[0];
            Assert.Equal(SelfEscapedString.STRING, tok.TokenID);
            Assert.Equal("'et voilà'", tokens[0].Value);
        }
Пример #16
0
        public void TestSameIntValuesError()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <SameIntValuesError> >());

            Assert.True(lexerRes.IsError);
            Assert.Null(lexerRes.Result);
            Assert.Single(lexerRes.Errors);
            Assert.Equal(ErrorCodes.LEXER_SAME_VALUE_USED_MANY_TIME, lexerRes.Errors.First().Code);
        }
Пример #17
0
        public void TestInnerMultiComment()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsToken> >());

            Assert.False(lexerRes.IsError);
            var lexer = lexerRes.Result as GenericLexer <CommentsToken>;


            var dump = lexer.ToString();

            var code = @"1
2 /* inner */ 3
4
            ";

            var r = lexer.Tokenize(code);

            Assert.True(r.IsOk);
            var tokens = r.Tokens;

            Assert.Equal(6, tokens.Count);

            var token1 = tokens[0];
            var token2 = tokens[1];
            var token3 = tokens[2];
            var token4 = tokens[3];
            var token5 = tokens[4];


            Assert.Equal(CommentsToken.INT, token1.TokenID);
            Assert.Equal("1", token1.Value);
            Assert.Equal(0, token1.Position.Line);
            Assert.Equal(0, token1.Position.Column);

            Assert.Equal(CommentsToken.INT, token2.TokenID);
            Assert.Equal("2", token2.Value);
            Assert.Equal(1, token2.Position.Line);
            Assert.Equal(0, token2.Position.Column);

            Assert.Equal(CommentsToken.COMMENT, token3.TokenID);
            Assert.Equal(@" inner ", token3.Value);
            Assert.Equal(1, token3.Position.Line);
            Assert.Equal(2, token3.Position.Column);

            Assert.Equal(CommentsToken.INT, token4.TokenID);
            Assert.Equal("3", token4.Value);
            Assert.Equal(1, token4.Position.Line);
            Assert.Equal(14, token4.Position.Column);

            Assert.Equal(CommentsToken.INT, token5.TokenID);
            Assert.Equal("4", token5.Value);
            Assert.Equal(2, token5.Position.Line);
            Assert.Equal(0, token5.Position.Column);
        }
Пример #18
0
        public void TestBadEmptyStringDelimiter()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEmptyStringDelimiter> >());

            Assert.True(lexerRes.IsError);
            Assert.Single(lexerRes.Errors);
            var error = lexerRes.Errors[0];

            Assert.Equal(ErrorLevel.FATAL, error.Level);
            Assert.Equal(ErrorCodes.LEXER_STRING_DELIMITER_MUST_BE_1_CHAR, error.Code);
        }
        public void TestBadEscapeStringDelimiterTooLong()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEscapeStringDelimiterTooLong> >());

            Assert.True(lexerRes.IsError);
            Assert.Single(lexerRes.Errors);
            var error = lexerRes.Errors[0];

            Assert.Equal(ErrorLevel.FATAL, error.Level);
            Assert.Contains("must be 1 character length", error.Message);
        }
        public void Setup()
        {
            content = File.ReadAllText("test.json");

            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <JsonTokenGeneric> >());

            if (lexerRes != null)
            {
                BenchedLexer = lexerRes.Result;
            }
        }
        public void TestBadEscapeStringDelimiterLetter()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEscapeStringDelimiterLetter> >());

            Assert.True(lexerRes.IsError);
            Assert.Single(lexerRes.Errors);
            var error = lexerRes.Errors[0];

            Assert.Equal(ErrorLevel.FATAL, error.Level);
            Assert.Contains("can not start with a letter", error.Message);
        }
Пример #22
0
        public void TestBadLetterStringDelimiter()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadLetterStringDelimiter> >());

            Assert.True(lexerRes.IsError);
            Assert.Single(lexerRes.Errors);
            var error = lexerRes.Errors[0];

            Assert.Equal(ErrorLevel.FATAL, error.Level);
            Assert.Equal(ErrorCodes.LEXER_STRING_DELIMITER_CANNOT_BE_LETTER_OR_DIGIT, error.Code);
        }
Пример #23
0
        private static void TestDoubleExponent()
        {
            var lex = LexerBuilder.BuildLexer <DoubleExponent>(AddExponentExtension);

            if (lex.IsOk)
            {
                var one = lex.Result.Tokenize("2.0E+2");
                ;
                var two = lex.Result.Tokenize("4.0e-2");
                ;
            }
        }
        protected GeneratorArgumentParser(string input)
        {
            _input = input;
            var buildResult = LexerBuilder.BuildLexer(new BuildResult <ILexer <GeneratorTokenType> >());

            if (buildResult.IsError)
            {
                throw new ApplicationException("Lexer failed to build");
            }

            _lexer = buildResult.Result;
        }
Пример #25
0
        private static void benchLexer()
        {
            var content = File.ReadAllText("test.json");

            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <JsonTokenGeneric> >());
            ILexer <JsonTokenGeneric> BenchedLexer = null;

            if (lexerRes != null)
            {
                BenchedLexer = lexerRes.Result;
                BenchedLexer.Tokenize(content);
            }
        }
Пример #26
0
        public CanParseGenerics()
        {
            var lexer = LexerBuilder.Build()
                        .Match("nr", @"\d+", match => new Number {
                Value = int.Parse(match.Value)
            })
                        .Match("txt", @".+", match => new Text {
                Value = match.Value
            });

            _subject = ParserBuilder.Build(lexer)
                       .Match <Number, Text>((nr, t) => new KeyValuePair <Number, Text>(nr, t), MatchOptions.AtLeastOne)
                       .Create <KeyValuePair <Number, Text> >();
        }
Пример #27
0
        public void MixedErrors()
        {
            var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError10> >());

            Assert.True(lexerRes10.IsError);
            Assert.Equal(4, lexerRes10.Errors.Count);
            var expectedErrors = new[]
            {
                "too many comment lexem", "too many multi-line comment lexem", "too many single-line comment lexem",
                "comment lexem can't be used together with single-line or multi-line comment lexems"
            };

            foreach (var expectedError in expectedErrors)
            {
                Assert.True(lexerRes10.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
            }

            expectedErrors = new[] { "too many multi-line comment lexem", "too many single-line comment lexem" };

            var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError9> >());

            Assert.True(lexerRes9.IsError);
            Assert.Equal(2, lexerRes9.Errors.Count);
            foreach (var expectedError in expectedErrors)
            {
                Assert.True(lexerRes9.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
            }

            expectedErrors = new[] { "too many multi-line comment lexem", "comment lexem can't be used together with single-line or multi-line comment lexems" };

            var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError8> >());

            Assert.True(lexerRes8.IsError);
            Assert.Equal(2, lexerRes8.Errors.Count);
            foreach (var expectedError in expectedErrors)
            {
                Assert.True(lexerRes8.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
            }

            expectedErrors = new[] { "too many single-line comment lexem", "comment lexem can't be used together with single-line or multi-line comment lexems" };
            var lexerRes7 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError7> >());

            Assert.True(lexerRes7.IsError);
            Assert.Equal(2, lexerRes7.Errors.Count);
            foreach (var expectedError in expectedErrors)
            {
                Assert.True(lexerRes7.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
            }
        }
        public void TestAlphaNumId()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <AlphaNumId> >());

            Assert.False(lexerRes.IsError);
            var lexer = lexerRes.Result;
            var r     = lexer.Tokenize("alpha123");

            Assert.True(r.IsOk);
            Assert.Equal(2, r.Tokens.Count);
            var tok = r.Tokens[0];

            Assert.Equal(AlphaNumId.ID, tok.TokenID);
            Assert.Equal("alpha123", tok.StringWithoutQuotes);
        }
        public void TestIgnoreEOL()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <IgnoreEOL> >());

            Assert.False(lexerRes.IsError);
            var lexer = lexerRes.Result;
            var r     = lexer.Tokenize(" \n ");

            Assert.True(r.IsOk);
            Assert.Equal(2, r.Tokens.Count);
            var tok = r.Tokens[0];

            Assert.Equal(IgnoreEOL.EOL, tok.TokenID);
            Assert.Equal("\n", tok.Value);
        }
        public void TestWhiteSpace()
        {
            var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <WhiteSpace> >());

            Assert.False(lexerRes.IsError);
            var lexer = lexerRes.Result;
            var r     = lexer.Tokenize(" \t ");

            Assert.True(r.IsOk);
            Assert.Equal(2, r.Tokens.Count);
            var tok = r.Tokens[0];

            Assert.Equal(WhiteSpace.TAB, tok.TokenID);
            Assert.Equal("\t", tok.Value);
        }