public static void TestManyString() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <ManyString> >()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; var string1 = "\"hello \\\"world \""; var expectString1 = "\"hello \"world \""; var string2 = "'that''s it'"; var expectString2 = "'that's it'"; var source1 = $"{string1} {string2}"; var r = lexer.Tokenize(source1); Assert.True(r.IsOk); Assert.Equal(3, r.Tokens.Count); var tok1 = r.Tokens[0]; Assert.Equal(ManyString.STRING, tok1.TokenID); Assert.Equal(expectString1, tok1.Value); Assert.Equal('"', tok1.StringDelimiter); var tok2 = r.Tokens[1]; Assert.Equal(ManyString.STRING, tok2.TokenID); Assert.Equal(expectString2, tok2.Value); Assert.Equal('\'', tok2.StringDelimiter); }
public static void Issue218() { var lexerResult = LexerBuilder.BuildLexer <Token218>(); Assert.True(lexerResult.IsOk); var lexer = lexerResult.Result; var result = lexer.Tokenize("a = 0.0;"); Assert.True(result.IsOk); var tokens = result.Tokens; var dump = string.Join(" ", tokens.Select(x => x.ToString())); int n = 0; int count = 0; while ((n = dump.IndexOf("<<EOS>>", n)) != -1) { n++; count++; } Console.WriteLine(dump); Assert.Equal(1, count); var eoss = tokens.Where(x => x.IsEOS); Assert.Single(eoss); }
private static void Main(string[] args) { if (args.Length < 1 || args.Any(arg => !File.Exists(arg))) { Console.WriteLine( $"Usage {Path.GetFileNameWithoutExtension(Assembly.GetExecutingAssembly().Location)} codefile.llama [, codefile2.llama, ...]" ); } var time = Stopwatch.StartNew(); var sourceFilePath = args[0]; var source = string.Join("\n", args.Select(File.ReadAllText)); var lexer = LexerBuilder.BuildLlamaLexer(); var parser = new ParseContext(LlamaParseStore.Instance, lexer, source); var document = parser.ReadNode <LlamaDocument>(); Console.WriteLine($"Parsing time: {time.Elapsed.TotalMilliseconds:F2} ms"); time.Restart(); var functionDeclarations = document.Functions.Select(fun => fun.Declaration).ToArray(); var compiler = new Compiler(new LlamaCompilerStore(), new LinkerFactory(), document.Imports, functionDeclarations); foreach (var functionImplementation in document.Functions) { compiler.AddFunction(functionImplementation); } var codeBlob = compiler.Code; var linker = (Linker)compiler.LinkingInfo; Console.WriteLine($"Compilation time: {time.Elapsed.TotalMilliseconds:F2} ms"); time.Restart(); var pe32PlusExeBuilder = new ExecutableBuilder { Subsystem = Subsystem.WindowsCui // Change to WindowsGui to make a Window application (without console) }; linker.LinkPreBuild(pe32PlusExeBuilder); var mainOffset = linker.GetCodeOffsetOfKnownFunction("Main"); if (mainOffset < 0) { Console.WriteLine("No main function"); return; } pe32PlusExeBuilder.AddAdditionalSection("A\nB\t\0", 200, SectionCharacteristics.MemWrite | SectionCharacteristics.MemRead); var peResult = pe32PlusExeBuilder.Build((uint)codeBlob.Length, (uint)mainOffset); codeBlob.CopyTo(peResult.GetCodeSectionBuffer()); linker.LinkPostBuild(peResult); Console.WriteLine($"Linking time: {time.Elapsed.TotalMilliseconds:F2} ms"); time.Restart(); using var fileStream = File.Open(Path.ChangeExtension(sourceFilePath, "exe"), FileMode.Create); peResult.Finish(fileStream); }
/// <summary> /// Handles construction of the elements yielded by the /// <see cref="ParserBuilder"/> with the <paramref name="input"/> /// provided. /// </summary> /// <param name="input">The <see cref="Tuple{T1, T2, T3}"/> which provides the /// <see cref="ParserCompiler"/>, <see cref="TokenSymbolBuilder"/> and /// <see cref="IIntermediateAssembly"/> necessary to perform the operation.</param> /// <returns>A <see cref="Tuple{T1, T2, T3}"/> which yields the /// <see cref="IIntermediateInterfaceType"/>, <see cref="IIntermediateClassType"/> /// for the result parser of the language, and /// <see cref="LexerBuilder"/> of the language which denotes the class /// and interface of the tokenizer.</returns> public Tuple <IIntermediateInterfaceType, IIntermediateClassType, LexerBuilder> Build(Tuple <ParserCompiler, TokenSymbolBuilder, IIntermediateAssembly, RegularLanguageDFAState> input) { this.compiler = input.Item1; this._tokenSymbolBuilder = input.Item2; this._assembly = input.Item3; this._identityManager = (IIntermediateCliManager)this._assembly.IdentityManager; this._parserInterface = _assembly.DefaultNamespace.Parts.Add().Interfaces.Add("I{0}", compiler.Source.Options.ParserName); this._parserClass = _assembly.DefaultNamespace.Parts.Add().Classes.Add(compiler.Source.Options.ParserName); this._parserClass.Assembly.ScopeCoercions.Add("System.Linq"); this._genericSymbolStreamBuilder = new GenericSymbolStreamBuilder(); this._symbolStreamBuilder = new SymbolStreamBuilder(); this._lexerBuilder = new LexerBuilder(); this._parserClass.AccessLevel = AccessLevelModifiers.Internal; this._parserInterface.AccessLevel = AccessLevelModifiers.Public; this._lexerCore = input.Item4; this._parserState = this._parserClass.Fields.Add(new TypedName("_state", _assembly.IdentityManager.ObtainTypeReference(RuntimeCoreType.Int32))); this._parserState.AccessLevel = AccessLevelModifiers.Private; this._parserStateProp = this._parserInterface.Properties.Add(new TypedName("State", this._parserState.FieldType), true, false); this._parserStatePropImpl = this._parserClass.Properties.Add(new TypedName("State", this._parserState.FieldType), true, true); this._parserStatePropImpl.AccessLevel = AccessLevelModifiers.Public; this._parserStatePropImpl.SetMethod.AccessLevel = AccessLevelModifiers.Private; this._parserStatePropImpl.GetMethod.Return(this._parserState.GetReference()); this._parserStatePropImpl.SetMethod.Assign(this._parserState.GetReference(), this._parserStatePropImpl.SetMethod.ValueParameter.GetReference()); this._parserClass.ImplementedInterfaces.ImplementInterfaceQuick(this._parserInterface); return(Tuple.Create(this.ParserInterface, this.ParserClass, this._lexerBuilder)); }
public override bool TryMatchCurrent(LexerBuilder lexer) { var prev = lexer.Previous; var first = lexer.Current; // check for keywords if (lexer.IsDelimiter(prev, true) && char.IsLetter(first)) { // can be a keyword... var sb = new StringBuilder(); sb.Append(lexer.Current); while (!lexer.EndOfInput && char.IsLetter(lexer.PeekNext())) { sb.Append(lexer.Current); } if (keywords.Contains(sb.ToString())) { if (!lexer.EndOfInput) { lexer.RollbackBy(1); } lexer.Commit(); return(true); } return(false); } else { return(false); } }
public void TestExtensions() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <Extensions> >(), ExtendedGenericLexer.AddExtension); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result as GenericLexer <Extensions>; Assert.NotNull(lexer); var tokens = lexer.Tokenize("20.02.2018 3.14").ToList(); Assert.Equal(3, tokens.Count); Assert.Equal(Extensions.DATE, tokens[0].TokenID); Assert.Equal("20.02.2018", tokens[0].Value); Assert.Equal(Extensions.DOUBLE, tokens[1].TokenID); Assert.Equal("3.14", tokens[1].Value); tokens = lexer.Tokenize("'that''s it'").ToList(); Assert.Equal(2, tokens.Count); var tok = tokens[0]; Assert.Equal(Extensions.CHAINE, tok.TokenID); Assert.Equal("'that's it'", tokens[0].Value); tokens = lexer.Tokenize("'et voilà'").ToList(); Assert.Equal(2, tokens.Count); tok = tokens[0]; Assert.Equal(Extensions.CHAINE, tok.TokenID); Assert.Equal("'et voilà'", tokens[0].Value); }
public override bool TryMatchCurrent(LexerBuilder lexer) { if (lexer.Current == '/') { lexer.PeekNext(); if (lexer.Current == '/') { // line comment. read to end of line or file. do { lexer.Commit(); lexer.PeekNext(); }while (!lexer.EndOrNewLine); return(true); } else if (lexer.Current == '*') { // block comment, read until end of file or closing '*/' lexer.PeekNext(); do { lexer.PeekNext(); lexer.Commit(); }while (!lexer.EndOfInput && !(lexer.Current == '/' && lexer.Previous == '*')); return(true); } } return(false); }
public void Test__Numeric() { // Arrange var lexer = LexerBuilder.New <BaseNode>() .Init() .WithAmbiguityResolverEnum(AmbiguityResolverEnum.FirstMatch) .NilNode(() => new NilNode()) .WithHandler(new Regex("[\\d]+"), opt => opt.Yield(opt.Buffer, new NumericNode { Value = decimal.Parse(opt.Buffer) })) .WithHandler(new Regex("[+-\\/*]"), opt => opt.Yield(opt.Buffer, new OpNode { Operator = opt.Buffer[0] })) .WithHandler(new Regex("[\\s]+"), opt => opt.Dispose(opt.Buffer)) .WithHandler(new Regex("[\\w]+"), opt => opt.SetBuffer(string.Empty)) .WithHandler(new Regex("$"), opt => opt.Stop()) .FinalizeHandlers() .Build(); // Act var nodes = lexer.Process(new MemoryStream(Encoding.Default.GetBytes("12 * 23 + 34 / 45 - 56 Error 12 * 23 + 34 / 45 - 56"))); // Assert Assert.Equal(18, nodes.Count()); }
public void TestCharTokenDelimiterConflict() { var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CharTokensConflicts> >()); Assert.True(res.IsError); Assert.Equal(2, res.Errors.Count); }
public void TestIssue114() { var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <Issue114> >()); Assert.False(res.IsError); var lexer = res.Result as GenericLexer <Issue114>; var r = lexer?.Tokenize("// /&"); Assert.True(r.IsError); Assert.Equal('&', r.Error.UnexpectedChar); r = lexer?.Tokenize("/&"); Assert.Equal('&', r.Error.UnexpectedChar); r = lexer?.Tokenize("&/"); Assert.True(r.IsError); Assert.Equal('&', r.Error.UnexpectedChar); r = lexer?.Tokenize("// &"); Assert.True(r.IsError); Assert.Equal('&', r.Error.UnexpectedChar); }
private static void TestChars() { var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CharTokens> >()); if (res.IsOk) { var lexer = res.Result as GenericLexer <CharTokens>; var dump = lexer.ToString(); var graph = lexer.ToGraphViz(); Console.WriteLine(graph); var source = "'\\''"; Console.WriteLine(source); var res2 = lexer.Tokenize(source); Console.WriteLine($"{res2.IsOk} - {res2.Tokens[0].Value}"); var sourceU = "'\\u0066'"; var res3 = lexer.Tokenize(sourceU); Console.WriteLine($"{res3.IsOk} - {res3.Tokens[0].Value}"); } else { var errors = string.Join('\n', res.Errors.Select(e => e.Level + " - " + e.Message).ToList()); Console.WriteLine("error building lexer : "); Console.WriteLine(errors); } }
public void TestCustomIdWithOther() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <CustomId> >()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; var r = lexer.Tokenize("_b_ -C-"); Assert.True(r.IsOk); Assert.Equal(5, r.Tokens.Count); var tok1 = r.Tokens[0]; Assert.Equal(CustomId.OTHER, tok1.TokenID); Assert.Equal("_", tok1.Value); var tok2 = r.Tokens[1]; Assert.Equal(CustomId.ID, tok2.TokenID); Assert.Equal("b_", tok2.Value); var tok3 = r.Tokens[2]; Assert.Equal(CustomId.OTHER, tok3.TokenID); Assert.Equal("-", tok3.Value); var tok4 = r.Tokens[3]; Assert.Equal(CustomId.ID, tok4.TokenID); Assert.Equal("C-", tok4.Value); }
public void TestIssue210() { var lexResult = LexerBuilder.BuildLexer <Issue210Token>(Issue210Extensions.AddExtensions); Assert.True(lexResult.IsOk); var lexer = lexResult.Result; Assert.NotNull(lexer); var r = lexer.Tokenize("?"); Assert.True(r.IsOk); var l = ToTokens2(r); Assert.Equal("QMARK[?]EOF[]", l); r = lexer.Tokenize("?special?"); Assert.True(r.IsOk); l = ToTokens2(r); Assert.Equal("SPECIAL[special]EOF[]", l); r = lexer.Tokenize("x?x"); Assert.True(r.IsOk); l = ToTokens2(r); Assert.Equal("X[x]QMARK[?]X[x]EOF[]", l); r = lexer.Tokenize("??"); Assert.True(r.IsOk); l = ToTokens2(r); Assert.Equal("SPECIAL[]EOF[]", l); }
public void TestIssue114() { var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <Issue114> >()); Assert.False(res.IsError); var lexer = res.Result as GenericLexer <Issue114>; var error = Assert.Throws <LexerException>(() => { lexer?.Tokenize("// /&").ToList(); }); Assert.Equal('&', error.Error.UnexpectedChar); error = Assert.Throws <LexerException>(() => { lexer?.Tokenize("/&").ToList(); }); Assert.Equal('&', error.Error.UnexpectedChar); error = Assert.Throws <LexerException>(() => { lexer?.Tokenize("&/").ToList(); }); Assert.Equal('&', error.Error.UnexpectedChar); error = Assert.Throws <LexerException>(() => { lexer?.Tokenize("// &").ToList(); }); Assert.Equal('&', error.Error.UnexpectedChar); }
public void TestSelfEscapedString() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <SelfEscapedString> >()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result as GenericLexer <SelfEscapedString>; Assert.NotNull(lexer); var r = lexer.Tokenize("'that''s it'"); Assert.True(r.IsOk); var tokens = r.Tokens; Assert.Equal(2, tokens.Count); var tok = tokens[0]; Assert.Equal(SelfEscapedString.STRING, tok.TokenID); Assert.Equal("'that's it'", tokens[0].Value); r = lexer.Tokenize("'et voilà'"); Assert.True(r.IsOk); tokens = r.Tokens; Assert.Equal(2, tokens.Count); tok = tokens[0]; Assert.Equal(SelfEscapedString.STRING, tok.TokenID); Assert.Equal("'et voilà'", tokens[0].Value); }
public void TestSameIntValuesError() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <SameIntValuesError> >()); Assert.True(lexerRes.IsError); Assert.Null(lexerRes.Result); Assert.Single(lexerRes.Errors); Assert.Equal(ErrorCodes.LEXER_SAME_VALUE_USED_MANY_TIME, lexerRes.Errors.First().Code); }
public void TestInnerMultiComment() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsToken> >()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result as GenericLexer <CommentsToken>; var dump = lexer.ToString(); var code = @"1 2 /* inner */ 3 4 "; var r = lexer.Tokenize(code); Assert.True(r.IsOk); var tokens = r.Tokens; Assert.Equal(6, tokens.Count); var token1 = tokens[0]; var token2 = tokens[1]; var token3 = tokens[2]; var token4 = tokens[3]; var token5 = tokens[4]; Assert.Equal(CommentsToken.INT, token1.TokenID); Assert.Equal("1", token1.Value); Assert.Equal(0, token1.Position.Line); Assert.Equal(0, token1.Position.Column); Assert.Equal(CommentsToken.INT, token2.TokenID); Assert.Equal("2", token2.Value); Assert.Equal(1, token2.Position.Line); Assert.Equal(0, token2.Position.Column); Assert.Equal(CommentsToken.COMMENT, token3.TokenID); Assert.Equal(@" inner ", token3.Value); Assert.Equal(1, token3.Position.Line); Assert.Equal(2, token3.Position.Column); Assert.Equal(CommentsToken.INT, token4.TokenID); Assert.Equal("3", token4.Value); Assert.Equal(1, token4.Position.Line); Assert.Equal(14, token4.Position.Column); Assert.Equal(CommentsToken.INT, token5.TokenID); Assert.Equal("4", token5.Value); Assert.Equal(2, token5.Position.Line); Assert.Equal(0, token5.Position.Column); }
public void TestBadEmptyStringDelimiter() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEmptyStringDelimiter> >()); Assert.True(lexerRes.IsError); Assert.Single(lexerRes.Errors); var error = lexerRes.Errors[0]; Assert.Equal(ErrorLevel.FATAL, error.Level); Assert.Equal(ErrorCodes.LEXER_STRING_DELIMITER_MUST_BE_1_CHAR, error.Code); }
public void TestBadEscapeStringDelimiterTooLong() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEscapeStringDelimiterTooLong> >()); Assert.True(lexerRes.IsError); Assert.Single(lexerRes.Errors); var error = lexerRes.Errors[0]; Assert.Equal(ErrorLevel.FATAL, error.Level); Assert.Contains("must be 1 character length", error.Message); }
public void Setup() { content = File.ReadAllText("test.json"); var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <JsonTokenGeneric> >()); if (lexerRes != null) { BenchedLexer = lexerRes.Result; } }
public void TestBadEscapeStringDelimiterLetter() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEscapeStringDelimiterLetter> >()); Assert.True(lexerRes.IsError); Assert.Single(lexerRes.Errors); var error = lexerRes.Errors[0]; Assert.Equal(ErrorLevel.FATAL, error.Level); Assert.Contains("can not start with a letter", error.Message); }
public void TestBadLetterStringDelimiter() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadLetterStringDelimiter> >()); Assert.True(lexerRes.IsError); Assert.Single(lexerRes.Errors); var error = lexerRes.Errors[0]; Assert.Equal(ErrorLevel.FATAL, error.Level); Assert.Equal(ErrorCodes.LEXER_STRING_DELIMITER_CANNOT_BE_LETTER_OR_DIGIT, error.Code); }
private static void TestDoubleExponent() { var lex = LexerBuilder.BuildLexer <DoubleExponent>(AddExponentExtension); if (lex.IsOk) { var one = lex.Result.Tokenize("2.0E+2"); ; var two = lex.Result.Tokenize("4.0e-2"); ; } }
protected GeneratorArgumentParser(string input) { _input = input; var buildResult = LexerBuilder.BuildLexer(new BuildResult <ILexer <GeneratorTokenType> >()); if (buildResult.IsError) { throw new ApplicationException("Lexer failed to build"); } _lexer = buildResult.Result; }
private static void benchLexer() { var content = File.ReadAllText("test.json"); var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <JsonTokenGeneric> >()); ILexer <JsonTokenGeneric> BenchedLexer = null; if (lexerRes != null) { BenchedLexer = lexerRes.Result; BenchedLexer.Tokenize(content); } }
public CanParseGenerics() { var lexer = LexerBuilder.Build() .Match("nr", @"\d+", match => new Number { Value = int.Parse(match.Value) }) .Match("txt", @".+", match => new Text { Value = match.Value }); _subject = ParserBuilder.Build(lexer) .Match <Number, Text>((nr, t) => new KeyValuePair <Number, Text>(nr, t), MatchOptions.AtLeastOne) .Create <KeyValuePair <Number, Text> >(); }
public void MixedErrors() { var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError10> >()); Assert.True(lexerRes10.IsError); Assert.Equal(4, lexerRes10.Errors.Count); var expectedErrors = new[] { "too many comment lexem", "too many multi-line comment lexem", "too many single-line comment lexem", "comment lexem can't be used together with single-line or multi-line comment lexems" }; foreach (var expectedError in expectedErrors) { Assert.True(lexerRes10.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError)); } expectedErrors = new[] { "too many multi-line comment lexem", "too many single-line comment lexem" }; var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError9> >()); Assert.True(lexerRes9.IsError); Assert.Equal(2, lexerRes9.Errors.Count); foreach (var expectedError in expectedErrors) { Assert.True(lexerRes9.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError)); } expectedErrors = new[] { "too many multi-line comment lexem", "comment lexem can't be used together with single-line or multi-line comment lexems" }; var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError8> >()); Assert.True(lexerRes8.IsError); Assert.Equal(2, lexerRes8.Errors.Count); foreach (var expectedError in expectedErrors) { Assert.True(lexerRes8.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError)); } expectedErrors = new[] { "too many single-line comment lexem", "comment lexem can't be used together with single-line or multi-line comment lexems" }; var lexerRes7 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError7> >()); Assert.True(lexerRes7.IsError); Assert.Equal(2, lexerRes7.Errors.Count); foreach (var expectedError in expectedErrors) { Assert.True(lexerRes7.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError)); } }
public void TestAlphaNumId() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <AlphaNumId> >()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; var r = lexer.Tokenize("alpha123"); Assert.True(r.IsOk); Assert.Equal(2, r.Tokens.Count); var tok = r.Tokens[0]; Assert.Equal(AlphaNumId.ID, tok.TokenID); Assert.Equal("alpha123", tok.StringWithoutQuotes); }
public void TestIgnoreEOL() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <IgnoreEOL> >()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; var r = lexer.Tokenize(" \n "); Assert.True(r.IsOk); Assert.Equal(2, r.Tokens.Count); var tok = r.Tokens[0]; Assert.Equal(IgnoreEOL.EOL, tok.TokenID); Assert.Equal("\n", tok.Value); }
public void TestWhiteSpace() { var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <WhiteSpace> >()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; var r = lexer.Tokenize(" \t "); Assert.True(r.IsOk); Assert.Equal(2, r.Tokens.Count); var tok = r.Tokens[0]; Assert.Equal(WhiteSpace.TAB, tok.TokenID); Assert.Equal("\t", tok.Value); }