Beispiel #1
0
        private static void Main(string[] args)
        {
            //TestCompiler();

            if (args.Length == 0)
            {
                Console.WriteLine("No llex or alx file specified");
                return;
            }
            else if (!File.Exists(args[0]))
            {
                Console.WriteLine("Invalid llex file {0}", args[0]);
                return;
            }

            Console.WriteLine("Generating lexer...");

            string outputFile = args.Length > 1 ? args[1] : new FileInfo(args[0]).FullName + ".cs";

            var lexer = args[0].ToLower().EndsWith(".alx") ?
                        AlxFile.Interpret(args[0]) :
                        LLexFile.Interpret(args[0]);

            File.WriteAllText(outputFile, lexer);

            Console.WriteLine("Lexer written to {0}", outputFile);
        }
Beispiel #2
0
        public static string Interpret(string file)
        {
            var interpreter = new AphidInterpreter();
            interpreter.InterpretFile(file);
            var retVal = interpreter.GetReturnValue();
            var llexFile = new LLexFile();
            retVal.Bind(llexFile);
            var tokenTable = new TokenTable();
            tokenTable.Ignore = llexFile.Ignore;
            var nameInfo = LLexNameInfo.Parse(llexFile.Name);

            int z = 0;
            var modeTable = llexFile.Modes.ToDictionary(x => x.Mode, x => z++);
            var tokenTypes = new List<string>();

            foreach (var mode in llexFile.Modes)
            {
                tokenTable.SetMode(modeTable[mode.Mode]);
                foreach (var token in mode.Tokens)
                {
                    if (token.Regex != null)
                    {
                        var regexLexer = new RegexLexer(token.Regex);
                        var tokens = regexLexer.GetTokens();
                        var parser = new RegexParser(tokens.ToArray());
                        var ast = parser.Parse();
                        var compiler = new RegexCompiler(ast);
                        var strings = compiler.ExpandRegex();

                        foreach (var l in strings)
                        {
                            if (token.Code != null)
                            {
                                tokenTable.AddLexemeCode(l, token.Code);
                                continue;
                            }

                            if (!tokenTypes.Contains(token.TokenType))
                                tokenTypes.Add(token.TokenType);

                            if (!string.IsNullOrEmpty(token.NewMode))
                            {
                                tokenTable.Add(l, token.TokenType, modeTable[token.NewMode]);
                            }
                            else
                            {
                                tokenTable.Add(l, token.TokenType);
                            }
                        }
                    }
                    else if (token.Code != null)
                    {
                        tokenTable.AddCode(token.Code);
                    }
                }

                foreach (var keyword in mode.Keywords)
                {
                    tokenTable.AddKeyword(keyword);
                    var t = keyword + "Keyword";
                    if (mode.KeywordTail != null)
                    {
                        tokenTable.AddLexemeCode(keyword, mode.KeywordTail.Replace("{Keyword}", t));
                    }
                    else
                    {
                        tokenTable.Add(keyword, t);
                    }
                }

                if (!string.IsNullOrEmpty(mode.KeywordDefault))
                {
                    var k = mode.Keywords
                        .SelectMany(x => Enumerable
                            .Range(1, x.Length - 1)
                            .Select(y => x.Remove(y))
                            .ToArray())
                        .Distinct()
                        .ToArray();

                    foreach (var i in k)
                    {
                        if (tokenTable.Lists.Any(x => x.Value.Any(y => y.Lexeme == i)))
                        {
                            continue;
                        }
                        tokenTable.AddLexemeCode(i, mode.KeywordDefault);
                    }
                }
            }

            var generator = new LexerGenerator(tokenTable) { IgnoreCase = llexFile.IgnoreCase };
            var lexer = generator.Generate();

            return lexer
                .Replace("{Lexer}", nameInfo.LexerName)
                .Replace("{Token}", nameInfo.TokenName)
                .Replace("{TokenType}", nameInfo.TokenTypeName)
                .Replace("{LexerNamespace}", nameInfo.Namespace);
        }
Beispiel #3
0
        public static string Interpret(string file)
        {
            var interpreter = new AphidInterpreter();

            interpreter.InterpretFile(file);
            var retVal   = interpreter.GetReturnValue();
            var llexFile = new LLexFile();

            retVal.Bind(llexFile);
            var tokenTable = new TokenTable();

            tokenTable.Ignore = llexFile.Ignore;
            var nameInfo = LLexNameInfo.Parse(llexFile.Name);

            int z          = 0;
            var modeTable  = llexFile.Modes.ToDictionary(x => x.Mode, x => z++);
            var tokenTypes = new List <string>();

            foreach (var mode in llexFile.Modes)
            {
                tokenTable.SetMode(modeTable[mode.Mode]);
                foreach (var token in mode.Tokens)
                {
                    if (token.Regex != null)
                    {
                        var regexLexer = new RegexLexer(token.Regex);
                        var tokens     = regexLexer.GetTokens();
                        var parser     = new RegexParser(tokens.ToArray());
                        var ast        = parser.Parse();
                        var compiler   = new RegexCompiler(ast);
                        var strings    = compiler.ExpandRegex();

                        foreach (var l in strings)
                        {
                            if (token.Code != null)
                            {
                                tokenTable.AddLexemeCode(l, token.Code);
                                continue;
                            }

                            if (!tokenTypes.Contains(token.TokenType))
                            {
                                tokenTypes.Add(token.TokenType);
                            }

                            if (!string.IsNullOrEmpty(token.NewMode))
                            {
                                tokenTable.Add(l, token.TokenType, modeTable[token.NewMode]);
                            }
                            else
                            {
                                tokenTable.Add(l, token.TokenType);
                            }
                        }
                    }
                    else if (token.Code != null)
                    {
                        tokenTable.AddCode(token.Code);
                    }
                }

                foreach (var keyword in mode.Keywords)
                {
                    tokenTable.AddKeyword(keyword);
                    var t = keyword + "Keyword";
                    if (mode.KeywordTail != null)
                    {
                        tokenTable.AddLexemeCode(keyword, mode.KeywordTail.Replace("{Keyword}", t));
                    }
                    else
                    {
                        tokenTable.Add(keyword, t);
                    }
                }

                if (!string.IsNullOrEmpty(mode.KeywordDefault))
                {
                    var k = mode.Keywords
                            .SelectMany(x => Enumerable
                                        .Range(1, x.Length - 1)
                                        .Select(y => x.Remove(y))
                                        .ToArray())
                            .Distinct()
                            .ToArray();

                    foreach (var i in k)
                    {
                        if (tokenTable.Lists.Any(x => x.Value.Any(y => y.Lexeme == i)))
                        {
                            continue;
                        }
                        tokenTable.AddLexemeCode(i, mode.KeywordDefault);
                    }
                }
            }

            var generator = new LexerGenerator(tokenTable);
            var lexer     = generator.Generate();


            return(lexer
                   .Replace("{Lexer}", nameInfo.LexerName)
                   .Replace("{Token}", nameInfo.TokenName)
                   .Replace("{TokenType}", nameInfo.TokenTypeName)
                   .Replace("{LexerNamespace}", nameInfo.Namespace));
        }
Beispiel #4
0
        public static string From(AphidObject retVal)
        {
            var llexFile = new LLexFile();

            retVal.Bind(llexFile);
            var tokenTable = new TokenTable {
                Ignore = llexFile.Ignore
            };
            var nameInfo = LLexNameInfo.Parse(llexFile.Name);

            var z          = 0;
            var modeTable  = llexFile.Modes.ToDictionary(x => x.Mode, x => z++);
            var tokenTypes = new List <string>();

            foreach (var mode in llexFile.Modes)
            {
                tokenTable.SetMode(modeTable[mode.Mode]);
                foreach (var token in mode.Tokens)
                {
                    if (token.Regex != null)
                    {
                        var regexLexer = new RegexLexer(token.Regex);
                        var tokens     = regexLexer.GetTokens();
                        var parser     = new RegexParser(tokens.ToArray());
                        var ast        = parser.Parse();
                        var compiler   = new RegexCompiler(ast);
                        var strings    = compiler.ExpandRegex();

                        foreach (var l in strings)
                        {
                            if (token.Code != null)
                            {
                                tokenTable.AddLexemeCode(l, token.Code);
                                continue;
                            }

                            if (!tokenTypes.Contains(token.TokenType))
                            {
                                tokenTypes.Add(token.TokenType);
                            }

                            if (!string.IsNullOrEmpty(token.NewMode))
                            {
                                tokenTable.Add(l, token.TokenType, modeTable[token.NewMode]);
                            }
                            else
                            {
                                tokenTable.Add(l, token.TokenType);
                            }
                        }
                    }
                    else if (token.Code != null)
                    {
                        tokenTable.AddCode(token.Code);
                    }
                    else if (token.TokenType != null)
                    {
                        tokenTable.AddCode("return {TokenType}." + token.TokenType + ";\r\n");
                    }
                    else
                    {
                        throw new NotImplementedException(
                                  "Token with no regex, code, or type not supported.");
                    }
                }

                foreach (var keyword in mode.Keywords ?? Array.Empty <string>())
                {
                    tokenTable.AddKeyword(keyword);
                    var t = keyword + "Keyword";
                    if (mode.KeywordTail != null)
                    {
                        tokenTable.AddLexemeCode(keyword, mode.KeywordTail.Replace("{Keyword}", t));
                    }
                    else
                    {
                        tokenTable.Add(keyword, t);
                    }
                }

                if (!string.IsNullOrEmpty(mode.KeywordDefault))
                {
                    var k = mode.Keywords
                            .SelectMany(x => Enumerable
                                        .Range(1, x.Length - 1)
                                        .Select(y => x.Remove(y))
                                        .ToArray())
                            .Distinct()
                            .ToArray();

                    foreach (var i in k)
                    {
                        if (tokenTable.Lists.Any(x => x.Value.Any(y => y.Lexeme == i)))
                        {
                            continue;
                        }

                        tokenTable.AddLexemeCode(i, mode.KeywordDefault);
                    }
                }
            }

            var generator = new LexerGenerator(tokenTable)
            {
                IgnoreCase = llexFile.IgnoreCase
            };
            var lexer = generator.Generate();

            return(lexer
                   .Replace("{Lexer}", nameInfo.LexerName)
                   .Replace("{Token}", nameInfo.TokenName)
                   .Replace("{TokenType}", nameInfo.TokenTypeName)
                   .Replace("{LexerNamespace}", nameInfo.Namespace));
        }