Lexer for Python.
Exemplo n.º 1
0
 private void RunTest(string pyExp)
 {
     var lexer =  new Lexer("foo.py", new StringReader(pyExp));
     var parser = new Parser("foo.py", lexer);
     var exp = parser.test();
     exp.Accept(new ExpNameDiscovery(syms));
 }
Exemplo n.º 2
0
 public void Translate(string filename, TextReader input, TextWriter output)
 {
     Debug.Print("Translating module {0} in namespace {1}", moduleName, nmspace);
     var lex = new Lexer(filename, input);
     var par = new Parser(filename, lex);
     var stm = par.Parse();
     var unt = new CodeCompileUnit();
     var gen = new CodeGenerator(unt, nmspace, Path.GetFileNameWithoutExtension(moduleName));
     var xlt = new ModuleTranslator(gen);
     xlt.Translate(stm);
     var pvd = new CSharpCodeProvider();
     pvd.GenerateCodeFromCompileUnit(unt, output, new CodeGeneratorOptions { });
 }
Exemplo n.º 3
0
        /// <summary>
        /// Returns the syntax tree for <paramref name="path" />. May find and/or create a
        /// cached copy in the mem cache or the disk cache.
        /// 
        /// <param name="path">Absolute path to a source file.</param>
        /// <returns>The AST, or <code>null</code> if the parse failed for any reason</returns>
        /// </summary>
        public Module getAST(string path)
        {
            // Cache stores null value if the parse failed.
            Module module;
            if (cache.TryGetValue(path, out module))
            {
                return module;
            }

            // Might be cached on disk but not in memory.
            module = GetSerializedModule(path);
            if (module != null)
            {
                LOG.Verbose("Reusing " + path);
                cache[path] = module;
                return module;
            }

            module = null;
            try
            {
                LOG.Verbose("parsing " + path);
                var lexer = new Lexer(path, fs.CreateStreamReader(path));
                var parser = new Parser(path, lexer);
                var moduleStmts = parser.Parse().ToList();
                int posStart = 0;
                int posEnd = 0;
                if (moduleStmts.Count > 0)
                {
                    posStart = moduleStmts[0].Start;
                    posEnd = moduleStmts.Last().End;
                }
                module = new Module(
                    analyzer.ModuleName(path),
                    new SuiteStatement(moduleStmts, path, posStart, posEnd),
                    path, posStart, posEnd);
            }
            finally
            {
                cache[path] = module;  // may be null
            }
            return module;
        }
Exemplo n.º 4
0
 public Parser(string filename, Lexer lexer)
 {
     this.filename = filename;
     this.lexer = lexer;
 }
Exemplo n.º 5
0
 private void Lex(string str, params TokenType[] tokens)
 {
     StringReader rdr = new StringReader(str);
     lexer = new Lexer("foo.py", rdr);
     Token tok = new Token();
     foreach (var exp in tokens)
     {
         tok = lexer.Get();
         Assert.AreEqual(exp, tok.Type);
     }
     Assert.AreEqual(TokenType.EOF, tok.Type);
 }
Exemplo n.º 6
0
 private Token Lex(string str)
 {
     StringReader rdr = new StringReader(str);
     lexer = new Lexer("foo.py", rdr);
     return lexer.Get();
 }
Exemplo n.º 7
0
 public void Setup()
 {
     lexer = null;
 }
Exemplo n.º 8
0
 public void Setup()
 {
     lexer = null;
 }