Ejemplo n.º 1
0
 public CoverageParserLogger(Parser/*!*/ parser, TextWriter/*!*/ output)
 {
     Assert.NotNull(parser, output);
     _parser = parser;
     _output = output;
     _rules = new Stack<int>();
 }
Ejemplo n.º 2
0
        public DefaultParserLogger(Parser/*!*/ parser, int verbosity, TextWriter/*!*/ output) {
            Assert.NotNull(parser, output);
            Debug.Assert(verbosity > 0 && verbosity <= 2);

            _parser = parser;
            _logVerbosity = verbosity;
            _log = output;
        }
Ejemplo n.º 3
0
        public void Parse()
        {
            var srcunit = HostingHelpers.GetSourceUnit(source);
            var parser = new IronRuby.Compiler.Parser(errorSink);
            ast = parser.Parse(srcunit, new RubyCompilerOptions(), errorSink);

            var transformer = new Transformer.Transformer(ast);
            transformations = transformer.Transform();
        }
Ejemplo n.º 4
0
        public void Parse()
        {
            var srcunit = HostingHelpers.GetSourceUnit(source);
            var parser  = new IronRuby.Compiler.Parser(errorSink);

            ast = parser.Parse(srcunit, new RubyCompilerOptions(), errorSink);

            var transformer = new Transformer.Transformer(ast);

            transformations = transformer.Transform();
        }
Ejemplo n.º 5
0
		public SourceUnitTree CreateAst(string fileName, ITextBuffer fileContent)
		{
			if (scriptEngine == null) {
				scriptEngine = Ruby.CreateEngine();
			}
			
			RubyContext rubyContext = HostingHelpers.GetLanguageContext(scriptEngine) as RubyContext;
			sourceUnit = rubyContext.CreateFileUnit(fileName, fileContent.Text);
			RubyCompilerSink sink = new RubyCompilerSink();
			RubyCompilerOptions compilerOptions = new RubyCompilerOptions((RubyOptions)rubyContext.Options);
			Parser parser = new Parser();
			return parser.Parse(sourceUnit, compilerOptions, sink);
		}
Ejemplo n.º 6
0
        /*!*/
        internal static LeftValue MakeLeftValue(int kind, Parser/*!*/ parser, string name, SourceSpan location)
        {
            switch (kind) {
                case Identifier:
                    return parser.CurrentScope.ResolveOrAddVariable(name, location);

                case Instance:
                    return new InstanceVariable(name, location);

                case Global:
                    return new GlobalVariable(name, location);

                case Constant:
                    return new ConstantVariable(name, location);

                case Class:
                    return new ClassVariable(name, location);

                case Nil:
                    return parser.CannotAssignError("nil", location);

                case Self:
                    return parser.CannotAssignError("self", location);

                case True:
                    return parser.CannotAssignError("true", location);

                case False:
                    return parser.CannotAssignError("false", location);

                case File:
                    return parser.CannotAssignError("__FILE__", location);

                case Line:
                    return parser.CannotAssignError("__LINE__", location);

                case Encoding:
                    return parser.CannotAssignError("__ENCODING__", location);
            }

            return null;
        }
Ejemplo n.º 7
0
        internal static Expression/*!*/ MakeRead(int kind, Parser/*!*/ parser, string name, SourceSpan location) {
            switch (kind) {
                case Identifier:
                    return (Expression)parser.CurrentScope.ResolveVariable(name) ?? new MethodCall(null, name, null, null, location);

                case Instance:
                    return new InstanceVariable(name, location);

                case Global:
                    return new GlobalVariable(name, location);

                case Constant:
                    return new ConstantVariable(name, location);

                case Class:
                    return new ClassVariable(name, location);

                case Nil:
                    return Literal.Nil(location);

                case Self:
                    return new SelfReference(location);

                case True:
                    return Literal.True(location);

                case False:
                    return Literal.False(location);

                case File:
                    return new FileLiteral(location);

                case Line:
                    return Literal.Integer(parser.Tokenizer.TokenSpan.Start.Line, location);

                case Encoding:
                    return new EncodingExpression(location);
            }

            throw Assert.Unreachable;
        }
Ejemplo n.º 8
0
        private void ParserLoggingTest() {
#if DEBUG
            string source = "def foo(a); end";
            var sourceUnit = Context.CreateSnippet(source, SourceCodeKind.Statements);
            var options = new RubyCompilerOptions();

            string temp = Path.Combine(Path.GetTempPath(), "RubyParser");
            Console.WriteLine("> see {0}", temp);
            Directory.CreateDirectory(temp);

            Parser parser = new Parser();
            using (TextWriter writer = File.CreateText(Path.Combine(temp, "default.log"))) {
                DefaultParserLogger.Attach(parser, writer);
                parser.Parse(sourceUnit, options, ErrorSink.Null);
            }

            using (TextWriter writer = File.CreateText(Path.Combine(temp, "tables.csv"))) {
                parser.DumpTables(writer);
            }

            using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) {
                for (int i = 0; i < parser.Rules.Length; i++) {
                    writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i));
                }
            }

            parser = new Parser();
            using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) {
                for (int i = 0; i < parser.Rules.Length; i++) {
                    writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i));
                }
            }

            using (TextWriter writer = File.CreateText(Path.Combine(temp, "second_order.log"))) {
                parser.EnableLogging(new CoverageParserLogger(parser, writer));
                parser.Parse(sourceUnit, options, ErrorSink.Null);
            }
#endif
        }
Ejemplo n.º 9
0
 public static void Attach(Parser/*!*/ parser, TextWriter/*!*/ output) {
     parser.EnableLogging(new DefaultParserLogger(parser, 1, output));
 }
Ejemplo n.º 10
0
 public static void Attach(Parser/*!*/ parser) {
     Attach(parser, Console.Out);
 }
Ejemplo n.º 11
0
 public static void Attach(Parser/*!*/ parser)
 {
     Attach(parser, Driver.Output);
 }
Ejemplo n.º 12
0
        public void Scenario_RubyScopeParsing() {
            LoggingErrorSink log = new LoggingErrorSink();

            SourceUnitTree p;
            SourceUnit unit;

            unit = Context.CreateSnippet(@"
                class c << G
                end
            ", SourceCodeKind.File);

            p = new Parser().Parse(unit, new RubyCompilerOptions(), log);
            Assert(p == null && log.FatalErrorCount == 1);
            log.ClearCounters();

            unit = Context.CreateSnippet(@"
                def goo(&b)
                    end

                    class C
                        def foo()
                           x.goo() { 
                                goo() {
                                    class << M
                                        def bar()
                                            goo() {
                                            }
                                        end
                                    end
                                }
                           } 
                        end
                    end

                BEGIN { goo1() { } }
                END { goo2() { } } 
            ", SourceCodeKind.File);

            p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null);
            Assert(p != null && !log.AnyError);
            log.ClearCounters();

            unit = Context.CreateSnippet(@"
                for x in array do
                    goo()
                end
            ", SourceCodeKind.File);

            p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null);
            Assert(p != null && !log.AnyError);
            log.ClearCounters();
        }
Ejemplo n.º 13
0
        private void AstLocations1() {
            // DumpExpression uses private reflection:
            if (_driver.PartialTrust) return;

            var sourceUnit = Context.CreateSnippet(@"
def add a,b
  a + b
end

add 1, 1
add 'foo', 'bar'
", SourceCodeKind.Expression);

            var options = new RubyCompilerOptions();
            var parser = new Parser();
            var tokens = new List<KeyValuePair<SourceSpan, Tokens>>();

            parser.TokenSink = (token, span) => { tokens.Add(new KeyValuePair<SourceSpan, Tokens>(span, token)); };
            var ast = parser.Parse(sourceUnit, options, Context.RuntimeErrorSink);

            const int Id = 0x12345678;

            var lambda = CallSiteTracer.Transform<Func<Scope, LanguageContext, object>>(ast, sourceUnit, options, Id);
            var code = new RubyScriptCode(lambda, sourceUnit);

            var locations = new List<int>();
            CallSiteTracer.Register((context, args, result, id, location) => {
                locations.Add(location);
                Debug.Assert(id == Id);
                Debug.Assert(location > 0);

                //Console.WriteLine("-- {0} ---------", location);
                //Console.WriteLine(this);
                //Console.WriteLine(AstUtils.DumpExpression(result.Restrictions.ToExpression()));
                //Console.WriteLine();
                //Console.WriteLine(AstUtils.DumpExpression(result.Expression));
                //Console.WriteLine("----------------");
            });

            code.Run();

            Debug.Assert(locations.Count == 4 && locations[0] == 31 && locations[1] == 19 && locations[2] == 41 && locations[3] == 19);
        }
Ejemplo n.º 14
0
        private void AssertTokens(string/*!*/ source, params Tokens[] expected) {
            var tokens = new List<Tokens>();
            var parser = new Parser() {
                TokenSink = (token, span) => tokens.Add(token)
            };

            parser.Parse(Context.CreateSnippet(source, SourceCodeKind.AutoDetect), new RubyCompilerOptions(), ErrorSink.Null);
            Assert(tokens.ToArray().ValueEquals(expected));
        }
Ejemplo n.º 15
0
        public Tokenizer(bool verbatim, ErrorSink/*!*/ errorSink) {
            ContractUtils.RequiresNotNull(errorSink, "errorSink");

            _bigIntParser = new BignumParser();
            _errorSink = errorSink;
            _sourceUnit = null;
            _parser = null;
            _verbatim = verbatim;
            _compatibility = RubyCompatibility.Default;
//            _buffer = null;
            _initialLocation = SourceLocation.Invalid;
            _tokenSpan = SourceSpan.Invalid;
            _tokenValue = new TokenValue();
            _bufferPos = 0;
            
            // TODO:
            _input = null;
        }
Ejemplo n.º 16
0
        private void Benchmark(List<string>/*!*/ files) {
            
            var sources = new List<SourceUnit>();
            Stopwatch readTime = new Stopwatch();
            long totalSize = 0;
            readTime.Start();
            foreach (string path in files) {
                try {
                    byte[] data = File.ReadAllBytes(path);
                    sources.Add(_context.CreateSourceUnit(new BinaryContentProvider(data), path, Encoding.Default, SourceCodeKind.File));
                    totalSize += data.Length;
                } catch (Exception) {
                    Console.WriteLine("Error: {0}", path);
                }
            }
            readTime.Stop();

            Console.WriteLine("Read: {0} kB in {1}", totalSize / 1024, readTime.Elapsed);
            
#if F
            Stopwatch tokenizeTime = new Stopwatch();
            tokenizeTime.Start();
            foreach (var source in sources) {
                try {
                    var tokenizer = new Tokenizer();
                    tokenizer.Initialize(source);

                    Tokens token;
                    do {
                        token = tokenizer.GetNextToken();
                    } while (token != Tokens.EndOfFile);
                } catch (Exception) {
                    Console.WriteLine("Tokenization error: {0}", source.Path);
                    break;
                }
            }
            tokenizeTime.Stop();
#endif
            //var stackSizes = new Dictionary<int, int>();

            var options = new RubyCompilerOptions();
            Stopwatch parseTime = new Stopwatch();
            Stopwatch transformTime = new Stopwatch();
            foreach (var source in sources) {
                try {
                    parseTime.Start();
                    var parser = new Parser();
                    var rubyTree = parser.Parse(source, options, ErrorSink.Null);
                    //int mt;
                    //stackSizes[parser.StackMaxTop] = stackSizes.TryGetValue(parser.StackMaxTop, out mt) ? mt + 1 : 1;
                    parseTime.Stop();
#if F
                    if (rubyTree != null) {
                        transformTime.Start();
                        var lambda = _context.TransformTree<DlrMainCallTarget>(rubyTree, source, options);
                        transformTime.Stop();
                    } else {
                        Console.WriteLine("SyntaxError: {0}", source.Path);
                    }
#endif
                } catch (Exception e) {
                    Console.WriteLine("{0}: {1}: {2}", e.GetType().Name, source.Path, e.Message);
                    break;
                }
            }

          //  Console.WriteLine("Tokenize:        {0}", tokenizeTime.Elapsed);
            Console.WriteLine("Parse:           {0}", parseTime.Elapsed);
            //Console.WriteLine("Idf/Kwd/Loc: {0}/{1}/{2}", Tokenizer.IdfLength, Tokenizer.KwdLength, Tokenizer.LocLength);
           // Console.WriteLine("Transform:       {0}", transformTime.Elapsed);

            //PerfTrack.DumpHistogram(Parser.Reductions);
            //PerfTrack.DumpHistogram(stackSizes);
        }
Ejemplo n.º 17
0
        private void DumpFile(string/*!*/ path) {
            _log.WriteLine(path);

            try {
                string fullPath = Path.GetFullPath(path);
                string root = Path.GetPathRoot(fullPath);
                string outputPath = Path.ChangeExtension(Path.Combine(_targetDir, fullPath.Substring(root.Length).TrimEnd('\'', '/')), ".txt");
                Directory.CreateDirectory(Path.GetDirectoryName(outputPath));

                using (TextWriter output = File.CreateText(outputPath)) {

                    output.WriteLine(fullPath);
                    output.WriteLine();
                    output.WriteLine("Tokens:");
                    output.WriteLine();

                    ErrorLog errors = new ErrorLog();
                    Parser parser = new Parser();

                    parser.TokenSink = delegate(Tokens token, SourceSpan span) {
                        DumpTokenDetail(output, parser.Tokenizer, token);
                    };

                    if (_logProductions) {
#if DEBUG
                        parser.EnableLogging(new CoverageParserLogger(parser, _parserLog));
#endif
                    }

                    _currentSourceFile = path;

                    SourceUnitTree ast = null;
                    try {
                        ast = parser.Parse(_context.CreateFileUnit(path), new RubyCompilerOptions(), errors);
                    } catch (Exception e) {
                        WriteException(e.ToString());
                    }

                    output.WriteLine();

                    if (errors.ErrorCount + errors.FatalErrorCount + errors.WarningCount > 0) {
                        output.WriteLine();
                        output.WriteLine("Errors:");

                        foreach (string error in errors.Errors) {
                            output.WriteLine(error);
                        }
                    } else {
                        Debug.Assert(ast != null);
                        DumpRegularExpressions(ast);
                    }

                    output.WriteLine(".");
                }
            } catch (Exception e) {
                _log.WriteLine("!{0}", e.Message);
            } finally {
                _currentSourceFile = null;
                _regexLog.Flush();
                _parserLog.Flush();
            }
        }
Ejemplo n.º 18
0
 internal Tokenizer(Parser/*!*/ parser)
     : this(false) {
     _parser = parser;
 }