public CoverageParserLogger(Parser/*!*/ parser, TextWriter/*!*/ output) { Assert.NotNull(parser, output); _parser = parser; _output = output; _rules = new Stack<int>(); }
public DefaultParserLogger(Parser/*!*/ parser, int verbosity, TextWriter/*!*/ output) { Assert.NotNull(parser, output); Debug.Assert(verbosity > 0 && verbosity <= 2); _parser = parser; _logVerbosity = verbosity; _log = output; }
public void Parse() { var srcunit = HostingHelpers.GetSourceUnit(source); var parser = new IronRuby.Compiler.Parser(errorSink); ast = parser.Parse(srcunit, new RubyCompilerOptions(), errorSink); var transformer = new Transformer.Transformer(ast); transformations = transformer.Transform(); }
public SourceUnitTree CreateAst(string fileName, ITextBuffer fileContent) { if (scriptEngine == null) { scriptEngine = Ruby.CreateEngine(); } RubyContext rubyContext = HostingHelpers.GetLanguageContext(scriptEngine) as RubyContext; sourceUnit = rubyContext.CreateFileUnit(fileName, fileContent.Text); RubyCompilerSink sink = new RubyCompilerSink(); RubyCompilerOptions compilerOptions = new RubyCompilerOptions((RubyOptions)rubyContext.Options); Parser parser = new Parser(); return parser.Parse(sourceUnit, compilerOptions, sink); }
/*!*/ internal static LeftValue MakeLeftValue(int kind, Parser/*!*/ parser, string name, SourceSpan location) { switch (kind) { case Identifier: return parser.CurrentScope.ResolveOrAddVariable(name, location); case Instance: return new InstanceVariable(name, location); case Global: return new GlobalVariable(name, location); case Constant: return new ConstantVariable(name, location); case Class: return new ClassVariable(name, location); case Nil: return parser.CannotAssignError("nil", location); case Self: return parser.CannotAssignError("self", location); case True: return parser.CannotAssignError("true", location); case False: return parser.CannotAssignError("false", location); case File: return parser.CannotAssignError("__FILE__", location); case Line: return parser.CannotAssignError("__LINE__", location); case Encoding: return parser.CannotAssignError("__ENCODING__", location); } return null; }
internal static Expression/*!*/ MakeRead(int kind, Parser/*!*/ parser, string name, SourceSpan location) { switch (kind) { case Identifier: return (Expression)parser.CurrentScope.ResolveVariable(name) ?? new MethodCall(null, name, null, null, location); case Instance: return new InstanceVariable(name, location); case Global: return new GlobalVariable(name, location); case Constant: return new ConstantVariable(name, location); case Class: return new ClassVariable(name, location); case Nil: return Literal.Nil(location); case Self: return new SelfReference(location); case True: return Literal.True(location); case False: return Literal.False(location); case File: return new FileLiteral(location); case Line: return Literal.Integer(parser.Tokenizer.TokenSpan.Start.Line, location); case Encoding: return new EncodingExpression(location); } throw Assert.Unreachable; }
private void ParserLoggingTest() { #if DEBUG string source = "def foo(a); end"; var sourceUnit = Context.CreateSnippet(source, SourceCodeKind.Statements); var options = new RubyCompilerOptions(); string temp = Path.Combine(Path.GetTempPath(), "RubyParser"); Console.WriteLine("> see {0}", temp); Directory.CreateDirectory(temp); Parser parser = new Parser(); using (TextWriter writer = File.CreateText(Path.Combine(temp, "default.log"))) { DefaultParserLogger.Attach(parser, writer); parser.Parse(sourceUnit, options, ErrorSink.Null); } using (TextWriter writer = File.CreateText(Path.Combine(temp, "tables.csv"))) { parser.DumpTables(writer); } using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) { for (int i = 0; i < parser.Rules.Length; i++) { writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i)); } } parser = new Parser(); using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) { for (int i = 0; i < parser.Rules.Length; i++) { writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i)); } } using (TextWriter writer = File.CreateText(Path.Combine(temp, "second_order.log"))) { parser.EnableLogging(new CoverageParserLogger(parser, writer)); parser.Parse(sourceUnit, options, ErrorSink.Null); } #endif }
public static void Attach(Parser/*!*/ parser, TextWriter/*!*/ output) { parser.EnableLogging(new DefaultParserLogger(parser, 1, output)); }
public static void Attach(Parser/*!*/ parser) { Attach(parser, Console.Out); }
public static void Attach(Parser/*!*/ parser) { Attach(parser, Driver.Output); }
public void Scenario_RubyScopeParsing() { LoggingErrorSink log = new LoggingErrorSink(); SourceUnitTree p; SourceUnit unit; unit = Context.CreateSnippet(@" class c << G end ", SourceCodeKind.File); p = new Parser().Parse(unit, new RubyCompilerOptions(), log); Assert(p == null && log.FatalErrorCount == 1); log.ClearCounters(); unit = Context.CreateSnippet(@" def goo(&b) end class C def foo() x.goo() { goo() { class << M def bar() goo() { } end end } } end end BEGIN { goo1() { } } END { goo2() { } } ", SourceCodeKind.File); p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null); Assert(p != null && !log.AnyError); log.ClearCounters(); unit = Context.CreateSnippet(@" for x in array do goo() end ", SourceCodeKind.File); p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null); Assert(p != null && !log.AnyError); log.ClearCounters(); }
private void AstLocations1() { // DumpExpression uses private reflection: if (_driver.PartialTrust) return; var sourceUnit = Context.CreateSnippet(@" def add a,b a + b end add 1, 1 add 'foo', 'bar' ", SourceCodeKind.Expression); var options = new RubyCompilerOptions(); var parser = new Parser(); var tokens = new List<KeyValuePair<SourceSpan, Tokens>>(); parser.TokenSink = (token, span) => { tokens.Add(new KeyValuePair<SourceSpan, Tokens>(span, token)); }; var ast = parser.Parse(sourceUnit, options, Context.RuntimeErrorSink); const int Id = 0x12345678; var lambda = CallSiteTracer.Transform<Func<Scope, LanguageContext, object>>(ast, sourceUnit, options, Id); var code = new RubyScriptCode(lambda, sourceUnit); var locations = new List<int>(); CallSiteTracer.Register((context, args, result, id, location) => { locations.Add(location); Debug.Assert(id == Id); Debug.Assert(location > 0); //Console.WriteLine("-- {0} ---------", location); //Console.WriteLine(this); //Console.WriteLine(AstUtils.DumpExpression(result.Restrictions.ToExpression())); //Console.WriteLine(); //Console.WriteLine(AstUtils.DumpExpression(result.Expression)); //Console.WriteLine("----------------"); }); code.Run(); Debug.Assert(locations.Count == 4 && locations[0] == 31 && locations[1] == 19 && locations[2] == 41 && locations[3] == 19); }
private void AssertTokens(string/*!*/ source, params Tokens[] expected) { var tokens = new List<Tokens>(); var parser = new Parser() { TokenSink = (token, span) => tokens.Add(token) }; parser.Parse(Context.CreateSnippet(source, SourceCodeKind.AutoDetect), new RubyCompilerOptions(), ErrorSink.Null); Assert(tokens.ToArray().ValueEquals(expected)); }
public Tokenizer(bool verbatim, ErrorSink/*!*/ errorSink) { ContractUtils.RequiresNotNull(errorSink, "errorSink"); _bigIntParser = new BignumParser(); _errorSink = errorSink; _sourceUnit = null; _parser = null; _verbatim = verbatim; _compatibility = RubyCompatibility.Default; // _buffer = null; _initialLocation = SourceLocation.Invalid; _tokenSpan = SourceSpan.Invalid; _tokenValue = new TokenValue(); _bufferPos = 0; // TODO: _input = null; }
private void Benchmark(List<string>/*!*/ files) { var sources = new List<SourceUnit>(); Stopwatch readTime = new Stopwatch(); long totalSize = 0; readTime.Start(); foreach (string path in files) { try { byte[] data = File.ReadAllBytes(path); sources.Add(_context.CreateSourceUnit(new BinaryContentProvider(data), path, Encoding.Default, SourceCodeKind.File)); totalSize += data.Length; } catch (Exception) { Console.WriteLine("Error: {0}", path); } } readTime.Stop(); Console.WriteLine("Read: {0} kB in {1}", totalSize / 1024, readTime.Elapsed); #if F Stopwatch tokenizeTime = new Stopwatch(); tokenizeTime.Start(); foreach (var source in sources) { try { var tokenizer = new Tokenizer(); tokenizer.Initialize(source); Tokens token; do { token = tokenizer.GetNextToken(); } while (token != Tokens.EndOfFile); } catch (Exception) { Console.WriteLine("Tokenization error: {0}", source.Path); break; } } tokenizeTime.Stop(); #endif //var stackSizes = new Dictionary<int, int>(); var options = new RubyCompilerOptions(); Stopwatch parseTime = new Stopwatch(); Stopwatch transformTime = new Stopwatch(); foreach (var source in sources) { try { parseTime.Start(); var parser = new Parser(); var rubyTree = parser.Parse(source, options, ErrorSink.Null); //int mt; //stackSizes[parser.StackMaxTop] = stackSizes.TryGetValue(parser.StackMaxTop, out mt) ? mt + 1 : 1; parseTime.Stop(); #if F if (rubyTree != null) { transformTime.Start(); var lambda = _context.TransformTree<DlrMainCallTarget>(rubyTree, source, options); transformTime.Stop(); } else { Console.WriteLine("SyntaxError: {0}", source.Path); } #endif } catch (Exception e) { Console.WriteLine("{0}: {1}: {2}", e.GetType().Name, source.Path, e.Message); break; } } // Console.WriteLine("Tokenize: {0}", tokenizeTime.Elapsed); Console.WriteLine("Parse: {0}", parseTime.Elapsed); //Console.WriteLine("Idf/Kwd/Loc: {0}/{1}/{2}", Tokenizer.IdfLength, Tokenizer.KwdLength, Tokenizer.LocLength); // Console.WriteLine("Transform: {0}", transformTime.Elapsed); //PerfTrack.DumpHistogram(Parser.Reductions); //PerfTrack.DumpHistogram(stackSizes); }
private void DumpFile(string/*!*/ path) { _log.WriteLine(path); try { string fullPath = Path.GetFullPath(path); string root = Path.GetPathRoot(fullPath); string outputPath = Path.ChangeExtension(Path.Combine(_targetDir, fullPath.Substring(root.Length).TrimEnd('\'', '/')), ".txt"); Directory.CreateDirectory(Path.GetDirectoryName(outputPath)); using (TextWriter output = File.CreateText(outputPath)) { output.WriteLine(fullPath); output.WriteLine(); output.WriteLine("Tokens:"); output.WriteLine(); ErrorLog errors = new ErrorLog(); Parser parser = new Parser(); parser.TokenSink = delegate(Tokens token, SourceSpan span) { DumpTokenDetail(output, parser.Tokenizer, token); }; if (_logProductions) { #if DEBUG parser.EnableLogging(new CoverageParserLogger(parser, _parserLog)); #endif } _currentSourceFile = path; SourceUnitTree ast = null; try { ast = parser.Parse(_context.CreateFileUnit(path), new RubyCompilerOptions(), errors); } catch (Exception e) { WriteException(e.ToString()); } output.WriteLine(); if (errors.ErrorCount + errors.FatalErrorCount + errors.WarningCount > 0) { output.WriteLine(); output.WriteLine("Errors:"); foreach (string error in errors.Errors) { output.WriteLine(error); } } else { Debug.Assert(ast != null); DumpRegularExpressions(ast); } output.WriteLine("."); } } catch (Exception e) { _log.WriteLine("!{0}", e.Message); } finally { _currentSourceFile = null; _regexLog.Flush(); _parserLog.Flush(); } }
internal Tokenizer(Parser/*!*/ parser) : this(false) { _parser = parser; }