public AssertTokenizer(Tests/*!*/ tests) { _log = new LoggingErrorSink(); _tests = tests; _context = tests.Context; DefaultEncoding = RubyEncoding.UTF8; Compatibility = tests.Context.RubyOptions.Compatibility; }
static int Main(string[] args) { SourceUnit unit = new SourceCodeUnit(RB, File.ReadAllText(args[0])); LoggingErrorSink log = new LoggingErrorSink(); new Parser().Parse(new CompilerContext(unit, new RubyCompilerOptions(), log)); return log.ErrorCount + log.FatalErrorCount; }
static int Main(string[] args) { SourceUnit unit = new SourceCodeUnit(RB, File.ReadAllText(args[0])); LoggingErrorSink log = new LoggingErrorSink(); new Parser().Parse(new CompilerContext(unit, new RubyCompilerOptions(), log)); return(log.ErrorCount + log.FatalErrorCount); }
public void Scenario_RubyScopeParsing() { LoggingErrorSink log = new LoggingErrorSink(); SourceUnitTree p; SourceUnit unit; unit = Context.CreateSnippet(@" class c << G end ", SourceCodeKind.File); p = new Parser().Parse(unit, new RubyCompilerOptions(), log); Assert(p == null && log.FatalErrorCount == 1); log.ClearCounters(); unit = Context.CreateSnippet(@" def goo(&b) end class C def foo() x.goo() { goo() { class << M def bar() goo() { } end end } } end end BEGIN { goo1() { } } END { goo2() { } } ", SourceCodeKind.File); p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null); Assert(p != null && !log.AnyError); log.ClearCounters(); unit = Context.CreateSnippet(@" for x in array do goo() end ", SourceCodeKind.File); p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null); Assert(p != null && !log.AnyError); log.ClearCounters(); }
public void Scenario_RubyTokenizer1() { LoggingErrorSink log = new LoggingErrorSink(); List<Tokens> tokens; tokens = GetRubyTokens(log, "foo (while (f do end) do end) do end"); Assert(tokens.Count == 14); Assert(tokens[0] == Tokens.Identifier); Assert(tokens[1] == Tokens.LparenArg); Assert(tokens[2] == Tokens.While); Assert(tokens[3] == Tokens.LeftParen); Assert(tokens[4] == Tokens.Identifier); Assert(tokens[5] == Tokens.Do); Assert(tokens[6] == Tokens.End); Assert(tokens[7] == (Tokens)')'); Assert(tokens[8] == Tokens.LoopDo); Assert(tokens[9] == Tokens.End); Assert(tokens[10] == (Tokens)')'); Assert(tokens[11] == Tokens.BlockDo); Assert(tokens[12] == Tokens.End); Assert(tokens[13] == Tokens.EndOfFile); log.Errors.Clear(); tokens = GetRubyTokens(log, "print 'foo'"); Assert(log.Errors.Count == 0); Assert(tokens.Count == 5 && tokens[0] == Tokens.Identifier && tokens[1] == Tokens.StringBeg && tokens[2] == Tokens.StringContent && tokens[3] == Tokens.StringEnd && tokens[4] == Tokens.EndOfFile); Assert(log.Errors.Count == 0); tokens = GetRubyTokens(log, "print '"); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Error ); Assert(tokens.Count == 4 && tokens[0] == Tokens.Identifier && tokens[1] == Tokens.StringBeg && tokens[2] == Tokens.StringEnd && tokens[3] == Tokens.EndOfFile); }
public void Tokenizer1() { LoggingErrorSink log = new LoggingErrorSink(); List<Tokens> tokens; tokens = GetRubyTokens(log, "foo (while (f do end) do end) do end"); Assert(tokens.Count == 14); Assert(tokens[0] == Tokens.Identifier); Assert(tokens[1] == Tokens.LeftArgParenthesis); Assert(tokens[2] == Tokens.While); Assert(tokens[3] == Tokens.LeftExprParenthesis); Assert(tokens[4] == Tokens.Identifier); Assert(tokens[5] == Tokens.Do); Assert(tokens[6] == Tokens.End); Assert(tokens[7] == Tokens.RightParenthesis); Assert(tokens[8] == Tokens.LoopDo); Assert(tokens[9] == Tokens.End); Assert(tokens[10] == Tokens.RightParenthesis); Assert(tokens[11] == Tokens.BlockDo); Assert(tokens[12] == Tokens.End); Assert(tokens[13] == Tokens.EndOfFile); log.Errors.Clear(); tokens = GetRubyTokens(log, "print 'foo'"); Assert(log.Errors.Count == 0); Assert(tokens.Count == 5 && tokens[0] == Tokens.Identifier && tokens[1] == Tokens.StringBegin && tokens[2] == Tokens.StringContent && tokens[3] == Tokens.StringEnd && tokens[4] == Tokens.EndOfFile); Assert(log.Errors.Count == 0); tokens = GetRubyTokens(log, "print '"); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Error ); Assert(tokens.Count == 4 && tokens[0] == Tokens.Identifier && tokens[1] == Tokens.StringBegin && tokens[2] == Tokens.StringEnd && tokens[3] == Tokens.EndOfFile); log.Errors.Clear(); tokens = GetRubyTokens(log, "f (1,2) {}"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.LeftArgParenthesis, Tokens.Integer, Tokens.Comma, Tokens.Integer, Tokens.RightParenthesis, Tokens.LeftBlockArgBrace, Tokens.RightBrace, Tokens.EndOfFile, })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Warning ); log.Errors.Clear(); tokens = GetRubyTokens(log, "f \"a#{g.h /x/}\" do\nend"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.StringBegin, Tokens.StringContent, Tokens.StringEmbeddedCodeBegin, Tokens.Identifier, Tokens.Dot, Tokens.Identifier, Tokens.RegexpBegin, Tokens.StringContent, Tokens.RegexpEnd, Tokens.StringEmbeddedCodeEnd, Tokens.StringEnd, Tokens.BlockDo, Tokens.End, Tokens.EndOfFile })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Warning ); log.Errors.Clear(); }
private void TokenizerErrors1() { LoggingErrorSink log = new LoggingErrorSink(); var tokens = GetRubyTokens(log, "def ["); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Def, Tokens.LeftIndexingBracket, })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.FatalError ); log.Errors.Clear(); tokens = GetRubyTokens(log, "${"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Dollar, })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.FatalError ); log.Errors.Clear(); }
public void Parser1() { LoggingErrorSink log = new LoggingErrorSink(); List<Tokens> tokens; tokens = GetRubyTokens(log, "foo (while (f do end) do end) do end"); Assert(tokens.Count == 14); Assert(tokens[0] == Tokens.Identifier); Assert(tokens[1] == Tokens.LeftArgParenthesis); Assert(tokens[2] == Tokens.While); Assert(tokens[3] == Tokens.LeftExprParenthesis); Assert(tokens[4] == Tokens.Identifier); Assert(tokens[5] == Tokens.Do); Assert(tokens[6] == Tokens.End); Assert(tokens[7] == Tokens.RightParenthesis); Assert(tokens[8] == Tokens.LoopDo); Assert(tokens[9] == Tokens.End); Assert(tokens[10] == Tokens.RightParenthesis); Assert(tokens[11] == Tokens.BlockDo); Assert(tokens[12] == Tokens.End); Assert(tokens[13] == Tokens.EndOfFile); log.Errors.Clear(); tokens = GetRubyTokens(log, "print 'foo'"); Assert(log.Errors.Count == 0); Assert(tokens.Count == 5 && tokens[0] == Tokens.Identifier && tokens[1] == Tokens.StringBegin && tokens[2] == Tokens.StringContent && tokens[3] == Tokens.StringEnd && tokens[4] == Tokens.EndOfFile); Assert(log.Errors.Count == 0); tokens = GetRubyTokens(log, "print '"); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Error ); Assert(tokens.Count == 4 && tokens[0] == Tokens.Identifier && tokens[1] == Tokens.StringBegin && tokens[2] == Tokens.StringEnd && tokens[3] == Tokens.EndOfFile); log.Errors.Clear(); tokens = GetRubyTokens(log, "f \"a#{g.h /x/}\" do\nend"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.StringBegin, Tokens.StringContent, Tokens.StringEmbeddedCodeBegin, Tokens.Identifier, Tokens.Dot, Tokens.Identifier, Tokens.RegexpBegin, Tokens.StringContent, Tokens.RegexpEnd, Tokens.StringEmbeddedCodeEnd, Tokens.StringEnd, Tokens.BlockDo, Tokens.End, Tokens.EndOfFile })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Warning ); log.Errors.Clear(); // 1.9: tokens = GetRubyTokens(log, "def !@\nend"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Def, Tokens.Bang, Tokens.NewLine, Tokens.End, Tokens.EndOfFile })); Assert(log.Errors.Count == 0); // 1.9: tokens = GetRubyTokens(log, "puts <<L, foo\n .bar\nL\n .baz"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.StringBegin, Tokens.StringContent, Tokens.StringEnd, Tokens.Comma, Tokens.Identifier, Tokens.Dot, Tokens.Identifier, Tokens.EndOfFile, })); Assert(log.Errors.Count == 0); // EXPR_VALUE is included in IsBeginLexicalState (discriminates left bracket kind): tokens = GetRubyTokens(log, "x = 1 ? [] : []"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.Assignment, Tokens.Integer, Tokens.QuestionMark, Tokens.LeftBracket, Tokens.RightBracket, Tokens.Colon, Tokens.LeftBracket, Tokens.RightBracket, Tokens.EndOfFile, })); Assert(log.Errors.Count == 0); // Default block parameter is different from default method parameter. // Block parameter doesn't allow using binary expressions etc. due to ambiguity with binary OR operator. tokens = GetRubyTokens(log, "lambda {|x=1|}"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.LeftBlockBrace, Tokens.Pipe, Tokens.Identifier, Tokens.Assignment, Tokens.Integer, Tokens.Pipe, Tokens.RightBrace, Tokens.EndOfFile, })); Assert(log.Errors.Count == 0); tokens = GetRubyTokens(log, "lambda { ->(){} }"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.LeftBlockBrace, Tokens.Lambda, Tokens.LeftParenthesis, Tokens.RightParenthesis, Tokens.LeftLambdaBrace, Tokens.RightBrace, Tokens.RightBrace, Tokens.EndOfFile, })); Assert(log.Errors.Count == 0); tokens = GetRubyTokens(log, "\"#{->{ }}\"\n-> do\nend"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.StringBegin, Tokens.StringEmbeddedCodeBegin, Tokens.Lambda, Tokens.LeftLambdaBrace, Tokens.RightBrace, Tokens.StringEmbeddedCodeEnd, Tokens.StringEnd, Tokens.NewLine, Tokens.Lambda, Tokens.LambdaDo, Tokens.NewLine, Tokens.End, Tokens.EndOfFile, })); Assert(log.Errors.Count == 0); }
private void ParserErrors1() { LoggingErrorSink log = new LoggingErrorSink(); var tokens = GetRubyTokens(log, "def ["); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Def, Tokens.LeftIndexingBracket, })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.FatalError ); log.Errors.Clear(); tokens = GetRubyTokens(log, "${"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Dollar, })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.FatalError ); log.Errors.Clear(); // Error: block argument should not be given (yield as a command): tokens = GetRubyTokens(log, "a[yield 1, &x]"); Assert(tokens.ToArray().ValueEquals(new[] { Tokens.Identifier, Tokens.LeftIndexingBracket, Tokens.Yield, Tokens.Integer, Tokens.Comma, Tokens.BlockReference, Tokens.Identifier, Tokens.RightBracket, Tokens.EndOfFile, })); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Error ); log.Errors.Clear(); // restrictions on jump statements: foreach (var src in new[] { "a[return 1, &x]", "puts if return 1", "1 && return 1", "(a ? return 1 : 1) while true", "a while return", "1 + return", "return / 1", "a = return", "return.foo = bar", "a = 1 rescue return 2", }) { tokens = GetRubyTokens(log, src); Assert(log.Errors.Count > 0 && log.Errors.Exists((e) => e.Severity == Severity.FatalError)); log.Errors.Clear(); } // Error: Duplicate parameter name: tokens = GetRubyTokens(log, "lambda { |x,x| }"); Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Error); log.Errors.Clear(); // Underscore may be duplicated: tokens = GetRubyTokens(log, "lambda { |_,_| }"); Assert(log.Errors.Count == 0); log.Errors.Clear(); }
public AssertTokenizer(RubyContext/*!*/ context) { _log = new LoggingErrorSink(); _context = context; }