public AssertTokenizer(Tests/*!*/ tests) {
_log = new LoggingErrorSink();
_tests = tests;
_context = tests.Context;
DefaultEncoding = RubyEncoding.UTF8;
Compatibility = tests.Context.RubyOptions.Compatibility;
}
static int Main(string[] args)
{
SourceUnit unit = new SourceCodeUnit(RB, File.ReadAllText(args[0]));
LoggingErrorSink log = new LoggingErrorSink();
new Parser().Parse(new CompilerContext(unit, new RubyCompilerOptions(), log));
return log.ErrorCount + log.FatalErrorCount;
}
static int Main(string[] args)
{
SourceUnit unit = new SourceCodeUnit(RB, File.ReadAllText(args[0]));
LoggingErrorSink log = new LoggingErrorSink();
new Parser().Parse(new CompilerContext(unit, new RubyCompilerOptions(), log));
return(log.ErrorCount + log.FatalErrorCount);
}
public void Scenario_RubyScopeParsing() {
LoggingErrorSink log = new LoggingErrorSink();
SourceUnitTree p;
SourceUnit unit;
unit = Context.CreateSnippet(@"
class c << G
end
", SourceCodeKind.File);
p = new Parser().Parse(unit, new RubyCompilerOptions(), log);
Assert(p == null && log.FatalErrorCount == 1);
log.ClearCounters();
unit = Context.CreateSnippet(@"
def goo(&b)
end
class C
def foo()
x.goo() {
goo() {
class << M
def bar()
goo() {
}
end
end
}
}
end
end
BEGIN { goo1() { } }
END { goo2() { } }
", SourceCodeKind.File);
p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null);
Assert(p != null && !log.AnyError);
log.ClearCounters();
unit = Context.CreateSnippet(@"
for x in array do
goo()
end
", SourceCodeKind.File);
p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null);
Assert(p != null && !log.AnyError);
log.ClearCounters();
}
public void Scenario_RubyTokenizer1() {
LoggingErrorSink log = new LoggingErrorSink();
List<Tokens> tokens;
tokens = GetRubyTokens(log, "foo (while (f do end) do end) do end");
Assert(tokens.Count == 14);
Assert(tokens[0] == Tokens.Identifier);
Assert(tokens[1] == Tokens.LparenArg);
Assert(tokens[2] == Tokens.While);
Assert(tokens[3] == Tokens.LeftParen);
Assert(tokens[4] == Tokens.Identifier);
Assert(tokens[5] == Tokens.Do);
Assert(tokens[6] == Tokens.End);
Assert(tokens[7] == (Tokens)')');
Assert(tokens[8] == Tokens.LoopDo);
Assert(tokens[9] == Tokens.End);
Assert(tokens[10] == (Tokens)')');
Assert(tokens[11] == Tokens.BlockDo);
Assert(tokens[12] == Tokens.End);
Assert(tokens[13] == Tokens.EndOfFile);
log.Errors.Clear();
tokens = GetRubyTokens(log, "print 'foo'");
Assert(log.Errors.Count == 0);
Assert(tokens.Count == 5 &&
tokens[0] == Tokens.Identifier &&
tokens[1] == Tokens.StringBeg &&
tokens[2] == Tokens.StringContent &&
tokens[3] == Tokens.StringEnd &&
tokens[4] == Tokens.EndOfFile);
Assert(log.Errors.Count == 0);
tokens = GetRubyTokens(log, "print '");
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.Error
);
Assert(tokens.Count == 4 &&
tokens[0] == Tokens.Identifier &&
tokens[1] == Tokens.StringBeg &&
tokens[2] == Tokens.StringEnd &&
tokens[3] == Tokens.EndOfFile);
}
public void Tokenizer1() {
LoggingErrorSink log = new LoggingErrorSink();
List<Tokens> tokens;
tokens = GetRubyTokens(log, "foo (while (f do end) do end) do end");
Assert(tokens.Count == 14);
Assert(tokens[0] == Tokens.Identifier);
Assert(tokens[1] == Tokens.LeftArgParenthesis);
Assert(tokens[2] == Tokens.While);
Assert(tokens[3] == Tokens.LeftExprParenthesis);
Assert(tokens[4] == Tokens.Identifier);
Assert(tokens[5] == Tokens.Do);
Assert(tokens[6] == Tokens.End);
Assert(tokens[7] == Tokens.RightParenthesis);
Assert(tokens[8] == Tokens.LoopDo);
Assert(tokens[9] == Tokens.End);
Assert(tokens[10] == Tokens.RightParenthesis);
Assert(tokens[11] == Tokens.BlockDo);
Assert(tokens[12] == Tokens.End);
Assert(tokens[13] == Tokens.EndOfFile);
log.Errors.Clear();
tokens = GetRubyTokens(log, "print 'foo'");
Assert(log.Errors.Count == 0);
Assert(tokens.Count == 5 &&
tokens[0] == Tokens.Identifier &&
tokens[1] == Tokens.StringBegin &&
tokens[2] == Tokens.StringContent &&
tokens[3] == Tokens.StringEnd &&
tokens[4] == Tokens.EndOfFile);
Assert(log.Errors.Count == 0);
tokens = GetRubyTokens(log, "print '");
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.Error
);
Assert(tokens.Count == 4 &&
tokens[0] == Tokens.Identifier &&
tokens[1] == Tokens.StringBegin &&
tokens[2] == Tokens.StringEnd &&
tokens[3] == Tokens.EndOfFile);
log.Errors.Clear();
tokens = GetRubyTokens(log, "f (1,2) {}");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.LeftArgParenthesis,
Tokens.Integer,
Tokens.Comma,
Tokens.Integer,
Tokens.RightParenthesis,
Tokens.LeftBlockArgBrace,
Tokens.RightBrace,
Tokens.EndOfFile,
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.Warning
);
log.Errors.Clear();
tokens = GetRubyTokens(log, "f \"a#{g.h /x/}\" do\nend");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.StringBegin,
Tokens.StringContent,
Tokens.StringEmbeddedCodeBegin,
Tokens.Identifier,
Tokens.Dot,
Tokens.Identifier,
Tokens.RegexpBegin,
Tokens.StringContent,
Tokens.RegexpEnd,
Tokens.StringEmbeddedCodeEnd,
Tokens.StringEnd,
Tokens.BlockDo,
Tokens.End,
Tokens.EndOfFile
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.Warning
);
log.Errors.Clear();
}
private void TokenizerErrors1() {
LoggingErrorSink log = new LoggingErrorSink();
var tokens = GetRubyTokens(log, "def [");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Def,
Tokens.LeftIndexingBracket,
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.FatalError
);
log.Errors.Clear();
tokens = GetRubyTokens(log, "${");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Dollar,
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.FatalError
);
log.Errors.Clear();
}
public void Parser1() {
LoggingErrorSink log = new LoggingErrorSink();
List<Tokens> tokens;
tokens = GetRubyTokens(log, "foo (while (f do end) do end) do end");
Assert(tokens.Count == 14);
Assert(tokens[0] == Tokens.Identifier);
Assert(tokens[1] == Tokens.LeftArgParenthesis);
Assert(tokens[2] == Tokens.While);
Assert(tokens[3] == Tokens.LeftExprParenthesis);
Assert(tokens[4] == Tokens.Identifier);
Assert(tokens[5] == Tokens.Do);
Assert(tokens[6] == Tokens.End);
Assert(tokens[7] == Tokens.RightParenthesis);
Assert(tokens[8] == Tokens.LoopDo);
Assert(tokens[9] == Tokens.End);
Assert(tokens[10] == Tokens.RightParenthesis);
Assert(tokens[11] == Tokens.BlockDo);
Assert(tokens[12] == Tokens.End);
Assert(tokens[13] == Tokens.EndOfFile);
log.Errors.Clear();
tokens = GetRubyTokens(log, "print 'foo'");
Assert(log.Errors.Count == 0);
Assert(tokens.Count == 5 &&
tokens[0] == Tokens.Identifier &&
tokens[1] == Tokens.StringBegin &&
tokens[2] == Tokens.StringContent &&
tokens[3] == Tokens.StringEnd &&
tokens[4] == Tokens.EndOfFile);
Assert(log.Errors.Count == 0);
tokens = GetRubyTokens(log, "print '");
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.Error
);
Assert(tokens.Count == 4 &&
tokens[0] == Tokens.Identifier &&
tokens[1] == Tokens.StringBegin &&
tokens[2] == Tokens.StringEnd &&
tokens[3] == Tokens.EndOfFile);
log.Errors.Clear();
tokens = GetRubyTokens(log, "f \"a#{g.h /x/}\" do\nend");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.StringBegin,
Tokens.StringContent,
Tokens.StringEmbeddedCodeBegin,
Tokens.Identifier,
Tokens.Dot,
Tokens.Identifier,
Tokens.RegexpBegin,
Tokens.StringContent,
Tokens.RegexpEnd,
Tokens.StringEmbeddedCodeEnd,
Tokens.StringEnd,
Tokens.BlockDo,
Tokens.End,
Tokens.EndOfFile
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.Warning
);
log.Errors.Clear();
// 1.9:
tokens = GetRubyTokens(log, "def !@\nend");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Def,
Tokens.Bang,
Tokens.NewLine,
Tokens.End,
Tokens.EndOfFile
}));
Assert(log.Errors.Count == 0);
// 1.9:
tokens = GetRubyTokens(log, "puts <<L, foo\n .bar\nL\n .baz");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.StringBegin,
Tokens.StringContent,
Tokens.StringEnd,
Tokens.Comma,
Tokens.Identifier,
Tokens.Dot,
Tokens.Identifier,
Tokens.EndOfFile,
}));
Assert(log.Errors.Count == 0);
// EXPR_VALUE is included in IsBeginLexicalState (discriminates left bracket kind):
tokens = GetRubyTokens(log, "x = 1 ? [] : []");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.Assignment,
Tokens.Integer,
Tokens.QuestionMark,
Tokens.LeftBracket,
Tokens.RightBracket,
Tokens.Colon,
Tokens.LeftBracket,
Tokens.RightBracket,
Tokens.EndOfFile,
}));
Assert(log.Errors.Count == 0);
// Default block parameter is different from default method parameter.
// Block parameter doesn't allow using binary expressions etc. due to ambiguity with binary OR operator.
tokens = GetRubyTokens(log, "lambda {|x=1|}");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.LeftBlockBrace,
Tokens.Pipe,
Tokens.Identifier,
Tokens.Assignment,
Tokens.Integer,
Tokens.Pipe,
Tokens.RightBrace,
Tokens.EndOfFile,
}));
Assert(log.Errors.Count == 0);
tokens = GetRubyTokens(log, "lambda { ->(){} }");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.LeftBlockBrace,
Tokens.Lambda,
Tokens.LeftParenthesis,
Tokens.RightParenthesis,
Tokens.LeftLambdaBrace,
Tokens.RightBrace,
Tokens.RightBrace,
Tokens.EndOfFile,
}));
Assert(log.Errors.Count == 0);
tokens = GetRubyTokens(log, "\"#{->{ }}\"\n-> do\nend");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.StringBegin,
Tokens.StringEmbeddedCodeBegin,
Tokens.Lambda,
Tokens.LeftLambdaBrace,
Tokens.RightBrace,
Tokens.StringEmbeddedCodeEnd,
Tokens.StringEnd,
Tokens.NewLine,
Tokens.Lambda,
Tokens.LambdaDo,
Tokens.NewLine,
Tokens.End,
Tokens.EndOfFile,
}));
Assert(log.Errors.Count == 0);
}
private void ParserErrors1() {
LoggingErrorSink log = new LoggingErrorSink();
var tokens = GetRubyTokens(log, "def [");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Def,
Tokens.LeftIndexingBracket,
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.FatalError
);
log.Errors.Clear();
tokens = GetRubyTokens(log, "${");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Dollar,
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.FatalError
);
log.Errors.Clear();
// Error: block argument should not be given (yield as a command):
tokens = GetRubyTokens(log, "a[yield 1, &x]");
Assert(tokens.ToArray().ValueEquals(new[] {
Tokens.Identifier,
Tokens.LeftIndexingBracket,
Tokens.Yield,
Tokens.Integer,
Tokens.Comma,
Tokens.BlockReference,
Tokens.Identifier,
Tokens.RightBracket,
Tokens.EndOfFile,
}));
Assert(log.Errors.Count == 1 &&
log.Errors[0].Severity == Severity.Error
);
log.Errors.Clear();
// restrictions on jump statements:
foreach (var src in new[] {
"a[return 1, &x]",
"puts if return 1",
"1 && return 1",
"(a ? return 1 : 1) while true",
"a while return",
"1 + return",
"return / 1",
"a = return",
"return.foo = bar",
"a = 1 rescue return 2",
}) {
tokens = GetRubyTokens(log, src);
Assert(log.Errors.Count > 0 && log.Errors.Exists((e) => e.Severity == Severity.FatalError));
log.Errors.Clear();
}
// Error: Duplicate parameter name:
tokens = GetRubyTokens(log, "lambda { |x,x| }");
Assert(log.Errors.Count == 1 && log.Errors[0].Severity == Severity.Error);
log.Errors.Clear();
// Underscore may be duplicated:
tokens = GetRubyTokens(log, "lambda { |_,_| }");
Assert(log.Errors.Count == 0);
log.Errors.Clear();
}
public AssertTokenizer(RubyContext/*!*/ context) {
_log = new LoggingErrorSink();
_context = context;
}
