public CoverageParserLogger(Parser/*!*/ parser, TextWriter/*!*/ output)
{
Assert.NotNull(parser, output);
_parser = parser;
_output = output;
_rules = new Stack<int>();
}
public DefaultParserLogger(Parser/*!*/ parser, int verbosity, TextWriter/*!*/ output) {
Assert.NotNull(parser, output);
Debug.Assert(verbosity > 0 && verbosity <= 2);
_parser = parser;
_logVerbosity = verbosity;
_log = output;
}
public void Parse()
{
var srcunit = HostingHelpers.GetSourceUnit(source);
var parser = new IronRuby.Compiler.Parser(errorSink);
ast = parser.Parse(srcunit, new RubyCompilerOptions(), errorSink);
var transformer = new Transformer.Transformer(ast);
transformations = transformer.Transform();
}
public void Parse()
{
var srcunit = HostingHelpers.GetSourceUnit(source);
var parser = new IronRuby.Compiler.Parser(errorSink);
ast = parser.Parse(srcunit, new RubyCompilerOptions(), errorSink);
var transformer = new Transformer.Transformer(ast);
transformations = transformer.Transform();
}
public SourceUnitTree CreateAst(string fileName, ITextBuffer fileContent)
{
if (scriptEngine == null) {
scriptEngine = Ruby.CreateEngine();
}
RubyContext rubyContext = HostingHelpers.GetLanguageContext(scriptEngine) as RubyContext;
sourceUnit = rubyContext.CreateFileUnit(fileName, fileContent.Text);
RubyCompilerSink sink = new RubyCompilerSink();
RubyCompilerOptions compilerOptions = new RubyCompilerOptions((RubyOptions)rubyContext.Options);
Parser parser = new Parser();
return parser.Parse(sourceUnit, compilerOptions, sink);
}
/*!*/
internal static LeftValue MakeLeftValue(int kind, Parser/*!*/ parser, string name, SourceSpan location)
{
switch (kind) {
case Identifier:
return parser.CurrentScope.ResolveOrAddVariable(name, location);
case Instance:
return new InstanceVariable(name, location);
case Global:
return new GlobalVariable(name, location);
case Constant:
return new ConstantVariable(name, location);
case Class:
return new ClassVariable(name, location);
case Nil:
return parser.CannotAssignError("nil", location);
case Self:
return parser.CannotAssignError("self", location);
case True:
return parser.CannotAssignError("true", location);
case False:
return parser.CannotAssignError("false", location);
case File:
return parser.CannotAssignError("__FILE__", location);
case Line:
return parser.CannotAssignError("__LINE__", location);
case Encoding:
return parser.CannotAssignError("__ENCODING__", location);
}
return null;
}
internal static Expression/*!*/ MakeRead(int kind, Parser/*!*/ parser, string name, SourceSpan location) {
switch (kind) {
case Identifier:
return (Expression)parser.CurrentScope.ResolveVariable(name) ?? new MethodCall(null, name, null, null, location);
case Instance:
return new InstanceVariable(name, location);
case Global:
return new GlobalVariable(name, location);
case Constant:
return new ConstantVariable(name, location);
case Class:
return new ClassVariable(name, location);
case Nil:
return Literal.Nil(location);
case Self:
return new SelfReference(location);
case True:
return Literal.True(location);
case False:
return Literal.False(location);
case File:
return new FileLiteral(location);
case Line:
return Literal.Integer(parser.Tokenizer.TokenSpan.Start.Line, location);
case Encoding:
return new EncodingExpression(location);
}
throw Assert.Unreachable;
}
private void ParserLoggingTest() {
#if DEBUG
string source = "def foo(a); end";
var sourceUnit = Context.CreateSnippet(source, SourceCodeKind.Statements);
var options = new RubyCompilerOptions();
string temp = Path.Combine(Path.GetTempPath(), "RubyParser");
Console.WriteLine("> see {0}", temp);
Directory.CreateDirectory(temp);
Parser parser = new Parser();
using (TextWriter writer = File.CreateText(Path.Combine(temp, "default.log"))) {
DefaultParserLogger.Attach(parser, writer);
parser.Parse(sourceUnit, options, ErrorSink.Null);
}
using (TextWriter writer = File.CreateText(Path.Combine(temp, "tables.csv"))) {
parser.DumpTables(writer);
}
using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) {
for (int i = 0; i < parser.Rules.Length; i++) {
writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i));
}
}
parser = new Parser();
using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) {
for (int i = 0; i < parser.Rules.Length; i++) {
writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i));
}
}
using (TextWriter writer = File.CreateText(Path.Combine(temp, "second_order.log"))) {
parser.EnableLogging(new CoverageParserLogger(parser, writer));
parser.Parse(sourceUnit, options, ErrorSink.Null);
}
#endif
}
public static void Attach(Parser/*!*/ parser, TextWriter/*!*/ output) {
parser.EnableLogging(new DefaultParserLogger(parser, 1, output));
}
public static void Attach(Parser/*!*/ parser) {
Attach(parser, Console.Out);
}
public static void Attach(Parser/*!*/ parser)
{
Attach(parser, Driver.Output);
}
public void Scenario_RubyScopeParsing() {
LoggingErrorSink log = new LoggingErrorSink();
SourceUnitTree p;
SourceUnit unit;
unit = Context.CreateSnippet(@"
class c << G
end
", SourceCodeKind.File);
p = new Parser().Parse(unit, new RubyCompilerOptions(), log);
Assert(p == null && log.FatalErrorCount == 1);
log.ClearCounters();
unit = Context.CreateSnippet(@"
def goo(&b)
end
class C
def foo()
x.goo() {
goo() {
class << M
def bar()
goo() {
}
end
end
}
}
end
end
BEGIN { goo1() { } }
END { goo2() { } }
", SourceCodeKind.File);
p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null);
Assert(p != null && !log.AnyError);
log.ClearCounters();
unit = Context.CreateSnippet(@"
for x in array do
goo()
end
", SourceCodeKind.File);
p = new Parser().Parse(unit, new RubyCompilerOptions(), ErrorSink.Null);
Assert(p != null && !log.AnyError);
log.ClearCounters();
}
private void AstLocations1() {
// DumpExpression uses private reflection:
if (_driver.PartialTrust) return;
var sourceUnit = Context.CreateSnippet(@"
def add a,b
a + b
end
add 1, 1
add 'foo', 'bar'
", SourceCodeKind.Expression);
var options = new RubyCompilerOptions();
var parser = new Parser();
var tokens = new List<KeyValuePair<SourceSpan, Tokens>>();
parser.TokenSink = (token, span) => { tokens.Add(new KeyValuePair<SourceSpan, Tokens>(span, token)); };
var ast = parser.Parse(sourceUnit, options, Context.RuntimeErrorSink);
const int Id = 0x12345678;
var lambda = CallSiteTracer.Transform<Func<Scope, LanguageContext, object>>(ast, sourceUnit, options, Id);
var code = new RubyScriptCode(lambda, sourceUnit);
var locations = new List<int>();
CallSiteTracer.Register((context, args, result, id, location) => {
locations.Add(location);
Debug.Assert(id == Id);
Debug.Assert(location > 0);
//Console.WriteLine("-- {0} ---------", location);
//Console.WriteLine(this);
//Console.WriteLine(AstUtils.DumpExpression(result.Restrictions.ToExpression()));
//Console.WriteLine();
//Console.WriteLine(AstUtils.DumpExpression(result.Expression));
//Console.WriteLine("----------------");
});
code.Run();
Debug.Assert(locations.Count == 4 && locations[0] == 31 && locations[1] == 19 && locations[2] == 41 && locations[3] == 19);
}
private void AssertTokens(string/*!*/ source, params Tokens[] expected) {
var tokens = new List<Tokens>();
var parser = new Parser() {
TokenSink = (token, span) => tokens.Add(token)
};
parser.Parse(Context.CreateSnippet(source, SourceCodeKind.AutoDetect), new RubyCompilerOptions(), ErrorSink.Null);
Assert(tokens.ToArray().ValueEquals(expected));
}
public Tokenizer(bool verbatim, ErrorSink/*!*/ errorSink) {
ContractUtils.RequiresNotNull(errorSink, "errorSink");
_bigIntParser = new BignumParser();
_errorSink = errorSink;
_sourceUnit = null;
_parser = null;
_verbatim = verbatim;
_compatibility = RubyCompatibility.Default;
// _buffer = null;
_initialLocation = SourceLocation.Invalid;
_tokenSpan = SourceSpan.Invalid;
_tokenValue = new TokenValue();
_bufferPos = 0;
// TODO:
_input = null;
}
private void Benchmark(List<string>/*!*/ files) {
var sources = new List<SourceUnit>();
Stopwatch readTime = new Stopwatch();
long totalSize = 0;
readTime.Start();
foreach (string path in files) {
try {
byte[] data = File.ReadAllBytes(path);
sources.Add(_context.CreateSourceUnit(new BinaryContentProvider(data), path, Encoding.Default, SourceCodeKind.File));
totalSize += data.Length;
} catch (Exception) {
Console.WriteLine("Error: {0}", path);
}
}
readTime.Stop();
Console.WriteLine("Read: {0} kB in {1}", totalSize / 1024, readTime.Elapsed);
#if F
Stopwatch tokenizeTime = new Stopwatch();
tokenizeTime.Start();
foreach (var source in sources) {
try {
var tokenizer = new Tokenizer();
tokenizer.Initialize(source);
Tokens token;
do {
token = tokenizer.GetNextToken();
} while (token != Tokens.EndOfFile);
} catch (Exception) {
Console.WriteLine("Tokenization error: {0}", source.Path);
break;
}
}
tokenizeTime.Stop();
#endif
//var stackSizes = new Dictionary<int, int>();
var options = new RubyCompilerOptions();
Stopwatch parseTime = new Stopwatch();
Stopwatch transformTime = new Stopwatch();
foreach (var source in sources) {
try {
parseTime.Start();
var parser = new Parser();
var rubyTree = parser.Parse(source, options, ErrorSink.Null);
//int mt;
//stackSizes[parser.StackMaxTop] = stackSizes.TryGetValue(parser.StackMaxTop, out mt) ? mt + 1 : 1;
parseTime.Stop();
#if F
if (rubyTree != null) {
transformTime.Start();
var lambda = _context.TransformTree<DlrMainCallTarget>(rubyTree, source, options);
transformTime.Stop();
} else {
Console.WriteLine("SyntaxError: {0}", source.Path);
}
#endif
} catch (Exception e) {
Console.WriteLine("{0}: {1}: {2}", e.GetType().Name, source.Path, e.Message);
break;
}
}
// Console.WriteLine("Tokenize: {0}", tokenizeTime.Elapsed);
Console.WriteLine("Parse: {0}", parseTime.Elapsed);
//Console.WriteLine("Idf/Kwd/Loc: {0}/{1}/{2}", Tokenizer.IdfLength, Tokenizer.KwdLength, Tokenizer.LocLength);
// Console.WriteLine("Transform: {0}", transformTime.Elapsed);
//PerfTrack.DumpHistogram(Parser.Reductions);
//PerfTrack.DumpHistogram(stackSizes);
}
private void DumpFile(string/*!*/ path) {
_log.WriteLine(path);
try {
string fullPath = Path.GetFullPath(path);
string root = Path.GetPathRoot(fullPath);
string outputPath = Path.ChangeExtension(Path.Combine(_targetDir, fullPath.Substring(root.Length).TrimEnd('\'', '/')), ".txt");
Directory.CreateDirectory(Path.GetDirectoryName(outputPath));
using (TextWriter output = File.CreateText(outputPath)) {
output.WriteLine(fullPath);
output.WriteLine();
output.WriteLine("Tokens:");
output.WriteLine();
ErrorLog errors = new ErrorLog();
Parser parser = new Parser();
parser.TokenSink = delegate(Tokens token, SourceSpan span) {
DumpTokenDetail(output, parser.Tokenizer, token);
};
if (_logProductions) {
#if DEBUG
parser.EnableLogging(new CoverageParserLogger(parser, _parserLog));
#endif
}
_currentSourceFile = path;
SourceUnitTree ast = null;
try {
ast = parser.Parse(_context.CreateFileUnit(path), new RubyCompilerOptions(), errors);
} catch (Exception e) {
WriteException(e.ToString());
}
output.WriteLine();
if (errors.ErrorCount + errors.FatalErrorCount + errors.WarningCount > 0) {
output.WriteLine();
output.WriteLine("Errors:");
foreach (string error in errors.Errors) {
output.WriteLine(error);
}
} else {
Debug.Assert(ast != null);
DumpRegularExpressions(ast);
}
output.WriteLine(".");
}
} catch (Exception e) {
_log.WriteLine("!{0}", e.Message);
} finally {
_currentSourceFile = null;
_regexLog.Flush();
_parserLog.Flush();
}
}
internal Tokenizer(Parser/*!*/ parser)
: this(false) {
_parser = parser;
}