public static List<Tokens> GetRubyTokens(RubyContext context, ErrorSink log, string source, bool dumpTokens, bool dumpReductions) {
Parser parser = new Parser();
List<Tokens> tokens = new List<Tokens>();
if (dumpTokens) {
parser.Tokenizer.EnableLogging(1, Console.Out);
}
parser.TokenSink = delegate(Tokens token, SourceSpan span) {
tokens.Add(token);
};
#if DEBUG
if (dumpReductions) {
DefaultParserLogger.Attach(parser, Console.Out);
}
#endif
parser.Parse(context.CreateSnippet(source, SourceCodeKind.File), new RubyCompilerOptions(), log);
//tokenizer.Initialize(SourceUnit.CreateSnippet(RB, source));
//List<Tokens> tokens = new List<Tokens>();
//Tokens token;
//while ((token = tokenizer.GetNextToken()) != Tokens.EOF) {
// tokens.Add(token);
//}
return tokens;
}
private void ParserLoggingTest()
{
#if DEBUG
string source = "def foo(a); end";
var sourceUnit = Context.CreateSnippet(source, SourceCodeKind.Statements);
var options = new RubyCompilerOptions();
string temp = Path.Combine(Path.GetTempPath(), "RubyParser");
Console.WriteLine("> see {0}", temp);
Directory.CreateDirectory(temp);
Parser parser = new Parser();
using (TextWriter writer = File.CreateText(Path.Combine(temp, "default.log"))) {
DefaultParserLogger.Attach(parser, writer);
parser.Parse(sourceUnit, options, ErrorSink.Null);
}
using (TextWriter writer = File.CreateText(Path.Combine(temp, "tables.csv"))) {
parser.DumpTables(writer);
}
using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) {
for (int i = 0; i < parser.RuleLhsNonTerminals.Length; i++)
{
writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i));
}
}
parser = new Parser();
using (TextWriter writer = File.CreateText(Path.Combine(temp, "productions.txt"))) {
for (int i = 0; i < parser.RuleLhsNonTerminals.Length; i++)
{
writer.WriteLine("{0}\t{1}", i, parser.RuleToString(i));
}
}
using (TextWriter writer = File.CreateText(Path.Combine(temp, "second_order.log"))) {
parser.EnableLogging(new CoverageParserLogger(parser, writer));
parser.Parse(sourceUnit, options, ErrorSink.Null);
}
#endif
}