public AssertTokenizer/*!*/ Load(string/*!*/ source) {
Tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)");
_tokenizer = new Tokenizer(false, _log);
_tokenizer.Compatibility = _context.RubyOptions.Compatibility;
_tokenizer.Initialize(_context.CreateSnippet(source, SourceCodeKind.File));
_allTokens = new List<Tokens>();
_allValues = new List<object>();
return this;
}
public AssertTokenizer/*!*/ Load(byte[]/*!*/ source) {
Tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)");
_tokenizer = new Tokenizer(false, _log);
_tokenizer.Compatibility = _context.RubyOptions.Compatibility;
_tokenizer.Initialize(_context.CreateSourceUnit(
new BinaryContentProvider(source), null, BinaryEncoding.Instance, SourceCodeKind.File)
);
_allTokens = new List<Tokens>();
_allValues = new List<object>();
return this;
}
public void DumpTokenDetail(TextWriter/*!*/ output, Tokenizer/*!*/ tokenizer, Tokens token)
{
TokenValue value = tokenizer.TokenValue;
output.Write("{0}: ", Parser.GetTerminalName((int)token));
switch (token) {
default:
break;
case Tokens.Identifier:
output.Write(value.String);
break;
case Tokens.Float:
output.Write("{0}D", value.Double);
break;
case Tokens.Integer:
output.Write(value.Integer1);
break;
case Tokens.BigInteger:
output.Write("{0}BI", value.BigInteger.ToString(10));
break;
case Tokens.RegexpEnd:
output.Write("RegexOptions({0})", (RubyRegexOptions)value.Integer1);
break;
case Tokens.StringContent:
if (value.StringContent is string) {
output.Write("String(\"{0}\", {1})", Parser.EscapeString((string)value.StringContent), value.Encoding);
} else {
output.Write("String({0}), {1}", BitConverter.ToString((byte[])value.StringContent), value.Encoding);
}
break;
case Tokens.StringBegin:
case Tokens.RegexpBegin:
case Tokens.ShellStringBegin:
case Tokens.SymbolBegin:
output.Write(((Tokenizer.State)tokenizer.CurrentState).CurrentSequence);
break;
}
output.Write(' ');
output.Write(tokenizer.LexicalState);
output.WriteLine();
}
// This method uses the tokenizer to auto-detect the base type -- happens when agument to to_i is 0
private static object ParseInteger(RubyContext/*!*/ context, string/*!*/ str) {
bool isNegative = false;
if (str.Length == 0) {
return ScriptingRuntimeHelpers.Int32ToObject(0);
}
str = ParseSign(str, ref isNegative);
Tokenizer tokenizer = new Tokenizer(false, ErrorSink.Null);
tokenizer.Initialize(context.CreateSnippet(str, SourceCodeKind.File));
tokenizer.GetNextToken();
TokenValue value = tokenizer.TokenValue;
TokenValueType type = value.Type;
if (type == TokenValueType.Integer)
return ScriptingRuntimeHelpers.Int32ToObject(isNegative ? -value.Integer: value.Integer);
else if (type == TokenValueType.BigInteger)
return isNegative ? BigInteger.Negate(value.BigInteger) : value.BigInteger;
else
return ScriptingRuntimeHelpers.Int32ToObject(0);
}
private static int ParseInteger(RubyContext/*!*/ context, string/*!*/ str) {
bool isNegative = false;
if (str[0] == '-') {
isNegative = true;
str = str.Remove(0, 1);
}
Tokenizer tokenizer = new Tokenizer(false, ErrorSink.Null);
tokenizer.Initialize(context.CreateSnippet(str, SourceCodeKind.File));
tokenizer.GetNextToken();
TokenValue value = tokenizer.TokenValue;
TokenValueType type = value.Type;
tokenizer.GetNextToken();
TokenValueType nextType = tokenizer.TokenValue.Type;
// We are only interested in the whole string being a valid Integer
if (type == TokenValueType.Integer && nextType == TokenValueType.None) {
return isNegative ? -value.Integer : value.Integer;
} else {
throw RubyExceptions.CreateTypeConversionError("String", "Integer");
}
}
private static int ParseInteger(RubyContext/*!*/ context, string/*!*/ str) {
bool isNegative = false;
if (str[0] == '-') {
isNegative = true;
str = str.Remove(0, 1);
}
Tokenizer tokenizer = new Tokenizer();
tokenizer.Initialize(new StringReader(str));
Tokens token = tokenizer.GetNextToken();
TokenValue value = tokenizer.TokenValue;
Tokens nextToken = tokenizer.GetNextToken();
// We are only interested in the whole string being a valid Integer
if (token == Tokens.Integer && nextToken == Tokens.Integer) {
return isNegative ? -value.Integer1 : value.Integer1;
} else {
throw RubyExceptions.CreateTypeConversionError("String", "Integer");
}
}
public virtual List<TokenInfo> GetTokenInfos(string code)
{
var tokenizer = new Tokenizer(true);
tokenizer.Initialize(CreateSourceUnit(code));
return new List<TokenInfo>(tokenizer.ReadTokens(code.Length));
}
private void Benchmark(List<string>/*!*/ files) {
var sources = new List<SourceUnit>();
Stopwatch readTime = new Stopwatch();
long totalSize = 0;
readTime.Start();
foreach (string path in files) {
try {
byte[] data = File.ReadAllBytes(path);
sources.Add(_context.CreateSourceUnit(new BinaryContentProvider(data), path, Encoding.Default, SourceCodeKind.File));
totalSize += data.Length;
} catch (Exception) {
Console.WriteLine("Error: {0}", path);
}
}
readTime.Stop();
Console.WriteLine("Read: {0} kB in {1}", totalSize / 1024, readTime.Elapsed);
#if F
Stopwatch tokenizeTime = new Stopwatch();
tokenizeTime.Start();
foreach (var source in sources) {
try {
var tokenizer = new Tokenizer();
tokenizer.Initialize(source);
Tokens token;
do {
token = tokenizer.GetNextToken();
} while (token != Tokens.EndOfFile);
} catch (Exception) {
Console.WriteLine("Tokenization error: {0}", source.Path);
break;
}
}
tokenizeTime.Stop();
#endif
//var stackSizes = new Dictionary<int, int>();
var options = new RubyCompilerOptions();
Stopwatch parseTime = new Stopwatch();
Stopwatch transformTime = new Stopwatch();
foreach (var source in sources) {
try {
parseTime.Start();
var parser = new Parser();
var rubyTree = parser.Parse(source, options, ErrorSink.Null);
//int mt;
//stackSizes[parser.StackMaxTop] = stackSizes.TryGetValue(parser.StackMaxTop, out mt) ? mt + 1 : 1;
parseTime.Stop();
#if F
if (rubyTree != null) {
transformTime.Start();
var lambda = _context.TransformTree<DlrMainCallTarget>(rubyTree, source, options);
transformTime.Stop();
} else {
Console.WriteLine("SyntaxError: {0}", source.Path);
}
#endif
} catch (Exception e) {
Console.WriteLine("{0}: {1}: {2}", e.GetType().Name, source.Path, e.Message);
break;
}
}
// Console.WriteLine("Tokenize: {0}", tokenizeTime.Elapsed);
Console.WriteLine("Parse: {0}", parseTime.Elapsed);
//Console.WriteLine("Idf/Kwd/Loc: {0}/{1}/{2}", Tokenizer.IdfLength, Tokenizer.KwdLength, Tokenizer.LocLength);
// Console.WriteLine("Transform: {0}", transformTime.Elapsed);
//PerfTrack.DumpHistogram(Parser.Reductions);
//PerfTrack.DumpHistogram(stackSizes);
}
internal override Tokens Tokenize(Tokenizer/*!*/ tokenizer) {
return tokenizer.TokenizeString(this);
}
internal abstract Tokens Tokenize(Tokenizer/*!*/ tokenizer);
internal override Tokens Tokenize(Tokenizer/*!*/ tokenizer) {
return tokenizer.TokenizeHeredoc(this);
}
public AssertTokenizer/*!*/ Load(object/*!*/ source) { // source: byte[] or string
_tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)");
SourceUnit sourceUnit;
RubyEncoding encoding;
byte[] binarySource = source as byte[];
if (binarySource != null) {
encoding = RubyEncoding.Binary;
sourceUnit = _context.CreateSourceUnit(new BinaryContentProvider(binarySource), null, encoding.Encoding, SourceCodeKind.File);
} else {
encoding = DefaultEncoding;
sourceUnit = _context.CreateSnippet((string)source, SourceCodeKind.File);
}
_tokenizer = new Tokenizer(false, DummyVariableResolver.AllMethodNames) {
ErrorSink = _log,
Compatibility = Compatibility,
Encoding = encoding
};
_tokenizer.Initialize(sourceUnit);
_allTokens = new List<Tokens>();
_allValues = new List<object>();
return this;
}
public void DumpTokenDetail(TextWriter/*!*/ output, Tokenizer/*!*/ tokenizer, Tokens token) {
TokenValue value = tokenizer.TokenValue;
output.Write("{0}: ", Parser.TerminalToString((int)token));
switch (token) {
default:
break;
case Tokens.Float:
output.Write("{0}D", value.Double);
break;
case Tokens.Integer:
output.Write(value.Integer1);
break;
case Tokens.BigInteger:
output.Write("{0}BI", value.BigInteger.ToString(10));
break;
case Tokens.RegexpEnd:
output.Write("RegexOptions({0})", (RubyRegexOptions)value.Integer1);
break;
case Tokens.StringContent:
output.Write("String(\"{0}\")", Parser.EscapeString(value.String));
break;
case Tokens.StringBegin:
case Tokens.RegexpBegin:
case Tokens.ShellStringBegin:
case Tokens.SymbolBegin:
output.Write(value.StringTokenizer);
break;
}
output.Write(' ');
output.Write(tokenizer.LexicalState);
output.WriteLine();
}
public void DumpTokenDetail(TextWriter/*!*/ output, Tokenizer/*!*/ tokenizer, Tokens token) {
TokenValue value = tokenizer.TokenValue;
output.Write("{0}: ", Parser.TerminalToString((int)token));
switch (value.Type) {
case TokenValueType.None:
break;
case TokenValueType.Double:
output.Write("{0}D", value.Double);
break;
case TokenValueType.Integer:
output.Write(value.Integer);
break;
case TokenValueType.BigInteger:
output.Write("{0}BI", value.BigInteger.ToString(10));
break;
case TokenValueType.RegexOptions:
output.Write("RegexOptions({0})", (RubyRegexOptions)value.Integer);
break;
case TokenValueType.String:
output.Write("String(\"{0}\")", Parser.EscapeString(value.String));
break;
case TokenValueType.StringTokenizer:
output.Write(value.StringTokenizer);
break;
}
output.Write(' ');
output.Write(tokenizer.LexicalState);
output.WriteLine();
}
