public AssertTokenizer/*!*/ Load(string/*!*/ source) { Tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)"); _tokenizer = new Tokenizer(false, _log); _tokenizer.Compatibility = _context.RubyOptions.Compatibility; _tokenizer.Initialize(_context.CreateSnippet(source, SourceCodeKind.File)); _allTokens = new List<Tokens>(); _allValues = new List<object>(); return this; }
public AssertTokenizer/*!*/ Load(byte[]/*!*/ source) { Tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)"); _tokenizer = new Tokenizer(false, _log); _tokenizer.Compatibility = _context.RubyOptions.Compatibility; _tokenizer.Initialize(_context.CreateSourceUnit( new BinaryContentProvider(source), null, BinaryEncoding.Instance, SourceCodeKind.File) ); _allTokens = new List<Tokens>(); _allValues = new List<object>(); return this; }
public void DumpTokenDetail(TextWriter/*!*/ output, Tokenizer/*!*/ tokenizer, Tokens token) { TokenValue value = tokenizer.TokenValue; output.Write("{0}: ", Parser.GetTerminalName((int)token)); switch (token) { default: break; case Tokens.Identifier: output.Write(value.String); break; case Tokens.Float: output.Write("{0}D", value.Double); break; case Tokens.Integer: output.Write(value.Integer1); break; case Tokens.BigInteger: output.Write("{0}BI", value.BigInteger.ToString(10)); break; case Tokens.RegexpEnd: output.Write("RegexOptions({0})", (RubyRegexOptions)value.Integer1); break; case Tokens.StringContent: if (value.StringContent is string) { output.Write("String(\"{0}\", {1})", Parser.EscapeString((string)value.StringContent), value.Encoding); } else { output.Write("String({0}), {1}", BitConverter.ToString((byte[])value.StringContent), value.Encoding); } break; case Tokens.StringBegin: case Tokens.RegexpBegin: case Tokens.ShellStringBegin: case Tokens.SymbolBegin: output.Write(((Tokenizer.State)tokenizer.CurrentState).CurrentSequence); break; } output.Write(' '); output.Write(tokenizer.LexicalState); output.WriteLine(); }
// This method uses the tokenizer to auto-detect the base type -- happens when agument to to_i is 0 private static object ParseInteger(RubyContext/*!*/ context, string/*!*/ str) { bool isNegative = false; if (str.Length == 0) { return ScriptingRuntimeHelpers.Int32ToObject(0); } str = ParseSign(str, ref isNegative); Tokenizer tokenizer = new Tokenizer(false, ErrorSink.Null); tokenizer.Initialize(context.CreateSnippet(str, SourceCodeKind.File)); tokenizer.GetNextToken(); TokenValue value = tokenizer.TokenValue; TokenValueType type = value.Type; if (type == TokenValueType.Integer) return ScriptingRuntimeHelpers.Int32ToObject(isNegative ? -value.Integer: value.Integer); else if (type == TokenValueType.BigInteger) return isNegative ? BigInteger.Negate(value.BigInteger) : value.BigInteger; else return ScriptingRuntimeHelpers.Int32ToObject(0); }
private static int ParseInteger(RubyContext/*!*/ context, string/*!*/ str) { bool isNegative = false; if (str[0] == '-') { isNegative = true; str = str.Remove(0, 1); } Tokenizer tokenizer = new Tokenizer(false, ErrorSink.Null); tokenizer.Initialize(context.CreateSnippet(str, SourceCodeKind.File)); tokenizer.GetNextToken(); TokenValue value = tokenizer.TokenValue; TokenValueType type = value.Type; tokenizer.GetNextToken(); TokenValueType nextType = tokenizer.TokenValue.Type; // We are only interested in the whole string being a valid Integer if (type == TokenValueType.Integer && nextType == TokenValueType.None) { return isNegative ? -value.Integer : value.Integer; } else { throw RubyExceptions.CreateTypeConversionError("String", "Integer"); } }
private static int ParseInteger(RubyContext/*!*/ context, string/*!*/ str) { bool isNegative = false; if (str[0] == '-') { isNegative = true; str = str.Remove(0, 1); } Tokenizer tokenizer = new Tokenizer(); tokenizer.Initialize(new StringReader(str)); Tokens token = tokenizer.GetNextToken(); TokenValue value = tokenizer.TokenValue; Tokens nextToken = tokenizer.GetNextToken(); // We are only interested in the whole string being a valid Integer if (token == Tokens.Integer && nextToken == Tokens.Integer) { return isNegative ? -value.Integer1 : value.Integer1; } else { throw RubyExceptions.CreateTypeConversionError("String", "Integer"); } }
public virtual List<TokenInfo> GetTokenInfos(string code) { var tokenizer = new Tokenizer(true); tokenizer.Initialize(CreateSourceUnit(code)); return new List<TokenInfo>(tokenizer.ReadTokens(code.Length)); }
private void Benchmark(List<string>/*!*/ files) { var sources = new List<SourceUnit>(); Stopwatch readTime = new Stopwatch(); long totalSize = 0; readTime.Start(); foreach (string path in files) { try { byte[] data = File.ReadAllBytes(path); sources.Add(_context.CreateSourceUnit(new BinaryContentProvider(data), path, Encoding.Default, SourceCodeKind.File)); totalSize += data.Length; } catch (Exception) { Console.WriteLine("Error: {0}", path); } } readTime.Stop(); Console.WriteLine("Read: {0} kB in {1}", totalSize / 1024, readTime.Elapsed); #if F Stopwatch tokenizeTime = new Stopwatch(); tokenizeTime.Start(); foreach (var source in sources) { try { var tokenizer = new Tokenizer(); tokenizer.Initialize(source); Tokens token; do { token = tokenizer.GetNextToken(); } while (token != Tokens.EndOfFile); } catch (Exception) { Console.WriteLine("Tokenization error: {0}", source.Path); break; } } tokenizeTime.Stop(); #endif //var stackSizes = new Dictionary<int, int>(); var options = new RubyCompilerOptions(); Stopwatch parseTime = new Stopwatch(); Stopwatch transformTime = new Stopwatch(); foreach (var source in sources) { try { parseTime.Start(); var parser = new Parser(); var rubyTree = parser.Parse(source, options, ErrorSink.Null); //int mt; //stackSizes[parser.StackMaxTop] = stackSizes.TryGetValue(parser.StackMaxTop, out mt) ? mt + 1 : 1; parseTime.Stop(); #if F if (rubyTree != null) { transformTime.Start(); var lambda = _context.TransformTree<DlrMainCallTarget>(rubyTree, source, options); transformTime.Stop(); } else { Console.WriteLine("SyntaxError: {0}", source.Path); } #endif } catch (Exception e) { Console.WriteLine("{0}: {1}: {2}", e.GetType().Name, source.Path, e.Message); break; } } // Console.WriteLine("Tokenize: {0}", tokenizeTime.Elapsed); Console.WriteLine("Parse: {0}", parseTime.Elapsed); //Console.WriteLine("Idf/Kwd/Loc: {0}/{1}/{2}", Tokenizer.IdfLength, Tokenizer.KwdLength, Tokenizer.LocLength); // Console.WriteLine("Transform: {0}", transformTime.Elapsed); //PerfTrack.DumpHistogram(Parser.Reductions); //PerfTrack.DumpHistogram(stackSizes); }
internal override Tokens Tokenize(Tokenizer/*!*/ tokenizer) { return tokenizer.TokenizeString(this); }
internal abstract Tokens Tokenize(Tokenizer/*!*/ tokenizer);
internal override Tokens Tokenize(Tokenizer/*!*/ tokenizer) { return tokenizer.TokenizeHeredoc(this); }
public AssertTokenizer/*!*/ Load(object/*!*/ source) { // source: byte[] or string _tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)"); SourceUnit sourceUnit; RubyEncoding encoding; byte[] binarySource = source as byte[]; if (binarySource != null) { encoding = RubyEncoding.Binary; sourceUnit = _context.CreateSourceUnit(new BinaryContentProvider(binarySource), null, encoding.Encoding, SourceCodeKind.File); } else { encoding = DefaultEncoding; sourceUnit = _context.CreateSnippet((string)source, SourceCodeKind.File); } _tokenizer = new Tokenizer(false, DummyVariableResolver.AllMethodNames) { ErrorSink = _log, Compatibility = Compatibility, Encoding = encoding }; _tokenizer.Initialize(sourceUnit); _allTokens = new List<Tokens>(); _allValues = new List<object>(); return this; }
public void DumpTokenDetail(TextWriter/*!*/ output, Tokenizer/*!*/ tokenizer, Tokens token) { TokenValue value = tokenizer.TokenValue; output.Write("{0}: ", Parser.TerminalToString((int)token)); switch (token) { default: break; case Tokens.Float: output.Write("{0}D", value.Double); break; case Tokens.Integer: output.Write(value.Integer1); break; case Tokens.BigInteger: output.Write("{0}BI", value.BigInteger.ToString(10)); break; case Tokens.RegexpEnd: output.Write("RegexOptions({0})", (RubyRegexOptions)value.Integer1); break; case Tokens.StringContent: output.Write("String(\"{0}\")", Parser.EscapeString(value.String)); break; case Tokens.StringBegin: case Tokens.RegexpBegin: case Tokens.ShellStringBegin: case Tokens.SymbolBegin: output.Write(value.StringTokenizer); break; } output.Write(' '); output.Write(tokenizer.LexicalState); output.WriteLine(); }
public void DumpTokenDetail(TextWriter/*!*/ output, Tokenizer/*!*/ tokenizer, Tokens token) { TokenValue value = tokenizer.TokenValue; output.Write("{0}: ", Parser.TerminalToString((int)token)); switch (value.Type) { case TokenValueType.None: break; case TokenValueType.Double: output.Write("{0}D", value.Double); break; case TokenValueType.Integer: output.Write(value.Integer); break; case TokenValueType.BigInteger: output.Write("{0}BI", value.BigInteger.ToString(10)); break; case TokenValueType.RegexOptions: output.Write("RegexOptions({0})", (RubyRegexOptions)value.Integer); break; case TokenValueType.String: output.Write("String(\"{0}\")", Parser.EscapeString(value.String)); break; case TokenValueType.StringTokenizer: output.Write(value.StringTokenizer); break; } output.Write(' '); output.Write(tokenizer.LexicalState); output.WriteLine(); }