/// <summary> /// Initialize /// </summary> public ParserBase(Context context) { _context = context; _parseErrors = new List<LangException>(); _lexer = new Lexer(""); _lexer.SetContext(_context); }
/// <summary> /// Convert the script to a series of tokens. /// </summary> /// <param name="script">The script content or file name</param> /// <param name="isFile">Whether or not the script supplied is a filename or actual script content</param> /// <returns></returns> public List<TokenData> ToTokens(string script, bool isFile) { List<TokenData> tokens = null; if (isFile) { script = ReadFile(script); } Lexer lexer = null; if (_parser == null) { lexer = new Lexer(script); lexer.SetContext(_context); } else { lexer = _parser.Lexer; lexer.Init(script); } //lexer.DiagnosticData.Reset(); Execute(() => { tokens = lexer.Tokenize(); }, () => string.Format("Last token: {0}, Line : {1}, Pos : {2} ", lexer.LastToken.Text, lexer.State.Line, lexer.State.LineCharPosition)); //Console.WriteLine("total : " + lexer.DiagnosticData.TotalTokens); //Console.WriteLine("total white space: " + lexer.DiagnosticData.TotalWhiteSpaceTokens); //Console.WriteLine("total new line : " + lexer.DiagnosticData.TotalNewLineTokens); return tokens; }