/// <summary> /// Finds the object that best matches the phrase /// </summary> /// <param name="phrase">The words to match</param> /// <param name="index">The first word in phrase to match</param> /// <param name="score">The score for the best match: 0 if no match[out]</param> /// <param name="numWordsMatch">The number of words that match [out]</param> /// <param name="bestMatch">The object that best matches [out]</param> /// <param name="seen">Set of objects already examined</param> public void match(Lexer lexer, ref double score, ref int numWordsMatch, ref object bestMatch, ref LexState lexerState, Dictionary<object,object> seen) { // Skip if we've already seen this object if (seen.ContainsKey(this)) return; // Add ourself to prevent cycles seen[this] = this; // edge.match(lexer, ref score, ref numWordsMatch, ref bestMatch, ref lexerState, seen); #if true /// We only check to see if the gate matches the phrase if (null != edge.gate) edge.gate.match(lexer, ref score, ref numWordsMatch, ref bestMatch, ref lexerState, seen); // Should only match if no door, or door is open if (null != edge.sink && (null == edge.gate || isOpen(edge.gate))) { // we only want match -- matchInContext checks the stuff from the parents.. edge.sink.match(lexer, ref score, ref numWordsMatch, ref bestMatch, ref lexerState, seen); } #endif }
public static void LuaXLexError(LexState ls, CharPtr msg, int token) { CharPtr buff = new char[MAXSRC]; LuaOChunkID(buff, GetStr(ls.source), MAXSRC); msg = LuaOPushFString(ls.L, "%s:%d: %s", buff, ls.linenumber, msg); if (token != 0) LuaOPushFString(ls.L, "%s near " + LUA_QS, msg, TextToken(ls, token)); LuaDThrow(ls.L, LUA_ERRSYNTAX); }
/// <summary> /// Finds the object that best directly matches the phrase. /// /// This naively finds the single best match. Future is to return a list /// </summary> /// <param name="lexer">The text being matched</param> /// <param name="numWordsMatch">The number of words that match [out]</param> /// <returns>The object that matches; null on error</returns> object matchInContext(ZObject addressedTo, Lexer lexer, out int numWordsMatch, ref LexState lexerState) { // A table of already examined objects var seen = new Dictionary<object,object>(); // The score board for the search var score = 0.0; // The score for the best match: 0 if no match numWordsMatch = 1; // The number of words that match: Minimum of 1 object bestMatch = null;// The object that best matches matchInContext(addressedTo, lexer, ref score, ref numWordsMatch, ref bestMatch, ref lexerState, seen); // Return the best match return bestMatch; }
/// <summary> /// Finds the object that best matches the phrase /// </summary> /// <param name="phrase">The words to match</param> /// <param name="index">The first word in phrase to match</param> /// <param name="score">The score for the best match: 0 if no match[out]</param> /// <param name="numWordsMatch">The number of words that match [out]</param> /// <param name="bestMatch">The object that best matches [out]</param> /// <param name="seen">Set of objects already examined</param> internal static void match(Edge<ZObject> edge, Lexer lexer, ref double score, ref int numWordsMatch, ref object bestMatch, ref LexState lexerState, Dictionary<object,object> seen) { /// We only check to see if the gate matches the phrase if (null != edge.gate) edge.gate.match(lexer, ref score, ref numWordsMatch, ref bestMatch, ref lexerState, seen); // Should only match if no door, or door is open if (null != edge.sink && (null == edge.gate /*|| isOpen(gate)*/)) { // we only want match -- matchInContext checks the stuff from the parents.. edge.sink.match(lexer, ref score, ref numWordsMatch, ref bestMatch, ref lexerState, seen); } }
/*============================================================*/ /* GRAMMAR RULES */ /*============================================================*/ private static void field(LexState ls, expdesc v) { /* field . ['.' | ':'] NAME */ FuncState fs = ls.fs; expdesc key = new expdesc(); LuaKExp2AnyReg(fs, v); LuaXNext(ls); /* skip the dot or colon */ checkname(ls, key); LuaKIndexed(fs, v, key); }
private static int exp1(LexState ls) { expdesc e = new expdesc(); int k; expr(ls, e); k = (int)e.k; LuaKExp2NextReg(ls.fs, e); return k; }
private static void expr(LexState ls, expdesc v) { subexpr(ls, v, 0); }
private static void new_localvar(LexState ls, TString name, int n) { FuncState fs = ls.fs; luaY_checklimit(fs, fs.nactvar+n+1, LUAI_MAXVARS, "local variables"); fs.actvar[fs.nactvar+n] = (ushort)registerlocalvar(ls, name); }
private static int llex(LexState ls, SemInfo seminfo) { luaZ_resetbuffer(ls.buff); for (; ;) { switch (ls.current) { case '\n': case '\r': { inclinenumber(ls); continue; } case '-': { next(ls); if (ls.current != '-') { return('-'); } /* else is a comment */ next(ls); if (ls.current == '[') { int sep = skip_sep(ls); luaZ_resetbuffer(ls.buff); /* `skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(ls, null, sep); /* long comment */ luaZ_resetbuffer(ls.buff); continue; } } /* else short comment */ while (!currIsNewline(ls) && ls.current != EOZ) { next(ls); } continue; } // EDIT: C-style comments from http://lua-users.org/files/wiki_insecure/power_patches/5.2/cppcomt.diff -- Icedream case '/': { /* '/' or '/''/' (line comment) or '/''*' (long comment) */ next(ls); if (ls.current == '/') { /* line comment */ next(ls); while (!currIsNewline(ls) && ls.current != EOZ) { next(ls); } } else if (ls.current == '*') { /* long comment */ next(ls); int last = 0; while (ls.current != EOZ) { if (last == '*' && ls.current == '/') { break; } last = ls.current; next(ls); /* skip until closing marker (or end of file) */ } if (ls.current == EOZ) { luaX_lexerror(ls, "unfinished long comment", (int)RESERVED.TK_EOS); } else { next(ls); } } else { return('/'); } break; } // END EDIT case '[': { int sep = skip_sep(ls); if (sep >= 0) { read_long_string(ls, seminfo, sep); return((int)RESERVED.TK_STRING); } else if (sep == -1) { return('['); } else { luaX_lexerror(ls, "invalid long string delimiter", (int)RESERVED.TK_STRING); } } break; case '=': { next(ls); if (ls.current != '=') { return('='); } else { next(ls); return((int)RESERVED.TK_EQ); } } case '<': { // either <, <=, or << next(ls); if (ls.current == '=') { next(ls); return((int)RESERVED.TK_LE); } else { return('<'); } } case '>': { // either >, >=, or >> next(ls); if (ls.current == '=') { next(ls); return((int)RESERVED.TK_GE); } else { return('>'); } } case '~': { next(ls); if (ls.current != '=') { return('~'); } else { next(ls); return((int)RESERVED.TK_NE); } } case '"': case '\'': { read_string(ls, ls.current, seminfo); return((int)RESERVED.TK_STRING); } case '.': { save_and_next(ls); if (check_next(ls, ".") != 0) { if (check_next(ls, ".") != 0) { return((int)RESERVED.TK_DOTS); /* ... */ } else { return((int)RESERVED.TK_CONCAT); /* .. */ } } else if (!isdigit(ls.current)) { return('.'); } else { read_numeral(ls, seminfo); return((int)RESERVED.TK_NUMBER); } } case EOZ: { return((int)RESERVED.TK_EOS); } default: { if (isspace(ls.current)) { lua_assert(!currIsNewline(ls)); next(ls); continue; } else if (isdigit(ls.current)) { read_numeral(ls, seminfo); return((int)RESERVED.TK_NUMBER); } else if (isalpha(ls.current) || ls.current == '_') { /* identifier or reserved word */ TString ts; do { save_and_next(ls); } while (isalnum(ls.current) || ls.current == '_'); ts = luaX_newstring(ls, luaZ_buffer(ls.buff), luaZ_bufflen(ls.buff)); if (ts.tsv.reserved > 0) /* reserved word? */ { return(ts.tsv.reserved - 1 + FIRST_RESERVED); } else { seminfo.ts = ts; return((int)RESERVED.TK_NAME); } } else { int c = ls.current; next(ls); return(c); /* single-char tokens (+ - / ...) */ } } } } }
private static int funcname(LexState ls, expdesc v) { /* funcname . NAME {field} [`:' NAME] */ int needself = 0; singlevar(ls, v); while (ls.t.token == '.') field(ls, v); if (ls.t.token == ':') { needself = 1; field(ls, v); } return needself; }
private static void listfield(LexState ls, ConsControl cc) { expr(ls, cc.v); luaY_checklimit(ls.fs, cc.na, MAXINT, "items in a constructor"); cc.na++; cc.tostore++; }
private static void read_long_string(LexState ls, SemInfo seminfo, int sep) { //int cont = 0; //(void)(cont); /* avoid warnings when `cont' is not used */ save_and_next(ls); /* skip 2nd `[' */ if (currIsNewline(ls)) /* string starts with a newline? */ { inclinenumber(ls); /* skip it */ } for (;;) { switch (ls.current) { case EOZ: luaX_lexerror(ls, (seminfo != null) ? "unfinished long string" : "unfinished long comment", (int)RESERVED.TK_EOS); break; /* to avoid warnings */ #if LUA_COMPAT_LSTR case '[': { if (skip_sep(ls) == sep) { save_and_next(ls); /* skip 2nd `[' */ cont++; #if LUA_COMPAT_LSTR if (sep == 0) { luaX_lexerror(ls, "nesting of [[...]] is deprecated", '['); } #endif } break; } #endif case ']': if (skip_sep(ls) == sep) { save_and_next(ls); /* skip 2nd `]' */ //#if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2 // cont--; // if (sep == 0 && cont >= 0) break; //#endif goto endloop; } break; case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); if (seminfo == null) { luaZ_resetbuffer(ls.buff); /* avoid wasting space */ } break; default: { if (seminfo != null) { save_and_next(ls); } else { next(ls); } } break; } } endloop: if (seminfo != null) { seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + (2 + sep), (uint)(luaZ_bufflen(ls.buff) - 2 * (2 + sep))); } }
static void read_string(LexState ls, int del, SemInfo seminfo) { save_and_next(ls); while (ls.current != del) { switch (ls.current) { case EOZ: luaX_lexerror(ls, "unfinished string", (int)RESERVED.TK_EOS); continue; /* to avoid warnings */ case '\n': case '\r': luaX_lexerror(ls, "unfinished string", (int)RESERVED.TK_STRING); continue; /* to avoid warnings */ case '\\': { int c; next(ls); /* do not save the `\' */ switch (ls.current) { case 'a': c = '\a'; break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\v'; break; case '\n': /* go through */ case '\r': save(ls, '\n'); inclinenumber(ls); continue; case EOZ: continue; /* will raise an error next loop */ default: { if (!isdigit(ls.current)) { save_and_next(ls); /* handles \\, \", \', and \? */ } else /* \xxx */ { int i = 0; c = 0; do { c = 10 * c + (ls.current - '0'); next(ls); } while (++i < 3 && isdigit(ls.current)); if (c > System.Byte.MaxValue) { luaX_lexerror(ls, "escape sequence too large", (int)RESERVED.TK_STRING); } save(ls, c); } continue; } } save(ls, c); next(ls); continue; } default: save_and_next(ls); break; } } save_and_next(ls); /* skip delimiter */ seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + 1, luaZ_bufflen(ls.buff) - 2); }
public static void LuaXSyntaxError(LexState ls, CharPtr msg) { LuaXLexError(ls, msg, ls.t.token); }
public static void luaX_syntaxerror(LexState ls, CharPtr msg) { luaX_lexerror(ls, msg, ls.t.token); }
static void read_string(LexState ls, int del, SemInfo seminfo) { save_and_next(ls); while (ls.current != del) { switch (ls.current) { case EOZ: lexerror(ls, "unfinished string", (int)RESERVED.TK_EOS); continue; /* to avoid warnings */ case '\n': case '\r': lexerror(ls, "unfinished string", (int)RESERVED.TK_STRING); continue; /* to avoid warnings */ case '\\': { int c; next(ls); /* do not save the `\' */ switch (ls.current) { case 'a': c = '\a'; break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\v'; break; case 'x': c = readhexaesc(ls); break; case '\n': /* go through */ case '\r': save(ls, '\n'); inclinenumber(ls); continue; case EOZ: continue; /* will raise an error next loop */ default: { if (lisdigit(ls.current) == 0) { c = ls.current; /* handles \\, \", \', and \? */ } else /* digital escape \ddd */ { c = readdecesc(ls); } break; } } next(ls); save(ls, c); continue; } default: save_and_next(ls); break; //FIXME:added } } save_and_next(ls); /* skip delimiter */ seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + 1, luaZ_bufflen(ls.buff) - 2); }
public List <Token> GetTokens() { m_iSourceLine = 0; m_iSourceChar = 0; m_lexState = LexState.Space; String strToken = null; List <Token> listTokens = new List <Token>(); while (!EndOfSource) { String strSourceLine = m_listSourceLines[m_iSourceLine]; char ch = ReadChar(); switch (m_lexState) { case LexState.Space: switch (ch) { case ' ': case '\t': case '\r': case '\n': break; // ignore whitespace case '{': listTokens.Add(new Token(TokenType.LeftBrace, "{", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case '}': listTokens.Add(new Token(TokenType.RightBrace, "}", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case '(': listTokens.Add(new Token(TokenType.LeftPar, "(", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case ')': listTokens.Add(new Token(TokenType.RightPar, ")", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case '[': listTokens.Add(new Token(TokenType.LeftBracket, "[", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case ']': listTokens.Add(new Token(TokenType.RightBracket, "]", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case '.': listTokens.Add(new Token(TokenType.Period, ".", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case ',': listTokens.Add(new Token(TokenType.Comma, ",", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case ';': listTokens.Add(new Token(TokenType.SemiColon, ";", m_iSourceLine, m_iSourceChar, strSourceLine)); break; case '=': m_lexState = LexState.AssignOrEqual; break; case '+': m_lexState = LexState.PlusOrIncrementOrAssignPlus; break; case '-': m_lexState = LexState.MinusOrDecrementOrAssignMinus; break; case '*': m_lexState = LexState.MultiplyOrAssignMultiply; break; case '/': m_lexState = LexState.CommentOrDivideOrAssignDivide; break; case '^': m_lexState = LexState.PowerOrAssignPower; break; case '%': m_lexState = LexState.ModuloOrAssignModulo; break; case '&': m_lexState = LexState.And; break; case '|': m_lexState = LexState.Or; break; case '!': m_lexState = LexState.NotOrNotEqual; break; case '>': m_lexState = LexState.GreaterOrGreaterEqual; break; case '<': m_lexState = LexState.LessOrLessEqual; break; case '\"': strToken = ""; m_lexState = LexState.String; break; case ':': listTokens.Add(new Token(TokenType.Colon, ":", m_iSourceLine, m_iSourceChar, strSourceLine)); break; default: if (ch == '_' || char.IsLetter(ch)) { m_lexState = LexState.IdentifierOrKeyword; strToken = "" + ch; } else if (char.IsDigit(ch)) { strToken = "" + ch; m_lexState = LexState.IntegerOrFloat; } else { ThrowInvalidCharacterException(ch); } break; } break; case LexState.CommentOrDivideOrAssignDivide: switch (ch) { case '/': m_lexState = LexState.LineComment; break; case '*': m_lexState = LexState.BlockCommentStart; break; case '=': listTokens.Add(new Token(TokenType.AssignDivide, "/=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; break; default: listTokens.Add(new Token(TokenType.Divide, "/", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; break; } break; case LexState.LineComment: if (ch == '\n') { m_lexState = LexState.Space; } break; case LexState.BlockCommentStart: if (ch == '*') { m_lexState = LexState.BlockCommentEnd; } break; case LexState.BlockCommentEnd: if (ch == '/') { m_lexState = LexState.Space; } else { m_lexState = LexState.BlockCommentStart; } break; case LexState.AssignOrEqual: if (ch == '=') { listTokens.Add(new Token(TokenType.Equal, "==", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Assign, "=", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.PlusOrIncrementOrAssignPlus: if (ch == '+') { listTokens.Add(new Token(TokenType.Increment, "++", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else if (ch == '=') { listTokens.Add(new Token(TokenType.AssignPlus, "+=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Plus, "+", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.MinusOrDecrementOrAssignMinus: if (ch == '-') { listTokens.Add(new Token(TokenType.Decrement, "--", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else if (ch == '=') { listTokens.Add(new Token(TokenType.AssignMinus, "-=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Minus, "-", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.MultiplyOrAssignMultiply: if (ch == '=') { listTokens.Add(new Token(TokenType.AssignMultiply, "*=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Multiply, "*", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.PowerOrAssignPower: if (ch == '=') { listTokens.Add(new Token(TokenType.AssignPower, "^=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Power, "^", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.ModuloOrAssignModulo: if (ch == '=') { listTokens.Add(new Token(TokenType.AssignModulo, "%=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Modulo, "%", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.And: if (ch == '&') { listTokens.Add(new Token(TokenType.And, "&&", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { ThrowInvalidCharacterException(ch); } break; case LexState.Or: if (ch == '|') { listTokens.Add(new Token(TokenType.Or, "||", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { ThrowInvalidCharacterException(ch); } break; case LexState.NotOrNotEqual: if (ch == '=') { listTokens.Add(new Token(TokenType.NotEqual, "!=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Not, "!", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.GreaterOrGreaterEqual: if (ch == '=') { listTokens.Add(new Token(TokenType.GreaterOrEqual, ">=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Greater, ">", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.LessOrLessEqual: if (ch == '=') { listTokens.Add(new Token(TokenType.LessOrEqual, "<=", m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else { listTokens.Add(new Token(TokenType.Less, "<", m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.IdentifierOrKeyword: if (ch == '_' || char.IsLetterOrDigit(ch)) { strToken += ch; } else { TokenType tokenType; if (strToken == "include") { tokenType = TokenType.Include; } else if (strToken == "global") { tokenType = TokenType.Global; } else if (strToken == "var") { tokenType = TokenType.Var; } else if (strToken == "yield") { tokenType = TokenType.Yield; } else if (strToken == "wait") { tokenType = TokenType.Wait; } else if (strToken == "notify") { tokenType = TokenType.Notify; } else if (strToken == "lock") { tokenType = TokenType.Lock; } else if (strToken == "if") { tokenType = TokenType.If; } else if (strToken == "else") { tokenType = TokenType.Else; } else if (strToken == "while") { tokenType = TokenType.While; } else if (strToken == "for") { tokenType = TokenType.For; } else if (strToken == "foreach") { tokenType = TokenType.Foreach; } else if (strToken == "in") { tokenType = TokenType.In; } else if (strToken == "switch") { tokenType = TokenType.Switch; } else if (strToken == "case") { tokenType = TokenType.Case; } else if (strToken == "default") { tokenType = TokenType.Default; } else if (strToken == "break") { tokenType = TokenType.Break; } else if (strToken == "continue") { tokenType = TokenType.Continue; } else if (strToken == "function") { tokenType = TokenType.Function; } else if (strToken == "return") { tokenType = TokenType.Return; } else if (strToken == "thread") { tokenType = TokenType.Thread; } else if (strToken == "null") { tokenType = TokenType.Null; } else if (strToken == "true" || strToken == "false") { tokenType = TokenType.Boolean; } else { tokenType = TokenType.Identifier; } if (tokenType == TokenType.Boolean) { listTokens.Add(new Token(tokenType, strToken == "true", m_iSourceLine, m_iSourceChar, strSourceLine)); } else { listTokens.Add(new Token(tokenType, strToken, m_iSourceLine, m_iSourceChar, strSourceLine)); } UndoChar(); m_lexState = LexState.Space; } break; case LexState.String: if (ch == '\"') { listTokens.Add(new Token(TokenType.String, strToken, m_iSourceLine, m_iSourceChar, strSourceLine)); m_lexState = LexState.Space; } else if (ch == '\\') { m_lexState = LexState.StringEscape; } else if (ch == '\r' || ch == '\n') { throw new LexerException("String literal cannot span multiple lines.", m_iSourceLine, m_iSourceChar, m_listSourceLines[m_iSourceLine]); } else { strToken += ch; } break; case LexState.StringEscape: if (ch == '\\' || ch == '\"') { strToken += ch; m_lexState = LexState.String; } else if (ch == 't') { strToken += '\t'; m_lexState = LexState.String; } else if (ch == 'r') { strToken += '\r'; m_lexState = LexState.String; } else if (ch == 'n') { strToken += '\n'; m_lexState = LexState.String; } else { throw new LexerException( "Invalid string escape sequence '\\" + ch + "'.", m_iSourceLine, m_iSourceChar, m_listSourceLines[m_iSourceLine]); } break; case LexState.IntegerOrFloat: if (char.IsDigit(ch)) { strToken += ch; } else if (ch == '.') { strToken += ch; m_lexState = LexState.Float; } else { int iValue = int.Parse(strToken); listTokens.Add(new Token(TokenType.Integer, iValue, m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; case LexState.Float: if (char.IsDigit(ch)) { strToken += ch; } else { float fValue = float.Parse(strToken); listTokens.Add(new Token(TokenType.Float, fValue, m_iSourceLine, m_iSourceChar, strSourceLine)); UndoChar(); m_lexState = LexState.Space; } break; default: throw new LexerException("Unhandled lexer state."); } } if (m_lexState != LexState.Space) { throw new LexerException( "Unexpected end of source reached."); } return(listTokens); }
void DoBreak(WordVisitor visitor, char[] input, int start, int len, OnBreak onBreak) { //---------------------------------------- //simple break word/ num/ punc / space //similar to lexer function //---------------------------------------- LexState lexState = LexState.Init; int endBefore = start + len; char first = (char)1; char last = (char)255; for (int i = start; i < endBefore; ++i) { char c = input[i]; if (c < first || c > last) { //clear accum state if (i > start) { //some remaining data breakBounds.length = i - breakBounds.startIndex; // onBreak(breakBounds); // } visitor.State = VisitorState.OutOfRangeChar; return; } switch (lexState) { case LexState.Init: { //check char if (c == '\r') { //check next if '\n' if (i < endBefore - 1) { if (input[i + 1] == '\n') { //this is '\r\n' linebreak breakBounds.startIndex = i; breakBounds.length = 2; breakBounds.kind = WordKind.NewLine; // onBreak(breakBounds); // breakBounds.length = 0; lexState = LexState.Init; i++; continue; } } else { //sinple \r? //to whitespace? lexState = LexState.Whitespace; breakBounds.startIndex = i; } } else if (c == '\n') { breakBounds.startIndex = i; breakBounds.length = 1; breakBounds.kind = WordKind.NewLine; // onBreak(breakBounds); // breakBounds.length = 0; lexState = LexState.Init; continue; } else if (char.IsLetter(c)) { //just collect breakBounds.startIndex = i; breakBounds.kind = WordKind.Text; lexState = LexState.Text; } else if (char.IsNumber(c)) { breakBounds.startIndex = i; breakBounds.kind = WordKind.Number; lexState = LexState.Number; } else if (char.IsWhiteSpace(c)) { //we collect whitespace breakBounds.startIndex = i; breakBounds.kind = WordKind.Whitespace; lexState = LexState.Whitespace; } else if (char.IsPunctuation(c) || char.IsSymbol(c)) { //for eng - if (c == '-') { //check next char is number or not if (i < endBefore - 1 && char.IsNumber(input[i + 1])) { breakBounds.startIndex = i; breakBounds.kind = WordKind.Number; lexState = LexState.Number; continue; } } breakBounds.startIndex = i; breakBounds.length = 1; breakBounds.kind = WordKind.Punc; //we not collect punc onBreak(breakBounds); // breakBounds.startIndex += 1; breakBounds.length = 0; lexState = LexState.Init; continue; } else { throw new System.NotSupportedException(); } } break; case LexState.Number: { //in number state if (!char.IsNumber(c) && c != '.') { //if number then continue collect //if not //flush current state breakBounds.length = i - breakBounds.startIndex; breakBounds.kind = WordKind.Number; // onBreak(breakBounds); // breakBounds.length = 0; lexState = LexState.Init; goto case LexState.Init; } } break; case LexState.Text: { if (!char.IsLetter(c) && !char.IsNumber(c)) { //flush breakBounds.length = i - breakBounds.startIndex; breakBounds.kind = WordKind.Text; // onBreak(breakBounds); // breakBounds.length = 0; lexState = LexState.Init; goto case LexState.Init; } } break; case LexState.Whitespace: { if (!char.IsWhiteSpace(c)) { breakBounds.length = i - breakBounds.startIndex; breakBounds.kind = WordKind.Whitespace; // onBreak(breakBounds); // breakBounds.length = 0; lexState = LexState.Init; goto case LexState.Init; } } break; } } if (lexState != LexState.Init && breakBounds.startIndex < start + len) { //some remaining data breakBounds.length = (start + len) - breakBounds.startIndex; // onBreak(breakBounds); // } visitor.State = VisitorState.End; }
private static void forlist(LexState ls, TString indexname) { /* forlist . NAME {,NAME} IN explist1 forbody */ FuncState fs = ls.fs; expdesc e = new expdesc(); int nvars = 0; int line; int base_ = fs.freereg; /* create control variables */ new_localvarliteral(ls, "(for generator)", nvars++); new_localvarliteral(ls, "(for state)", nvars++); new_localvarliteral(ls, "(for control)", nvars++); /* create declared variables */ new_localvar(ls, indexname, nvars++); while (testnext(ls, ',') != 0) new_localvar(ls, str_checkname(ls), nvars++); checknext(ls, (int)RESERVED.TK_IN); line = ls.linenumber; adjust_assign(ls, 3, explist1(ls, e), e); LuaKCheckStack(fs, 3); /* extra space to call generator */ forbody(ls, base_, line, nvars - 3, 0); }
private static int llex(LexState ls, SemInfo seminfo) { luaZ_resetbuffer(ls.buff); for (;;) { switch (ls.current) { case '\n': case '\r': { inclinenumber(ls); continue; } case '-': { next(ls); if (ls.current != '-') { return('-'); } /* else is a comment */ next(ls); if (ls.current == '[') { int sep = skip_sep(ls); luaZ_resetbuffer(ls.buff); /* `skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(ls, null, sep); /* long comment */ luaZ_resetbuffer(ls.buff); continue; } } /* else short comment */ while (!currIsNewline(ls) && ls.current != EOZ) { next(ls); } continue; } case '[': { int sep = skip_sep(ls); if (sep >= 0) { read_long_string(ls, seminfo, sep); return((int)RESERVED.TK_STRING); } else if (sep == -1) { return('['); } else { luaX_lexerror(ls, "invalid long string delimiter", (int)RESERVED.TK_STRING); } } break; case '=': { next(ls); if (ls.current != '=') { return('='); } else { next(ls); return((int)RESERVED.TK_EQ); } } case '<': { next(ls); if (ls.current != '=') { return('<'); } else { next(ls); return((int)RESERVED.TK_LE); } } case '>': { next(ls); if (ls.current != '=') { return('>'); } else { next(ls); return((int)RESERVED.TK_GE); } } case '~': { next(ls); if (ls.current != '=') { return('~'); } else { next(ls); return((int)RESERVED.TK_NE); } } case '"': case '\'': { read_string(ls, ls.current, seminfo); return((int)RESERVED.TK_STRING); } case '.': { save_and_next(ls); if (check_next(ls, ".") != 0) { if (check_next(ls, ".") != 0) { return((int)RESERVED.TK_DOTS); /* ... */ } else { return((int)RESERVED.TK_CONCAT); /* .. */ } } else if (!isdigit(ls.current)) { return('.'); } else { read_numeral(ls, seminfo); return((int)RESERVED.TK_NUMBER); } } case EOZ: { return((int)RESERVED.TK_EOS); } default: { if (isspace(ls.current)) { lua_assert(!currIsNewline(ls)); next(ls); continue; } else if (isdigit(ls.current)) { read_numeral(ls, seminfo); return((int)RESERVED.TK_NUMBER); } else if (isalpha(ls.current) || ls.current == '_') { /* identifier or reserved word */ TString ts; do { save_and_next(ls); } while (isalnum(ls.current) || ls.current == '_'); ts = luaX_newstring(ls, luaZ_buffer(ls.buff), luaZ_bufflen(ls.buff)); if (ts.tsv.reserved > 0) /* reserved word? */ { return(ts.tsv.reserved - 1 + FIRST_RESERVED); } else { seminfo.ts = ts; return((int)RESERVED.TK_NAME); } } else { int c = ls.current; next(ls); return(c); /* single-char tokens (+ - / ...) */ } } } } }
private static void forstat(LexState ls, int line) { /* forstat . FOR (fornum | forlist) END */ FuncState fs = ls.fs; TString varname; BlockCnt bl = new BlockCnt(); enterblock(fs, bl, 1); /* scope for loop and control variables */ LuaXNext(ls); /* skip `for' */ varname = str_checkname(ls); /* first variable name */ switch (ls.t.token) { case '=': fornum(ls, varname, line); break; case ',': case (int)RESERVED.TK_IN: forlist(ls, varname); break; default: LuaXSyntaxError(ls, LUA_QL("=") + " or " + LUA_QL("in") + " expected"); break; } check_match(ls, (int)RESERVED.TK_END, (int)RESERVED.TK_FOR, line); leaveblock(fs); /* loop scope (`break' jumps to this point) */ }
public static void next(LexState ls) { ls.current = zgetc(ls.z); }
private static void ifstat(LexState ls, int line) { /* ifstat . IF cond THEN block {ELSEIF cond THEN block} [ELSE block] END */ FuncState fs = ls.fs; int flist; int escapelist = NO_JUMP; flist = test_then_block(ls); /* IF cond THEN block */ while (ls.t.token == (int)RESERVED.TK_ELSEIF) { LuaKConcat(fs, ref escapelist, LuaKJump(fs)); LuaKPatchToHere(fs, flist); flist = test_then_block(ls); /* ELSEIF cond THEN block */ } if (ls.t.token == (int)RESERVED.TK_ELSE) { LuaKConcat(fs, ref escapelist, LuaKJump(fs)); LuaKPatchToHere(fs, flist); LuaXNext(ls); /* skip ELSE (after patch, for correct line info) */ block(ls); /* `else' part */ } else LuaKConcat(fs, ref escapelist, flist); LuaKPatchToHere(fs, escapelist); check_match(ls, (int)RESERVED.TK_END, (int)RESERVED.TK_IF, line); }
internal static IEnumerable <Token> Analyze(Stream stream, Encoding encoding) { using StreamReader reader = new StreamReader(stream, encoding, false, -1, true); StringBuilder sb = new StringBuilder(); LexState state = LexState.Empty; int row = 1; int column = 0; while (true) { LexChoice choice; Int32 nextCharInt = reader.Peek(); char nextChar = Char.MinValue; LexCharType nextCharType = LexCharType.Unknown; Debug.Assert((sb.Length == 0) == (state == LexState.Empty)); if (nextCharInt == -1) // EOL { if (state == LexState.Empty) { choice = LexChoice.Terminate; } else if (state == LexState.String || state == LexState.StringEscaping) { throw new Exception("Unexpected end of line. The string is not closed."); } else { choice = LexChoice.PeekReturn; } } else { nextChar = (char)nextCharInt; ++column; if (nextChar == '\n') { column = 0; ++row; } nextCharType = GetCharType(nextChar); switch (state) // a 5 by 6 table { case LexState.LetterOrDigitOrUnderscore when nextCharType == LexCharType.Unknown: case LexState.Empty when nextCharType == LexCharType.Unknown: case LexState.Punctuation when nextCharType == LexCharType.Unknown: throw new Exception("Unrecognized character \"" + nextChar.ToString() + "\""); case LexState.Empty when nextCharType == LexCharType.WhiteSpace: choice = LexChoice.Drop; break; case LexState.LetterOrDigitOrUnderscore when nextCharType == LexCharType.LetterOrDigitOrUnderscore: case LexState.Empty when nextCharType == LexCharType.QuotationMark: case LexState.Empty when nextCharType == LexCharType.Backslash: case LexState.Empty when nextCharType == LexCharType.LetterOrDigitOrUnderscore: case LexState.Empty when nextCharType == LexCharType.OtherPunctuation: case LexState.String when nextCharType == LexCharType.Unknown: case LexState.String when nextCharType == LexCharType.WhiteSpace: case LexState.String when nextCharType == LexCharType.LetterOrDigitOrUnderscore: case LexState.String when nextCharType == LexCharType.OtherPunctuation: choice = LexChoice.ReadAppend; break; case LexState.String when nextCharType == LexCharType.QuotationMark: choice = LexChoice.ReadAppendReturn; break; case LexState.String when nextCharType == LexCharType.Backslash: choice = LexChoice.ReadAppendStringEscapeIn; break; case LexState.StringEscaping when nextCharType == LexCharType.Unknown: case LexState.StringEscaping when nextCharType == LexCharType.WhiteSpace: case LexState.StringEscaping when nextCharType == LexCharType.Backslash: case LexState.StringEscaping when nextCharType == LexCharType.QuotationMark: case LexState.StringEscaping when nextCharType == LexCharType.LetterOrDigitOrUnderscore: case LexState.StringEscaping when nextCharType == LexCharType.OtherPunctuation: choice = LexChoice.ReadAppendStringEscapeOut; break; case LexState.Punctuation when nextCharType == LexCharType.WhiteSpace: case LexState.LetterOrDigitOrUnderscore when nextCharType == LexCharType.WhiteSpace: choice = LexChoice.DropReturn; break; case LexState.Punctuation when nextCharType == LexCharType.LetterOrDigitOrUnderscore: case LexState.LetterOrDigitOrUnderscore when nextCharType == LexCharType.OtherPunctuation: case LexState.LetterOrDigitOrUnderscore when nextCharType == LexCharType.Backslash: case LexState.LetterOrDigitOrUnderscore when nextCharType == LexCharType.QuotationMark: choice = LexChoice.PeekReturn; break; case LexState.Punctuation when nextCharType == LexCharType.Backslash: case LexState.Punctuation when nextCharType == LexCharType.QuotationMark: case LexState.Punctuation when nextCharType == LexCharType.OtherPunctuation: // consider '&' and '&&' if (GetTokenType(sb.ToString()) != null) { if (GetTokenType(sb.ToString() + nextChar) != null) { choice = LexChoice.ReadAppend; } else { choice = LexChoice.PeekReturn; } } else { choice = LexChoice.ReadAppend; } break; default: throw CommonException.AssertFailedException(); } } switch (choice) { case LexChoice.PeekReturn: yield return(GetToken(sb.ToString(), row, column)); _ = sb.Clear(); state = LexState.Empty; break; case LexChoice.DropReturn: _ = reader.Read(); yield return(GetToken(sb.ToString(), row, column)); _ = sb.Clear(); state = LexState.Empty; break; case LexChoice.Drop: _ = reader.Read(); break; case LexChoice.ReadAppendStringEscapeOut: case LexChoice.ReadAppendReturn: case LexChoice.ReadAppendStringEscapeIn: case LexChoice.ReadAppend: _ = reader.Read(); if (state == LexState.Empty) { switch (nextCharType) { case LexCharType.QuotationMark: state = LexState.String; break; case LexCharType.LetterOrDigitOrUnderscore: state = LexState.LetterOrDigitOrUnderscore; break; case LexCharType.Backslash: case LexCharType.OtherPunctuation: state = LexState.Punctuation; break; default: throw CommonException.AssertFailedException(); } } _ = sb.Append(nextChar); if (choice == LexChoice.ReadAppendReturn) { yield return(GetToken(sb.ToString(), row, column)); _ = sb.Clear(); state = LexState.Empty; } else if (choice == LexChoice.ReadAppendStringEscapeIn) { Debug.Assert(state == LexState.String); state = LexState.StringEscaping; } else if (choice == LexChoice.ReadAppendStringEscapeOut) { Debug.Assert(state == LexState.StringEscaping); state = LexState.String; } break; case LexChoice.Terminate: yield return(GetEOL(row, column)); yield break; default: throw CommonException.AssertFailedException(); } } }
private static void localstat(LexState ls) { /* stat . LOCAL NAME {`,' NAME} [`=' explist1] */ int nvars = 0; int nexps; expdesc e = new expdesc(); do { new_localvar(ls, str_checkname(ls), nvars++); } while (testnext(ls, ',') != 0); if (testnext(ls, '=') != 0) nexps = explist1(ls, e); else { e.k = expkind.VVOID; nexps = 0; } adjust_assign(ls, nvars, nexps, e); adjustlocalvars(ls, nvars); }
public static void SaveAndNext(LexState ls) { Save(ls, ls.current); Next(ls); }
/* }====================================================================== */ private static void parlist(LexState ls) { /* parlist . [ param { `,' param } ] */ FuncState fs = ls.fs; Proto f = fs.f; int nparams = 0; f.is_vararg = 0; if (ls.t.token != ')') { /* is `parlist' not empty? */ do { switch (ls.t.token) { case (int)RESERVED.TK_NAME: { /* param . NAME */ new_localvar(ls, str_checkname(ls), nparams++); break; } case (int)RESERVED.TK_DOTS: { /* param . `...' */ LuaXNext(ls); #if LUA_COMPAT_VARARG /* use `arg' as default name */ new_localvarliteral(ls, "arg", nparams++); f.is_vararg = VARARG_HASARG | VARARG_NEEDSARG; #endif f.is_vararg |= VARARG_ISVARARG; break; } default: LuaXSyntaxError(ls, "<name> or " + LUA_QL("...") + " expected"); break; } } while ((f.is_vararg==0) && (testnext(ls, ',')!=0)); } adjustlocalvars(ls, nparams); f.numparams = CastByte(fs.nactvar - (f.is_vararg & VARARG_HASARG)); LuaKReserveRegs(fs, fs.nactvar); /* reserve register for parameters */ }
public static void /*l_noret*/ luaX_syntaxerror(LexState ls, CharPtr msg) { lexerror(ls, msg, ls.t.token); }
private static void error_expected(LexState ls, int token) { LuaXSyntaxError(ls, LuaOPushFString(ls.L, LUA_QS + " expected", LuaXTokenToString(ls, token))); }
public static int next(LexState ls) { ls.current = zgetc(ls.z); return(ls.current); }
private static int explist1(LexState ls, expdesc v) { /* explist1 . expr { `,' expr } */ int n = 1; /* at least one expression */ expr(ls, v); while (testnext(ls, ',') != 0) { LuaKExp2NextReg(ls.fs, v); expr(ls, v); n++; } return n; }
public static bool currIsNewline(LexState ls) { return(ls.current == '\n' || ls.current == '\r'); }
private static void exprstat(LexState ls) { /* stat . func | assignment */ FuncState fs = ls.fs; LHS_assign v = new LHS_assign(); primaryexp(ls, v.v); if (v.v.k == expkind.VCALL) /* stat . func */ SETARG_C(GetCode(fs, v.v), 1); /* call statement uses no results */ else { /* stat . assignment */ v.prev = null; assignment(ls, v, 1); } }
private static void read_long_string(LexState ls, SemInfo seminfo, int sep) { int line = ls.linenumber; /* initial line (for error message) */ save_and_next(ls); /* skip 2nd `[' */ if (currIsNewline(ls)) /* string starts with a newline? */ { inclinenumber(ls); /* skip it */ } for (;;) { switch (ls.current) { case EOZ: { /* error */ CharPtr what = (seminfo != null ? "string" : "comment"); CharPtr msg = luaO_pushfstring(ls.L, "unfinished long %s (starting at line %d)", what, line); lexerror(ls, msg, (int)RESERVED.TK_EOS); break; /* to avoid warnings */ } case ']': { if (skip_sep(ls) == sep) { save_and_next(ls); /* skip 2nd `]' */ goto endloop; } break; } case '\n': case '\r': { save(ls, '\n'); inclinenumber(ls); if (seminfo == null) { luaZ_resetbuffer(ls.buff); /* avoid wasting space */ } break; } default: { if (seminfo != null) { save_and_next(ls); } else { next(ls); } } break; } } endloop: if (seminfo != null) { seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + (2 + sep), (uint)(luaZ_bufflen(ls.buff) - 2 * (2 + sep))); } }
private static void forbody(LexState ls, int base_, int line, int nvars, int isnum) { /* forbody . DO block */ BlockCnt bl = new BlockCnt(); FuncState fs = ls.fs; int prep, endfor; adjustlocalvars(ls, 3); /* control variables */ checknext(ls, (int)RESERVED.TK_DO); prep = (isnum != 0) ? LuaKCodeAsBx(fs, OpCode.OP_FORPREP, base_, NO_JUMP) : LuaKJump(fs); enterblock(fs, bl, 0); /* scope for declared variables */ adjustlocalvars(ls, nvars); LuaKReserveRegs(fs, nvars); block(ls); leaveblock(fs); /* end of scope for declared variables */ LuaKPatchToHere(fs, prep); endfor = (isnum!=0) ? LuaKCodeAsBx(fs, OpCode.OP_FORLOOP, base_, NO_JUMP) : LuaKCodeABC(fs, OpCode.OP_TFORLOOP, base_, 0, nvars); LuaKFixLine(fs, line); /* pretend that `OP_FOR' starts the loop */ LuaKPatchList(fs, ((isnum!=0) ? endfor : LuaKJump(fs)), prep + 1); }
public static void save_and_next(LexState ls) { save(ls, ls.current); next(ls); }
private static void fornum(LexState ls, TString varname, int line) { /* fornum . NAME = exp1,exp1[,exp1] forbody */ FuncState fs = ls.fs; int base_ = fs.freereg; new_localvarliteral(ls, "(for index)", 0); new_localvarliteral(ls, "(for limit)", 1); new_localvarliteral(ls, "(for step)", 2); new_localvar(ls, varname, 3); checknext(ls, '='); exp1(ls); /* initial value */ checknext(ls, ','); exp1(ls); /* limit */ if (testnext(ls, ',') != 0) exp1(ls); /* optional step */ else { /* default step = 1 */ LuaKCodeABx(fs, OpCode.OP_LOADK, fs.freereg, LuaKNumberK(fs, 1)); LuaKReserveRegs(fs, 1); } forbody(ls, base_, line, 1, 1); }
private static int gethexa(LexState ls) { save_and_next(ls); esccheck(ls, lisxdigit(ls.current), "hexadecimal digit expected"); return(luaO_hexavalue(ls.current)); }
private static void funcargs(LexState ls, expdesc f) { FuncState fs = ls.fs; expdesc args = new expdesc(); int base_, nparams; int line = ls.linenumber; switch (ls.t.token) { case '(': { /* funcargs . `(' [ explist1 ] `)' */ if (line != ls.lastline) LuaXSyntaxError(ls,"ambiguous syntax (function call x new statement)"); LuaXNext(ls); if (ls.t.token == ')') /* arg list is empty? */ args.k = expkind.VVOID; else { explist1(ls, args); LuaKSetMultRet(fs, args); } check_match(ls, ')', '(', line); break; } case '{': { /* funcargs . constructor */ constructor(ls, args); break; } case (int)RESERVED.TK_STRING: { /* funcargs . STRING */ codestring(ls, args, ls.t.seminfo.ts); LuaXNext(ls); /* must use `seminfo' before `next' */ break; } default: { LuaXSyntaxError(ls, "function arguments expected"); return; } } LuaAssert(f.k == expkind.VNONRELOC); base_ = f.u.s.info; /* base_ register for call */ if (hasmultret(args.k) != 0) nparams = LUA_MULTRET; /* open call */ else { if (args.k != expkind.VVOID) LuaKExp2NextReg(fs, args); /* close last argument */ nparams = fs.freereg - (base_+1); } init_exp(f, expkind.VCALL, LuaKCodeABC(fs, OpCode.OP_CALL, base_, nparams + 1, 2)); LuaKFixLine(fs, line); fs.freereg = base_+1; /* call remove function and arguments and leaves (unless changed) one result */ }
static void read_string(LexState ls, int del, SemInfo seminfo) { save_and_next(ls); /* keep delimiter (for error messages) */ while (ls.current != del) { switch (ls.current) { case EOZ: lexerror(ls, "unfinished string", (int)RESERVED.TK_EOS); break; /* to avoid warnings */ case '\n': case '\r': lexerror(ls, "unfinished string", (int)RESERVED.TK_STRING); break; /* to avoid warnings */ case '\\': { /* escape sequences */ int c; /* final character to be saved */ save_and_next(ls); /* keep '\\' for error messages */ switch (ls.current) { case 'a': c = '\a'; goto read_save; case 'b': c = '\b'; goto read_save; case 'f': c = '\f'; goto read_save; case 'n': c = '\n'; goto read_save; case 'r': c = '\r'; goto read_save; case 't': c = '\t'; goto read_save; case 'v': c = '\v'; goto read_save; case 'x': c = readhexaesc(ls); goto read_save; case 'u': utf8esc(ls); goto no_save; case '\n': case '\r': inclinenumber(ls); c = '\n'; goto only_save; case '\\': case '\"': case '\'': c = ls.current; goto read_save; case EOZ: goto no_save; /* will raise an error next loop */ case 'z': { /* zap following span of spaces */ luaZ_buffremove(ls.buff, 1); /* remove '\\' */ next(ls); /* skip the 'z' */ while (lisspace(ls.current) != 0) { if (currIsNewline(ls)) { inclinenumber(ls); } else { next(ls); } } goto no_save; } default: { esccheck(ls, lisdigit(ls.current), "invalid escape sequence"); c = readdecesc(ls); /* digital escape \ddd */ goto only_save; } } read_save: next(ls); /* go through */ only_save: luaZ_buffremove(ls.buff, 1); /* remove '\\' */ save(ls, c); /* go through */ no_save : break; } default: save_and_next(ls); break; //FIXME:added } } save_and_next(ls); /* skip delimiter */ seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + 1, luaZ_bufflen(ls.buff) - 2); }
private static void funcstat(LexState ls, int line) { /* funcstat . FUNCTION funcname body */ int needself; expdesc v = new expdesc(), b = new expdesc(); LuaXNext(ls); /* skip FUNCTION */ needself = funcname(ls, v); body(ls, b, needself, line); LuaKStoreVar(ls.fs, v, b); LuaKFixLine(ls.fs, line); /* definition `happens' in the first line */ }
private static int llex(LexState ls, SemInfo seminfo) { luaZ_resetbuffer(ls.buff); for (;;) { switch (ls.current) { case '\n': case '\r': { /* line breaks */ inclinenumber(ls); break; } case ' ': case '\f': case '\t': case '\v': { /* spaces */ next(ls); break; } case '-': { /* '-' or '--' (comment) */ next(ls); if (ls.current != '-') { return('-'); } /* else is a comment */ next(ls); if (ls.current == '[') /* long comment? */ { int sep = skip_sep(ls); luaZ_resetbuffer(ls.buff); /* `skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(ls, null, sep); /* skip long comment */ luaZ_resetbuffer(ls.buff); /* previous call may dirty the buff. */ break; } } /* else short comment */ while (!currIsNewline(ls) && ls.current != EOZ) { next(ls); /* skip until end of line (or end of file) */ } break; } case '[': { /* long string or simply '[' */ int sep = skip_sep(ls); if (sep >= 0) { read_long_string(ls, seminfo, sep); return((int)RESERVED.TK_STRING); } else if (sep == -1) { return('['); } else { lexerror(ls, "invalid long string delimiter", (int)RESERVED.TK_STRING); } } break; case '=': { next(ls); if (0 != check_next1(ls, '=')) { return((int)RESERVED.TK_EQ); } else { return('='); } } case '<': { next(ls); if (0 != check_next1(ls, '=')) { return((int)RESERVED.TK_LE); } else if (0 != check_next1(ls, '<')) { return((int)RESERVED.TK_SHL); } else { return('<'); } } case '>': { next(ls); if (0 != check_next1(ls, '=')) { return((int)RESERVED.TK_GE); } else if (0 != check_next1(ls, '>')) { return((int)RESERVED.TK_SHR); } else { return('>'); } } case '/': { next(ls); if (0 != check_next1(ls, '/')) { return((int)RESERVED.TK_IDIV); } else { return('/'); } } case '~': { next(ls); if (0 != check_next1(ls, '=')) { return((int)RESERVED.TK_NE); } else { return('~'); } } case ':': { next(ls); if (0 != check_next1(ls, ':')) { return((int)RESERVED.TK_DBCOLON); } else { return(':'); } } case '"': case '\'': { /* short literal strings */ read_string(ls, ls.current, seminfo); return((int)RESERVED.TK_STRING); } case '.': { /* '.', '..', '...', or number */ save_and_next(ls); if (0 != check_next1(ls, '.')) { if (0 != check_next1(ls, '.')) { return((int)RESERVED.TK_DOTS); /* '...' */ } else { return((int)RESERVED.TK_CONCAT); /* '..' */ } } else if (0 == lisdigit(ls.current)) { return('.'); } else { return(read_numeral(ls, seminfo)); } } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { return(read_numeral(ls, seminfo)); } case EOZ: { return((int)RESERVED.TK_EOS); } default: { if (lislalpha(ls.current) != 0) /* identifier or reserved word? */ { TString ts; do { save_and_next(ls); } while (lislalnum(ls.current) != 0); ts = luaX_newstring(ls, luaZ_buffer(ls.buff), luaZ_bufflen(ls.buff)); seminfo.ts = ts; if (isreserved(ts)) /* reserved word? */ { return(ts.tsv.extra - 1 + FIRST_RESERVED); } else { return((int)RESERVED.TK_NAME); } } else /* single-char tokens (+ - / ...) */ { int c = ls.current; next(ls); return(c); } } } } }
private static void leavelevel(LexState ls) { ls.L.nCcalls--; }
public static int luaX_lookahead(LexState ls) { lua_assert(ls.lookahead.token == (int)RESERVED.TK_EOS); ls.lookahead.token = llex(ls, ls.lookahead.seminfo); return(ls.lookahead.token); }
private static void localfunc(LexState ls) { expdesc v = new expdesc(), b = new expdesc(); FuncState fs = ls.fs; new_localvar(ls, str_checkname(ls), 0); init_exp(v, expkind.VLOCAL, fs.freereg); LuaKReserveRegs(fs, 1); adjustlocalvars(ls, 1); body(ls, b, 0, ls.linenumber); LuaKStoreVar(fs, v, b); /* debug information will only see the variable after this point! */ getlocvar(fs, fs.nactvar - 1).startpc = fs.pc; }
private static void primaryexp(LexState ls, expdesc v) { /* primaryexp . prefixexp { `.' NAME | `[' exp `]' | `:' NAME funcargs | funcargs } */ FuncState fs = ls.fs; prefixexp(ls, v); for (;;) { switch (ls.t.token) { case '.': { /* field */ field(ls, v); break; } case '[': { /* `[' exp1 `]' */ expdesc key = new expdesc(); LuaKExp2AnyReg(fs, v); yindex(ls, key); LuaKIndexed(fs, v, key); break; } case ':': { /* `:' NAME funcargs */ expdesc key = new expdesc(); LuaXNext(ls); checkname(ls, key); LuaKSelf(fs, v, key); funcargs(ls, v); break; } case '(': case (int)RESERVED.TK_STRING: case '{': { /* funcargs */ LuaKExp2NextReg(fs, v); funcargs(ls, v); break; } default: return; } } }
public static void check_condition(LexState ls, bool c, CharPtr msg) { if (!(c)) LuaXSyntaxError(ls, msg); }
static void read_string(LexState ls, int del, SemInfo seminfo) { save_and_next(ls); /* keep delimiter (for error messages) */ while (ls.current != del) { switch (ls.current) { case EOZ: lexerror(ls, "unfinished string", (int)RESERVED.TK_EOS); break; /* to avoid warnings */ case '\n': case '\r': lexerror(ls, "unfinished string", (int)RESERVED.TK_STRING); break; /* to avoid warnings */ case '\\': { /* escape sequences */ int c; /* final character to be saved */ next(ls); /* do not save the `\' */ switch (ls.current) { case 'a': c = '\a'; break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\v'; break; case 'x': c = readhexaesc(ls); break; case '\n': /* go through */ case '\r': save(ls, '\n'); inclinenumber(ls); continue; case '\\': case '\"': case '\'': c = ls.current; break; case EOZ: continue; /* will raise an error next loop */ case 'z': { /* zap following span of spaces */ next(ls); /* skip the 'z' */ while (lisspace(ls.current) != 0) { if (currIsNewline(ls)) { inclinenumber(ls); } else { next(ls); } } continue; /* do not save 'c' */ } default: { if (lisdigit(ls.current) == 0) { escerror(ls, new int[] { ls.current }, 1, "invalid escape sequence"); //FIXME:changed, new int[]{} } /* digital escape \ddd */ c = readdecesc(ls); break; } } next(ls); save(ls, c); break; } default: save_and_next(ls); break; //FIXME:added } } save_and_next(ls); /* skip delimiter */ seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + 1, luaZ_bufflen(ls.buff) - 2); }
private static void open_func(LexState ls, FuncState fs) { LuaState L = ls.L; Proto f = LuaFNewProto(L); fs.f = f; fs.prev = ls.fs; /* linked list of funcstates */ fs.ls = ls; fs.L = L; ls.fs = fs; fs.pc = 0; fs.lasttarget = -1; fs.jpc = NO_JUMP; fs.freereg = 0; fs.nk = 0; fs.np = 0; fs.nlocvars = 0; fs.nactvar = 0; fs.bl = null; f.source = ls.source; f.maxstacksize = 2; /* registers 0/1 are always valid */ fs.h = luaH_new(L, 0, 0); /* anchor table of constants and prototype (to avoid being collected) */ SetHValue2S(L, L.top, fs.h); IncrTop(L); SetPTValue2S(L, L.top, f); IncrTop(L); }
private static void enterlevel(LexState ls) { if (++ls.L.nCcalls > LUAI_MAXCCALLS) LuaXLexError(ls, "chunk has too many syntax levels", 0); }
/* ** {====================================================================== ** Expression parsing ** ======================================================================= */ private static void prefixexp(LexState ls, expdesc v) { /* prefixexp . NAME | '(' expr ')' */ switch (ls.t.token) { case '(': { int line = ls.linenumber; LuaXNext(ls); expr(ls, v); check_match(ls, ')', '(', line); LuaKDischargeVars(ls.fs, v); return; } case (int)RESERVED.TK_NAME: { singlevar(ls, v); return; } default: { LuaXSyntaxError(ls, "unexpected symbol"); return; } } }
public static void LuaXLookAhead(LexState ls) { LuaAssert(ls.lookahead.token == (int)RESERVED.TK_EOS); ls.lookahead.token = LLex(ls, ls.lookahead.seminfo); }
private static void pushclosure(LexState ls, FuncState func, expdesc v) { FuncState fs = ls.fs; Proto f = fs.f; int oldsize = f.sizep; int i; LuaMGrowVector(ls.L, ref f.p, fs.np, ref f.sizep, MAXARG_Bx, "constant table overflow"); while (oldsize < f.sizep) f.p[oldsize++] = null; f.p[fs.np++] = func.f; LuaCObjBarrier(ls.L, f, func.f); init_exp(v, expkind.VRELOCABLE, LuaKCodeABx(fs, OpCode.OP_CLOSURE, 0, fs.np - 1)); for (i=0; i<func.f.nups; i++) { OpCode o = ((int)func.upvalues[i].k == (int)expkind.VLOCAL) ? OpCode.OP_MOVE : OpCode.OP_GETUPVAL; LuaKCodeABC(fs, o, 0, func.upvalues[i].info, 0); } }
/// <summary> /// 词法分析 得到TOKEN列表 /// </summary> /// <param name="jtsql"></param> /// <returns></returns> internal static List <Token> Tokenize(string jtsql) { // 空 if (null == jtsql) { throw new ArgumentNullException(); } // 结果 List <Token> token_list = new List <Token>(); // 上一个Token Token last_token = new Token(TokenType.Default, -1, -1, 1, string.Empty); // 当前状态 LexState now = LexState.Default; // 字符串转为字符数组 char[] jtsql_chars = jtsql.ToCharArray(); // 长度 long jtsql_chars_length = jtsql_chars.GetLongLength(0); // Token起始索引 long temp_token_start_index = 0; // 当前行数 long line = 1; // 遍历字符 for (var i = 0; i < jtsql_chars_length; i++) { // 当前字符 char c = jtsql_chars[i]; // 进入新状态标记 bool is_entering_new_state = false; // 产生新token标志 bool is_producing_new_token = false; // 新token类型 TokenType new_token_type = TokenType.Default; // 根据当前状态找不同的字符 switch (now) { #region 普通状态 // 普通状态 可以进入任何Token状态 case LexState.Default: if (c.IsSlash()) { // 进入状态标记 is_entering_new_state = true; // 注释起始 斜杠 / now = LexState.CommentStartSlash; } else if (c.IsDollar()) { // 进入状态标记 is_entering_new_state = true; // JS内嵌SQL起始 $ now = LexState.Dolllar; } else if (c.IsGreaterThan()) { // 进入状态标记 is_entering_new_state = true; // JS内嵌SQL结束 > now = LexState.GreaterThan; } else if (c.IsCurlyBracketLeft()) { // 进入状态标记 is_entering_new_state = true; // SQL内嵌JS起始 { now = LexState.CurlyBracketLeft; } else if (c.IsCurlyBracketRight()) { // 进入状态标记 is_entering_new_state = true; // SQL内嵌JS结束 } now = LexState.CurlyBracketRight; } // 进入了新状态 if (is_entering_new_state) { // 记录进入状态时的索引 temp_token_start_index = i; } break; #endregion #region 注释状态 case LexState.CommentStartSlash: if (c.IsSlash()) { // 进入状态标记 is_entering_new_state = true; // 行注释 第二个斜杠 / now = LexState.InlineCommentSecondSlash; } else if (c.IsAsterisk()) { // 进入状态标记 is_entering_new_state = true; // 块注释起始 星号 * now = LexState.BlockCommentStartAsterisk; } else { // 进入注释状态失败 返回到普通状态 now = LexState.Default; } break; case LexState.InlineCommentSecondSlash: if (c.IsNewLine()) { // 产生新token标志 is_producing_new_token = true; new_token_type = TokenType.Comment; } break; case LexState.BlockCommentStartAsterisk: if (c.IsAsterisk()) { // 块注释结束 星号 * now = LexState.BlockCommentEndAsterisk; } break; case LexState.BlockCommentEndAsterisk: if (c.IsSlash()) { // 产生新token标志 is_producing_new_token = true; new_token_type = TokenType.Comment; } else if (c.IsAsterisk()) { // 还是星号 什么也不做 接着找 } else { // 重新寻找星号 * now = LexState.BlockCommentStartAsterisk; } break; #endregion #region 内嵌状态 $< > case LexState.Dolllar: if (c.IsLessThan()) { // 产生新token标志 is_producing_new_token = true; new_token_type = TokenType.SqlInJsStart; } else { // 返回普通状态 now = LexState.Default; } break; case LexState.GreaterThan: if (c.IsBlank() && !c.IsNewLine()) { // 空白符 什么也不做 } else if (c.IsSemicolon() || c.IsNewLine() || c.IsCurlyBracketRight()) { // 右花括号、分号和换行 产生新token is_producing_new_token = true; new_token_type = TokenType.SqlInJsEnd; // 这个字符实际上是提前读取 因此须重新判断 goto prdc_new_token_then_re_loop; } else { // 返回普通状态 now = LexState.Default; // 需要重新判断当前字符 goto re_loop; } break; #endregion #region 内嵌状态 {{ }} case LexState.CurlyBracketLeft: if (c.IsCurlyBracketLeft()) { // 产生新token标志 is_producing_new_token = true; new_token_type = TokenType.JsInSqlStart; } else { // 返回普通状态 now = LexState.Default; } break; case LexState.CurlyBracketRight: if (c.IsCurlyBracketRight()) { // 产生新token标志 is_producing_new_token = true; new_token_type = TokenType.JsInSqlEnd; } else { // 返回普通状态 now = LexState.Default; } break; #endregion default: break; } // 产生新token ProduceNewToken(ref now, ref last_token, is_producing_new_token, jtsql, token_list, last_token, temp_token_start_index, i, line, new_token_type); // 行数 if (c.IsNewLine()) { line++; } // 下次循环 next_loop: continue; // 重进本次循环 re_loop: // 回退索引 i--; continue; // 产生新token 然后重进本次循环 prdc_new_token_then_re_loop: // 回退索引 i--; // 产生新token ProduceNewToken(ref now, ref last_token, is_producing_new_token, jtsql, token_list, last_token, temp_token_start_index, i, line, new_token_type); continue; } return(token_list); }