static int TokenizeUndefined(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { token = null; return tokStart; }
static int TokenizeSlash(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { string ts = tenv.Source; if (ts[tokStart + 1] == '/') { return TokenizeLine(ctx, out token, tenv, tokStart, thunk); } else if (ts[tokStart + 1] == '*') { return TokenizeComment(ctx, out token, tenv, tokStart, thunk); } return TokenizeOperator(ctx, out token, tenv, tokStart, thunk); }
static int TokenizeSymbol(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { int pos = tokStart; string ts = tenv.Source; while (pos < ts.Length && IsSymbolic(ts[pos])) ++pos; token = new Token(TokenType.SYMBOL, ts.Substring(tokStart, pos - tokStart), tokStart); return pos; }
static int TokenizeOneCharOperator(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { token = new Token(TokenType.OPERATOR, tenv.Source.Substring(tokStart, 1), tokStart); return ++tokStart; }
static int TokenizeOperator(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { int pos = tokStart; string ts = tenv.Source; while (pos < ts.Length) { switch (ts[pos]) { case '<': case '>': case '@': case '$': case '#': case '+': case '-': case '*': case '%': case '/': case '=': case '&': case '?': case ':': case '.': case '^': case '!': case '~': case '|': ++pos; continue; } break; } token = new Token(TokenType.OPERATOR, ts.Substring(tokStart, pos - tokStart), tokStart); return pos; }
static int TokenizeNextline(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { int pos = tokStart; string ts = tenv.Source; if (pos < ts.Length) { if (ts[pos] == '\r') { ++pos; } if (ts[pos] == '\n') { ++pos; } } tenv.Line.LineNumber += 1; tenv.Bol = pos; return TokenizeIndent(ctx, out token, tenv, pos, thunk); }
static int TokenizeNumber(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { int pos = tokStart; bool dotAppeared = false; string ts = tenv.Source; while (pos < ts.Length) { char ch = ts[pos++]; if (ch == '_') continue; // nothing if (ch == '.') { if (!IsNumChar(ts[pos])) { --pos; break; } dotAppeared = true; continue; } if ((ch == 'e' || ch == 'E') && (ts[pos] == '+' || ts[pos] == '-')) { pos++; continue; } if (!IsAlphaOrNum(ch)) { --pos; break; } } string str = ts.Substring(tokStart, pos - tokStart).Replace("_", ""); if (dotAppeared) { token = new Token(TokenType.FLOAT, str, tokStart); } else { token = new Token(TokenType.INT, str, tokStart); } return pos; // next }
static int TokenizeLine(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { string ts = tenv.Source; int pos = tokStart; while (ts[pos] != '\n' || ts[pos] != '\r') ++pos; token = null; return pos; }
static int TokenizeIndent(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { int pos = tokStart; string ts = tenv.Source; int indent = 0; if (pos < ts.Length) { char ch = ts[pos++]; if (ch == '\t') { indent += tenv.TabWidth; } else if (ch == ' ') { indent += 1; } else { --pos; } } token = null; return pos; }
static int TokenizeDoubleQuote(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { string ts = tenv.Source; char ch = '\0'; char prev = '"'; int pos = tokStart + 1; token = null; while (pos < ts.Length) { ch = ts[pos++]; if (ch == '\n' || ch == '\r') { break; } if (ch == '"' && prev != '\\') { token = new Token(TokenType.TEXT, ts.Substring(tokStart + 1, (pos - 1) - (tokStart + 1)), tokStart + 1); return pos; } prev = ch; } return pos - 1; }
static int TokenizeComment(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { string ts = tenv.Source; int pos = tokStart + 2; char ch = '\0'; char prev = '\0'; int level = 1; token = null; while (pos < ts.Length) { ch = ts[pos++]; if (ch == '\r') { tenv.Line.LineNumber += 1; if (ts[pos] == '\n') ++pos; } else if (ch == '\n') { tenv.Line.LineNumber += 1; } if (prev == '*' && ch == '/') { level--; if (level == 0) return pos; } else if (prev == '/' && ch == '*') { level++; } prev = ch; } return pos; }
static int TokenizeBlock(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk) { string ts = tenv.Source; char ch = '\0'; int pos = tokStart + 1; int level = 1; FTokenizer[] fmat = tenv.TokenizerMatrix; token = null; while (pos < ts.Length) { ch = ts[pos]; if (ch == '}') { level--; if (level == 0) { token = new Token(TokenType.CODE, ts.Substring(tokStart + 1, pos - 1 - tokStart), tokStart + 1); return pos + 1; } pos++; } else if (ch == '{') { level++; pos++; } else { var f = fmat[(int)charTypeMatrix[ch]]; pos = f(ctx, out token, tenv, pos, null); } } return pos; }