Пример #1
0
 static int TokenizeUndefined(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
 {
     token = null;
     return tokStart;
 }
Пример #2
0
 static int TokenizeSlash(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
 {
     string ts = tenv.Source;
     if (ts[tokStart + 1] == '/')
     {
         return TokenizeLine(ctx, out token, tenv, tokStart, thunk);
     }
     else if (ts[tokStart + 1] == '*')
     {
         return TokenizeComment(ctx, out token, tenv, tokStart, thunk);
     }
     return TokenizeOperator(ctx, out token, tenv, tokStart, thunk);
 }
Пример #3
0
        static int TokenizeSymbol(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            int pos = tokStart;
            string ts = tenv.Source;

            while (pos < ts.Length && IsSymbolic(ts[pos]))
                ++pos;

            token = new Token(TokenType.SYMBOL, ts.Substring(tokStart, pos - tokStart), tokStart);
            return pos;
        }
Пример #4
0
 static int TokenizeOneCharOperator(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
 {
     token = new Token(TokenType.OPERATOR, tenv.Source.Substring(tokStart, 1), tokStart);
     return ++tokStart;
 }
Пример #5
0
        static int TokenizeOperator(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            int pos = tokStart;
            string ts = tenv.Source;

            while (pos < ts.Length)
            {
                switch (ts[pos])
                {
                    case '<':
                    case '>':
                    case '@':
                    case '$':
                    case '#':
                    case '+':
                    case '-':
                    case '*':
                    case '%':
                    case '/':
                    case '=':
                    case '&':
                    case '?':
                    case ':':
                    case '.':
                    case '^':
                    case '!':
                    case '~':
                    case '|':
                        ++pos;
                        continue;
                }
                break;
            }
            token = new Token(TokenType.OPERATOR, ts.Substring(tokStart, pos - tokStart), tokStart);
            return pos;
        }
Пример #6
0
        static int TokenizeNextline(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            int pos = tokStart;
            string ts = tenv.Source;

            if (pos < ts.Length)
            {
                if (ts[pos] == '\r')
                {
                    ++pos;
                }
                if (ts[pos] == '\n')
                {
                    ++pos;
                }
            }

            tenv.Line.LineNumber += 1;
            tenv.Bol = pos;
            return TokenizeIndent(ctx, out token, tenv, pos, thunk);
        }
Пример #7
0
        static int TokenizeNumber(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            int pos = tokStart;
            bool dotAppeared = false;
            string ts = tenv.Source;

            while (pos < ts.Length)
            {
                char ch = ts[pos++];
                if (ch == '_')
                    continue; // nothing
                if (ch == '.')
                {
                    if (!IsNumChar(ts[pos]))
                    {
                        --pos;
                        break;
                    }
                    dotAppeared = true;
                    continue;
                }
                if ((ch == 'e' || ch == 'E') && (ts[pos] == '+' || ts[pos] == '-'))
                {
                    pos++;
                    continue;
                }
                if (!IsAlphaOrNum(ch))
                {
                    --pos;
                    break;
                }
            }

            string str = ts.Substring(tokStart, pos - tokStart).Replace("_", "");
            if (dotAppeared)
            {
                token = new Token(TokenType.FLOAT, str, tokStart);
            }
            else
            {
                token = new Token(TokenType.INT, str, tokStart);
            }
            return pos;  // next
        }
Пример #8
0
 static int TokenizeLine(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
 {
     string ts = tenv.Source;
     int pos = tokStart;
     while (ts[pos] != '\n' || ts[pos] != '\r')
         ++pos;
     token = null;
     return pos;
 }
Пример #9
0
        static int TokenizeIndent(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            int pos = tokStart;
            string ts = tenv.Source;
            int indent = 0;

            if (pos < ts.Length)
            {
                char ch = ts[pos++];
                if (ch == '\t')
                {
                    indent += tenv.TabWidth;
                }
                else if (ch == ' ')
                {
                    indent += 1;
                }
                else
                {
                    --pos;
                }
            }
            token = null;

            return pos;
        }
Пример #10
0
        static int TokenizeDoubleQuote(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            string ts = tenv.Source;
            char ch = '\0';
            char prev = '"';
            int pos = tokStart + 1;

            token = null;

            while (pos < ts.Length)
            {
                ch = ts[pos++];
                if (ch == '\n' || ch == '\r')
                {
                    break;
                }
                if (ch == '"' && prev != '\\')
                {
                    token = new Token(TokenType.TEXT, ts.Substring(tokStart + 1, (pos - 1) - (tokStart + 1)), tokStart + 1);
                    return pos;
                }
                prev = ch;
            }
            return pos - 1;
        }
Пример #11
0
        static int TokenizeComment(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            string ts = tenv.Source;
            int pos = tokStart + 2;
            char ch = '\0';
            char prev = '\0';
            int level = 1;
            token = null;

            while (pos < ts.Length)
            {
                ch = ts[pos++];
                if (ch == '\r')
                {
                    tenv.Line.LineNumber += 1;
                    if (ts[pos] == '\n')
                        ++pos;
                }
                else if (ch == '\n')
                {
                    tenv.Line.LineNumber += 1;
                }
                if (prev == '*' && ch == '/')
                {
                    level--;
                    if (level == 0)
                        return pos;
                }
                else if (prev == '/' && ch == '*')
                {
                    level++;
                }
                prev = ch;
            }

            return pos;
        }
Пример #12
0
        static int TokenizeBlock(Context ctx, out Token token, TokenizerEnvironment tenv, int tokStart, KMethod thunk)
        {
            string ts = tenv.Source;
            char ch = '\0';
            int pos = tokStart + 1;
            int level = 1;
            FTokenizer[] fmat = tenv.TokenizerMatrix;

            token = null;

            while (pos < ts.Length)
            {
                ch = ts[pos];
                if (ch == '}')
                {
                    level--;
                    if (level == 0)
                    {
                        token = new Token(TokenType.CODE, ts.Substring(tokStart + 1, pos - 1 - tokStart), tokStart + 1);
                        return pos + 1;
                    }
                    pos++;
                }
                else if (ch == '{')
                {
                    level++;
                    pos++;
                }
                else
                {
                    var f = fmat[(int)charTypeMatrix[ch]];
                    pos = f(ctx, out token, tenv, pos, null);
                }
            }
            return pos;
        }