示例#1
0
 public double GetNumberValue()
 {
     if (this.Type == TokenType.Number)
     {
         return(LexerUtils.ParseNumber(this));
     }
     else if (this.Type == TokenType.Number_Hex)
     {
         return(LexerUtils.ParseHexInteger(this));
     }
     else if (this.Type == TokenType.Number_HexFloat)
     {
         return(LexerUtils.ParseHexFloat(this));
     }
     else
     {
         throw new NotSupportedException("GetNumberValue is supported only on numeric tokens");
     }
 }
示例#2
0
        public static string ReadHexProgressive(string s, ref double d, out int digits)
        {
            digits = 0;

            for (int i = 0; i < s.Length; i++)
            {
                char c = s[i];

                if (LexerUtils.CharIsHexDigit(c))
                {
                    int v = LexerUtils.HexDigit2Value(c);
                    d *= 16.0;
                    d += v;
                    ++digits;
                }
                else
                {
                    return(s.Substring(i));
                }
            }

            return(string.Empty);
        }
示例#3
0
        private Token ReadSimpleStringToken(int fromLine, int fromCol)
        {
            StringBuilder text      = new StringBuilder(32);
            char          separator = CursorChar();

            for (char c = CursorCharNext(); CursorNotEof(); c = CursorCharNext())
            {
redo_Loop:

                if (c == '\\')
                {
                    text.Append(c);
                    c = CursorCharNext();
                    text.Append(c);

                    if (c == '\r')
                    {
                        c = CursorCharNext();
                        if (c == '\n')
                        {
                            text.Append(c);
                        }
                        else
                        {
                            goto redo_Loop;
                        }
                    }
                    else if (c == 'z')
                    {
                        c = CursorCharNext();

                        if (char.IsWhiteSpace(c))
                        {
                            SkipWhiteSpace();
                        }

                        c = CursorChar();

                        goto redo_Loop;
                    }
                }
                else if (c == '\n' || c == '\r')
                {
                    throw new SyntaxErrorException(
                              CreateToken(TokenType.Invalid, fromLine, fromCol),
                              "unfinished string near '{0}'", text.ToString());
                }
                else if (c == separator)
                {
                    CursorCharNext();
                    Token t = CreateToken(TokenType.String, fromLine, fromCol);
                    t.Text = LexerUtils.UnescapeLuaString(t, text.ToString());
                    return(t);
                }
                else
                {
                    text.Append(c);
                }
            }

            throw new SyntaxErrorException(
                      CreateToken(TokenType.Invalid, fromLine, fromCol),
                      "unfinished string near '{0}'", text.ToString())
                  {
                      IsPrematureStreamTermination = true
                  };
        }
示例#4
0
        private Token ReadNumberToken(int fromLine, int fromCol, bool leadingDot)
        {
            StringBuilder text = new StringBuilder(32);

            //INT : Digit+
            //HEX : '0' [xX] HexDigit+
            //FLOAT : Digit+ '.' Digit* ExponentPart?
            //		| '.' Digit+ ExponentPart?
            //		| Digit+ ExponentPart
            //HEX_FLOAT : '0' [xX] HexDigit+ '.' HexDigit* HexExponentPart?
            //			| '0' [xX] '.' HexDigit+ HexExponentPart?
            //			| '0' [xX] HexDigit+ HexExponentPart
            //
            // ExponentPart : [eE] [+-]? Digit+
            // HexExponentPart : [pP] [+-]? Digit+

            bool isHex               = false;
            bool dotAdded            = false;
            bool exponentPart        = false;
            bool exponentSignAllowed = false;

            if (leadingDot)
            {
                text.Append("0.");
            }
            else if (CursorChar() == '0')
            {
                text.Append(CursorChar());
                char secondChar = CursorCharNext();

                if (secondChar == 'x' || secondChar == 'X')
                {
                    isHex = true;
                    text.Append(CursorChar());
                    CursorCharNext();
                }
            }

            for (char c = CursorChar(); CursorNotEof(); c = CursorCharNext())
            {
                if (exponentSignAllowed && (c == '+' || c == '-'))
                {
                    exponentSignAllowed = false;
                    text.Append(c);
                }
                else if (LexerUtils.CharIsDigit(c))
                {
                    text.Append(c);
                }
                else if (c == '.' && !dotAdded)
                {
                    dotAdded = true;
                    text.Append(c);
                }
                else if (LexerUtils.CharIsHexDigit(c) && isHex && !exponentPart)
                {
                    text.Append(c);
                }
                else if (c == 'e' || c == 'E' || (isHex && (c == 'p' || c == 'P')))
                {
                    text.Append(c);
                    exponentPart        = true;
                    exponentSignAllowed = true;
                    dotAdded            = true;
                }
                else
                {
                    break;
                }
            }

            TokenType numberType = TokenType.Number;

            if (isHex && (dotAdded || exponentPart))
            {
                numberType = TokenType.Number_HexFloat;
            }
            else if (isHex)
            {
                numberType = TokenType.Number_Hex;
            }

            string tokenStr = text.ToString();

            return(CreateToken(numberType, fromLine, fromCol, tokenStr));
        }
示例#5
0
        private string ReadLongString(int fromLine, int fromCol, string startpattern, string subtypeforerrors)
        {
            // here we are at the first '=' or second '['
            StringBuilder text        = new StringBuilder(1024);
            string        end_pattern = "]";

            if (startpattern == null)
            {
                for (char c = CursorChar(); ; c = CursorCharNext())
                {
                    if (c == '\0' || !CursorNotEof())
                    {
                        throw new SyntaxErrorException(
                                  CreateToken(TokenType.Invalid, fromLine, fromCol),
                                  "unfinished long {0} near '<eof>'", subtypeforerrors)
                              {
                                  IsPrematureStreamTermination = true
                              };
                    }
                    else if (c == '=')
                    {
                        end_pattern += "=";
                    }
                    else if (c == '[')
                    {
                        end_pattern += "]";
                        break;
                    }
                    else
                    {
                        throw new SyntaxErrorException(
                                  CreateToken(TokenType.Invalid, fromLine, fromCol),
                                  "invalid long {0} delimiter near '{1}'", subtypeforerrors, c)
                              {
                                  IsPrematureStreamTermination = true
                              };
                    }
                }
            }
            else
            {
                end_pattern = startpattern.Replace('[', ']');
            }


            for (char c = CursorCharNext(); ; c = CursorCharNext())
            {
                if (c == '\r')                 // XXI century and we still debate on how a newline is made. throw new DeveloperExtremelyAngryException.
                {
                    continue;
                }

                if (c == '\0' || !CursorNotEof())
                {
                    throw new SyntaxErrorException(
                              CreateToken(TokenType.Invalid, fromLine, fromCol),
                              "unfinished long {0} near '{1}'", subtypeforerrors, text.ToString())
                          {
                              IsPrematureStreamTermination = true
                          };
                }
                else if (c == ']' && CursorMatches(end_pattern))
                {
                    for (int i = 0; i < end_pattern.Length; i++)
                    {
                        CursorCharNext();
                    }

                    return(LexerUtils.AdjustLuaLongString(text.ToString()));
                }
                else
                {
                    text.Append(c);
                }
            }
        }
示例#6
0
        private Token ReadToken()
        {
            SkipWhiteSpace();

            int fromLine = m_Line;
            int fromCol  = m_Col;

            if (!CursorNotEof())
            {
                return(CreateToken(TokenType.Eof, fromLine, fromCol, "<eof>"));
            }

            char c = CursorChar();

            switch (c)
            {
            case '|':
                CursorCharNext();
                return(CreateToken(TokenType.Lambda, fromLine, fromCol, "|"));

            case ';':
                CursorCharNext();
                return(CreateToken(TokenType.SemiColon, fromLine, fromCol, ";"));

            case '=':
                return(PotentiallyDoubleCharOperator('=', TokenType.Op_Assignment, TokenType.Op_Equal, fromLine, fromCol));

            case '<':
                return(PotentiallyDoubleCharOperator('=', TokenType.Op_LessThan, TokenType.Op_LessThanEqual, fromLine, fromCol));

            case '>':
                return(PotentiallyDoubleCharOperator('=', TokenType.Op_GreaterThan, TokenType.Op_GreaterThanEqual, fromLine, fromCol));

            case '~':
            case '!':
                if (CursorCharNext() != '=')
                {
                    throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", c);
                }

                CursorCharNext();
                return(CreateToken(TokenType.Op_NotEqual, fromLine, fromCol, "~="));

            case '.':
            {
                char next = CursorCharNext();
                if (next == '.')
                {
                    return(PotentiallyDoubleCharOperator('.', TokenType.Op_Concat, TokenType.VarArgs, fromLine, fromCol));
                }
                else if (LexerUtils.CharIsDigit(next))
                {
                    return(ReadNumberToken(fromLine, fromCol, true));
                }
                else
                {
                    return(CreateToken(TokenType.Dot, fromLine, fromCol, "."));
                }
            }

            case '+':
                return(CreateSingleCharToken(TokenType.Op_Add, fromLine, fromCol));

            case '-':
            {
                char next = CursorCharNext();
                if (next == '-')
                {
                    return(ReadComment(fromLine, fromCol));
                }
                else
                {
                    return(CreateToken(TokenType.Op_MinusOrSub, fromLine, fromCol, "-"));
                }
            }

            case '*':
                return(CreateSingleCharToken(TokenType.Op_Mul, fromLine, fromCol));

            case '/':
                return(CreateSingleCharToken(TokenType.Op_Div, fromLine, fromCol));

            case '%':
                return(CreateSingleCharToken(TokenType.Op_Mod, fromLine, fromCol));

            case '^':
                return(CreateSingleCharToken(TokenType.Op_Pwr, fromLine, fromCol));

            case '$':
                return(PotentiallyDoubleCharOperator('{', TokenType.Op_Dollar, TokenType.Brk_Open_Curly_Shared, fromLine, fromCol));

            case '#':
                if (m_Cursor == 0 && m_Code.Length > 1 && m_Code[1] == '!')
                {
                    return(ReadHashBang(fromLine, fromCol));
                }

                return(CreateSingleCharToken(TokenType.Op_Len, fromLine, fromCol));

            case '[':
            {
                char next = CursorCharNext();
                if (next == '=' || next == '[')
                {
                    string str = ReadLongString(fromLine, fromCol, null, "string");
                    return(CreateToken(TokenType.String_Long, fromLine, fromCol, str));
                }
                return(CreateToken(TokenType.Brk_Open_Square, fromLine, fromCol, "["));
            }

            case ']':
                return(CreateSingleCharToken(TokenType.Brk_Close_Square, fromLine, fromCol));

            case '(':
                return(CreateSingleCharToken(TokenType.Brk_Open_Round, fromLine, fromCol));

            case ')':
                return(CreateSingleCharToken(TokenType.Brk_Close_Round, fromLine, fromCol));

            case '{':
                return(CreateSingleCharToken(TokenType.Brk_Open_Curly, fromLine, fromCol));

            case '}':
                return(CreateSingleCharToken(TokenType.Brk_Close_Curly, fromLine, fromCol));

            case ',':
                return(CreateSingleCharToken(TokenType.Comma, fromLine, fromCol));

            case ':':
                return(PotentiallyDoubleCharOperator(':', TokenType.Colon, TokenType.DoubleColon, fromLine, fromCol));

            case '"':
            case '\'':
                return(ReadSimpleStringToken(fromLine, fromCol));

            case '\0':
                throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar())
                      {
                          IsPrematureStreamTermination = true
                      };

            default:
            {
                if (char.IsLetter(c) || c == '_')
                {
                    string name = ReadNameToken();
                    return(CreateNameToken(name, fromLine, fromCol));
                }
                else if (LexerUtils.CharIsDigit(c))
                {
                    return(ReadNumberToken(fromLine, fromCol, false));
                }
            }

                throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar());
            }
        }