private Token ReadNumberToken(int fromLine, int fromCol, bool leadingDot) { StringBuilder text = new StringBuilder(32); //INT : Digit+ //HEX : '0' [xX] HexDigit+ //FLOAT : Digit+ '.' Digit* ExponentPart? // | '.' Digit+ ExponentPart? // | Digit+ ExponentPart //HEX_FLOAT : '0' [xX] HexDigit+ '.' HexDigit* HexExponentPart? // | '0' [xX] '.' HexDigit+ HexExponentPart? // | '0' [xX] HexDigit+ HexExponentPart // // ExponentPart : [eE] [+-]? Digit+ // HexExponentPart : [pP] [+-]? Digit+ bool isHex = false; bool dotAdded = false; bool exponentPart = false; bool exponentSignAllowed = false; if (leadingDot) { text.Append("0."); } else if (CursorChar() == '0') { text.Append(CursorChar()); char secondChar = CursorCharNext(); if (secondChar == 'x' || secondChar == 'X') { isHex = true; text.Append(CursorChar()); CursorCharNext(); } } for (char c = CursorChar(); CursorNotEof(); c = CursorCharNext()) { if (exponentSignAllowed && (c == '+' || c == '-')) { exponentSignAllowed = false; text.Append(c); } else if (LexerUtils.CharIsDigit(c)) { text.Append(c); } else if (c == '.' && !dotAdded) { dotAdded = true; text.Append(c); } else if (LexerUtils.CharIsHexDigit(c) && isHex && !exponentPart) { text.Append(c); } else if (c == 'e' || c == 'E' || (isHex && (c == 'p' || c == 'P'))) { text.Append(c); exponentPart = true; exponentSignAllowed = true; dotAdded = true; } else { break; } } TokenType numberType = TokenType.Number; if (isHex && (dotAdded || exponentPart)) { numberType = TokenType.Number_HexFloat; } else if (isHex) { numberType = TokenType.Number_Hex; } string tokenStr = text.ToString(); return(CreateToken(numberType, fromLine, fromCol, tokenStr)); }
private Token ReadToken() { SkipWhiteSpace(); int fromLine = m_Line; int fromCol = m_Col; if (!CursorNotEof()) { return(CreateToken(TokenType.Eof, fromLine, fromCol, "<eof>")); } char c = CursorChar(); switch (c) { case '|': CursorCharNext(); return(CreateToken(TokenType.Lambda, fromLine, fromCol, "|")); case ';': CursorCharNext(); return(CreateToken(TokenType.SemiColon, fromLine, fromCol, ";")); case '=': return(PotentiallyDoubleCharOperator('=', TokenType.Op_Assignment, TokenType.Op_Equal, fromLine, fromCol)); case '<': return(PotentiallyDoubleCharOperator('=', TokenType.Op_LessThan, TokenType.Op_LessThanEqual, fromLine, fromCol)); case '>': return(PotentiallyDoubleCharOperator('=', TokenType.Op_GreaterThan, TokenType.Op_GreaterThanEqual, fromLine, fromCol)); case '~': case '!': if (CursorCharNext() != '=') { throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", c); } CursorCharNext(); return(CreateToken(TokenType.Op_NotEqual, fromLine, fromCol, "~=")); case '.': { char next = CursorCharNext(); if (next == '.') { return(PotentiallyDoubleCharOperator('.', TokenType.Op_Concat, TokenType.VarArgs, fromLine, fromCol)); } else if (LexerUtils.CharIsDigit(next)) { return(ReadNumberToken(fromLine, fromCol, true)); } else { return(CreateToken(TokenType.Dot, fromLine, fromCol, ".")); } } case '+': return(CreateSingleCharToken(TokenType.Op_Add, fromLine, fromCol)); case '-': { char next = CursorCharNext(); if (next == '-') { return(ReadComment(fromLine, fromCol)); } else { return(CreateToken(TokenType.Op_MinusOrSub, fromLine, fromCol, "-")); } } case '*': return(CreateSingleCharToken(TokenType.Op_Mul, fromLine, fromCol)); case '/': return(CreateSingleCharToken(TokenType.Op_Div, fromLine, fromCol)); case '%': return(CreateSingleCharToken(TokenType.Op_Mod, fromLine, fromCol)); case '^': return(CreateSingleCharToken(TokenType.Op_Pwr, fromLine, fromCol)); case '$': return(PotentiallyDoubleCharOperator('{', TokenType.Op_Dollar, TokenType.Brk_Open_Curly_Shared, fromLine, fromCol)); case '#': if (m_Cursor == 0 && m_Code.Length > 1 && m_Code[1] == '!') { return(ReadHashBang(fromLine, fromCol)); } return(CreateSingleCharToken(TokenType.Op_Len, fromLine, fromCol)); case '[': { char next = CursorCharNext(); if (next == '=' || next == '[') { string str = ReadLongString(fromLine, fromCol, null, "string"); return(CreateToken(TokenType.String_Long, fromLine, fromCol, str)); } return(CreateToken(TokenType.Brk_Open_Square, fromLine, fromCol, "[")); } case ']': return(CreateSingleCharToken(TokenType.Brk_Close_Square, fromLine, fromCol)); case '(': return(CreateSingleCharToken(TokenType.Brk_Open_Round, fromLine, fromCol)); case ')': return(CreateSingleCharToken(TokenType.Brk_Close_Round, fromLine, fromCol)); case '{': return(CreateSingleCharToken(TokenType.Brk_Open_Curly, fromLine, fromCol)); case '}': return(CreateSingleCharToken(TokenType.Brk_Close_Curly, fromLine, fromCol)); case ',': return(CreateSingleCharToken(TokenType.Comma, fromLine, fromCol)); case ':': return(PotentiallyDoubleCharOperator(':', TokenType.Colon, TokenType.DoubleColon, fromLine, fromCol)); case '"': case '\'': return(ReadSimpleStringToken(fromLine, fromCol)); case '\0': throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar()) { IsPrematureStreamTermination = true }; default: { if (char.IsLetter(c) || c == '_') { string name = ReadNameToken(); return(CreateNameToken(name, fromLine, fromCol)); } else if (LexerUtils.CharIsDigit(c)) { return(ReadNumberToken(fromLine, fromCol, false)); } } throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar()); } }