private Token ReadNumberToken(int fromLine, int fromCol, bool leadingDot)
{
StringBuilder text = new StringBuilder(32);
//INT : Digit+
//HEX : '0' [xX] HexDigit+
//FLOAT : Digit+ '.' Digit* ExponentPart?
// | '.' Digit+ ExponentPart?
// | Digit+ ExponentPart
//HEX_FLOAT : '0' [xX] HexDigit+ '.' HexDigit* HexExponentPart?
// | '0' [xX] '.' HexDigit+ HexExponentPart?
// | '0' [xX] HexDigit+ HexExponentPart
//
// ExponentPart : [eE] [+-]? Digit+
// HexExponentPart : [pP] [+-]? Digit+
bool isHex = false;
bool dotAdded = false;
bool exponentPart = false;
bool exponentSignAllowed = false;
if (leadingDot)
{
text.Append("0.");
}
else if (CursorChar() == '0')
{
text.Append(CursorChar());
char secondChar = CursorCharNext();
if (secondChar == 'x' || secondChar == 'X')
{
isHex = true;
text.Append(CursorChar());
CursorCharNext();
}
}
for (char c = CursorChar(); CursorNotEof(); c = CursorCharNext())
{
if (exponentSignAllowed && (c == '+' || c == '-'))
{
exponentSignAllowed = false;
text.Append(c);
}
else if (LexerUtils.CharIsDigit(c))
{
text.Append(c);
}
else if (c == '.' && !dotAdded)
{
dotAdded = true;
text.Append(c);
}
else if (LexerUtils.CharIsHexDigit(c) && isHex && !exponentPart)
{
text.Append(c);
}
else if (c == 'e' || c == 'E' || (isHex && (c == 'p' || c == 'P')))
{
text.Append(c);
exponentPart = true;
exponentSignAllowed = true;
dotAdded = true;
}
else
{
break;
}
}
TokenType numberType = TokenType.Number;
if (isHex && (dotAdded || exponentPart))
{
numberType = TokenType.Number_HexFloat;
}
else if (isHex)
{
numberType = TokenType.Number_Hex;
}
string tokenStr = text.ToString();
return(CreateToken(numberType, fromLine, fromCol, tokenStr));
}
private Token ReadToken()
{
SkipWhiteSpace();
int fromLine = m_Line;
int fromCol = m_Col;
if (!CursorNotEof())
{
return(CreateToken(TokenType.Eof, fromLine, fromCol, "<eof>"));
}
char c = CursorChar();
switch (c)
{
case '|':
CursorCharNext();
return(CreateToken(TokenType.Lambda, fromLine, fromCol, "|"));
case ';':
CursorCharNext();
return(CreateToken(TokenType.SemiColon, fromLine, fromCol, ";"));
case '=':
return(PotentiallyDoubleCharOperator('=', TokenType.Op_Assignment, TokenType.Op_Equal, fromLine, fromCol));
case '<':
return(PotentiallyDoubleCharOperator('=', TokenType.Op_LessThan, TokenType.Op_LessThanEqual, fromLine, fromCol));
case '>':
return(PotentiallyDoubleCharOperator('=', TokenType.Op_GreaterThan, TokenType.Op_GreaterThanEqual, fromLine, fromCol));
case '~':
case '!':
if (CursorCharNext() != '=')
{
throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", c);
}
CursorCharNext();
return(CreateToken(TokenType.Op_NotEqual, fromLine, fromCol, "~="));
case '.':
{
char next = CursorCharNext();
if (next == '.')
{
return(PotentiallyDoubleCharOperator('.', TokenType.Op_Concat, TokenType.VarArgs, fromLine, fromCol));
}
else if (LexerUtils.CharIsDigit(next))
{
return(ReadNumberToken(fromLine, fromCol, true));
}
else
{
return(CreateToken(TokenType.Dot, fromLine, fromCol, "."));
}
}
case '+':
return(CreateSingleCharToken(TokenType.Op_Add, fromLine, fromCol));
case '-':
{
char next = CursorCharNext();
if (next == '-')
{
return(ReadComment(fromLine, fromCol));
}
else
{
return(CreateToken(TokenType.Op_MinusOrSub, fromLine, fromCol, "-"));
}
}
case '*':
return(CreateSingleCharToken(TokenType.Op_Mul, fromLine, fromCol));
case '/':
return(CreateSingleCharToken(TokenType.Op_Div, fromLine, fromCol));
case '%':
return(CreateSingleCharToken(TokenType.Op_Mod, fromLine, fromCol));
case '^':
return(CreateSingleCharToken(TokenType.Op_Pwr, fromLine, fromCol));
case '$':
return(PotentiallyDoubleCharOperator('{', TokenType.Op_Dollar, TokenType.Brk_Open_Curly_Shared, fromLine, fromCol));
case '#':
if (m_Cursor == 0 && m_Code.Length > 1 && m_Code[1] == '!')
{
return(ReadHashBang(fromLine, fromCol));
}
return(CreateSingleCharToken(TokenType.Op_Len, fromLine, fromCol));
case '[':
{
char next = CursorCharNext();
if (next == '=' || next == '[')
{
string str = ReadLongString(fromLine, fromCol, null, "string");
return(CreateToken(TokenType.String_Long, fromLine, fromCol, str));
}
return(CreateToken(TokenType.Brk_Open_Square, fromLine, fromCol, "["));
}
case ']':
return(CreateSingleCharToken(TokenType.Brk_Close_Square, fromLine, fromCol));
case '(':
return(CreateSingleCharToken(TokenType.Brk_Open_Round, fromLine, fromCol));
case ')':
return(CreateSingleCharToken(TokenType.Brk_Close_Round, fromLine, fromCol));
case '{':
return(CreateSingleCharToken(TokenType.Brk_Open_Curly, fromLine, fromCol));
case '}':
return(CreateSingleCharToken(TokenType.Brk_Close_Curly, fromLine, fromCol));
case ',':
return(CreateSingleCharToken(TokenType.Comma, fromLine, fromCol));
case ':':
return(PotentiallyDoubleCharOperator(':', TokenType.Colon, TokenType.DoubleColon, fromLine, fromCol));
case '"':
case '\'':
return(ReadSimpleStringToken(fromLine, fromCol));
case '\0':
throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar())
{
IsPrematureStreamTermination = true
};
default:
{
if (char.IsLetter(c) || c == '_')
{
string name = ReadNameToken();
return(CreateNameToken(name, fromLine, fromCol));
}
else if (LexerUtils.CharIsDigit(c))
{
return(ReadNumberToken(fromLine, fromCol, false));
}
}
throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar());
}
}