public double GetNumberValue()
{
if (this.Type == TokenType.Number)
{
return(LexerUtils.ParseNumber(this));
}
else if (this.Type == TokenType.Number_Hex)
{
return(LexerUtils.ParseHexInteger(this));
}
else if (this.Type == TokenType.Number_HexFloat)
{
return(LexerUtils.ParseHexFloat(this));
}
else
{
throw new NotSupportedException("GetNumberValue is supported only on numeric tokens");
}
}
public static string ReadHexProgressive(string s, ref double d, out int digits)
{
digits = 0;
for (int i = 0; i < s.Length; i++)
{
char c = s[i];
if (LexerUtils.CharIsHexDigit(c))
{
int v = LexerUtils.HexDigit2Value(c);
d *= 16.0;
d += v;
++digits;
}
else
{
return(s.Substring(i));
}
}
return(string.Empty);
}
private Token ReadSimpleStringToken(int fromLine, int fromCol)
{
StringBuilder text = new StringBuilder(32);
char separator = CursorChar();
for (char c = CursorCharNext(); CursorNotEof(); c = CursorCharNext())
{
redo_Loop:
if (c == '\\')
{
text.Append(c);
c = CursorCharNext();
text.Append(c);
if (c == '\r')
{
c = CursorCharNext();
if (c == '\n')
{
text.Append(c);
}
else
{
goto redo_Loop;
}
}
else if (c == 'z')
{
c = CursorCharNext();
if (char.IsWhiteSpace(c))
{
SkipWhiteSpace();
}
c = CursorChar();
goto redo_Loop;
}
}
else if (c == '\n' || c == '\r')
{
throw new SyntaxErrorException(
CreateToken(TokenType.Invalid, fromLine, fromCol),
"unfinished string near '{0}'", text.ToString());
}
else if (c == separator)
{
CursorCharNext();
Token t = CreateToken(TokenType.String, fromLine, fromCol);
t.Text = LexerUtils.UnescapeLuaString(t, text.ToString());
return(t);
}
else
{
text.Append(c);
}
}
throw new SyntaxErrorException(
CreateToken(TokenType.Invalid, fromLine, fromCol),
"unfinished string near '{0}'", text.ToString())
{
IsPrematureStreamTermination = true
};
}
private Token ReadNumberToken(int fromLine, int fromCol)
{
StringBuilder text = new StringBuilder(32);
//INT : Digit+
//HEX : '0' [xX] HexDigit+
//FLOAT : Digit+ '.' Digit* ExponentPart?
// | '.' Digit+ ExponentPart?
// | Digit+ ExponentPart
//HEX_FLOAT : '0' [xX] HexDigit+ '.' HexDigit* HexExponentPart?
// | '0' [xX] '.' HexDigit+ HexExponentPart?
// | '0' [xX] HexDigit+ HexExponentPart
//
// ExponentPart : [eE] [+-]? Digit+
// HexExponentPart : [pP] [+-]? Digit+
bool isHex = false;
bool dotAdded = false;
bool exponentPart = false;
bool exponentSignAllowed = false;
text.Append(CursorChar());
char secondChar = CursorCharNext();
if (secondChar == 'x' || secondChar == 'X')
{
isHex = true;
text.Append(CursorChar());
CursorCharNext();
}
for (char c = CursorChar(); CursorNotEof(); c = CursorCharNext())
{
if (exponentSignAllowed && (c == '+' || c == '-'))
{
exponentSignAllowed = false;
text.Append(c);
}
else if (char.IsDigit(c))
{
text.Append(c);
}
else if (c == '.' && !dotAdded)
{
dotAdded = true;
text.Append(c);
}
else if (LexerUtils.CharIsHexDigit(c) && isHex && !exponentPart)
{
text.Append(c);
}
else if (c == 'e' || c == 'E' || (isHex && (c == 'p' || c == 'P')))
{
text.Append(c);
exponentPart = true;
exponentSignAllowed = true;
dotAdded = true;
}
else
{
break;
}
}
TokenType numberType = TokenType.Number;
if (isHex && (dotAdded || exponentPart))
{
numberType = TokenType.Number_HexFloat;
}
else if (isHex)
{
numberType = TokenType.Number_Hex;
}
return(CreateToken(numberType, fromLine, fromCol, text.ToString()));
}
private string ReadLongString(int fromLine, int fromCol, string startpattern, string subtypeforerrors)
{
// here we are at the first '=' or second '['
StringBuilder text = new StringBuilder(1024);
string end_pattern = "]";
if (startpattern == null)
{
for (char c = CursorChar(); ; c = CursorCharNext())
{
if (c == '\0' || !CursorNotEof())
{
throw new SyntaxErrorException(
CreateToken(TokenType.Invalid, fromLine, fromCol),
"unfinished long {0} near '<eof>'", subtypeforerrors)
{
IsPrematureStreamTermination = true
};
}
else if (c == '=')
{
end_pattern += "=";
}
else if (c == '[')
{
end_pattern += "]";
break;
}
else
{
throw new SyntaxErrorException(
CreateToken(TokenType.Invalid, fromLine, fromCol),
"invalid long {0} delimiter near '{1}'", subtypeforerrors, c)
{
IsPrematureStreamTermination = true
};
}
}
}
else
{
end_pattern = startpattern.Replace('[', ']');
}
for (char c = CursorCharNext(); ; c = CursorCharNext())
{
if (c == '\r') // XXI century and we still debate on how a newline is made. throw new DeveloperExtremelyAngryException.
{
continue;
}
if (c == '\0' || !CursorNotEof())
{
throw new SyntaxErrorException(
CreateToken(TokenType.Invalid, fromLine, fromCol),
"unfinished long {0} near '{1}'", subtypeforerrors, text.ToString())
{
IsPrematureStreamTermination = true
};
}
else if (c == ']' && CursorMatches(end_pattern))
{
for (int i = 0; i < end_pattern.Length; i++)
{
CursorCharNext();
}
return(LexerUtils.AdjustLuaLongString(text.ToString()));
}
else
{
text.Append(c);
}
}
}
private Token ReadToken()
{
SkipWhiteSpace();
int fromLine = m_Line;
int fromCol = m_Col;
if (!CursorNotEof())
{
return(CreateToken(TokenType.Eof, fromLine, fromCol, "<eof>"));
}
char c = CursorChar();
switch (c)
{
case '|':
CursorCharNext();
return(CreateToken(TokenType.Lambda, fromLine, fromCol, "|"));
case ';':
CursorCharNext();
return(CreateToken(TokenType.SemiColon, fromLine, fromCol, ";"));
case '=':
return(PotentiallyDoubleCharOperator('=', TokenType.Op_Assignment, TokenType.Op_Equal, fromLine, fromCol));
case '<':
return(PotentiallyDoubleCharOperator('=', TokenType.Op_LessThan, TokenType.Op_LessThanEqual, fromLine, fromCol));
case '>':
return(PotentiallyDoubleCharOperator('=', TokenType.Op_GreaterThan, TokenType.Op_GreaterThanEqual, fromLine, fromCol));
case '~':
case '!':
if (CursorCharNext() != '=')
{
throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", c);
}
CursorCharNext();
return(CreateToken(TokenType.Op_NotEqual, fromLine, fromCol, "~="));
case '.':
if (CursorCharNext() == '.')
{
return(PotentiallyDoubleCharOperator('.', TokenType.Op_Concat, TokenType.VarArgs, fromLine, fromCol));
}
else
{
return(CreateToken(TokenType.Dot, fromLine, fromCol, "."));
}
case '+':
return(CreateSingleCharToken(TokenType.Op_Add, fromLine, fromCol));
case '-':
{
char next = CursorCharNext();
if (next == '-')
{
return(ReadComment(fromLine, fromCol));
}
else
{
return(CreateToken(TokenType.Op_MinusOrSub, fromLine, fromCol, "-"));
}
}
case '*':
return(CreateSingleCharToken(TokenType.Op_Mul, fromLine, fromCol));
case '/':
return(CreateSingleCharToken(TokenType.Op_Div, fromLine, fromCol));
case '%':
return(CreateSingleCharToken(TokenType.Op_Mod, fromLine, fromCol));
case '^':
return(CreateSingleCharToken(TokenType.Op_Pwr, fromLine, fromCol));
case '$':
return(PotentiallyDoubleCharOperator('{', TokenType.Op_Dollar, TokenType.Brk_Open_Curly_Shared, fromLine, fromCol));
case '#':
if (m_Cursor == 0 && m_Code.Length > 1 && m_Code[1] == '!')
{
return(ReadHashBang(fromLine, fromCol));
}
return(CreateSingleCharToken(TokenType.Op_Len, fromLine, fromCol));
case '[':
{
char next = CursorCharNext();
if (next == '=' || next == '[')
{
string str = ReadLongString(fromLine, fromCol, null, "string");
return(CreateToken(TokenType.String_Long, fromLine, fromCol, str));
}
return(CreateToken(TokenType.Brk_Open_Square, fromLine, fromCol, "["));
}
case ']':
return(CreateSingleCharToken(TokenType.Brk_Close_Square, fromLine, fromCol));
case '(':
return(CreateSingleCharToken(TokenType.Brk_Open_Round, fromLine, fromCol));
case ')':
return(CreateSingleCharToken(TokenType.Brk_Close_Round, fromLine, fromCol));
case '{':
return(CreateSingleCharToken(TokenType.Brk_Open_Curly, fromLine, fromCol));
case '}':
return(CreateSingleCharToken(TokenType.Brk_Close_Curly, fromLine, fromCol));
case ',':
return(CreateSingleCharToken(TokenType.Comma, fromLine, fromCol));
case ':':
return(PotentiallyDoubleCharOperator(':', TokenType.Colon, TokenType.DoubleColon, fromLine, fromCol));
case '"':
case '\'':
return(ReadSimpleStringToken(fromLine, fromCol));
case '\0':
throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar())
{
IsPrematureStreamTermination = true
};
default:
{
if (char.IsLetter(c) || c == '_')
{
string name = ReadNameToken();
return(CreateNameToken(name, fromLine, fromCol));
}
else if (LexerUtils.CharIsDigit(c))
{
return(ReadNumberToken(fromLine, fromCol));
}
}
throw new SyntaxErrorException(CreateToken(TokenType.Invalid, fromLine, fromCol), "unexpected symbol near '{0}'", CursorChar());
}
}
private Token ReadNumberToken(int fromLine, int fromCol, bool leadingDot)
{
//INT : Digit+
//HEX : '0' [xX] HexDigit+
//FLOAT : Digit+ '.' Digit* ExponentPart?
// | '.' Digit+ ExponentPart?
// | Digit+ ExponentPart
//HEX_FLOAT : '0' [xX] HexDigit+ '.' HexDigit* HexExponentPart?
// | '0' [xX] '.' HexDigit+ HexExponentPart?
// | '0' [xX] HexDigit+ HexExponentPart
//
// ExponentPart : [eE] [+-]? Digit+
// HexExponentPart : [pP] [+-]? Digit+
using (var sb = ZString.CreateStringBuilder())
{
bool isHex = false;
bool dotAdded = false;
bool exponentPart = false;
bool exponentSignAllowed = false;
if (leadingDot)
{
sb.Append("0.");
}
else if (this.CursorChar() == '0')
{
sb.Append(this.CursorChar());
char secondChar = this.CursorCharNext();
if (secondChar == 'x' || secondChar == 'X')
{
isHex = true;
sb.Append(this.CursorChar());
this.CursorCharNext();
}
}
for (char c = this.CursorChar(); this.CursorNotEof(); c = this.CursorCharNext())
{
if (exponentSignAllowed && (c == '+' || c == '-'))
{
exponentSignAllowed = false;
sb.Append(c);
}
else if (LexerUtils.CharIsDigit(c))
{
sb.Append(c);
}
else if (c == '.' && !dotAdded)
{
dotAdded = true;
sb.Append(c);
}
else if (LexerUtils.CharIsHexDigit(c) && isHex && !exponentPart)
{
sb.Append(c);
}
else if (c == 'e' || c == 'E' || (isHex && (c == 'p' || c == 'P')))
{
sb.Append(c);
exponentPart = true;
exponentSignAllowed = true;
dotAdded = true;
}
else
{
break;
}
}
var numberType = TokenType.Number;
if (isHex && (dotAdded || exponentPart))
{
numberType = TokenType.Number_HexFloat;
}
else if (isHex)
{
numberType = TokenType.Number_Hex;
}
return(this.CreateToken(numberType, fromLine, fromCol, sb.ToString()));
}
}
