public Lexeme GetLexeme()
{
char current;
// strip white space and comments
SkipWhiteSpaceAndComments();
current = inputStream.Next();
if (Char.IsDigit(current))
{
return(RecognizeNumericConstant(current));
}
else if (Char.IsLetter(current))
{
return(RecognizeIdentifierOrKeyWord(current));
}
else if (IsSymbol(current))
{
return(RecognizeOperator(current));
}
return(lexemeFactory.Create(LexemeType.Unknown));
}
public Token NextToken(Action <int, string> onError)
{
char cur = stream.Next();
if (Utils.IsCarriageReturn(cur))
{
return(new Token(TokenType.Skip));
}
if (Utils.IsWhiteSpace(cur))
{
return(new Token(TokenType.Whitespace));
}
if (Utils.IsTab(cur))
{
return(new Token(TokenType.Tab));
}
if (Utils.IsNewLine(cur))
{
return(new Token(TokenType.NewLine));
}
switch (cur)
{
case '=':
return(Match('=') ? Match('=') ? new WordToken("===", TokenType.StrictEqual)
: new WordToken("==", TokenType.EqualEqual)
: new Token(TokenType.Equal));
case '!':
return(Match('=') ? Match('=') ? new WordToken("!==", TokenType.StrictNotEqual)
: new WordToken("!=", TokenType.NotEqual)
: new Token(TokenType.Bang));
case '>':
return(Match('=') ? new WordToken(">=", TokenType.GreaterEqual)
: Match('>') ? Match('=') ? new WordToken(">>=", TokenType.RightShiftEqual)
: new WordToken(">>", TokenType.RightShift)
: new Token(TokenType.Greater));
case '<':
return(Match('=') ? new WordToken("<=", TokenType.LessEqual)
: Match('<') ? Match('=') ? new WordToken("<<=", TokenType.LeftShiftEqual)
: new WordToken("<<", TokenType.LeftShift)
: new Token(TokenType.Less));
case '+':
return(Match('+') ? new WordToken("++", TokenType.PlusPlus)
: Match('=') ? new WordToken("+=", TokenType.PlusEqual)
: new Token(TokenType.Plus));
case '-':
return(Match('-') ? new WordToken("--", TokenType.MinusMinus)
: Match('=') ? new WordToken("-=", TokenType.MinusEqual)
: new Token(TokenType.Minus));
case '*':
return(Match('=') ? new WordToken("*=", TokenType.StarEqual)
: new Token(TokenType.Star));
case '/':
return(Match('/') ? ParseOneLineComment()
: Match('*') ? ParseMultilineToken(onError)
: Match('=') ? new WordToken("/=", TokenType.DivideEqual)
: new Token(TokenType.Division));
case '%':
return(Match('=') ? new WordToken("%=", TokenType.ModuloEqual)
: new Token(TokenType.Modulo));
case '^':
return(Match('=') ? new WordToken("^=", TokenType.BitwiseXorEqual)
: new Token(TokenType.BitwiseXor));
case '|':
return(Match('=') ? new WordToken("|=", TokenType.BitwiseOrEqual)
: Match('|') ? new WordToken("||", TokenType.LogicalOr)
: new Token(TokenType.BitwiseOr));
case '&':
return(Match('=') ? new WordToken("&=", TokenType.BitwiseAndEqual)
: Match('&') ? new WordToken("&&", TokenType.LogicalAnd)
: new Token(TokenType.BitwiseAnd));
default:
break;
}
if (cur < 127 && Enum.IsDefined(typeof(TokenType), (int)cur))
{
return(new Token((TokenType)cur));
}
if (Utils.IsStringSingleQuote(cur) || Utils.IsStringDoubleQuote(cur))
{
return(ParseSingleQuoteString(cur, onError));
}
if (char.IsDigit(cur))
{
return(ParseNumber(cur));
}
if (char.IsLetter(cur))
{
return(ParseReservedOrIdentifier(cur));
}
onError(stream.Line, UNIDENTIFIED_SYMBOL);
return(new WordToken(cur.ToString(), TokenType.Error));
}