private bool StringState(char c, out LexedToken token)
{
token = null;
if (c == '\\')
{
// we gonna get the escaped character from the next state
StateFunction = EscapeStringState;
// consume this slash
return(true);
}
else if (c == '"')
{
// return the built string
token = new LexedToken(LexedTokenType.String, currentValue.ToString());
// consume this closing quotes
return(true);
}
// build up the string
currentValue.Append(c);
return(true);
}
private bool NumberState(char c, out LexedToken token)
{
token = null;
if (char.IsDigit(c))
{
// consume this digit
currentValue.Append(c);
return(true);
}
else if (!hasDecimalPlace && c == '.')
{
hasDecimalPlace = true;
// our first decimal point, consume it
currentValue.Append(c);
return(true);
}
else
{
// we end it here since we found some other non-digit
token = new LexedToken(LexedTokenType.Number, currentValue.ToString());
// don't consume the character
return(false);
}
}
private bool LetterState(char c, out LexedToken token)
{
token = null;
if (char.IsLetterOrDigit(c) || c == '_')
{
// bumped into a letter/digit, so we build the string further
currentValue.Append(c);
// consume it
return(true);
}
else
{
// we hit some other random character
// so we end our processing here and determine what we have
string value = currentValue.ToString();
if (keywords.Contains(value))
{
token = new LexedToken(LexedTokenType.Keyword, value);
}
else
{
token = new LexedToken(LexedTokenType.Identifier, value);
}
// and we don't consume the character in case it is a start of a new token
return(false);
}
}
public LexedToken GetNextToken()
{
LexedToken token = null;
bool consumeChar = false;
while (!isEOF)
{
// line number calculation
if (currentChar == '\n')
{
lineNumber++;
positionNumber = 0;
}
consumeChar = StateFunction(currentChar, out token);
// we consume character only if the last
// call asked us to do so
if (consumeChar)
{
NextChar();
}
if (token != null)
{
// we managed to grab a token, so we should clear our state and return
// the token
ClearState();
token.PositionNumber = currentLexedPosition;
token.LineNumber = currentLineNumber;
return(token);
}
// we didn't get any token
// perhaps it was a state change
}
// we reached EOF, we see if we can get anything out of our current state?
if (currentValue.Length > 0)
{
StateFunction('\0', out token);
if (token != null)
{
ClearState();
token.PositionNumber = currentLexedPosition;
token.LineNumber = currentLineNumber;
}
return(token);
}
else
{
// nah, nothing left
return(null);
}
}
private bool EscapeStringState(char c, out LexedToken token)
{
token = null;
if (escapeLookup.ContainsKey(c))
{
// we support this escape character
currentValue.Append(escapeLookup[c]);
// set the state back to string builder
StateFunction = StringState;
// consume this character
return(true);
}
else
{
// return an invalid token and consume this
MarkPosition();
token = new LexedToken(LexedTokenType.Invalid, "\\" + c.ToString());
return(true);
}
}
/// <summary>
/// The initial state
/// </summary>
private bool RootState(char c, out LexedToken token)
{
token = null;
if (char.IsWhiteSpace(c)) // skip whitespaces
{
return(true);
}
if (char.IsLetter(c) || c == '_')
{
MarkPosition();
// we bumped into a letter
currentValue.Append(c);
// so we change to an appropriate state
StateFunction = LetterState;
// consume this character
return(true);
}
else if (char.IsDigit(c))
{
MarkPosition();
// we got a first digit, let's not consume it first and let the
// state function do that
StateFunction = NumberState;
return(false);
}
else if (c == '"')
{
MarkPosition();
// the start of a string
StateFunction = StringState;
// consume this character
return(true);
}
else if (c == '+')
{
token = SingularToken(LexedTokenType.OperatorPlus, c);
return(true);
}
else if (c == '-')
{
token = SingularToken(LexedTokenType.OperatorMinus, c);
return(true);
}
else if (c == '*')
{
token = SingularToken(LexedTokenType.OperatorMultiply, c);
return(true);
}
else if (c == '/')
{
token = SingularToken(LexedTokenType.OperatorDivide, c);
return(true);
}
else if (c == '(')
{
token = SingularToken(LexedTokenType.OpenParen, c);
return(true);
}
else if (c == ')')
{
token = SingularToken(LexedTokenType.CloseParen, c);
return(true);
}
else if (c == '{')
{
token = SingularToken(LexedTokenType.OpenBraces, c);
return(true);
}
else if (c == '}')
{
token = SingularToken(LexedTokenType.CloseBraces, c);
return(true);
}
else if (c == ';')
{
token = SingularToken(LexedTokenType.Semicolon, c);
return(true);
}
else if (c == '=')
{
MarkPosition();
// TODO: Transition to a state to get the ==
currentValue.Append(c);
token = new LexedToken(LexedTokenType.OperatorEqual, currentValue.ToString());
return(true);
}
MarkPosition();
currentValue.Append(c);
token = new LexedToken(LexedTokenType.Invalid, currentValue.ToString());
return(true);
}