private void readChar()
{
readTillNextChar('\'');
if (ct.Length != 1)
{
ErrorExceptionHandler.Throw(LexerException.InvalidToken("Can only store one char", ct, lc));
}
}
public Tuple <int, T> Next()
{
state = lexer.GetInitialState();
lexeme.Clear();
while (true)
{
int peek = source.Peek();
// Replace EOF with 0, or we will read outside of the table.
if (peek == -1)
{
// If reading the end of file and the lexeme is empty, return end of stream token
// If the lexeme isn't empty, it must try to find out whatever it is in the lexeme.
if (lexeme.Length == 0)
{
return(new Tuple <int, T>(lexer.endOfInputTokenNumber, default(T)));
}
peek = 0;
}
var c = (char)peek;
TState nextState = lexer.GetNextState(state, c);
var reachedTermination = lexer.ReachedTermination(nextState);
if (reachedTermination)
{
// We have reached termination
// Two possibilities, current state accepts, if so return token ID
// else there is an error
var action = lexer.GetAction(state);
if (action != null && lexeme.Length > 0)
{
// If tokennumber is int.MinValue it is an ignored token, like typically whitespace.
// In that case, dont return, continue lexing with the reset parser to get the next token.
if (action.Item1 == int.MinValue)
{
// Reset state
state = lexer.GetInitialState();
// Clear lexeme
lexeme.Clear();
}
else
{
// Token completed. Return it
return(new Tuple <int, T>(action.Item1,
action.Item2 == null ? default(T) : action.Item2(lexeme.ToString())));
}
}
else
{
// We get here if there is no action at the state where the lexer cannot continue given the input.
// This is fail.
var lexerException =
new LexerException(string.Format("Invalid character '{0}'",
c == '\0' ? "NULL" : c.ToString()))
{
LineContents = currentLine.ToString(),
LineNumber = lineNumber
};
throw lexerException;
}
}
else
{
// Peek is still last char. If we are going to be switching lines
// add to the line number and clear the current line buffer
if (c == '\n')
{
lineNumber++;
currentLine.Clear();
}
else
{
currentLine.Append(c);
}
// Machine has not terminated.
// Switch states, append character to lexeme.
state = nextState;
lexeme.Append(c);
source.Read();
}
}
}