/// <summary>
/// Gets the next token in the input
/// </summary>
/// <param name="contexts">The current applicable contexts</param>
/// <returns>The next token in the input</returns>
internal override TokenKernel GetNextToken(IContextProvider contexts)
{
if (isDollarEmitted)
{
return(new TokenKernel(Symbol.SID_EPSILON, -1));
}
while (true)
{
TokenMatch match = RunDFA(inputIndex);
if (!match.IsSuccess)
{
// failed to match, retry with error handling
match = RunDFAOnError(inputIndex);
}
if (match.IsSuccess)
{
if (match.state == 0)
{
// this is the dollar terminal, at the end of the input
// the index of the $ symbol is always 1
isDollarEmitted = true;
return(new TokenKernel(Symbol.SID_DOLLAR, tokens.Add(1, inputIndex, 0)));
}
else
{
// matched something
int tIndex = GetTerminalFor(match.state, contexts);
int tID = symTerminals[tIndex].ID;
if (tID == separatorID)
{
inputIndex += match.length;
continue;
}
else
{
TokenKernel token = new TokenKernel(tID, tokens.Add(tIndex, inputIndex, match.length));
inputIndex += match.length;
return(token);
}
}
}
else
{
inputIndex += match.length;
}
}
}
/// <summary>
/// Gets the next token in the input
/// </summary>
/// <param name="contexts">The current applicable contexts</param>
/// <returns>The next token in the input</returns>
internal override TokenKernel GetNextToken(IContextProvider contexts)
{
if (tokens.Size == 0)
{
// this is the first call to this method, prefetch the tokens
FindTokens();
tokenIndex = 0;
}
// no more tokens? return epsilon
if (tokenIndex >= tokens.Size)
{
return(new TokenKernel(Symbol.SID_EPSILON, 0));
}
TokenKernel result = new TokenKernel(tokens.GetSymbol(tokenIndex).ID, tokenIndex);
tokenIndex++;
return(result);
}