/// <summary> /// Gets the next token in the input /// </summary> /// <param name="contexts">The current applicable contexts</param> /// <returns>The next token in the input</returns> internal override TokenKernel GetNextToken(IContextProvider contexts) { if (isDollarEmitted) { return(new TokenKernel(Symbol.SID_EPSILON, -1)); } while (true) { TokenMatch match = RunDFA(inputIndex); if (!match.IsSuccess) { // failed to match, retry with error handling match = RunDFAOnError(inputIndex); } if (match.IsSuccess) { if (match.state == 0) { // this is the dollar terminal, at the end of the input // the index of the $ symbol is always 1 isDollarEmitted = true; return(new TokenKernel(Symbol.SID_DOLLAR, tokens.Add(1, inputIndex, 0))); } else { // matched something int tIndex = GetTerminalFor(match.state, contexts); int tID = symTerminals[tIndex].ID; if (tID == separatorID) { inputIndex += match.length; continue; } else { TokenKernel token = new TokenKernel(tID, tokens.Add(tIndex, inputIndex, match.length)); inputIndex += match.length; return(token); } } } else { inputIndex += match.length; } } }
/// <summary> /// Gets the next token in the input /// </summary> /// <param name="contexts">The current applicable contexts</param> /// <returns>The next token in the input</returns> internal override TokenKernel GetNextToken(IContextProvider contexts) { if (tokens.Size == 0) { // this is the first call to this method, prefetch the tokens FindTokens(); tokenIndex = 0; } // no more tokens? return epsilon if (tokenIndex >= tokens.Size) { return(new TokenKernel(Symbol.SID_EPSILON, 0)); } TokenKernel result = new TokenKernel(tokens.GetSymbol(tokenIndex).ID, tokenIndex); tokenIndex++; return(result); }