protected override void NextToken() { //1. Check if there are buffered tokens if (Context.BufferedTokens.Count > 0) { Context.CurrentToken = Context.BufferedTokens.Pop(); return; } //2. Skip whitespace. We don't need to check for EOF: at EOF we start getting 0-char, so we'll get out automatically while (Grammar.WhitespaceChars.IndexOf(SourceStream.PreviewChar) >= 0) { SourceStream.PreviewPosition++; } //3. That's the token start, calc location (line and column) SourceStream.MoveLocationToPreviewPosition(); //4. Check for EOF if (SourceStream.EOF()) { Context.CurrentToken = new Token(Grammar.Eof, SourceStream.Location, string.Empty, Grammar.Eof.Name);; return; } //5. Actually scan the source text and construct a new token ScanToken(); }//method