/// <summary> /// Lex an identifier. /// </summary> private Token LexIdent() { bool fVerbatim = false; if (ChCur == '@') { fVerbatim = true; ChNext(); } NormStr nstr = LexIdentCore(ref fVerbatim); if (nstr == null) { // Error already reported. return(null); } if (!fVerbatim) { KeyWordTable.KeyWordKind kind; if (_lex._kwt.IsKeyWord(nstr, out kind)) { return(KeyToken.CreateKeyWord(GetSpan(), nstr.Value.ToString(), kind.Kind, kind.IsContextKeyWord)); } } return(new IdentToken(GetSpan(), nstr.Value.ToString())); }
/// <summary> /// Called to lex a punctuator (operator). Asserts the current character lex type /// is LexCharType.Punc. /// </summary> private Token LexPunc() { int cchPunc = 0; TokKind tidPunc = TokKind.None; _sb.Length = 0; _sb.Append(ChCur); for (; ;) { TokKind tidCur; NormStr nstr = _lex._pool.Add(_sb); if (!_lex._kwt.IsPunctuator(nstr, out tidCur)) { break; } if (tidCur != TokKind.None) { // This is a real punctuator, not just a prefix. tidPunc = tidCur; cchPunc = _sb.Length; } char ch = ChPeek(_sb.Length); if (!LexCharUtils.IsPunc(ch)) { break; } _sb.Append(ch); } if (cchPunc == 0) { return(LexError()); } while (--cchPunc >= 0) { ChNext(); } return(KeyToken.Create(GetSpan(), tidPunc)); }
/// <summary> /// Lex a comment. /// </summary> private Token LexComment() { Contracts.Assert(ChCur == '/'); int ichErr = _cursor.IchCur; switch (ChPeek(1)) { default: return(LexPunc()); case '/': // Single line comment. ChNext(); _sb.Length = 0; _sb.Append("//"); for (; ;) { if (LexCharUtils.IsLineTerm(ChNext()) || Eof) { return(new CommentToken(GetSpan(), _sb.ToString(), 0)); } _sb.Append(ChCur); } case '*': /* block comment */ ChNext(); _sb.Length = 0; _sb.Append("/*"); ChNext(); int lines = 0; for (; ;) { if (Eof) { ReportError(ichErr, _cursor.IchCur, ErrId.UnterminatedComment); break; } char ch = ChCur; if (LexCharUtils.IsLineTerm(ch)) { ch = LexLineTerm(_sb); lines++; } else { ChNext(); } _sb.Append(ch); if (ch == '*' && ChCur == '/') { _sb.Append('/'); ChNext(); break; } } // We support comment keywords. KeyWordTable.KeyWordKind kind; NormStr nstr = _lex._pool.Add(_sb); if (_lex._kwt.IsKeyWord(nstr, out kind)) { return(KeyToken.CreateKeyWord(GetSpan(), nstr.ToString(), kind.Kind, kind.IsContextKeyWord)); } return(new CommentToken(GetSpan(), _sb.ToString(), lines)); } }