/// <summary>
/// Lex an identifier.
/// </summary>
private Token LexIdent()
{
bool fVerbatim = false;
if (ChCur == '@')
{
fVerbatim = true;
ChNext();
}
NormStr nstr = LexIdentCore(ref fVerbatim);
if (nstr == null)
{
// Error already reported.
return(null);
}
if (!fVerbatim)
{
KeyWordTable.KeyWordKind kind;
if (_lex._kwt.IsKeyWord(nstr, out kind))
{
return(KeyToken.CreateKeyWord(GetSpan(), nstr.Value.ToString(), kind.Kind, kind.IsContextKeyWord));
}
}
return(new IdentToken(GetSpan(), nstr.Value.ToString()));
}
/// <summary>
/// Called to lex a punctuator (operator). Asserts the current character lex type
/// is LexCharType.Punc.
/// </summary>
private Token LexPunc()
{
int cchPunc = 0;
TokKind tidPunc = TokKind.None;
_sb.Length = 0;
_sb.Append(ChCur);
for (; ;)
{
TokKind tidCur;
NormStr nstr = _lex._pool.Add(_sb);
if (!_lex._kwt.IsPunctuator(nstr, out tidCur))
{
break;
}
if (tidCur != TokKind.None)
{
// This is a real punctuator, not just a prefix.
tidPunc = tidCur;
cchPunc = _sb.Length;
}
char ch = ChPeek(_sb.Length);
if (!LexCharUtils.IsPunc(ch))
{
break;
}
_sb.Append(ch);
}
if (cchPunc == 0)
{
return(LexError());
}
while (--cchPunc >= 0)
{
ChNext();
}
return(KeyToken.Create(GetSpan(), tidPunc));
}
/// <summary>
/// Lex a comment.
/// </summary>
private Token LexComment()
{
Contracts.Assert(ChCur == '/');
int ichErr = _cursor.IchCur;
switch (ChPeek(1))
{
default:
return(LexPunc());
case '/':
// Single line comment.
ChNext();
_sb.Length = 0;
_sb.Append("//");
for (; ;)
{
if (LexCharUtils.IsLineTerm(ChNext()) || Eof)
{
return(new CommentToken(GetSpan(), _sb.ToString(), 0));
}
_sb.Append(ChCur);
}
case '*':
/* block comment */
ChNext();
_sb.Length = 0;
_sb.Append("/*");
ChNext();
int lines = 0;
for (; ;)
{
if (Eof)
{
ReportError(ichErr, _cursor.IchCur, ErrId.UnterminatedComment);
break;
}
char ch = ChCur;
if (LexCharUtils.IsLineTerm(ch))
{
ch = LexLineTerm(_sb);
lines++;
}
else
{
ChNext();
}
_sb.Append(ch);
if (ch == '*' && ChCur == '/')
{
_sb.Append('/');
ChNext();
break;
}
}
// We support comment keywords.
KeyWordTable.KeyWordKind kind;
NormStr nstr = _lex._pool.Add(_sb);
if (_lex._kwt.IsKeyWord(nstr, out kind))
{
return(KeyToken.CreateKeyWord(GetSpan(), nstr.ToString(), kind.Kind, kind.IsContextKeyWord));
}
return(new CommentToken(GetSpan(), _sb.ToString(), lines));
}
}
