/// <summary>
/// Lex a line termination character. Transforms CRLF into a single LF.
/// Updates the line mapping. When this "drops" a character and sb is not
/// null, it adds the character to sb. It does NOT add the returned character
/// to the sb.
/// </summary>
private char LexLineTerm(StringBuilder sb = null)
{
Contracts.Assert(LexCharUtils.StartKind(ChCur) == LexStartKind.LineTerm);
int ichMin = _cursor.IchCur;
if (ChCur == '\xD' && ChPeek(1) == '\xA')
{
if (sb != null)
{
sb.Append(ChCur);
}
ChNext();
}
char ch = ChCur;
ChNext();
if (_ichMinTok == ichMin)
{
// Not nested.
_queue.Enqueue(new NewLineToken(GetSpan(), false));
}
else
{
// Is nested.
_queue.Enqueue(new NewLineToken(GetTextSpan(ichMin, _cursor.IchCur), true));
}
_fLineStart = true;
return(ch);
}
/// <summary>
/// Skip over an error character. Always returns null.
/// REVIEW: Should we skip over multiple?
/// </summary>
private Token LexError()
{
_sb.Length = 0;
do
{
_sb.AppendFormat("{0}({1})", ChCur, LexCharUtils.GetUniEscape(ChCur));
} while (LexCharUtils.StartKind(ChNext()) == LexStartKind.None && !Eof);
return(new ErrorToken(GetSpan(), ErrId.BadChar, _sb.ToString()));
}
/// <summary>
/// Lex a sequence of spacing characters.
/// Always returns null.
/// </summary>
private Token LexSpace()
{
Contracts.Assert(LexCharUtils.StartKind(ChCur) == LexStartKind.Space);
while (LexCharUtils.IsSpace(ChNext()))
{
;
}
return(null);
}
private Token FetchToken()
{
Contracts.Assert(!Eof);
StartTok();
LexStartKind kind = LexCharUtils.StartKind(ChCur);
if (kind != LexStartKind.Space && kind != LexStartKind.PreProc)
{
_fLineStart = false;
}
switch (kind)
{
case LexStartKind.Punc:
return(LexPunc());
case LexStartKind.NumLit:
return(LexNumLit());
case LexStartKind.StrLit:
return(LexStrLit());
case LexStartKind.Verbatim:
if (ChPeek(1) == '"')
{
return(LexStrLit());
}
if (LexCharUtils.StartKind(ChPeek(1)) == LexStartKind.Ident)
{
return(LexIdent());
}
ChNext();
ReportError(ErrId.VerbatimLiteralExpected);
return(null);
case LexStartKind.Ident:
return(LexIdent());
case LexStartKind.Comment:
return(LexComment());
case LexStartKind.Space:
return(LexSpace());
case LexStartKind.LineTerm:
LexLineTerm();
return(null);
case LexStartKind.PreProc:
return(LexPreProc());
default:
return(LexError());
}
}
/// <summary>
/// Called to lex a numeric literal or a Dot token. Asserts the current
/// character lex type is LexCharType.NumLit.
/// </summary>
private Token LexNumLit()
{
Contracts.Assert(LexCharUtils.StartKind(ChCur) == LexStartKind.NumLit);
Contracts.Assert(LexCharUtils.IsDigit(ChCur) || ChCur == '.');
// A dot not followed by a digit is just a Dot. This is a very common case (hence first).
if (ChCur == '.' && !LexCharUtils.IsDigit(ChPeek(1)))
{
return(LexPunc());
}
// Check for a hex literal. Note that 0x followed by a non-hex-digit is really a 0 followed
// by an identifier.
if (ChCur == '0' && (ChPeek(1) == 'x' || ChPeek(1) == 'X') && LexCharUtils.IsHexDigit(ChPeek(2)))
{
// Advance to first hex digit.
ChNext();
ChNext();
return(LexHexInt());
}
// Decimal literal (possible floating point).
Contracts.Assert(LexCharUtils.IsDigit(ChCur) || ChCur == '.' && LexCharUtils.IsDigit(ChPeek(1)));
bool fExp = false;
bool fDot = ChCur == '.';
_sb.Length = 0;
_sb.Append(ChCur);
for (; ;)
{
if (ChNext() == '.')
{
if (fDot || !LexCharUtils.IsDigit(ChPeek(1)))
{
break;
}
fDot = true;
}
else if (!LexCharUtils.IsDigit(ChCur))
{
break;
}
_sb.Append(ChCur);
}
// Check for an exponent.
if (ChCur == 'e' || ChCur == 'E')
{
char chTmp = ChPeek(1);
if (LexCharUtils.IsDigit(chTmp) || (chTmp == '+' || chTmp == '-') && LexCharUtils.IsDigit(ChPeek(2)))
{
fExp = true;
_sb.Append(ChCur);
_sb.Append(ChNext());
while (LexCharUtils.IsDigit(chTmp = ChNext()))
{
_sb.Append(chTmp);
}
}
}
bool fReal = fDot || fExp;
char chSuf = LexRealSuffix(fReal);
if (fReal || chSuf != '\0')
{
return(LexRealNum(chSuf));
}
// Integer type.
return(LexDecInt(LexIntSuffix()));
}
