private LexResult LexString(string typeName)
{
Debug.Assert(Buffer.Peek(0) == '"' || Buffer.Peek(0) == '\'');
char quoteChar = Buffer.Peek();
Buffer.AdvanceColumn();
bool isComplete = false;
CppTokenKind kind = quoteChar == '\'' ? CppTokenKind.CharLiteral : CppTokenKind.StringLiteral;
int maxCount = (kind == CppTokenKind.CharLiteral) ? 1 : -1;
int minCount = (kind == CppTokenKind.CharLiteral) ? 1 : 0;
int count = 0;
while (!Buffer.IsEOF)
{
char first = Buffer.Peek();
char second = Buffer.Peek(1);
if (first == quoteChar)
{
isComplete = true;
break;
}
else if (first == '\\')
{
switch (second)
{
case '\'':
case '"':
case '?':
case '\\':
case 'a':
case 'b':
case 'f':
case 'n':
case 'e':
case 'r':
case 't':
case 'v':
{
Buffer.AdvanceColumns(2);
++count;
continue;
}
case 'x':
case 'X':
case 'u':
case 'U':
{
Buffer.AdvanceColumns(2);
if (SyntaxUtils.IsHex(Buffer.Peek()))
{
int len = 0;
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsHex(Buffer.Peek()))
{
break;
}
else
{
++len;
Buffer.AdvanceColumn();
}
}
}
else
{
AddError(Buffer.TextPosition, $"Unsupported hex escape character '{Buffer.Peek()}'!", typeName);
break;
}
++count;
continue;
}
default:
if (SyntaxUtils.IsOctal(second))
{
Buffer.AdvanceColumn();
while (!Buffer.IsEOF)
{
if (!SyntaxUtils.IsOctal(Buffer.Peek()))
{
break;
}
else
{
Buffer.AdvanceColumn();
}
}
++count;
continue;
}
else
{
AddError(Buffer.TextPosition, $"Not supported escape character '{Buffer.Peek()}'!", typeName);
break;
}
}
}
else if (SyntaxUtils.IsLineBreak(first))
{
break;
}
else if (char.IsWhiteSpace(first))
{
Buffer.AdvanceManual(first, second);
}
else
{
Buffer.AdvanceColumn();
}
++count;
}
// Skip over quote char
if (isComplete)
{
Debug.Assert(Buffer.Peek() == quoteChar);
Buffer.AdvanceColumn();
}
if (!isComplete)
{
AddError(Buffer.LexemeStart, $"Unterminated {typeName} literal!", typeName);
}
else
{
if (minCount > 0 && count < minCount)
{
AddError(Buffer.LexemeStart, $"Not enough characters for {typeName} literal, expect {minCount} but got {count}!", typeName);
}
else if (maxCount > -1 && (count > maxCount))
{
AddError(Buffer.LexemeStart, $"Too many characters for {typeName} literal, expect {maxCount} but got {count}!", typeName);
}
}
return(new LexResult(kind, isComplete));
}
private LexResult LexNumber()
{
Debug.Assert(SyntaxUtils.IsNumeric(Buffer.Peek()) || Buffer.Peek() == '.');
CppTokenKind kind;
char first = Buffer.Peek(0);
char second = Buffer.Peek(1);
bool dotSeen = false;
if (first == '0')
{
if (second == 'x' || second == 'X')
{
// Hex
kind = CppTokenKind.HexLiteral;
Buffer.AdvanceColumns(2); // Skip 0[xX]
}
else if (second == 'b' || second == 'B')
{
// Binary
kind = CppTokenKind.BinaryLiteral;
Buffer.AdvanceColumns(2); // Skip 0[bB]
}
else
{
// Octal
kind = CppTokenKind.OctalLiteral;
}
}
else if (first == '.')
{
Debug.Assert(SyntaxUtils.IsNumeric(second));
kind = CppTokenKind.IntegerFloatLiteral;
Buffer.AdvanceColumn();
dotSeen = true;
}
else
{
Debug.Assert(SyntaxUtils.IsNumeric(first));
kind = CppTokenKind.IntegerLiteral;
}
// @NOTE(final): We never set the DecimalHexLiteral kind initially,
// as every hex decimal always starts as a normal hex literal!
Debug.Assert(kind != CppTokenKind.HexadecimalFloatLiteral);
// First number part
int firstLiteralPos = Buffer.TextPosition.Index;
bool readNextLiteral = false;
do
{
readNextLiteral = false;
int s = Buffer.TextPosition.Index;
switch (kind)
{
case CppTokenKind.IntegerLiteral:
case CppTokenKind.IntegerFloatLiteral:
if (SyntaxUtils.IsNumeric(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsNumeric);
}
else
{
AddError(Buffer.TextPosition, $"Expect integer literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
case CppTokenKind.OctalLiteral:
if (SyntaxUtils.IsOctal(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsOctal);
}
else
{
AddError(Buffer.TextPosition, $"Expect octal literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
case CppTokenKind.HexLiteral:
if (SyntaxUtils.IsHex(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsHex);
}
else
{
AddError(Buffer.TextPosition, $"Expect hex literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
case CppTokenKind.BinaryLiteral:
if (SyntaxUtils.IsBinary(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsBinary);
}
else
{
AddError(Buffer.TextPosition, $"Expect binary literal, but got '{Buffer.Peek()}'", kind.ToString());
}
break;
default:
AddError(Buffer.TextPosition, $"Unsupported token kind '{kind}' for integer literal on {Buffer}", kind.ToString());
break;
}
bool hadIntegerLiteral = Buffer.TextPosition.Index > s;
if (kind != CppTokenKind.IntegerFloatLiteral && kind != CppTokenKind.HexadecimalFloatLiteral)
{
// @NOTE(final): Single quotes (') are allowed as separators for any non-decimal literal
char check0 = Buffer.Peek();
if (check0 == '\'')
{
if (!hadIntegerLiteral)
{
AddError(Buffer.TextPosition, $"Too many single quote escape in integer literal, expect any integer literal but got '{Buffer.Peek()}'", kind.ToString());
return(new LexResult(kind, false));
}
Buffer.AdvanceColumn();
readNextLiteral = true;
}
}
} while (!Buffer.IsEOF && readNextLiteral);
// Validate any literal after starting dot
if (dotSeen)
{
if (firstLiteralPos == Buffer.TextPosition.Index)
{
AddError(Buffer.TextPosition, $"Expect any integer literal after starting dot, but got '{Buffer.Peek()}'", kind.ToString());
return(new LexResult(kind, false));
}
}
// Dot separator
if ((!dotSeen) &&
((kind == CppTokenKind.IntegerLiteral) ||
(kind == CppTokenKind.HexLiteral) ||
(kind == CppTokenKind.OctalLiteral)
))
{
char check0 = Buffer.Peek();
if (check0 == '.')
{
dotSeen = true;
Buffer.AdvanceColumn();
if (kind == CppTokenKind.IntegerLiteral || kind == CppTokenKind.OctalLiteral)
{
kind = CppTokenKind.IntegerFloatLiteral;
}
else
{
Debug.Assert(kind == CppTokenKind.HexLiteral);
kind = CppTokenKind.HexadecimalFloatLiteral;
}
}
else if (SyntaxUtils.IsExponentPrefix(check0))
{
if (kind == CppTokenKind.IntegerLiteral || kind == CppTokenKind.OctalLiteral)
{
kind = CppTokenKind.IntegerFloatLiteral;
}
else
{
Debug.Assert(kind == CppTokenKind.HexLiteral);
kind = CppTokenKind.HexadecimalFloatLiteral;
}
}
}
// Decimal after dot separator
if ((kind != CppTokenKind.IntegerFloatLiteral) &&
(kind != CppTokenKind.HexadecimalFloatLiteral))
{
// Integer suffix
if (SyntaxUtils.IsIntegerSuffix(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsIntegerSuffix, 3);
}
}
else
{
if (kind == CppTokenKind.IntegerFloatLiteral)
{
// Float decimal
if (SyntaxUtils.IsNumeric(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsNumeric);
}
if (Buffer.Peek() == 'e' || Buffer.Peek() == 'E')
{
AdvanceExponent('e');
}
}
else
{
// Hex decimal
Debug.Assert(kind == CppTokenKind.HexadecimalFloatLiteral);
if (SyntaxUtils.IsHex(Buffer.Peek()))
{
Buffer.AdvanceColumnsWhile(SyntaxUtils.IsHex);
}
if (Buffer.Peek() == 'p' || Buffer.Peek() == 'P')
{
AdvanceExponent('e');
}
}
// Float suffix
if (SyntaxUtils.IsFloatSuffix(Buffer.Peek()))
{
Buffer.AdvanceColumn();
}
}
return(new LexResult(kind, true));
}
private static CppTokenKind ReadCharacterLiteral(TextStream stream)
{
char n = stream.Peek();
if (n == '\\')
{
stream.AdvanceColumn(); // Skip over backslash
char escapeChar = stream.Peek();
switch (escapeChar)
{
case '\'':
case '"':
case '?':
case '\\':
case 'a':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
case 'v':
stream.AdvanceColumn();
break;
case 'u':
case 'U':
{
int maxCount = escapeChar == 'u' ? 4 : 8;
stream.AdvanceColumn();
if (SyntaxUtils.IsNumeric(stream.Peek()))
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsNumeric, maxCount);
}
}
break;
case 'X':
{
// Hex escape
stream.AdvanceColumn();
if (SyntaxUtils.IsHex(stream.Peek()))
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsHex, 2);
}
}
break;
default:
{
// Octal escape
if (SyntaxUtils.IsOctal(escapeChar))
{
stream.AdvanceColumnsWhile(SyntaxUtils.IsOctal, 3);
}
}
break;
}
}
else
{
stream.AdvanceColumn();
}
if (stream.Peek() == '\'')
{
stream.AdvanceColumn();
}
return(CppTokenKind.CharLiteral);
}
