/// <summary>
/// Move to the next token
/// </summary>
/// <param name="Context">Flags indicating valid tokens in the current context</param>
/// <returns>True if the reader could move to the next token, false otherwise</returns>
public bool MoveNext(TokenReaderContext Context)
{
int LineIdx = TokenEndLocation.LineIdx;
int ColumnIdx = TokenEndLocation.ColumnIdx;
// Skip past the leading whitespace
TokenWhitespaceLocation = new TextLocation(LineIdx, ColumnIdx);
bool bHasWhitespace = SkipWhitespace(Text, ref LineIdx, ref ColumnIdx, Context.HasFlag(TokenReaderContext.IgnoreNewlines));
// Check we haven't reached the end of the buffer
TextLocation CurrentLocation = new TextLocation(LineIdx, ColumnIdx);
if (CurrentLocation >= EndLocation)
{
TokenLocation = EndLocation;
TokenEndLocation = EndLocation;
return(false);
}
// Read the token
TokenLocation = new TextLocation(LineIdx, ColumnIdx);
bool bResult = ReadToken(Text, ref LineIdx, ref ColumnIdx, bHasWhitespace? TokenFlags.HasLeadingSpace : TokenFlags.None, Context, out CurrentToken);
TokenEndLocation = new TextLocation(LineIdx, ColumnIdx);
return(bResult);
}
/// <summary>
/// Reads a single token from a text buffer
/// </summary>
/// <param name="Text">The text buffer to read from</param>
/// <param name="LineIdx">The current line index</param>
/// <param name="ColumnIdx">The current column index</param>
/// <param name="Flags">Flags for the new token</param>
/// <returns>The next token, or null at the end of the file</returns>
static bool ReadToken(TextBuffer Text, ref int LineIdx, ref int ColumnIdx, TokenFlags Flags, TokenReaderContext Context, out Token Result)
{
int StartLineIdx = LineIdx;
int StartColumnIdx = ColumnIdx;
char Character = Text.ReadCharacter(ref LineIdx, ref ColumnIdx);
if (Character == '\0')
{
Result = new Token("", TokenType.Placemarker, Flags);
return(false);
}
else if (Context.HasFlag(TokenReaderContext.TokenString))
{
// Raw token string until the end of the current line
StringBuilder Builder = new StringBuilder();
if (Character == '\n')
{
LineIdx = StartLineIdx;
ColumnIdx = StartColumnIdx;
}
else
{
Builder.Append(Character);
for (;;)
{
Character = Text[LineIdx, ColumnIdx];
if (Character == '\n')
{
break;
}
Builder.Append(Character);
if (!Text.MoveNext(ref LineIdx, ref ColumnIdx))
{
break;
}
}
}
Result = new Token(Builder.ToString().TrimEnd(), TokenType.StringOfTokens, Flags);
return(true);
}
else if (Character == '\'')
{
// Character literal
SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\'');
Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.CharacterLiteral, Flags);
return(true);
}
else if (Character == '\"')
{
// String literal
SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\"');
Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.StringLiteral, Flags);
return(true);
}
else if ((Character >= 'a' && Character <= 'z') || (Character >= 'A' && Character <= 'Z') || Character == '_')
{
// Identifier (or text literal with prefix)
for (;;)
{
Character = Text[LineIdx, ColumnIdx];
if ((Character < 'a' || Character > 'z') && (Character < 'A' || Character > 'Z') && (Character < '0' || Character > '9') && Character != '_' && Character != '$')
{
break;
}
Text.MoveNext(ref LineIdx, ref ColumnIdx);
}
// Check if it's a prefixed text literal
if (Character == '\'')
{
Text.MoveNext(ref LineIdx, ref ColumnIdx);
SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\'');
Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.CharacterLiteral, Flags);
return(true);
}
else if (Character == '\"')
{
Text.MoveNext(ref LineIdx, ref ColumnIdx);
SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\"');
Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.StringLiteral, Flags);
return(true);
}
else
{
Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.Identifier, Flags);
return(true);
}
}
else if ((Character >= '0' && Character <= '9') || (Character == '.' && (Text[LineIdx, ColumnIdx] >= '0' && Text[LineIdx, ColumnIdx] <= '9')))
{
// pp-number token
char LastCharacter = Character;
for (;;)
{
Character = Text[LineIdx, ColumnIdx];
if ((Character < 'a' || Character > 'z') && (Character < 'A' || Character > 'Z') && (Character < '0' || Character > '9') && Character != '_' && Character != '$' && Character != '\'')
{
if ((Character != '+' && Character != '-') || (LastCharacter != 'e' && LastCharacter != 'E'))
{
break;
}
}
LastCharacter = Text.ReadCharacter(ref LineIdx, ref ColumnIdx);
}
Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.NumericLiteral, Flags);
return(true);
}
else if (Character == '<' && Context.HasFlag(TokenReaderContext.IncludeDirective))
{
StringBuilder Builder = new StringBuilder("<");
while (Builder[Builder.Length - 1] != '>')
{
Builder.Append(Text[LineIdx, ColumnIdx]);
ColumnIdx++;
}
Result = new Token(Builder.ToString(), TokenType.SystemInclude, Flags);
return(true);
}
else
{
// Try to read a symbol
if (ColumnIdx > 0)
{
for (int Idx = 0; Idx < SymbolicTokens.Length; Idx++)
{
string SymbolicToken = SymbolicTokens[Idx];
for (int Length = 0; Text[LineIdx, ColumnIdx + Length - 1] == SymbolicToken[Length]; Length++)
{
if (Length + 1 == SymbolicToken.Length)
{
ColumnIdx += Length;
Result = new Token(SymbolicToken, TokenType.Symbol, Flags);
return(true);
}
}
}
}
// Otherwise just return a single character
TokenType Type;
switch (Character)
{
case '(':
Type = TokenType.LeftParen;
break;
case ')':
Type = TokenType.RightParen;
break;
case ',':
Type = TokenType.Comma;
break;
default:
Type = TokenType.Symbol;
break;
}
Result = new Token(Character.ToString(), Type, Flags);
return(true);
}
}
