/// <summary> /// Move to the next token /// </summary> /// <param name="Context">Flags indicating valid tokens in the current context</param> /// <returns>True if the reader could move to the next token, false otherwise</returns> public bool MoveNext(TokenReaderContext Context) { int LineIdx = TokenEndLocation.LineIdx; int ColumnIdx = TokenEndLocation.ColumnIdx; // Skip past the leading whitespace TokenWhitespaceLocation = new TextLocation(LineIdx, ColumnIdx); bool bHasWhitespace = SkipWhitespace(Text, ref LineIdx, ref ColumnIdx, Context.HasFlag(TokenReaderContext.IgnoreNewlines)); // Check we haven't reached the end of the buffer TextLocation CurrentLocation = new TextLocation(LineIdx, ColumnIdx); if (CurrentLocation >= EndLocation) { TokenLocation = EndLocation; TokenEndLocation = EndLocation; return(false); } // Read the token TokenLocation = new TextLocation(LineIdx, ColumnIdx); bool bResult = ReadToken(Text, ref LineIdx, ref ColumnIdx, bHasWhitespace? TokenFlags.HasLeadingSpace : TokenFlags.None, Context, out CurrentToken); TokenEndLocation = new TextLocation(LineIdx, ColumnIdx); return(bResult); }
/// <summary> /// Reads a single token from a text buffer /// </summary> /// <param name="Text">The text buffer to read from</param> /// <param name="LineIdx">The current line index</param> /// <param name="ColumnIdx">The current column index</param> /// <param name="Flags">Flags for the new token</param> /// <returns>The next token, or null at the end of the file</returns> static bool ReadToken(TextBuffer Text, ref int LineIdx, ref int ColumnIdx, TokenFlags Flags, TokenReaderContext Context, out Token Result) { int StartLineIdx = LineIdx; int StartColumnIdx = ColumnIdx; char Character = Text.ReadCharacter(ref LineIdx, ref ColumnIdx); if (Character == '\0') { Result = new Token("", TokenType.Placemarker, Flags); return(false); } else if (Context.HasFlag(TokenReaderContext.TokenString)) { // Raw token string until the end of the current line StringBuilder Builder = new StringBuilder(); if (Character == '\n') { LineIdx = StartLineIdx; ColumnIdx = StartColumnIdx; } else { Builder.Append(Character); for (;;) { Character = Text[LineIdx, ColumnIdx]; if (Character == '\n') { break; } Builder.Append(Character); if (!Text.MoveNext(ref LineIdx, ref ColumnIdx)) { break; } } } Result = new Token(Builder.ToString().TrimEnd(), TokenType.StringOfTokens, Flags); return(true); } else if (Character == '\'') { // Character literal SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\''); Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.CharacterLiteral, Flags); return(true); } else if (Character == '\"') { // String literal SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\"'); Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.StringLiteral, Flags); return(true); } else if ((Character >= 'a' && Character <= 'z') || (Character >= 'A' && Character <= 'Z') || Character == '_') { // Identifier (or text literal with prefix) for (;;) { Character = Text[LineIdx, ColumnIdx]; if ((Character < 'a' || Character > 'z') && (Character < 'A' || Character > 'Z') && (Character < '0' || Character > '9') && Character != '_' && Character != '$') { break; } Text.MoveNext(ref LineIdx, ref ColumnIdx); } // Check if it's a prefixed text literal if (Character == '\'') { Text.MoveNext(ref LineIdx, ref ColumnIdx); SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\''); Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.CharacterLiteral, Flags); return(true); } else if (Character == '\"') { Text.MoveNext(ref LineIdx, ref ColumnIdx); SkipTextLiteral(Text, ref LineIdx, ref ColumnIdx, '\"'); Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.StringLiteral, Flags); return(true); } else { Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.Identifier, Flags); return(true); } } else if ((Character >= '0' && Character <= '9') || (Character == '.' && (Text[LineIdx, ColumnIdx] >= '0' && Text[LineIdx, ColumnIdx] <= '9'))) { // pp-number token char LastCharacter = Character; for (;;) { Character = Text[LineIdx, ColumnIdx]; if ((Character < 'a' || Character > 'z') && (Character < 'A' || Character > 'Z') && (Character < '0' || Character > '9') && Character != '_' && Character != '$' && Character != '\'') { if ((Character != '+' && Character != '-') || (LastCharacter != 'e' && LastCharacter != 'E')) { break; } } LastCharacter = Text.ReadCharacter(ref LineIdx, ref ColumnIdx); } Result = new Token(Text.ExtractString(StartLineIdx, StartColumnIdx, LineIdx, ColumnIdx), TokenType.NumericLiteral, Flags); return(true); } else if (Character == '<' && Context.HasFlag(TokenReaderContext.IncludeDirective)) { StringBuilder Builder = new StringBuilder("<"); while (Builder[Builder.Length - 1] != '>') { Builder.Append(Text[LineIdx, ColumnIdx]); ColumnIdx++; } Result = new Token(Builder.ToString(), TokenType.SystemInclude, Flags); return(true); } else { // Try to read a symbol if (ColumnIdx > 0) { for (int Idx = 0; Idx < SymbolicTokens.Length; Idx++) { string SymbolicToken = SymbolicTokens[Idx]; for (int Length = 0; Text[LineIdx, ColumnIdx + Length - 1] == SymbolicToken[Length]; Length++) { if (Length + 1 == SymbolicToken.Length) { ColumnIdx += Length; Result = new Token(SymbolicToken, TokenType.Symbol, Flags); return(true); } } } } // Otherwise just return a single character TokenType Type; switch (Character) { case '(': Type = TokenType.LeftParen; break; case ')': Type = TokenType.RightParen; break; case ',': Type = TokenType.Comma; break; default: Type = TokenType.Symbol; break; } Result = new Token(Character.ToString(), Type, Flags); return(true); } }