/// <summary> /// Emits a token into the output stream. /// </summary> /// <param name="output">The output stream into which to emit the token.</param> /// <param name="token">The token to emit into the output stream.</param> /// <param name="lineIndex">The current line index.</param> /// <param name="columnIndex">The current column index.</param> /// <param name="options">The lexer's configurable options.</param> /// <returns>The number of characters by which to advance the lexer's position /// within the source string.</returns> private static Int32 EmitToken(IList <UvssLexerToken> output, UvssLexerToken token, ref Int32 lineIndex, ref Int32 columnIndex, UvssLexerOptions options) { HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options); output.Add(token); return(token.Text.Length); }
/// <summary> /// Adds a token to the end of the page, if space is available. /// </summary> /// <param name="token">The token to add to the end of the page.</param> /// <returns>true if the token was added to the page; otherwise, false.</returns> public Boolean Add(UvssLexerToken token) { if (count == Size) return false; tokens[count++] = token; return true; }
/// <summary> /// Adds a token to the end of the page, if space is available. /// </summary> /// <param name="token">The token to add to the end of the page.</param> /// <returns>true if the token was added to the page; otherwise, false.</returns> public Boolean Add(UvssLexerToken token) { if (count == Size) { return(false); } tokens[count++] = token; return(true); }
/// <summary> /// Emits the next token which is produced from the lexer's source text. /// </summary> /// <param name="token">The token that was emitted.</param> /// <returns>true if a token was emitted; otherwise, false.</returns> public Boolean Emit(out UvssLexerToken token) { Contract.Require(source, nameof(source)); Contract.Require(options, nameof(options)); if (position >= source.Length) { token = default(UvssLexerToken); return(false); } var tokenType = default(UvssLexerTokenType); var tokenText = default(String); var match = regexLexer.Match(source, position); if (match.Success) { if (!GetTokenInfoFromRegexMatch(match, out tokenType, out tokenText)) { var errorMessage = String.Format(UvssStrings.LexerInvalidToken, match.Value, lineIndex, columnIndex); throw new UvssLexerException(errorMessage, ErrorCodeInvalidToken, lineIndex, columnIndex); } ReadExtendedToken(source, position, ref tokenType, ref tokenText); var sourceOffset = position; var sourceLength = tokenText.Length; var sourceLine = lineIndex; var sourceColumn = columnIndex; token = new UvssLexerToken(tokenType, sourceOffset, sourceLength, sourceLine, sourceColumn, tokenText); } else { var sourceOffset = position; var sourceLength = 1; var sourceLine = lineIndex; var sourceColumn = columnIndex; token = new UvssLexerToken(UvssLexerTokenType.Unknown, sourceOffset, sourceLength, sourceLine, sourceColumn, source[position].ToString()); } HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options); position += token.Text.Length; return(true); }
/// <summary> /// Emits a token into the output stream. /// </summary> /// <param name="output">The output stream into which to emit the token.</param> /// <param name="token">The token to emit into the output stream.</param> /// <param name="lineIndex">The current line index.</param> /// <param name="columnIndex">The current column index.</param> /// <param name="options">The lexer's configurable options.</param> /// <returns>The number of characters by which to advance the lexer's position /// within the source string.</returns> private static Int32 EmitToken(IList<UvssLexerToken> output, UvssLexerToken token, ref Int32 lineIndex, ref Int32 columnIndex, UvssLexerOptions options) { HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options); output.Add(token); return token.Text.Length; }
/// <summary> /// Emits the next token which is produced from the lexer's source text. /// </summary> /// <param name="token">The token that was emitted.</param> /// <returns>true if a token was emitted; otherwise, false.</returns> public Boolean Emit(out UvssLexerToken token) { Contract.Require(source, nameof(source)); Contract.Require(options, nameof(options)); if (position >= source.Length) { token = default(UvssLexerToken); return false; } var tokenType = default(UvssLexerTokenType); var tokenText = default(String); var match = regexLexer.Match(source, position); if (match.Success) { if (!GetTokenInfoFromRegexMatch(match, out tokenType, out tokenText)) { var errorMessage = String.Format(UvssStrings.LexerInvalidToken, match.Value, lineIndex, columnIndex); throw new UvssLexerException(errorMessage, ErrorCodeInvalidToken, lineIndex, columnIndex); } ReadExtendedToken(source, position, ref tokenType, ref tokenText); var sourceOffset = position; var sourceLength = tokenText.Length; var sourceLine = lineIndex; var sourceColumn = columnIndex; token = new UvssLexerToken(tokenType, sourceOffset, sourceLength, sourceLine, sourceColumn, tokenText); } else { var sourceOffset = position; var sourceLength = 1; var sourceLine = lineIndex; var sourceColumn = columnIndex; token = new UvssLexerToken(UvssLexerTokenType.Unknown, sourceOffset, sourceLength, sourceLine, sourceColumn, source[position].ToString()); } HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options); position += token.Text.Length; return true; }