Пример #1
0
        /// <summary>
        /// Emits a token into the output stream.
        /// </summary>
        /// <param name="output">The output stream into which to emit the token.</param>
        /// <param name="token">The token to emit into the output stream.</param>
        /// <param name="lineIndex">The current line index.</param>
        /// <param name="columnIndex">The current column index.</param>
        /// <param name="options">The lexer's configurable options.</param>
        /// <returns>The number of characters by which to advance the lexer's position
        /// within the source string.</returns>
        private static Int32 EmitToken(IList <UvssLexerToken> output, UvssLexerToken token,
                                       ref Int32 lineIndex, ref Int32 columnIndex, UvssLexerOptions options)
        {
            HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options);
            output.Add(token);

            return(token.Text.Length);
        }
Пример #2
0
            /// <summary>
            /// Adds a token to the end of the page, if space is available.
            /// </summary>
            /// <param name="token">The token to add to the end of the page.</param>
            /// <returns>true if the token was added to the page; otherwise, false.</returns>
            public Boolean Add(UvssLexerToken token)
            {
                if (count == Size)
                    return false;

                tokens[count++] = token;

                return true;
            }
Пример #3
0
            /// <summary>
            /// Adds a token to the end of the page, if space is available.
            /// </summary>
            /// <param name="token">The token to add to the end of the page.</param>
            /// <returns>true if the token was added to the page; otherwise, false.</returns>
            public Boolean Add(UvssLexerToken token)
            {
                if (count == Size)
                {
                    return(false);
                }

                tokens[count++] = token;

                return(true);
            }
Пример #4
0
        /// <summary>
        /// Emits the next token which is produced from the lexer's source text.
        /// </summary>
        /// <param name="token">The token that was emitted.</param>
        /// <returns>true if a token was emitted; otherwise, false.</returns>
        public Boolean Emit(out UvssLexerToken token)
        {
            Contract.Require(source, nameof(source));
            Contract.Require(options, nameof(options));

            if (position >= source.Length)
            {
                token = default(UvssLexerToken);
                return(false);
            }

            var tokenType = default(UvssLexerTokenType);
            var tokenText = default(String);

            var match = regexLexer.Match(source, position);

            if (match.Success)
            {
                if (!GetTokenInfoFromRegexMatch(match, out tokenType, out tokenText))
                {
                    var errorMessage = String.Format(UvssStrings.LexerInvalidToken,
                                                     match.Value, lineIndex, columnIndex);
                    throw new UvssLexerException(errorMessage, ErrorCodeInvalidToken, lineIndex, columnIndex);
                }

                ReadExtendedToken(source, position, ref tokenType, ref tokenText);

                var sourceOffset = position;
                var sourceLength = tokenText.Length;
                var sourceLine   = lineIndex;
                var sourceColumn = columnIndex;

                token = new UvssLexerToken(tokenType,
                                           sourceOffset, sourceLength, sourceLine, sourceColumn, tokenText);
            }
            else
            {
                var sourceOffset = position;
                var sourceLength = 1;
                var sourceLine   = lineIndex;
                var sourceColumn = columnIndex;

                token = new UvssLexerToken(UvssLexerTokenType.Unknown,
                                           sourceOffset, sourceLength, sourceLine, sourceColumn, source[position].ToString());
            }

            HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options);
            position += token.Text.Length;

            return(true);
        }
Пример #5
0
        /// <summary>
        /// Emits a token into the output stream.
        /// </summary>
        /// <param name="output">The output stream into which to emit the token.</param>
        /// <param name="token">The token to emit into the output stream.</param>
        /// <param name="lineIndex">The current line index.</param>
        /// <param name="columnIndex">The current column index.</param>
        /// <param name="options">The lexer's configurable options.</param>
        /// <returns>The number of characters by which to advance the lexer's position 
        /// within the source string.</returns>
        private static Int32 EmitToken(IList<UvssLexerToken> output, UvssLexerToken token,
            ref Int32 lineIndex, ref Int32 columnIndex, UvssLexerOptions options)
        {
            HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options);
            output.Add(token);

            return token.Text.Length;
        }
Пример #6
0
        /// <summary>
        /// Emits the next token which is produced from the lexer's source text.
        /// </summary>
        /// <param name="token">The token that was emitted.</param>
        /// <returns>true if a token was emitted; otherwise, false.</returns>
        public Boolean Emit(out UvssLexerToken token)
        {
            Contract.Require(source, nameof(source));
            Contract.Require(options, nameof(options));

            if (position >= source.Length)
            {
                token = default(UvssLexerToken);
                return false;
            }

            var tokenType = default(UvssLexerTokenType);
            var tokenText = default(String);

            var match = regexLexer.Match(source, position);
            if (match.Success)
            {
                if (!GetTokenInfoFromRegexMatch(match, out tokenType, out tokenText))
                {
                    var errorMessage = String.Format(UvssStrings.LexerInvalidToken,
                        match.Value, lineIndex, columnIndex);
                    throw new UvssLexerException(errorMessage, ErrorCodeInvalidToken, lineIndex, columnIndex);
                }

                ReadExtendedToken(source, position, ref tokenType, ref tokenText);

                var sourceOffset = position;
                var sourceLength = tokenText.Length;
                var sourceLine = lineIndex;
                var sourceColumn = columnIndex;

                token = new UvssLexerToken(tokenType,
                    sourceOffset, sourceLength, sourceLine, sourceColumn, tokenText);
            }
            else
            {
                var sourceOffset = position;
                var sourceLength = 1;
                var sourceLine = lineIndex;
                var sourceColumn = columnIndex;

                token = new UvssLexerToken(UvssLexerTokenType.Unknown,
                    sourceOffset, sourceLength, sourceLine, sourceColumn, source[position].ToString());
            }

            HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options);
            position += token.Text.Length;

            return true;
        }