/// <summary>
/// Emits a token into the output stream.
/// </summary>
/// <param name="output">The output stream into which to emit the token.</param>
/// <param name="token">The token to emit into the output stream.</param>
/// <param name="lineIndex">The current line index.</param>
/// <param name="columnIndex">The current column index.</param>
/// <param name="options">The lexer's configurable options.</param>
/// <returns>The number of characters by which to advance the lexer's position
/// within the source string.</returns>
private static Int32 EmitToken(IList <UvssLexerToken> output, UvssLexerToken token,
ref Int32 lineIndex, ref Int32 columnIndex, UvssLexerOptions options)
{
HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options);
output.Add(token);
return(token.Text.Length);
}
/// <summary>
/// Adds a token to the end of the page, if space is available.
/// </summary>
/// <param name="token">The token to add to the end of the page.</param>
/// <returns>true if the token was added to the page; otherwise, false.</returns>
public Boolean Add(UvssLexerToken token)
{
if (count == Size)
{
return(false);
}
tokens[count++] = token;
return(true);
}
/// <summary>
/// Emits the next token which is produced from the lexer's source text.
/// </summary>
/// <param name="token">The token that was emitted.</param>
/// <returns>true if a token was emitted; otherwise, false.</returns>
public Boolean Emit(out UvssLexerToken token)
{
Contract.Require(source, nameof(source));
Contract.Require(options, nameof(options));
if (position >= source.Length)
{
token = default(UvssLexerToken);
return(false);
}
var tokenType = default(UvssLexerTokenType);
var tokenText = default(String);
var match = regexLexer.Match(source, position);
if (match.Success)
{
if (!GetTokenInfoFromRegexMatch(match, out tokenType, out tokenText))
{
var errorMessage = String.Format(UvssStrings.LexerInvalidToken,
match.Value, lineIndex, columnIndex);
throw new UvssLexerException(errorMessage, ErrorCodeInvalidToken, lineIndex, columnIndex);
}
ReadExtendedToken(source, position, ref tokenType, ref tokenText);
var sourceOffset = position;
var sourceLength = tokenText.Length;
var sourceLine = lineIndex;
var sourceColumn = columnIndex;
token = new UvssLexerToken(tokenType,
sourceOffset, sourceLength, sourceLine, sourceColumn, tokenText);
}
else
{
var sourceOffset = position;
var sourceLength = 1;
var sourceLine = lineIndex;
var sourceColumn = columnIndex;
token = new UvssLexerToken(UvssLexerTokenType.Unknown,
sourceOffset, sourceLength, sourceLine, sourceColumn, source[position].ToString());
}
HandleLineAndColumnTracking(token.Text, ref lineIndex, ref columnIndex, options);
position += token.Text.Length;
return(true);
}