public TextRendererTestContent(String text, TextParserOptions parserOptions = TextParserOptions.None)
{
Contract.RequireNotEmpty(text, nameof(text));
this.Text = text;
this.TextParserOptions = parserOptions;
}
/// <summary>
/// Retrieves a command token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeCommandToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var valid = false;
var start = ix++;
while (ix < input.Length)
{
if (Char.IsWhiteSpace(input[ix]))
{
break;
}
if (IsEndOfCommand(input, ix++))
{
valid = true;
break;
}
}
var sourceOffset = start;
var sourceLength = ix - start;
var segment = input.CreateStringSegmentFromSameSource(sourceOffset, sourceLength);
return(ParseLexerToken(valid ? LexedTokenType.Command : LexedTokenType.Word, segment, sourceOffset, sourceLength, options));
}
/// <summary>
/// Gets a value indicating whether the specified character is an escaped pipe.
/// </summary>
private static Boolean IsEscapedPipe <TSource>(TSource input, Int32 ix, TextParserOptions options)
where TSource : IStringSource <Char>
{
if ((options & TextParserOptions.IgnoreCommandCodes) == TextParserOptions.IgnoreCommandCodes)
{
return(false);
}
var c1 = input[ix];
if (c1 == '|')
{
if (ix + 1 == input.Length)
{
return(true);
}
var c2 = input[ix + 1];
if (c2 == '|' || Char.IsWhiteSpace(c2))
{
return(true);
}
}
return(false);
}
public TextRendererTestContent(String text, TextParserOptions parserOptions = TextParserOptions.None)
{
Contract.RequireNotEmpty(text, nameof(text));
this.Text = text;
this.TextParserOptions = parserOptions;
}
/// <summary>
/// Lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringBuilder"/> to parse.</param>
/// <param name="output">The parsed token stream.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
public void Parse(StringBuilder input, TextParserTokenStream output, TextParserOptions options = TextParserOptions.None)
{
Contract.Require(input, nameof(input));
Contract.Require(output, nameof(output));
output.Clear();
Parse(new StringSource(input), output, 0, input.Length, options);
}
/// <summary>
/// Incrementally lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="String"/> to parse.</param>
/// <param name="start">The index of the first character that was changed.</param>
/// <param name="count">The number of characters that were changed.</param>
/// <param name="result">The parsed token stream.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>An <see cref="IncrementalResult"/> structure that represents the result of the operation.</returns>
/// <remarks>Incremental parsing provides a performance benefit when relatively small changes are being made
/// to a large source text. Only tokens which are potentially influenced by changes within the specified substring
/// of the source text are re-parsed by this operation.</remarks>
public IncrementalResult ParseIncremental(String input, Int32 start, Int32 count, TextParserTokenStream result, TextParserOptions options = TextParserOptions.None)
{
Contract.Require(input, "input");
Contract.Require(result, "output");
Contract.EnsureRange(start >= 0, "start");
return ParseIncremental(new StringSource(input), start, count, result, options);
}
/// <summary>
/// Lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringBuilder"/> to parse.</param>
/// <param name="output">The parsed token stream.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
public void Parse(StringBuilder input, TextParserTokenStream output, TextParserOptions options = TextParserOptions.None)
{
Contract.Require(input, "input");
Contract.Require(output, "output");
output.Clear();
Parse(new StringSource(input), output, 0, input.Length, options);
}
/// <summary>
/// Gets a value indicating whether the specified character is an escaped pipe.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="ix">The index of the character to evaluate.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns><see langword="true"/> if the specified character is an escaped pipe; otherwise, <see langword="false"/>.</returns>
private static Boolean IsEscapedPipe(StringSource input, Int32 ix, TextParserOptions options)
{
if ((options & TextParserOptions.IgnoreCommandCodes) == TextParserOptions.IgnoreCommandCodes)
{
return(false);
}
return(input[ix] == '|' && (ix + 1 >= input.Length || input[ix + 1] == '|' || Char.IsWhiteSpace(input[ix + 1])));
}
/// <summary>
/// Retrieves an escaped pipe token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeEscapedPipeToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var start = ix++;
if (ix < input.Length && input[ix] == '|')
{
ix++;
}
var sourceOffset = start;
var sourceLength = 1;
return(ParseLexerToken(LexedTokenType.Pipe, "|", sourceOffset, sourceLength, options));
}
/// <summary>
/// Retrieves a word token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeWordToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var start = ix++;
while (ix < input.Length && !IsEndOfWord(input, ix))
{
ix++;
}
var sourceOffset = start;
var sourceLength = ix - start;
var segment = input.CreateStringSegmentFromSameSource(sourceOffset, sourceLength);
return(ParseLexerToken(LexedTokenType.Word, segment, sourceOffset, sourceLength, options));
}
/// <summary>
/// Retrieves a newline token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeNewlineToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var sourceLength = 1;
if (input[ix] == '\r' && ix + 1 < input.Length && input[ix + 1] == '\n')
{
sourceLength = 2;
}
var sourceOffset = ix;
ix += sourceLength;
var segment = input.CreateStringSegmentFromSameSource(sourceOffset, sourceLength);
return(ParseLexerToken(LexedTokenType.NewLine, segment, sourceOffset, sourceLength, options));
}
/// <summary>
/// Incrementally lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringBuilder"/> to parse.</param>
/// <param name="start">The index of the first character that was changed.</param>
/// <param name="count">The number of characters that were changed.</param>
/// <param name="result">The parsed token stream.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>An <see cref="IncrementalResult"/> structure that represents the result of the operation.</returns>
/// <remarks>Incremental parsing provides a performance benefit when relatively small changes are being made
/// to a large source text. Only tokens which are potentially influenced by changes within the specified substring
/// of the source text are re-parsed by this operation.</remarks>
public IncrementalResult ParseIncremental(StringBuilder input, Int32 start, Int32 count, TextParserTokenStream result, TextParserOptions options = TextParserOptions.None)
{
Contract.Require(input, nameof(input));
Contract.Require(result, nameof(result));
Contract.EnsureRange(start >= 0, nameof(start));
Contract.EnsureRange(count >= 0 && start + count <= input.Length, nameof(count));
return ParseIncremental(new StringSource(input), start, count, result, options);
}
/// <summary>
/// Incrementally lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringSource"/> to parse.</param>
/// <param name="start">The index of the first character that was changed.</param>
/// <param name="count">The number of characters that were changed.</param>
/// <param name="output">The parsed token stream.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>An <see cref="IncrementalResult"/> structure that represents the result of the operation.</returns>
/// <remarks>Incremental parsing provides a performance benefit when relatively small changes are being made
/// to a large source text. Only tokens which are potentially influenced by changes within the specified substring
/// of the source text are re-parsed by this operation.</remarks>
private IncrementalResult ParseIncremental(StringSource input, Int32 start, Int32 count, TextParserTokenStream output, TextParserOptions options = TextParserOptions.None)
{
var inputLengthOld = output.SourceText.Length;
var inputLengthNew = input.Length;
var inputLengthDiff = inputLengthNew - inputLengthOld;
Int32 ix1, ix2;
FindTokensInfluencedBySubstring(output, start, count - inputLengthDiff, out ix1, out ix2);
var token1 = output[ix1];
var token2 = output[ix2];
var invalidatedTokenCount = 1 + (ix2 - ix1);
output.RemoveRange(ix1, invalidatedTokenCount);
var lexStart = token1.SourceOffset;
var lexCount = inputLengthDiff + (token2.SourceOffset + token2.SourceLength) - lexStart;
var parserBuffer = incrementalParserBuffer.Value;
Parse(input, parserBuffer, lexStart, lexCount);
output.SourceText = input.CreateStringSegment();
output.InsertRange(ix1, parserBuffer);
var affectedOffset = ix1;
var affectedCount = parserBuffer.Count;
parserBuffer.Clear();
return new IncrementalResult(affectedOffset, affectedCount);
}
/// <summary>
/// Retrieves an escaped pipe token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeEscapedPipeToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var start = ix++;
if (ix < input.Length && input[ix] == '|')
ix++;
var sourceOffset = start;
var sourceLength = 1;
return ParseLexerToken(LexedTokenType.Pipe, "|", sourceOffset, sourceLength, options);
}
/// <summary>
/// Retrieves a command token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeCommandToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var valid = false;
var start = ix++;
while (ix < input.Length)
{
if (Char.IsWhiteSpace(input[ix]))
break;
if (IsEndOfCommand(input, ix++))
{
valid = true;
break;
}
}
var sourceOffset = start;
var sourceLength = ix - start;
var segment = input.CreateStringSegmentFromSameSource(sourceOffset, sourceLength);
return ParseLexerToken(valid ? LexedTokenType.Command : LexedTokenType.Word, segment, sourceOffset, sourceLength, options);
}
/// <summary>
/// Lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringSource"/> to parse.</param>
/// <param name="output">The parsed token stream.</param>
/// <param name="index">The index at which to begin parsing the input string.</param>
/// <param name="count">the number of characters to parse.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
private void Parse(StringSource input, TextParserTokenStream output, Int32 index, Int32 count, TextParserOptions options = TextParserOptions.None)
{
var bound = index + count;
while (index < bound)
{
if (IsStartOfNewline(input, index))
{
output.Add(ConsumeNewlineToken(input, options, ref index));
continue;
}
if (IsStartOfNonBreakingSpace(input, index))
{
output.Add(ConsumeNonBreakingSpaceToken(input, options, ref index));
continue;
}
if (IsStartOfBreakingSpace(input, index))
{
output.Add(ConsumeBreakingSpaceToken(input, options, ref index));
continue;
}
if (IsEscapedPipe(input, index, options))
{
output.Add(ConsumeEscapedPipeToken(input, options, ref index));
continue;
}
if (IsStartOfCommand(input, index))
{
output.Add(ConsumeCommandToken(input, options, ref index));
continue;
}
if (IsStartOfWord(input, index))
{
output.Add(ConsumeWordToken(input, options, ref index));
continue;
}
index++;
}
output.SourceText = input.CreateStringSegment();
output.ParserOptions = options;
}
/// <summary>
/// Incrementally lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringSource"/> to parse.</param>
/// <param name="start">The index of the first character that was changed.</param>
/// <param name="count">The number of characters that were changed.</param>
/// <param name="output">The parsed token stream.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>An <see cref="IncrementalResult"/> structure that represents the result of the operation.</returns>
/// <remarks>Incremental parsing provides a performance benefit when relatively small changes are being made
/// to a large source text. Only tokens which are potentially influenced by changes within the specified substring
/// of the source text are re-parsed by this operation.</remarks>
private IncrementalResult ParseIncremental(StringSource input, Int32 start, Int32 count, TextParserTokenStream output, TextParserOptions options = TextParserOptions.None)
{
var inputLengthOld = output.SourceText.Length;
var inputLengthNew = input.Length;
var inputLengthDiff = inputLengthNew - inputLengthOld;
Int32 ix1, ix2;
FindTokensInfluencedBySubstring(output, start, count - inputLengthDiff, out ix1, out ix2);
var token1 = output[ix1];
var token2 = output[ix2];
var invalidatedTokenCount = 1 + (ix2 - ix1);
output.RemoveRange(ix1, invalidatedTokenCount);
var lexStart = token1.SourceOffset;
var lexCount = inputLengthDiff + (token2.SourceOffset + token2.SourceLength) - lexStart;
var parserBuffer = incrementalParserBuffer.Value;
Parse(input, parserBuffer, lexStart, lexCount);
output.SourceText = input.CreateStringSegment();
output.InsertRange(ix1, parserBuffer);
var affectedOffset = ix1;
var affectedCount = parserBuffer.Count;
parserBuffer.Clear();
return(new IncrementalResult(affectedOffset, affectedCount));
}
/// <summary>
/// Retrieves a newline token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeNewlineToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var sourceLength = 1;
if (input[ix] == '\r' && ix + 1 < input.Length && input[ix + 1] == '\n')
sourceLength = 2;
var sourceOffset = ix;
ix += sourceLength;
var segment = input.CreateStringSegmentFromSameSource(sourceOffset, sourceLength);
return ParseLexerToken(LexedTokenType.NewLine, segment, sourceOffset, sourceLength, options);
}
/// <summary>
/// Parses a lexer token.
/// </summary>
/// <param name="tokenType">A <see cref="LexedTokenType"/> value specifying the type of token produced by the lexer.</param>
/// <param name="tokenText">The text associated with the lexer token.</param>
/// <param name="sourceOffset">The offset of the first character in the source text that produced the token.</param>
/// <param name="sourceLength">The number of characters in the source text that produced the token.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>The parsed token.</returns>
private static TextParserToken ParseLexerToken(LexedTokenType tokenType, StringSegment tokenText, Int32 sourceOffset, Int32 sourceLength, TextParserOptions options)
{
var isIgnoringCommandCodes = (options & TextParserOptions.IgnoreCommandCodes) == TextParserOptions.IgnoreCommandCodes;
if (tokenType == LexedTokenType.Command && !isIgnoringCommandCodes)
{
return(ParseCommandToken(tokenText, sourceOffset, sourceLength));
}
return(new TextParserToken(TextParserTokenType.Text, tokenText, sourceOffset, sourceLength,
tokenType == LexedTokenType.NonBreakingWhiteSpace));
}
/// <summary>
/// Incrementally lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringBuilder"/> to parse.</param>
/// <param name="start">The index of the first character that was changed.</param>
/// <param name="count">The number of characters that were changed.</param>
/// <param name="result">The parsed token stream.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>An <see cref="IncrementalResult"/> structure that represents the result of the operation.</returns>
/// <remarks>Incremental parsing provides a performance benefit when relatively small changes are being made
/// to a large source text. Only tokens which are potentially influenced by changes within the specified substring
/// of the source text are re-parsed by this operation.</remarks>
public IncrementalResult ParseIncremental(StringBuilder input, Int32 start, Int32 count, TextParserTokenStream result, TextParserOptions options = TextParserOptions.None)
{
Contract.Require(input, nameof(input));
Contract.Require(result, nameof(result));
Contract.EnsureRange(start >= 0, nameof(start));
Contract.EnsureRange(count >= 0 && start + count <= input.Length, nameof(count));
return(ParseIncremental(new StringSource(input), start, count, result, options));
}
/// <summary>
/// Parses a lexer token.
/// </summary>
/// <param name="tokenType">A <see cref="LexedTokenType"/> value specifying the type of token produced by the lexer.</param>
/// <param name="tokenText">The text associated with the lexer token.</param>
/// <param name="sourceOffset">The offset of the first character in the source text that produced the token.</param>
/// <param name="sourceLength">The number of characters in the source text that produced the token.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns>The parsed token.</returns>
private static TextParserToken ParseLexerToken(LexedTokenType tokenType, StringSegment tokenText, Int32 sourceOffset, Int32 sourceLength, TextParserOptions options)
{
var isIgnoringCommandCodes = (options & TextParserOptions.IgnoreCommandCodes) == TextParserOptions.IgnoreCommandCodes;
if (tokenType == LexedTokenType.Command && !isIgnoringCommandCodes)
return ParseCommandToken(tokenText, sourceOffset, sourceLength);
return new TextParserToken(TextParserTokenType.Text, tokenText, sourceOffset, sourceLength,
tokenType == LexedTokenType.NonBreakingWhiteSpace);
}
/// <summary>
/// Retrieves a word token from the input stream, beginning at the specified character.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="options">The parser options.</param>
/// <param name="ix">The index at which to begin consuming token characters.</param>
/// <returns>The token that was created.</returns>
private static TextParserToken ConsumeWordToken(StringSource input, TextParserOptions options, ref Int32 ix)
{
var start = ix++;
while (ix < input.Length && !IsEndOfWord(input, ix))
ix++;
var sourceOffset = start;
var sourceLength = ix - start;
var segment = input.CreateStringSegmentFromSameSource(sourceOffset, sourceLength);
return ParseLexerToken(LexedTokenType.Word, segment, sourceOffset, sourceLength, options);
}
/// <summary>
/// Retrieves a breaking white space token from the input stream, beginning at the specified character.
/// </summary>
private static TextParserToken ConsumeBreakingSpaceToken <TSource>(TSource input, TextParserOptions options, ref Int32 ix)
where TSource : ISegmentableStringSource
{
var start = ix++;
while (ix < input.Length && !IsEndOfBreakingSpace(input, ix))
{
ix++;
}
var sourceOffset = start;
var sourceLength = ix - start;
var segment = input.CreateStringSegmentFromSameOrigin(sourceOffset, sourceLength);
return(ParseLexerToken(LexedTokenType.BreakingWhiteSpace, segment, sourceOffset, sourceLength, options));
}
/// <summary>
/// Gets a value indicating whether the specified character is an escaped pipe.
/// </summary>
/// <param name="input">The input string.</param>
/// <param name="ix">The index of the character to evaluate.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
/// <returns><c>true</c> if the specified character is an escaped pipe; otherwise, <c>false</c>.</returns>
private static Boolean IsEscapedPipe(StringSource input, Int32 ix, TextParserOptions options)
{
if ((options & TextParserOptions.IgnoreCommandCodes) == TextParserOptions.IgnoreCommandCodes)
return false;
return input[ix] == '|' && (ix + 1 >= input.Length || input[ix + 1] == '|' || Char.IsWhiteSpace(input[ix + 1]));
}
/// <summary>
/// Lexes and parses the specified string.
/// </summary>
/// <param name="input">The <see cref="StringSource"/> to parse.</param>
/// <param name="output">The parsed token stream.</param>
/// <param name="index">The index at which to begin parsing the input string.</param>
/// <param name="count">the number of characters to parse.</param>
/// <param name="options">A set of <see cref="TextParserOptions"/> values that specify how the text should be parsed.</param>
private void Parse(StringSource input, TextParserTokenStream output, Int32 index, Int32 count, TextParserOptions options = TextParserOptions.None)
{
var bound = index + count;
while (index < bound)
{
if (IsStartOfNewline(input, index))
{
output.Add(ConsumeNewlineToken(input, options, ref index));
continue;
}
if (IsStartOfNonBreakingSpace(input, index))
{
output.Add(ConsumeNonBreakingSpaceToken(input, options, ref index));
continue;
}
if (IsStartOfBreakingSpace(input, index))
{
output.Add(ConsumeBreakingSpaceToken(input, options, ref index));
continue;
}
if (IsEscapedPipe(input, index, options))
{
output.Add(ConsumeEscapedPipeToken(input, options, ref index));
continue;
}
if (IsStartOfCommand(input, index))
{
output.Add(ConsumeCommandToken(input, options, ref index));
continue;
}
if (IsStartOfWord(input, index))
{
output.Add(ConsumeWordToken(input, options, ref index));
continue;
}
index++;
}
output.SourceText = input.CreateStringSegment();
output.ParserOptions = options;
}
public TextParser(string text, TextParserOptions options)
{
Text = text;
Options = options;
}
