public static TokenListParser <TKind, T> RightRec <TKind, T>(TokenListParser <TKind, T> head, Func <T, TokenListParser <TKind, T> > apply)
{
if (head == null)
{
throw new ArgumentNullException(nameof(head));
}
if (apply == null)
{
throw new ArgumentNullException(nameof(apply));
}
return(input =>
{
var parseResult = head(input);
if (!parseResult.HasValue)
{
return parseResult;
}
var result = parseResult.Value;
var tailResult = apply(result)(parseResult.Remainder);
while (true)
{
if (!tailResult.HasValue)
{
return TokenListParserResult.Value(result, input, tailResult.Remainder);
}
result = tailResult.Value;
tailResult = apply(result)(tailResult.Remainder);
}
});
}
public static TokenListParser <TTokenKind, U> SelectCatch <TTokenKind, T, U>(this TokenListParser <TTokenKind, T> parser, Func <T, U> trySelector, string errorMessage)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (trySelector == null)
{
throw new ArgumentNullException(nameof(trySelector));
}
if (errorMessage == null)
{
throw new ArgumentNullException(nameof(errorMessage));
}
return(input =>
{
var t = parser(input);
if (!t.HasValue)
{
return TokenListParserResult.CastEmpty <TTokenKind, T, U>(t);
}
try
{
var u = trySelector(t.Value);
return TokenListParserResult.Value(u, input, t.Remainder);
}
catch
{
return TokenListParserResult.Empty <TTokenKind, U>(input, errorMessage);
}
});
}
/// <summary>
/// Construct a parser that applies <paramref name="first"/>, provides the value to <paramref name="second"/> and returns the result.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <typeparam name="U">The type of the resulting value.</typeparam>
/// <param name="first">The first parser.</param>
/// <param name="second">The second parser.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, U> Then <TKind, T, U>(this TokenListParser <TKind, T> first, Func <T, TokenListParser <TKind, U> > second)
{
if (first == null)
{
throw new ArgumentNullException(nameof(first));
}
if (second == null)
{
throw new ArgumentNullException(nameof(second));
}
return(input =>
{
var rt = first(input);
if (!rt.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, T, U>(rt);
}
var ru = second(rt.Value)(rt.Remainder);
if (!ru.HasValue)
{
return ru;
}
return TokenListParserResult.Value(ru.Value, input, ru.Remainder);
});
}
/// <summary>
/// Construct a parser that evaluates the result of a previous parser and fails if <paramref name="predicate"/> returns false
/// for the result.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="predicate">The predicate to apply.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T> Where <TKind, T>(this TokenListParser <TKind, T> parser, Func <T, bool> predicate)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (predicate == null)
{
throw new ArgumentNullException(nameof(predicate));
}
return(input =>
{
var rt = parser(input);
if (!rt.HasValue)
{
return rt;
}
if (predicate(rt.Value))
{
return rt;
}
return TokenListParserResult.Empty <TKind, T>(input, "unsatisfied condition");
});
}
/// <summary>
/// Construct a parser that returns <paramref name="name"/> as its "expectation" if <paramref name="parser"/> fails.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="name">The name given to <paramref name="parser"/>.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T> Named <TKind, T>(this TokenListParser <TKind, T> parser, string name)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
return(input =>
{
var result = parser(input);
if (result.HasValue || result.Remainder != input)
{
return result;
}
// result.IsSubTokenError?
if (result.ErrorPosition.HasValue)
{
return TokenListParserResult.Empty <TKind, T>(result.Remainder, result.ErrorPosition, result.FormatErrorMessageFragment());
}
return TokenListParserResult.Empty <TKind, T>(result.Remainder, new[] { name });
});
}
/// <summary>
/// Construct a parser that tries first the <paramref name="lhs"/> parser, and if it fails, applies <paramref name="rhs"/>.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="lhs">The first parser to try.</param>
/// <param name="rhs">The second parser to try.</param>
/// <returns>The resulting parser.</returns>
/// <remarks>Or will fail if the first item partially matches this. To modify this behavior use <see cref="Try{TKind,T}(TokenListParser{TKind,T})"/>.</remarks>
public static TokenListParser <TKind, T> Or <TKind, T>(this TokenListParser <TKind, T> lhs, TokenListParser <TKind, T> rhs)
{
if (lhs == null)
{
throw new ArgumentNullException(nameof(lhs));
}
if (rhs == null)
{
throw new ArgumentNullException(nameof(rhs));
}
return(input =>
{
var first = lhs(input);
if (first.HasValue || !first.Backtrack && first.IsPartial(input))
{
return first;
}
var second = rhs(input);
if (second.HasValue)
{
return second;
}
return TokenListParserResult.CombineEmpty(first, second);
});
}
/// <summary>
/// Apply the character parser <paramref name="valueParser"/> to the span represented by the parsed token.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="U">The type of the resulting value.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="valueParser">A function that determines which character parser to apply.</param>
/// <returns>A parser that returns the result of parsing the token value.</returns>
public static TokenListParser <TKind, U> Apply <TKind, U>(this TokenListParser <TKind, Token <TKind> > parser, Func <Token <TKind>, TextParser <U> > valueParser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (valueParser == null)
{
throw new ArgumentNullException(nameof(valueParser));
}
return(input =>
{
var rt = parser(input);
if (!rt.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, Token <TKind>, U>(rt);
}
var uParser = valueParser(rt.Value);
var uResult = uParser.AtEnd()(rt.Value.Span);
if (!uResult.HasValue)
{
var message = $"invalid {Presentation.FormatExpectation(rt.Value.Kind)}, {uResult.FormatErrorMessageFragment()}";
return new TokenListParserResult <TKind, U>(input, uResult.Remainder.Position, message, null, uResult.Backtrack);
}
return TokenListParserResult.Value(uResult.Value, rt.Location, rt.Remainder);
});
}
/// <summary>
/// Constructs a parser that will fail if the given parser succeeds,
/// and will succeed if the given parser fails. In any case, it won't
/// consume any input. It's like a negative look-ahead in a regular expression.
/// </summary>
/// <typeparam name="T">The result type of the given parser.</typeparam>
/// <typeparam name="TKind">The kind of token being parsed.</typeparam>
/// <param name="parser">The parser to wrap</param>
/// <returns>A parser that is the negation of the given parser.</returns>
public static TokenListParser <TKind, Unit> Not <TKind, T>(TokenListParser <TKind, T> parser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(input =>
{
var result = parser(input);
if (result.HasValue)
{
// This is usually a success case for Not(), so the allocations here are a bit of a pity.
var current = input.ConsumeToken();
var last = result.Remainder.ConsumeToken();
if (current.HasValue)
{
var span = last.HasValue ?
current.Value.Span.Source.Substring(current.Value.Position.Absolute, last.Value.Position.Absolute - current.Value.Position.Absolute) :
current.Value.Span.Source.Substring(current.Value.Position.Absolute);
return TokenListParserResult.Empty <TKind, Unit>(input, $"unexpected successful parsing of {Presentation.FormatLiteral(Friendly.Clip(span, 12))}");
}
return TokenListParserResult.Empty <TKind, Unit>(input, "unexpected successful parsing");
}
return TokenListParserResult.Value(Unit.Value, input, input);
});
}
/// <summary>
/// Construct a parser that matches <paramref name="parser"/> zero or more times.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <returns>The resulting parser.</returns>
/// <remarks>Many will fail if any item partially matches this. To modify this behavior use <see cref="Try{TKind,T}(TokenListParser{TKind,T})"/>.</remarks>
public static TokenListParser <TKind, T[]> Many <TKind, T>(this TokenListParser <TKind, T> parser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(input =>
{
var result = new List <T>();
var @from = input;
var r = parser(input);
while (r.HasValue)
{
if (@from == r.Remainder) // Broken parser, not a failed parsing.
{
throw new ParseException($"Many() cannot be applied to zero-width parsers; value {r.Value} at position {r.Location.Position}.");
}
result.Add(r.Value);
@from = r.Remainder;
r = parser(r.Remainder);
}
if (!r.Backtrack && r.IsPartial(@from))
{
return TokenListParserResult.CastEmpty <TKind, T, T[]>(r);
}
return TokenListParserResult.Value(result.ToArray(), input, r.Remainder);
});
}
/// <summary>
/// Combine two empty results.
/// </summary>
/// <typeparam name="T">The source type.</typeparam>
/// <typeparam name="TKind">The kind of token.</typeparam>
/// <param name="first">The first value to combine.</param>
/// <param name="second">The second value to combine.</param>
/// <returns>A result of type <typeparamref name="T"/> carrying information from both results.</returns>
public static TokenListParserResult <TKind, T> CombineEmpty <TKind, T>(TokenListParserResult <TKind, T> first, TokenListParserResult <TKind, T> second)
{
if (first.Remainder != second.Remainder)
{
return(second);
}
var expectations = first.Expectations;
if (expectations == null)
{
expectations = second.Expectations;
}
else if (second.Expectations != null)
{
expectations = new string[first.Expectations.Length + second.Expectations.Length];
var i = 0;
for (; i < first.Expectations.Length; ++i)
{
expectations[i] = first.Expectations[i];
}
for (var j = 0; j < second.Expectations.Length; ++i, ++j)
{
expectations[i] = second.Expectations[j];
}
}
return(new TokenListParserResult <TKind, T>(second.Remainder, second.ErrorPosition, first.ErrorMessage, expectations, second.Backtrack));
}
/// <summary>
/// Parse a sequence of operands connected by left-associative operators.
/// </summary>
/// <typeparam name="T">The type being parsed.</typeparam>
/// <typeparam name="TOperator">The type of the operator.</typeparam>
/// <typeparam name="TKind">The kind of token being parsed.</typeparam>
/// <param name="operator">A parser matching operators.</param>
/// <param name="operand">A parser matching operands.</param>
/// <param name="apply">A function combining an operator and two operands into the result.</param>
/// <returns>The result of calling <paramref name="apply"/> successively on pairs of operands.</returns>
public static TokenListParser <TKind, T> Chain <TKind, T, TOperator>(
TokenListParser <TKind, TOperator> @operator,
TokenListParser <TKind, T> operand,
Func <TOperator, T, T, T> apply)
{
if (@operator == null)
{
throw new ArgumentNullException(nameof(@operator));
}
if (operand == null)
{
throw new ArgumentNullException(nameof(operand));
}
if (apply == null)
{
throw new ArgumentNullException(nameof(apply));
}
return(input =>
{
var parseResult = operand(input);
if (!parseResult.HasValue)
{
return parseResult;
}
var result = parseResult.Value;
var operatorResult = @operator(parseResult.Remainder);
while (operatorResult.HasValue || operatorResult.IsPartial(parseResult.Remainder))
{
// If operator read any input, but failed to read complete input, we return error
if (!operatorResult.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, TOperator, T>(operatorResult);
}
parseResult = operand(operatorResult.Remainder);
if (!parseResult.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, T, T>(parseResult);
}
result = apply(operatorResult.Value, result, parseResult.Value);
operatorResult = @operator(parseResult.Remainder);
}
return TokenListParserResult.Value(result, input, parseResult.Remainder);
});
}
/// <summary>
/// Parse a token of the kind <typeparamref name="TKind"/>.
/// </summary>
/// <typeparam name="TKind">The type of the token being matched.</typeparam>
/// <param name="kind">The kind of token to match.</param>
/// <returns>The matched token.</returns>
public static TokenListParser <TKind, Token <TKind> > EqualTo <TKind>(TKind kind)
{
var expectations = new[] { Presentation.FormatExpectation(kind) };
return(input =>
{
var next = input.ConsumeToken();
if (!next.HasValue || !next.Value.Kind.Equals(kind))
{
return TokenListParserResult.Empty <TKind, Token <TKind> >(input, expectations);
}
return next;
});
}
public void MessageWithExpectedTokensUsesTokenPresentation()
{
// Composing a complex parser which does not fit a LALR(1) grammar, one might need
// to have multiple look-ahead tokens. While it is possible to compose parsers with back-tracking,
// manual generated parsers are some times easier to construct. These parsers would like
// to report expectations using tokens, but still benefit from the annotations put on
// the tokens, to generated nicely formatted error messages. The following construct
// shows how to generate an empty result, which indicates which tokens are expected.
var emptyParseResult = TokenListParserResult.Empty <ArithmeticExpressionToken, string>(
new TokenList <ArithmeticExpressionToken>(),
new [] { ArithmeticExpressionToken.Times, ArithmeticExpressionToken.Zero });
// Empty result represent expectations using nice string representation taken from
// annotations of enum values of tokens
Assert.Equal(2, emptyParseResult.Expectations !.Length);
Assert.Equal("`*`", emptyParseResult.Expectations ![0]);
/// <summary>
/// Parse a sequence of tokens of the kind <typeparamref name="TKind"/>.
/// </summary>
/// <typeparam name="TKind">The type of the tokens being matched.</typeparam>
/// <param name="kinds">The kinds of token to match, once each in order.</param>
/// <returns>The matched tokens.</returns>
public static TokenListParser <TKind, Token <TKind>[]> Sequence <TKind>(params TKind[] kinds)
{
if (kinds == null)
{
throw new ArgumentNullException(nameof(kinds));
}
TokenListParser <TKind, Token <TKind>[]> result = input => TokenListParserResult.Value(new Token <TKind> [kinds.Length], input, input);
for (var i = 0; i < kinds.Length; ++i)
{
var token = EqualTo(kinds[i]);
var index = i;
result = result.Then(arr => token.Select(t => { arr[index] = t; return(arr); }));
}
return(result);
}
public static TokenListParser <TKind, Token <TKind> > NotEqualTo <TKind>(this IEnumerable <TKind> kinds)
{
var expectations = new[]
{
kinds.Select(k => k?.ToString() ?? string.Empty)
.Where(k => k.Length > 0)
.Join(", ")
};
return(input =>
{
var next = input.ConsumeToken();
if (!next.HasValue || kinds.Any(k => next.Value.Kind != null && next.Value.Kind.Equals(k)))
{
return TokenListParserResult.Empty <TKind, Token <TKind> >(input, expectations);
}
return next;
});
}
/// <summary>
/// Construct a parser that fails with error message <paramref name="errorMessage"/> when <paramref name="parser"/> fails.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="errorMessage">The error message.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T> Message <TKind, T>(this TokenListParser <TKind, T> parser, string errorMessage)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (errorMessage == null)
{
throw new ArgumentNullException(nameof(errorMessage));
}
return(input =>
{
var result = parser(input);
if (result.HasValue)
{
return result;
}
return TokenListParserResult.Empty <TKind, T>(result.Remainder, result.ErrorPosition, errorMessage);
});
}
private static TokenListParser <TKind, Token <TKind> > Matching <TKind>(Func <TKind, bool> predicate, string[] expectations)
{
if (predicate == null)
{
throw new ArgumentNullException(nameof(predicate));
}
if (expectations == null)
{
throw new ArgumentNullException(nameof(expectations));
}
return(input =>
{
var next = input.ConsumeToken();
if (!next.HasValue || !predicate(next.Value.Kind))
{
return TokenListParserResult.Empty <TKind, Token <TKind> >(input, expectations);
}
return next;
});
}
/// <summary>
/// Construct a parser that succeeds only if the source is at the end of input.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T> AtEnd <TKind, T>(this TokenListParser <TKind, T> parser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(input =>
{
var result = parser(input);
if (!result.HasValue)
{
return result;
}
if (result.Remainder.IsAtEnd)
{
return result;
}
return TokenListParserResult.Empty <TKind, T>(result.Remainder);
});
}
public static TokenListParser <TKind, TResult> ChainModified <TKind, TResult, TOperator, TModifier>(
TokenListParser <TKind, TOperator> @operator,
TokenListParser <TKind, TResult> operand,
TokenListParser <TKind, TModifier> modify,
Func <TOperator, TResult, TResult, TModifier, TResult> apply)
{
if (@operator == null)
{
throw new ArgumentNullException(nameof(@operator));
}
if (operand == null)
{
throw new ArgumentNullException(nameof(operand));
}
if (modify == null)
{
throw new ArgumentNullException(nameof(modify));
}
if (apply == null)
{
throw new ArgumentNullException(nameof(apply));
}
return(input =>
{
var parseResult = operand(input);
if (!parseResult.HasValue)
{
return parseResult;
}
var result = parseResult.Value;
var remainder = parseResult.Remainder;
var operatorResult = @operator(remainder);
while (operatorResult.HasValue || operatorResult.SubTokenErrorPosition.HasValue || remainder != operatorResult.Remainder)
{
// If operator read any input, but failed to read complete input, we return error
if (!operatorResult.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, TOperator, TResult>(operatorResult);
}
var operandResult = operand(operatorResult.Remainder);
remainder = operandResult.Remainder;
if (!operandResult.HasValue)
{
return operandResult;
}
var modifierResult = modify(remainder);
remainder = modifierResult.Remainder;
if (!modifierResult.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, TModifier, TResult>(modifierResult);
}
result = apply(operatorResult.Value, result, operandResult.Value, modifierResult.Value);
operatorResult = @operator(remainder);
}
return TokenListParserResult.Value(result, input, remainder);
});
}
/// <summary>
/// Construct a parser with a fixed value.
/// </summary>
/// <param name="value">The value returned by the parser.</param>
/// <typeparam name="T">The type of <paramref name="value"/>.</typeparam>
/// <typeparam name="TKind">The kind of token being parsed.</typeparam>
/// <returns>The parser.</returns>
public static TokenListParser <TKind, T> Return <TKind, T>(T value)
{
return(input => TokenListParserResult.Value(value, input, input));
}
public static T AssertSuccess <T>(TokenListParserResult <SelectToken, T> result)
{
Assert.True(result.HasValue, result.ToString());
return(result.Value);
}
public static void AssertFailed <T>(TokenListParserResult <SelectToken, T> result)
{
Assert.False(result.HasValue);
}
/// <summary>
/// Convert an empty result of one type into another.
/// </summary>
/// <typeparam name="TKind">The kind of token.</typeparam>
/// <typeparam name="T">The source type.</typeparam>
/// <typeparam name="U">The destination type.</typeparam>
/// <param name="result">The result to convert.</param>
/// <returns>The converted result.</returns>
public static TokenListParserResult <TKind, U> CastEmpty <TKind, T, U>(TokenListParserResult <TKind, T> result)
{
return(new TokenListParserResult <TKind, U>(result.Remainder, result.ErrorPosition, result.ErrorMessage, result.Expectations, result.Backtrack));
}
