public static FHXObject ParseTokens(List <Token> tokens)
{
Parser p = new TokenListParser(tokens);
FHXObject o = p.ParseAll();
return(o);
}
/// <summary>
/// Construct a parser that applies <paramref name="first"/>, provides the value to <paramref name="second"/> and returns the result.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <typeparam name="U">The type of the resulting value.</typeparam>
/// <param name="first">The first parser.</param>
/// <param name="second">The second parser.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, U> Then <TKind, T, U>(this TokenListParser <TKind, T> first, Func <T, TokenListParser <TKind, U> > second)
{
if (first == null)
{
throw new ArgumentNullException(nameof(first));
}
if (second == null)
{
throw new ArgumentNullException(nameof(second));
}
return(input =>
{
var rt = first(input);
if (!rt.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, T, U>(rt);
}
var ru = second(rt.Value)(rt.Remainder);
if (!ru.HasValue)
{
return ru;
}
return TokenListParserResult.Value(ru.Value, input, ru.Remainder);
});
}
/// <summary>
/// Constructs a parser that will fail if the given parser succeeds,
/// and will succeed if the given parser fails. In any case, it won't
/// consume any input. It's like a negative look-ahead in a regular expression.
/// </summary>
/// <typeparam name="T">The result type of the given parser.</typeparam>
/// <typeparam name="TKind">The kind of token being parsed.</typeparam>
/// <param name="parser">The parser to wrap</param>
/// <returns>A parser that is the negation of the given parser.</returns>
public static TokenListParser <TKind, Unit> Not <TKind, T>(TokenListParser <TKind, T> parser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(input =>
{
var result = parser(input);
if (result.HasValue)
{
// This is usually a success case for Not(), so the allocations here are a bit of a pity.
var current = input.ConsumeToken();
var last = result.Remainder.ConsumeToken();
if (current.HasValue)
{
var span = last.HasValue ?
current.Value.Span.Source.Substring(current.Value.Position.Absolute, last.Value.Position.Absolute - current.Value.Position.Absolute) :
current.Value.Span.Source.Substring(current.Value.Position.Absolute);
return TokenListParserResult.Empty <TKind, Unit>(input, $"unexpected successful parsing of {Presentation.FormatLiteral(Friendly.Clip(span, 12))}");
}
return TokenListParserResult.Empty <TKind, Unit>(input, "unexpected successful parsing");
}
return TokenListParserResult.Value(Unit.Value, input, input);
});
}
public static bool TryParseAll <T>(TokenListParser <ExpressionToken, T> parser, string source, out T value, out string error)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
var tokens = ExpressionTokenizer.TryTokenize(source);
if (!tokens.HasValue)
{
value = default;
error = tokens.ToString();
return(false);
}
var result = parser.AtEnd()(tokens.Value);
if (!result.HasValue)
{
value = default;
error = result.ToString();
return(false);
}
value = result.Value;
error = null;
return(true);
}
public static TokenListParser <TTokenKind, U> SelectCatch <TTokenKind, T, U>(this TokenListParser <TTokenKind, T> parser, Func <T, U> trySelector, string errorMessage)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (trySelector == null)
{
throw new ArgumentNullException(nameof(trySelector));
}
if (errorMessage == null)
{
throw new ArgumentNullException(nameof(errorMessage));
}
return(input =>
{
var t = parser(input);
if (!t.HasValue)
{
return TokenListParserResult.CastEmpty <TTokenKind, T, U>(t);
}
try
{
var u = trySelector(t.Value);
return TokenListParserResult.Value(u, input, t.Remainder);
}
catch
{
return TokenListParserResult.Empty <TTokenKind, U>(input, errorMessage);
}
});
}
private void AssertParse <T>(string input, string expected, TokenListParser <CToken, T> parser)
{
var tokenizer = Tokenizer.Create();
var ast = parser.Parse(tokenizer.Tokenize(input));
ast.ToString().Should().Be(expected);
}
public static TokenListParser <TKind, T> RightRec <TKind, T>(TokenListParser <TKind, T> head, Func <T, TokenListParser <TKind, T> > apply)
{
if (head == null)
{
throw new ArgumentNullException(nameof(head));
}
if (apply == null)
{
throw new ArgumentNullException(nameof(apply));
}
return(input =>
{
var parseResult = head(input);
if (!parseResult.HasValue)
{
return parseResult;
}
var result = parseResult.Value;
var tailResult = apply(result)(parseResult.Remainder);
while (true)
{
if (!tailResult.HasValue)
{
return TokenListParserResult.Value(result, input, tailResult.Remainder);
}
result = tailResult.Value;
tailResult = apply(result)(tailResult.Remainder);
}
});
}
public static void FailsWithMessage <TKind, T>(TokenListParser <TKind, T> parser, string input,
Tokenizer <TKind> tokenizer, string message)
{
var result = parser.TryParse(tokenizer.Tokenize(input));
Assert.Equal(message, result.ToString());
}
public static void SucceedsWith <T>(TokenListParser <char, T> parser, string input, T value)
{
Succeeds(parser, input, v =>
{
Assert.Equal(value, v);
});
}
/// <summary>
/// Parse a sequence of similar operands connected by left-associative operators.
/// </summary>
/// <typeparam name="T">The type being parsed.</typeparam>
/// <typeparam name="TOperator">The type of the operator.</typeparam>
/// <typeparam name="TKind">The kind of token being parsed.</typeparam>
/// <param name="operator">A parser matching operators.</param>
/// <param name="operand">A parser matching operands.</param>
/// <param name="apply">A function combining an operator and two operands into the result.</param>
/// <returns>The result of calling <paramref name="apply"/> successively on pairs of operands.</returns>
/// <seealso cref="Combinators.Chain{TKind, TResult,TOperator,TOperand}"/>
public static TokenListParser <TKind, T> Chain <TKind, T, TOperator>(
TokenListParser <TKind, TOperator> @operator,
TokenListParser <TKind, T> operand,
Func <TOperator, T, T, T> apply)
{
return(operand.Chain(@operator, operand, apply));
}
/// <summary>
/// Construct a parser that evaluates the result of a previous parser and fails if <paramref name="predicate"/> returns false
/// for the result.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="predicate">The predicate to apply.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T> Where <TKind, T>(this TokenListParser <TKind, T> parser, Func <T, bool> predicate)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (predicate == null)
{
throw new ArgumentNullException(nameof(predicate));
}
return(input =>
{
var rt = parser(input);
if (!rt.HasValue)
{
return rt;
}
if (predicate(rt.Value))
{
return rt;
}
return TokenListParserResult.Empty <TKind, T>(input, "unsatisfied condition");
});
}
public static void SucceedsWith(TokenListParser <char, Token <char> > parser, string input, char value)
{
Succeeds(parser, input, tok =>
{
Assert.Equal(value, tok.Kind);
});
}
private static T Parse <T>(TokenListParser <SelectToken, T> parser, string input)
{
var tokenizer = new SelectTokenizer();
var tokens = tokenizer.Tokenize(input);
return(parser.Parse(tokens));
}
public void SpeedTest()
{
var templateString = Properties.Resources.LoremIpsum;
var tokens = new[] { "{{FirstName}}", "{{LastName}}", "{{Email}}", "{{Color}}", "{{Music}}", "{{Sport}}" };
var templateParser = new TokenListParser(tokens);
var template = templateParser.GetTemplate(templateString);
using (var context = new PeopleDataDataContext())
{
var people = context.Peoples;
var elapsedTime = ExecutionTime(() =>
{
foreach (var person in people)
{
var values = new Dictionary <string, string>
{
{ "{{FirstName}}", person.FirstName },
{ "{{LastName}}", person.LastName },
{ "{{Email}}", person.Email },
{ "{{Color}}", person.Color },
{ "{{Music}}", person.Music },
{ "{{Sport}}", person.Sport }
};
var result = template.Render(values);
}
});
Console.WriteLine("Executed {0} template replacements in {1} seconds", people.Count(), elapsedTime.TotalSeconds);
Assert.IsTrue(elapsedTime < TimeSpan.FromSeconds(15));
}
}
public static TokenListParserResult <SelectToken, T> Parse <T>(TokenListParser <SelectToken, T> parser, string input)
{
var tokenizer = new SelectTokenizer();
var tokens = tokenizer.Tokenize(input);
return(parser.AtEnd().TryParse(tokens));
}
/// <summary>
/// Construct a parser that tries first the <paramref name="lhs"/> parser, and if it fails, applies <paramref name="rhs"/>.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="lhs">The first parser to try.</param>
/// <param name="rhs">The second parser to try.</param>
/// <returns>The resulting parser.</returns>
/// <remarks>Or will fail if the first item partially matches this. To modify this behavior use <see cref="Try{TKind,T}(TokenListParser{TKind,T})"/>.</remarks>
public static TokenListParser <TKind, T> Or <TKind, T>(this TokenListParser <TKind, T> lhs, TokenListParser <TKind, T> rhs)
{
if (lhs == null)
{
throw new ArgumentNullException(nameof(lhs));
}
if (rhs == null)
{
throw new ArgumentNullException(nameof(rhs));
}
return(input =>
{
var first = lhs(input);
if (first.HasValue || !first.Backtrack && first.IsPartial(input))
{
return first;
}
var second = rhs(input);
if (second.HasValue)
{
return second;
}
return TokenListParserResult.CombineEmpty(first, second);
});
}
/// <summary>
/// Construct a parser that returns <paramref name="name"/> as its "expectation" if <paramref name="parser"/> fails.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="name">The name given to <paramref name="parser"/>.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T> Named <TKind, T>(this TokenListParser <TKind, T> parser, string name)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
return(input =>
{
var result = parser(input);
if (result.HasValue || result.Remainder != input)
{
return result;
}
// result.IsSubTokenError?
if (result.ErrorPosition.HasValue)
{
return TokenListParserResult.Empty <TKind, T>(result.Remainder, result.ErrorPosition, result.FormatErrorMessageFragment());
}
return TokenListParserResult.Empty <TKind, T>(result.Remainder, new[] { name });
});
}
/// <summary>
/// Apply the character parser <paramref name="valueParser"/> to the span represented by the parsed token.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="U">The type of the resulting value.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="valueParser">A function that determines which character parser to apply.</param>
/// <returns>A parser that returns the result of parsing the token value.</returns>
public static TokenListParser <TKind, U> Apply <TKind, U>(this TokenListParser <TKind, Token <TKind> > parser, Func <Token <TKind>, TextParser <U> > valueParser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (valueParser == null)
{
throw new ArgumentNullException(nameof(valueParser));
}
return(input =>
{
var rt = parser(input);
if (!rt.HasValue)
{
return TokenListParserResult.CastEmpty <TKind, Token <TKind>, U>(rt);
}
var uParser = valueParser(rt.Value);
var uResult = uParser.AtEnd()(rt.Value.Span);
if (!uResult.HasValue)
{
var message = $"invalid {Presentation.FormatExpectation(rt.Value.Kind)}, {uResult.FormatErrorMessageFragment()}";
return new TokenListParserResult <TKind, U>(input, uResult.Remainder.Position, message, null, uResult.Backtrack);
}
return TokenListParserResult.Value(uResult.Value, rt.Location, rt.Remainder);
});
}
/// <summary>
/// Construct a parser that matches <paramref name="parser"/> zero or more times.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <returns>The resulting parser.</returns>
/// <remarks>Many will fail if any item partially matches this. To modify this behavior use <see cref="Try{TKind,T}(TokenListParser{TKind,T})"/>.</remarks>
public static TokenListParser <TKind, T[]> Many <TKind, T>(this TokenListParser <TKind, T> parser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(input =>
{
var result = new List <T>();
var @from = input;
var r = parser(input);
while (r.HasValue)
{
if (@from == r.Remainder) // Broken parser, not a failed parsing.
{
throw new ParseException($"Many() cannot be applied to zero-width parsers; value {r.Value} at position {r.Location.Position}.");
}
result.Add(r.Value);
@from = r.Remainder;
r = parser(r.Remainder);
}
if (!r.Backtrack && r.IsPartial(@from))
{
return TokenListParserResult.CastEmpty <TKind, T, T[]>(r);
}
return TokenListParserResult.Value(result.ToArray(), input, r.Remainder);
});
}
public static void SucceedsWithOne <T>(TokenListParser <char, T[]> parser, string input, T expectedResult)
{
Succeeds(parser, input, t =>
{
Assert.Single(t);
Assert.Equal(expectedResult, t.Single());
});
}
/// <summary>
/// Construct a parser that matches one or more instances of applying <paramref name="parser"/>.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T[]> AtLeastOnce <TKind, T>(this TokenListParser <TKind, T> parser)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(parser.Then(first => parser.Many().Select(rest => ArrayEnumerable.Cons(first, rest))));
}
/// <summary>
/// Construct a parser that matches zero or one instance of <paramref name="parser"/>, returning <paramref name="defaultValue"/> when
/// no match is possible.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="defaultValue">The default value</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T> OptionalOrDefault <TKind, T>(this TokenListParser <TKind, T> parser, T defaultValue = default(T))
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(parser.Or(Parse.Return <TKind, T>(defaultValue)));
}
/// <summary>
/// Construct a parser that applies the first, and returns <paramref name="value"/>.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <typeparam name="U">The type of the resulting value.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="value">The value to return.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, U> Value <TKind, T, U>(this TokenListParser <TKind, T> parser, U value)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(parser.IgnoreThen(Parse.Return <TKind, U>(value)));
}
/// <summary>
/// Tries to parse the input without throwing an exception upon failure.
/// </summary>
/// <typeparam name="TKind">The type of tokens consumed by the parser.</typeparam>
/// <typeparam name="T">The type of the result.</typeparam>
/// <param name="parser">The parser.</param>
/// <param name="input">The input.</param>
/// <returns>The result of the parser</returns>
/// <exception cref="ArgumentNullException">The parser or input is null.</exception>
public static TokenListParserResult <TKind, T> TryParse <TKind, T>(this TokenListParser <TKind, T> parser, TokenList <TKind> input)
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(parser(input));
}
/// <summary>
/// Construct a parser that matches zero or one instance of <paramref name="parser"/>.
/// </summary>
/// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam>
/// <typeparam name="T">The type of value being parsed.</typeparam>
/// <param name="parser">The parser.</param>
/// <returns>The resulting parser.</returns>
public static TokenListParser <TKind, T?> Optional <TKind, T>(this TokenListParser <TKind, T> parser)
where T : struct
{
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
return(parser.Select(t => (T?)t).Or(Parse.Return <TKind, T?>(null)));
}
private static void AssertParseResult <T>(string input, string expectation,
TokenListParser <SmcToken, T> transition) where T : ISyntax
{
var syntax = transition.Parse(Tokenizer.Create().Tokenize(input));
var formatter = new SyntaxFormatter();
syntax.Accept(formatter);
var formatted = formatter.Text;
Assert.Equal(expectation, formatted);
}
public TokenListTests()
{
Tokens = new Dictionary <string, string>
{
{ "quick", "slow" },
{ "fox", "cow" },
{ "lazy dog", "moon" }
};
TemplateParser = new TokenListParser(Tokens.Keys);
}
public void TestTokenInception()
{
var testString = "You mean, a token within a token";
var tokens = new Dictionary <string, string>
{
{ "token within", "dream within" },
{ "within a token", "within a dream" }
};
var templateParser = new TokenListParser(tokens.Keys);
var template = templateParser.GetTemplate(testString);
}
public static TokenListParser <TKind, U> OneOf <TKind, U, T1, T2>(
TokenListParser <TKind, T1> p1,
TokenListParser <TKind, T2> p2)
where T1 : U
where T2 : U
{
return((
p1.Cast <TKind, T1, U>()
).Or(
p2.Cast <TKind, T2, U>()
));
}
public static T MustParse <TToken, T>(this TokenListParser <TToken, T> parser, Tokenizer <TToken> tokenizer, string input)
{
var tokens = tokenizer.Tokenize(input);
var result = parser(tokens);
if (!result.HasValue)
{
throw new Exception(result.ToString());
}
return(result.Value);
}
