public static FHXObject ParseTokens(List <Token> tokens) { Parser p = new TokenListParser(tokens); FHXObject o = p.ParseAll(); return(o); }
/// <summary> /// Construct a parser that applies <paramref name="first"/>, provides the value to <paramref name="second"/> and returns the result. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <typeparam name="U">The type of the resulting value.</typeparam> /// <param name="first">The first parser.</param> /// <param name="second">The second parser.</param> /// <returns>The resulting parser.</returns> public static TokenListParser <TKind, U> Then <TKind, T, U>(this TokenListParser <TKind, T> first, Func <T, TokenListParser <TKind, U> > second) { if (first == null) { throw new ArgumentNullException(nameof(first)); } if (second == null) { throw new ArgumentNullException(nameof(second)); } return(input => { var rt = first(input); if (!rt.HasValue) { return TokenListParserResult.CastEmpty <TKind, T, U>(rt); } var ru = second(rt.Value)(rt.Remainder); if (!ru.HasValue) { return ru; } return TokenListParserResult.Value(ru.Value, input, ru.Remainder); }); }
/// <summary> /// Constructs a parser that will fail if the given parser succeeds, /// and will succeed if the given parser fails. In any case, it won't /// consume any input. It's like a negative look-ahead in a regular expression. /// </summary> /// <typeparam name="T">The result type of the given parser.</typeparam> /// <typeparam name="TKind">The kind of token being parsed.</typeparam> /// <param name="parser">The parser to wrap</param> /// <returns>A parser that is the negation of the given parser.</returns> public static TokenListParser <TKind, Unit> Not <TKind, T>(TokenListParser <TKind, T> parser) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } return(input => { var result = parser(input); if (result.HasValue) { // This is usually a success case for Not(), so the allocations here are a bit of a pity. var current = input.ConsumeToken(); var last = result.Remainder.ConsumeToken(); if (current.HasValue) { var span = last.HasValue ? current.Value.Span.Source.Substring(current.Value.Position.Absolute, last.Value.Position.Absolute - current.Value.Position.Absolute) : current.Value.Span.Source.Substring(current.Value.Position.Absolute); return TokenListParserResult.Empty <TKind, Unit>(input, $"unexpected successful parsing of {Presentation.FormatLiteral(Friendly.Clip(span, 12))}"); } return TokenListParserResult.Empty <TKind, Unit>(input, "unexpected successful parsing"); } return TokenListParserResult.Value(Unit.Value, input, input); }); }
public static bool TryParseAll <T>(TokenListParser <ExpressionToken, T> parser, string source, out T value, out string error) { if (source == null) { throw new ArgumentNullException(nameof(source)); } var tokens = ExpressionTokenizer.TryTokenize(source); if (!tokens.HasValue) { value = default; error = tokens.ToString(); return(false); } var result = parser.AtEnd()(tokens.Value); if (!result.HasValue) { value = default; error = result.ToString(); return(false); } value = result.Value; error = null; return(true); }
public static TokenListParser <TTokenKind, U> SelectCatch <TTokenKind, T, U>(this TokenListParser <TTokenKind, T> parser, Func <T, U> trySelector, string errorMessage) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } if (trySelector == null) { throw new ArgumentNullException(nameof(trySelector)); } if (errorMessage == null) { throw new ArgumentNullException(nameof(errorMessage)); } return(input => { var t = parser(input); if (!t.HasValue) { return TokenListParserResult.CastEmpty <TTokenKind, T, U>(t); } try { var u = trySelector(t.Value); return TokenListParserResult.Value(u, input, t.Remainder); } catch { return TokenListParserResult.Empty <TTokenKind, U>(input, errorMessage); } }); }
private void AssertParse <T>(string input, string expected, TokenListParser <CToken, T> parser) { var tokenizer = Tokenizer.Create(); var ast = parser.Parse(tokenizer.Tokenize(input)); ast.ToString().Should().Be(expected); }
public static TokenListParser <TKind, T> RightRec <TKind, T>(TokenListParser <TKind, T> head, Func <T, TokenListParser <TKind, T> > apply) { if (head == null) { throw new ArgumentNullException(nameof(head)); } if (apply == null) { throw new ArgumentNullException(nameof(apply)); } return(input => { var parseResult = head(input); if (!parseResult.HasValue) { return parseResult; } var result = parseResult.Value; var tailResult = apply(result)(parseResult.Remainder); while (true) { if (!tailResult.HasValue) { return TokenListParserResult.Value(result, input, tailResult.Remainder); } result = tailResult.Value; tailResult = apply(result)(tailResult.Remainder); } }); }
public static void FailsWithMessage <TKind, T>(TokenListParser <TKind, T> parser, string input, Tokenizer <TKind> tokenizer, string message) { var result = parser.TryParse(tokenizer.Tokenize(input)); Assert.Equal(message, result.ToString()); }
public static void SucceedsWith <T>(TokenListParser <char, T> parser, string input, T value) { Succeeds(parser, input, v => { Assert.Equal(value, v); }); }
/// <summary> /// Parse a sequence of similar operands connected by left-associative operators. /// </summary> /// <typeparam name="T">The type being parsed.</typeparam> /// <typeparam name="TOperator">The type of the operator.</typeparam> /// <typeparam name="TKind">The kind of token being parsed.</typeparam> /// <param name="operator">A parser matching operators.</param> /// <param name="operand">A parser matching operands.</param> /// <param name="apply">A function combining an operator and two operands into the result.</param> /// <returns>The result of calling <paramref name="apply"/> successively on pairs of operands.</returns> /// <seealso cref="Combinators.Chain{TKind, TResult,TOperator,TOperand}"/> public static TokenListParser <TKind, T> Chain <TKind, T, TOperator>( TokenListParser <TKind, TOperator> @operator, TokenListParser <TKind, T> operand, Func <TOperator, T, T, T> apply) { return(operand.Chain(@operator, operand, apply)); }
/// <summary> /// Construct a parser that evaluates the result of a previous parser and fails if <paramref name="predicate"/> returns false /// for the result. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <param name="parser">The parser.</param> /// <param name="predicate">The predicate to apply.</param> /// <returns>The resulting parser.</returns> public static TokenListParser <TKind, T> Where <TKind, T>(this TokenListParser <TKind, T> parser, Func <T, bool> predicate) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } if (predicate == null) { throw new ArgumentNullException(nameof(predicate)); } return(input => { var rt = parser(input); if (!rt.HasValue) { return rt; } if (predicate(rt.Value)) { return rt; } return TokenListParserResult.Empty <TKind, T>(input, "unsatisfied condition"); }); }
public static void SucceedsWith(TokenListParser <char, Token <char> > parser, string input, char value) { Succeeds(parser, input, tok => { Assert.Equal(value, tok.Kind); }); }
private static T Parse <T>(TokenListParser <SelectToken, T> parser, string input) { var tokenizer = new SelectTokenizer(); var tokens = tokenizer.Tokenize(input); return(parser.Parse(tokens)); }
public void SpeedTest() { var templateString = Properties.Resources.LoremIpsum; var tokens = new[] { "{{FirstName}}", "{{LastName}}", "{{Email}}", "{{Color}}", "{{Music}}", "{{Sport}}" }; var templateParser = new TokenListParser(tokens); var template = templateParser.GetTemplate(templateString); using (var context = new PeopleDataDataContext()) { var people = context.Peoples; var elapsedTime = ExecutionTime(() => { foreach (var person in people) { var values = new Dictionary <string, string> { { "{{FirstName}}", person.FirstName }, { "{{LastName}}", person.LastName }, { "{{Email}}", person.Email }, { "{{Color}}", person.Color }, { "{{Music}}", person.Music }, { "{{Sport}}", person.Sport } }; var result = template.Render(values); } }); Console.WriteLine("Executed {0} template replacements in {1} seconds", people.Count(), elapsedTime.TotalSeconds); Assert.IsTrue(elapsedTime < TimeSpan.FromSeconds(15)); } }
public static TokenListParserResult <SelectToken, T> Parse <T>(TokenListParser <SelectToken, T> parser, string input) { var tokenizer = new SelectTokenizer(); var tokens = tokenizer.Tokenize(input); return(parser.AtEnd().TryParse(tokens)); }
/// <summary> /// Construct a parser that tries first the <paramref name="lhs"/> parser, and if it fails, applies <paramref name="rhs"/>. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <param name="lhs">The first parser to try.</param> /// <param name="rhs">The second parser to try.</param> /// <returns>The resulting parser.</returns> /// <remarks>Or will fail if the first item partially matches this. To modify this behavior use <see cref="Try{TKind,T}(TokenListParser{TKind,T})"/>.</remarks> public static TokenListParser <TKind, T> Or <TKind, T>(this TokenListParser <TKind, T> lhs, TokenListParser <TKind, T> rhs) { if (lhs == null) { throw new ArgumentNullException(nameof(lhs)); } if (rhs == null) { throw new ArgumentNullException(nameof(rhs)); } return(input => { var first = lhs(input); if (first.HasValue || !first.Backtrack && first.IsPartial(input)) { return first; } var second = rhs(input); if (second.HasValue) { return second; } return TokenListParserResult.CombineEmpty(first, second); }); }
/// <summary> /// Construct a parser that returns <paramref name="name"/> as its "expectation" if <paramref name="parser"/> fails. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <param name="parser">The parser.</param> /// <param name="name">The name given to <paramref name="parser"/>.</param> /// <returns>The resulting parser.</returns> public static TokenListParser <TKind, T> Named <TKind, T>(this TokenListParser <TKind, T> parser, string name) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } if (name == null) { throw new ArgumentNullException(nameof(name)); } return(input => { var result = parser(input); if (result.HasValue || result.Remainder != input) { return result; } // result.IsSubTokenError? if (result.ErrorPosition.HasValue) { return TokenListParserResult.Empty <TKind, T>(result.Remainder, result.ErrorPosition, result.FormatErrorMessageFragment()); } return TokenListParserResult.Empty <TKind, T>(result.Remainder, new[] { name }); }); }
/// <summary> /// Apply the character parser <paramref name="valueParser"/> to the span represented by the parsed token. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="U">The type of the resulting value.</typeparam> /// <param name="parser">The parser.</param> /// <param name="valueParser">A function that determines which character parser to apply.</param> /// <returns>A parser that returns the result of parsing the token value.</returns> public static TokenListParser <TKind, U> Apply <TKind, U>(this TokenListParser <TKind, Token <TKind> > parser, Func <Token <TKind>, TextParser <U> > valueParser) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } if (valueParser == null) { throw new ArgumentNullException(nameof(valueParser)); } return(input => { var rt = parser(input); if (!rt.HasValue) { return TokenListParserResult.CastEmpty <TKind, Token <TKind>, U>(rt); } var uParser = valueParser(rt.Value); var uResult = uParser.AtEnd()(rt.Value.Span); if (!uResult.HasValue) { var message = $"invalid {Presentation.FormatExpectation(rt.Value.Kind)}, {uResult.FormatErrorMessageFragment()}"; return new TokenListParserResult <TKind, U>(input, uResult.Remainder.Position, message, null, uResult.Backtrack); } return TokenListParserResult.Value(uResult.Value, rt.Location, rt.Remainder); }); }
/// <summary> /// Construct a parser that matches <paramref name="parser"/> zero or more times. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <param name="parser">The parser.</param> /// <returns>The resulting parser.</returns> /// <remarks>Many will fail if any item partially matches this. To modify this behavior use <see cref="Try{TKind,T}(TokenListParser{TKind,T})"/>.</remarks> public static TokenListParser <TKind, T[]> Many <TKind, T>(this TokenListParser <TKind, T> parser) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } return(input => { var result = new List <T>(); var @from = input; var r = parser(input); while (r.HasValue) { if (@from == r.Remainder) // Broken parser, not a failed parsing. { throw new ParseException($"Many() cannot be applied to zero-width parsers; value {r.Value} at position {r.Location.Position}."); } result.Add(r.Value); @from = r.Remainder; r = parser(r.Remainder); } if (!r.Backtrack && r.IsPartial(@from)) { return TokenListParserResult.CastEmpty <TKind, T, T[]>(r); } return TokenListParserResult.Value(result.ToArray(), input, r.Remainder); }); }
public static void SucceedsWithOne <T>(TokenListParser <char, T[]> parser, string input, T expectedResult) { Succeeds(parser, input, t => { Assert.Single(t); Assert.Equal(expectedResult, t.Single()); }); }
/// <summary> /// Construct a parser that matches one or more instances of applying <paramref name="parser"/>. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <param name="parser">The parser.</param> /// <returns>The resulting parser.</returns> public static TokenListParser <TKind, T[]> AtLeastOnce <TKind, T>(this TokenListParser <TKind, T> parser) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } return(parser.Then(first => parser.Many().Select(rest => ArrayEnumerable.Cons(first, rest)))); }
/// <summary> /// Construct a parser that matches zero or one instance of <paramref name="parser"/>, returning <paramref name="defaultValue"/> when /// no match is possible. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <param name="parser">The parser.</param> /// <param name="defaultValue">The default value</param> /// <returns>The resulting parser.</returns> public static TokenListParser <TKind, T> OptionalOrDefault <TKind, T>(this TokenListParser <TKind, T> parser, T defaultValue = default(T)) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } return(parser.Or(Parse.Return <TKind, T>(defaultValue))); }
/// <summary> /// Construct a parser that applies the first, and returns <paramref name="value"/>. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <typeparam name="U">The type of the resulting value.</typeparam> /// <param name="parser">The parser.</param> /// <param name="value">The value to return.</param> /// <returns>The resulting parser.</returns> public static TokenListParser <TKind, U> Value <TKind, T, U>(this TokenListParser <TKind, T> parser, U value) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } return(parser.IgnoreThen(Parse.Return <TKind, U>(value))); }
/// <summary> /// Tries to parse the input without throwing an exception upon failure. /// </summary> /// <typeparam name="TKind">The type of tokens consumed by the parser.</typeparam> /// <typeparam name="T">The type of the result.</typeparam> /// <param name="parser">The parser.</param> /// <param name="input">The input.</param> /// <returns>The result of the parser</returns> /// <exception cref="ArgumentNullException">The parser or input is null.</exception> public static TokenListParserResult <TKind, T> TryParse <TKind, T>(this TokenListParser <TKind, T> parser, TokenList <TKind> input) { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } return(parser(input)); }
/// <summary> /// Construct a parser that matches zero or one instance of <paramref name="parser"/>. /// </summary> /// <typeparam name="TKind">The kind of the tokens being parsed.</typeparam> /// <typeparam name="T">The type of value being parsed.</typeparam> /// <param name="parser">The parser.</param> /// <returns>The resulting parser.</returns> public static TokenListParser <TKind, T?> Optional <TKind, T>(this TokenListParser <TKind, T> parser) where T : struct { if (parser == null) { throw new ArgumentNullException(nameof(parser)); } return(parser.Select(t => (T?)t).Or(Parse.Return <TKind, T?>(null))); }
private static void AssertParseResult <T>(string input, string expectation, TokenListParser <SmcToken, T> transition) where T : ISyntax { var syntax = transition.Parse(Tokenizer.Create().Tokenize(input)); var formatter = new SyntaxFormatter(); syntax.Accept(formatter); var formatted = formatter.Text; Assert.Equal(expectation, formatted); }
public TokenListTests() { Tokens = new Dictionary <string, string> { { "quick", "slow" }, { "fox", "cow" }, { "lazy dog", "moon" } }; TemplateParser = new TokenListParser(Tokens.Keys); }
public void TestTokenInception() { var testString = "You mean, a token within a token"; var tokens = new Dictionary <string, string> { { "token within", "dream within" }, { "within a token", "within a dream" } }; var templateParser = new TokenListParser(tokens.Keys); var template = templateParser.GetTemplate(testString); }
public static TokenListParser <TKind, U> OneOf <TKind, U, T1, T2>( TokenListParser <TKind, T1> p1, TokenListParser <TKind, T2> p2) where T1 : U where T2 : U { return(( p1.Cast <TKind, T1, U>() ).Or( p2.Cast <TKind, T2, U>() )); }
public static T MustParse <TToken, T>(this TokenListParser <TToken, T> parser, Tokenizer <TToken> tokenizer, string input) { var tokens = tokenizer.Tokenize(input); var result = parser(tokens); if (!result.HasValue) { throw new Exception(result.ToString()); } return(result.Value); }