public async Task TestTokenIteration(IList<Token> expectedTokens) { using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync())) { for (var i = 0; i < expectedTokens.Count; i++) { Assert.That(await tokenIterator.MoveNextAsync(), Is.True, "Move next i={0}", i); var current = await tokenIterator.CurrentAsync(); Assert.That(current, Is.EqualTo(expectedTokens[i]), "Current i={0} expected={1} is={2}", i, expectedTokens[i], current); var la = await tokenIterator.LookAheadAsync(); var expectedLA = i + 1 < expectedTokens.Count ? expectedTokens[i + 1] : Token.EOF; Assert.That(la, Is.EqualTo(expectedLA), "Lookahead i={0} expected={1} is={2}", i, expectedLA, la); } } }
public async Task TestSupportResetting(IList <Item> expectedTokens) { using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync())) { if (tokenIterator.SupportsResetting) { var isEmpty = true; while (await tokenIterator.MoveNextAsync()) { isEmpty = false; } tokenIterator.Reset(); Assert.That(tokenIterator.MoveNextAsync(), Is.EqualTo(isEmpty), "Has element after resetting is not empty"); } else { Assert.That(tokenIterator.Reset, Throws.InvalidOperationException); } } }
public async Task TestTokenIteration(IList <Item> expectedTokens) { using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync())) { for (var i = 0; i < expectedTokens.Count; i++) { Assert.That(await tokenIterator.MoveNextAsync(), Is.True, "Move next i={0}", i); var current = await tokenIterator.CurrentAsync(); Assert.That(current, Is.EqualTo(expectedTokens[i]), "Current i={0} expected={1} is={2}", i, expectedTokens[i], current); var la = await tokenIterator.LookAheadAsync(); var expectedLA = i + 1 < expectedTokens.Count ? expectedTokens[i + 1] : Item.EOF; Assert.That(la, Is.EqualTo(expectedLA), "Lookahead i={0} expected={1} is={2}", i, expectedLA, la); } } }
public async Task TestSupportResetting(IList<Token> expectedTokens) { using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync())) { if (tokenIterator.SupportsResetting) { var isEmpty = true; while (await tokenIterator.MoveNextAsync()) { isEmpty = false; } tokenIterator.Reset(); Assert.That(tokenIterator.MoveNextAsync(), Is.EqualTo(isEmpty), "Has element after resetting is not empty"); } else { Assert.That(tokenIterator.Reset, Throws.InvalidOperationException); } } }
private static async Task MainAsync(string[] args, CancellationToken token) { // // the following program produces a parse table for the following grammar // for infix expressions, and appropriately applies operator precedence of // the + - * / operators, otherwise evaluating the leftmost operations first // // S' -> e // e -> i // e -> ( e ) // e -> e * e // e -> e / e // e -> e + e // e -> e - e // var grammar = new Grammar( new[] {"S'", "e", "+", "-", "*", "/", "i", "(", ")"}, new PrecedenceGroup(Derivation.None, //S' -> e new Production(0, (_, x) => x[0], 1), //e -> i new Production(1, (_, x) => x.Length == 1 ? x[0] : null, 6), //e -> ( e ) new Production(1, (_, x) => x[1].ContentType == ContentType.Nested ? x[1].Nested : null, 7, 1, 8) ), new PrecedenceGroup(Derivation.LeftMost, //e -> e * e new Production(1, RewriteConstBinaryExpr, 1, 4, 1), //e -> e / e new Production(1, RewriteConstBinaryExpr, 1, 5, 1) ), // productions are left associative and bind less tightly than * or / new PrecedenceGroup(Derivation.LeftMost, //e -> e + e new Production(1, RewriteConstBinaryExpr, 1, 2, 1), //e -> e - e new Production(1, RewriteConstBinaryExpr, 1, 3, 1) ) ); // generate the parse table var parser = new Parser(grammar); var debugger = new Debug(parser, Console.Write, Console.Error.Write); debugger.DumpParseTable(); debugger.Flush(); var parseTime = System.Diagnostics.Stopwatch.StartNew(); #if DEBUG // (24 / 12) + 2 * (3-4) var inputSource = new[] { new Token(6, 2), new Token(2, "+"), new Token(7, "("), new Token(6, 0), new Token(4, "*"), new Token(6, int.MaxValue), new Token(8, ")") }; #else var inputSource = TestLarge(); #endif Token result; using (var tokenIterator = new AsyncLATokenIterator(inputSource.AsAsync())) { result = await parser.ParseInputAsync(tokenIterator, debugger, allowRewriting: true); } parseTime.Stop(); var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds); debugger.WriteFinalToken( string.Format("Accept ({0}): ", timeElapsed), string.Format("Error while parsing ({0}): ", timeElapsed), result); }
private static async Task MainAsync(string[] args, CancellationToken cancellationToken) { // // the following program produces a parse table for the following grammar // for infix expressions, and appropriately applies operator precedence of // the + - * / operators, otherwise evaluating the leftmost operations first // // S' -> e // e -> i // e -> ( e ) // e -> e * e // e -> e / e // e -> e + e // e -> e - e // var grammar = new Grammar( Symbols, new PrecedenceGroup(Derivation.None, //S' -> e new Production((int)SymbolId.Result, (_, x) => x[0], (int)SymbolId.Expr), //e -> i new Production((int)SymbolId.Expr, (_, x) => x.Length == 1 ? x[0] : null, (int)SymbolId.Integer), //e -> ( e ) new Production((int)SymbolId.Expr, (_, x) => x[1].ContentType == ContentType.Nested ? x[1].Nested : null, (int)SymbolId.LeftParen, (int)SymbolId.Expr, (int)SymbolId.RightParen) ), new PrecedenceGroup(Derivation.LeftMost, //e -> e * e new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Times, (int)SymbolId.Expr), //e -> e / e new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Divide, (int)SymbolId.Expr) ), // productions are left associative and bind less tightly than * or / new PrecedenceGroup(Derivation.LeftMost, //e -> e + e new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Plus, (int)SymbolId.Expr), //e -> e - e new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Minus, (int)SymbolId.Expr) ) ); // generate the parse table var parser = new Parser(grammar); var debugger = new Debug(parser, Console.Write, Console.Error.Write); debugger.DumpParseTable(); debugger.Flush(); var parseTime = System.Diagnostics.Stopwatch.StartNew(); Collection <PSParseError> syntaxErrors; var inputSource = PSParser.Tokenize("(24 / 12) + 2 * (3-4)", out syntaxErrors).Select(PSTokenToItem); Item result; using (var tokenIterator = new AsyncLATokenIterator(inputSource.AsAsync())) { result = await parser.ParseInputAsync(tokenIterator, debugger); } parseTime.Stop(); var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds); debugger.WriteFinalToken( string.Format("Accept ({0}): ", timeElapsed), string.Format("Error while parsing ({0}): ", timeElapsed), result); }
private static async Task MainAsync(string[] args, CancellationToken cancellationToken) { var grammar = new Grammar( Symbols, PG(Derivation.None, P(S.Ast, S.Syntax), PSL(S.Syntax, S.Rule), PSL(S.Syntax, S.Rule, S.Syntax), PR(S.Rule, (pLHS, pRHS) => Tuple.Create(pRHS[0].Content, pRHS[2]), S.RuleName, S.DefineOp, S.Clauses, S.RuleEnd) ), PG(Derivation.LeftMost, PSL(S.Clauses, S.Terms), PSL(S.Clauses, S.Terms, S.OrOp, S.Clauses), PL(S.Terms, MakeMetaTerm, S.Term), PL(S.Terms, MakeMetaTerm, S.Term, S.TermSep, S.Terms) ), PG(Derivation.None, P(S.Term, S.Literal), P(S.Term, S.RuleName), PR(S.Term, (pLHS, pRHS) => MakeMetaTerm(null)) ), PG(Derivation.LeftMost, PR(S.RuleName, MakeQuotedString, S.RuleLiteralLeft, S.RuleCharacters, S.RuleLiteralRight), PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter), PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter, S.RuleCharacters) ), PG(Derivation.LeftMost, PR(S.Literal, MakeQuotedString, S.DoubleQuote, S.TextInDoubleQuotes, S.DoubleQuote), PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes), PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes, S.TextInDoubleQuotes) ), PG(Derivation.LeftMost, PR(S.Literal, MakeQuotedString, S.SingleQuote, S.TextInSingleQuotes, S.SingleQuote), PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes), PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes, S.TextInSingleQuotes) ) ); // generate the parse table var parser = new Parser(grammar); var debugger = new Debug(parser, Console.Write, Console.Error.Write); var parseTime = System.Diagnostics.Stopwatch.StartNew(); Func <string, Regex> compileRegex = p => new Regex("^" + (p.StartsWith("[") || p.Length <= 2 ? p : (p.Substring(0, 2) + "(" + p.Substring(2) + "|$)")), RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture); Func <S, string, string, Tuple <int, Regex, string> > triple = (pSymbol, pPattern, pState) => Tuple.Create((int)pSymbol, compileRegex(pPattern), pState); Func <S, string, Tuple <int, Regex, string> > pair = (pSymbol, pPattern) => triple(pSymbol, pPattern, null); const string doubleQuoteState = "double"; const string singleQuoteState = "single"; const string doubleQuoteMarker = @"[""]"; const string singleQuoteMarker = "[']"; const string commonSet = "-A-Za-z0-9_"; const string symbolSet = @"| !#$%&()*+,./:;\\<=>?@[\]^`{|}~"; var lexerTable = new Dictionary <string, Tuple <int, Regex, string>[]> { { AsyncRegexLexer.RootState, new[] { triple(S.DoubleQuote, doubleQuoteMarker, doubleQuoteState), triple(S.SingleQuote, singleQuoteMarker, singleQuoteState), pair(S.RuleCharacter, "[" + commonSet + "]"), pair(S.DefineOp, "::="), pair(S.OrOp, "[|]"), pair(S.RuleLiteralLeft, "[<]"), pair(S.RuleLiteralRight, "[>]"), pair(S.RuleEnd, "[.]"), pair(S.TermSep, "[,]"), triple(S.EOL, @"[\r]?[\n]", AsyncRegexLexer.Ignore), triple(S.WS, @"[ \t]", AsyncRegexLexer.Ignore) } }, { doubleQuoteState, new[] { triple(S.DoubleQuote, doubleQuoteMarker, AsyncRegexLexer.PopState), pair(S.CharacterInDoubleQuotes, "[" + commonSet + symbolSet + "'" + "]") } }, { singleQuoteState, new[] { triple(S.SingleQuote, singleQuoteMarker, AsyncRegexLexer.PopState), pair(S.CharacterInSingleQuotes, "[" + commonSet + symbolSet + "\"" + "]") } } }; Item result; using (var charIterator = new AsyncLACharIterator(new StringReader(BNF))) using (var regexLexer = new AsyncRegexLexer(charIterator, lexerTable)) using (var tokenIterator = new AsyncLATokenIterator(regexLexer)) { result = await parser.ParseInputAsync(tokenIterator, debugger); } parseTime.Stop(); var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds); debugger.WriteFinalToken( string.Format("Accept ({0}): ", timeElapsed), string.Format("Error while parsing ({0}): ", timeElapsed), result); }