public async Task TestNormalisationAsync(string input) { // var expectedCount = input.Count(p => !char.IsSurrogate(p) && p != '\r'); using (var charReader = new AsyncLACharIterator(new StringReader(input), Capacity)) { var i = 0; while (i < input.Length) { int expectedCurrent; int expectedLA; input.CurrentAndLA(ref i, out expectedCurrent, out expectedLA); Assert.That(await charReader.MoveNextAsync(), Is.EqualTo(expectedCurrent != AsyncLACharIterator.EOF), "Move next i={0}", i); var current = await charReader.CurrentAsync(); Assert.That(current, Is.EqualTo(expectedCurrent), "Current codepoint i={0} expected={1} is={2}", i, expectedCurrent.DisplayUTF8(), current.DisplayUTF8()); var la = await charReader.LookAheadAsync(); Assert.That(la, Is.EqualTo(expectedLA), "Lookahead codepoint i={0} expected={1} is={2}", i, expectedLA.DisplayUTF8(), la.DisplayUTF8()); } } }
public void TestUninitialised() { using (var charReader = new AsyncLACharIterator(new StringReader(""), Capacity)) { // ReSharper disable AccessToDisposedClosure Assert.That(async() => await charReader.CurrentAsync(), Throws.InvalidOperationException); // ReSharper restore AccessToDisposedClosure } }
public void TestUninitialised() { using (var charReader = new AsyncLACharIterator(new StringReader(""), Capacity)) { // ReSharper disable AccessToDisposedClosure Assert.That(async () => await charReader.CurrentAsync(), Throws.InvalidOperationException); // ReSharper restore AccessToDisposedClosure } }
public async Task TestSupportResetting(string input) { using (var charReader = new AsyncLACharIterator(new StringReader(input), Capacity)) { if (charReader.SupportsResetting) { var isEmpty = true; while (await charReader.MoveNextAsync()) { isEmpty = false; } charReader.Reset(); Assert.That(charReader.MoveNextAsync(), Is.EqualTo(isEmpty), "Has element after resetting is not empty"); } else { Assert.That(charReader.Reset, Throws.InvalidOperationException); } } }
private static async Task MainAsync(string[] args, CancellationToken cancellationToken) { var grammar = new Grammar( Symbols, PG(Derivation.None, P(S.Ast, S.Syntax), PSL(S.Syntax, S.Rule), PSL(S.Syntax, S.Rule, S.Syntax), PR(S.Rule, (pLHS, pRHS) => Tuple.Create(pRHS[0].Content, pRHS[2]), S.RuleName, S.DefineOp, S.Clauses, S.RuleEnd) ), PG(Derivation.LeftMost, PSL(S.Clauses, S.Terms), PSL(S.Clauses, S.Terms, S.OrOp, S.Clauses), PL(S.Terms, MakeMetaTerm, S.Term), PL(S.Terms, MakeMetaTerm, S.Term, S.TermSep, S.Terms) ), PG(Derivation.None, P(S.Term, S.Literal), P(S.Term, S.RuleName), PR(S.Term, (pLHS, pRHS) => MakeMetaTerm(null)) ), PG(Derivation.LeftMost, PR(S.RuleName, MakeQuotedString, S.RuleLiteralLeft, S.RuleCharacters, S.RuleLiteralRight), PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter), PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter, S.RuleCharacters) ), PG(Derivation.LeftMost, PR(S.Literal, MakeQuotedString, S.DoubleQuote, S.TextInDoubleQuotes, S.DoubleQuote), PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes), PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes, S.TextInDoubleQuotes) ), PG(Derivation.LeftMost, PR(S.Literal, MakeQuotedString, S.SingleQuote, S.TextInSingleQuotes, S.SingleQuote), PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes), PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes, S.TextInSingleQuotes) ) ); // generate the parse table var parser = new Parser(grammar); var debugger = new Debug(parser, Console.Write, Console.Error.Write); var parseTime = System.Diagnostics.Stopwatch.StartNew(); Func <string, Regex> compileRegex = p => new Regex("^" + (p.StartsWith("[") || p.Length <= 2 ? p : (p.Substring(0, 2) + "(" + p.Substring(2) + "|$)")), RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture); Func <S, string, string, Tuple <int, Regex, string> > triple = (pSymbol, pPattern, pState) => Tuple.Create((int)pSymbol, compileRegex(pPattern), pState); Func <S, string, Tuple <int, Regex, string> > pair = (pSymbol, pPattern) => triple(pSymbol, pPattern, null); const string doubleQuoteState = "double"; const string singleQuoteState = "single"; const string doubleQuoteMarker = @"[""]"; const string singleQuoteMarker = "[']"; const string commonSet = "-A-Za-z0-9_"; const string symbolSet = @"| !#$%&()*+,./:;\\<=>?@[\]^`{|}~"; var lexerTable = new Dictionary <string, Tuple <int, Regex, string>[]> { { AsyncRegexLexer.RootState, new[] { triple(S.DoubleQuote, doubleQuoteMarker, doubleQuoteState), triple(S.SingleQuote, singleQuoteMarker, singleQuoteState), pair(S.RuleCharacter, "[" + commonSet + "]"), pair(S.DefineOp, "::="), pair(S.OrOp, "[|]"), pair(S.RuleLiteralLeft, "[<]"), pair(S.RuleLiteralRight, "[>]"), pair(S.RuleEnd, "[.]"), pair(S.TermSep, "[,]"), triple(S.EOL, @"[\r]?[\n]", AsyncRegexLexer.Ignore), triple(S.WS, @"[ \t]", AsyncRegexLexer.Ignore) } }, { doubleQuoteState, new[] { triple(S.DoubleQuote, doubleQuoteMarker, AsyncRegexLexer.PopState), pair(S.CharacterInDoubleQuotes, "[" + commonSet + symbolSet + "'" + "]") } }, { singleQuoteState, new[] { triple(S.SingleQuote, singleQuoteMarker, AsyncRegexLexer.PopState), pair(S.CharacterInSingleQuotes, "[" + commonSet + symbolSet + "\"" + "]") } } }; Item result; using (var charIterator = new AsyncLACharIterator(new StringReader(BNF))) using (var regexLexer = new AsyncRegexLexer(charIterator, lexerTable)) using (var tokenIterator = new AsyncLATokenIterator(regexLexer)) { result = await parser.ParseInputAsync(tokenIterator, debugger); } parseTime.Stop(); var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds); debugger.WriteFinalToken( string.Format("Accept ({0}): ", timeElapsed), string.Format("Error while parsing ({0}): ", timeElapsed), result); }