示例#1
0
        public async Task TestNormalisationAsync(string input)
        {
            // var expectedCount = input.Count(p => !char.IsSurrogate(p) && p != '\r');
            using (var charReader = new AsyncLACharIterator(new StringReader(input), Capacity))
            {
                var i = 0;
                while (i < input.Length)
                {
                    int expectedCurrent;
                    int expectedLA;
                    input.CurrentAndLA(ref i, out expectedCurrent, out expectedLA);
                    Assert.That(await charReader.MoveNextAsync(),
                                Is.EqualTo(expectedCurrent != AsyncLACharIterator.EOF),
                                "Move next i={0}", i);

                    var current = await charReader.CurrentAsync();

                    Assert.That(current, Is.EqualTo(expectedCurrent), "Current codepoint i={0} expected={1} is={2}",
                                i, expectedCurrent.DisplayUTF8(), current.DisplayUTF8());

                    var la = await charReader.LookAheadAsync();

                    Assert.That(la, Is.EqualTo(expectedLA), "Lookahead codepoint i={0} expected={1} is={2}",
                                i, expectedLA.DisplayUTF8(), la.DisplayUTF8());
                }
            }
        }
        public async Task TestNormalisationAsync(string input)
        {
            // var expectedCount = input.Count(p => !char.IsSurrogate(p) && p != '\r');
            using (var charReader = new AsyncLACharIterator(new StringReader(input), Capacity))
            {
                var i = 0;
                while (i < input.Length)
                {
                    int expectedCurrent;
                    int expectedLA;
                    input.CurrentAndLA(ref i, out expectedCurrent, out expectedLA);
                    Assert.That(await charReader.MoveNextAsync(),
                        Is.EqualTo(expectedCurrent != AsyncLACharIterator.EOF),
                        "Move next i={0}", i);

                    var current = await charReader.CurrentAsync();
                    Assert.That(current, Is.EqualTo(expectedCurrent), "Current codepoint i={0} expected={1} is={2}",
                        i, expectedCurrent.DisplayUTF8(),  current.DisplayUTF8());

                    var la = await charReader.LookAheadAsync();

                    Assert.That(la, Is.EqualTo(expectedLA), "Lookahead codepoint i={0} expected={1} is={2}",
                        i, expectedLA.DisplayUTF8(), la.DisplayUTF8());
                }
            }
        }
示例#3
0
        public void TestUninitialised()
        {
            using (var charReader = new AsyncLACharIterator(new StringReader(""), Capacity))
            {
// ReSharper disable AccessToDisposedClosure
                Assert.That(async() => await charReader.CurrentAsync(), Throws.InvalidOperationException);
// ReSharper restore AccessToDisposedClosure
            }
        }
        public void TestUninitialised()
        {
            using (var charReader = new AsyncLACharIterator(new StringReader(""), Capacity))
            {
// ReSharper disable AccessToDisposedClosure
                Assert.That(async () => await charReader.CurrentAsync(), Throws.InvalidOperationException);
// ReSharper restore AccessToDisposedClosure
            }
        }
示例#5
0
 public async Task TestSupportResetting(string input)
 {
     using (var charReader = new AsyncLACharIterator(new StringReader(input), Capacity))
     {
         if (charReader.SupportsResetting)
         {
             var isEmpty = true;
             while (await charReader.MoveNextAsync())
             {
                 isEmpty = false;
             }
             charReader.Reset();
             Assert.That(charReader.MoveNextAsync(), Is.EqualTo(isEmpty), "Has element after resetting is not empty");
         }
         else
         {
             Assert.That(charReader.Reset, Throws.InvalidOperationException);
         }
     }
 }
 public async Task TestSupportResetting(string input)
 {
     using (var charReader = new AsyncLACharIterator(new StringReader(input), Capacity))
     {
         if (charReader.SupportsResetting)
         {
             var isEmpty = true;
             while (await charReader.MoveNextAsync())
             {
                 isEmpty = false;
             }
             charReader.Reset();
             Assert.That(charReader.MoveNextAsync(), Is.EqualTo(isEmpty), "Has element after resetting is not empty");
         }
         else
         {
             Assert.That(charReader.Reset, Throws.InvalidOperationException);
         }
     }
 }
示例#7
0
        private static async Task MainAsync(string[] args, CancellationToken cancellationToken)
        {
            var grammar = new Grammar(
                Symbols,
                PG(Derivation.None,
                   P(S.Ast, S.Syntax),

                   PSL(S.Syntax, S.Rule),
                   PSL(S.Syntax, S.Rule, S.Syntax),

                   PR(S.Rule, (pLHS, pRHS) => Tuple.Create(pRHS[0].Content, pRHS[2]), S.RuleName, S.DefineOp, S.Clauses, S.RuleEnd)
                   ),
                PG(Derivation.LeftMost,
                   PSL(S.Clauses, S.Terms),
                   PSL(S.Clauses, S.Terms, S.OrOp, S.Clauses),

                   PL(S.Terms, MakeMetaTerm, S.Term),
                   PL(S.Terms, MakeMetaTerm, S.Term, S.TermSep, S.Terms)
                   ),
                PG(Derivation.None,
                   P(S.Term, S.Literal),
                   P(S.Term, S.RuleName),
                   PR(S.Term, (pLHS, pRHS) => MakeMetaTerm(null))
                   ),
                PG(Derivation.LeftMost,
                   PR(S.RuleName, MakeQuotedString, S.RuleLiteralLeft, S.RuleCharacters, S.RuleLiteralRight),
                   PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter),
                   PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter, S.RuleCharacters)
                   ),
                PG(Derivation.LeftMost,
                   PR(S.Literal, MakeQuotedString, S.DoubleQuote, S.TextInDoubleQuotes, S.DoubleQuote),
                   PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes),
                   PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes, S.TextInDoubleQuotes)
                   ),
                PG(Derivation.LeftMost,
                   PR(S.Literal, MakeQuotedString, S.SingleQuote, S.TextInSingleQuotes, S.SingleQuote),
                   PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes),
                   PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes, S.TextInSingleQuotes)
                   )
                );

            // generate the parse table
            var parser    = new Parser(grammar);
            var debugger  = new Debug(parser, Console.Write, Console.Error.Write);
            var parseTime = System.Diagnostics.Stopwatch.StartNew();

            Func <string, Regex> compileRegex =
                p => new Regex("^" + (p.StartsWith("[") || p.Length <= 2
                                          ? p
                                          : (p.Substring(0, 2) + "(" + p.Substring(2) + "|$)")),
                               RegexOptions.CultureInvariant |
                               RegexOptions.ExplicitCapture);

            Func <S, string, string, Tuple <int, Regex, string> > triple =
                (pSymbol, pPattern, pState) => Tuple.Create((int)pSymbol, compileRegex(pPattern), pState);

            Func <S, string, Tuple <int, Regex, string> > pair =
                (pSymbol, pPattern) => triple(pSymbol, pPattern, null);

            const string doubleQuoteState = "double";
            const string singleQuoteState = "single";

            const string doubleQuoteMarker = @"[""]";
            const string singleQuoteMarker = "[']";
            const string commonSet         = "-A-Za-z0-9_";
            const string symbolSet         = @"| !#$%&()*+,./:;\\<=>?@[\]^`{|}~";

            var lexerTable =
                new Dictionary <string, Tuple <int, Regex, string>[]>
            {
                {
                    AsyncRegexLexer.RootState, new[]
                    {
                        triple(S.DoubleQuote, doubleQuoteMarker, doubleQuoteState),
                        triple(S.SingleQuote, singleQuoteMarker, singleQuoteState),
                        pair(S.RuleCharacter, "[" + commonSet + "]"),
                        pair(S.DefineOp, "::="),
                        pair(S.OrOp, "[|]"),
                        pair(S.RuleLiteralLeft, "[<]"),
                        pair(S.RuleLiteralRight, "[>]"),
                        pair(S.RuleEnd, "[.]"),
                        pair(S.TermSep, "[,]"),
                        triple(S.EOL, @"[\r]?[\n]", AsyncRegexLexer.Ignore),
                        triple(S.WS, @"[ \t]", AsyncRegexLexer.Ignore)
                    }
                },
                {
                    doubleQuoteState, new[]
                    {
                        triple(S.DoubleQuote, doubleQuoteMarker, AsyncRegexLexer.PopState),
                        pair(S.CharacterInDoubleQuotes, "[" + commonSet + symbolSet + "'" + "]")
                    }
                },
                {
                    singleQuoteState, new[]
                    {
                        triple(S.SingleQuote, singleQuoteMarker, AsyncRegexLexer.PopState),
                        pair(S.CharacterInSingleQuotes, "[" + commonSet + symbolSet + "\"" + "]")
                    }
                }
            };
            Item result;

            using (var charIterator = new AsyncLACharIterator(new StringReader(BNF)))
                using (var regexLexer = new AsyncRegexLexer(charIterator, lexerTable))
                    using (var tokenIterator = new AsyncLATokenIterator(regexLexer))
                    {
                        result = await parser.ParseInputAsync(tokenIterator, debugger);
                    }
            parseTime.Stop();
            var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds);

            debugger.WriteFinalToken(
                string.Format("Accept ({0}): ", timeElapsed),
                string.Format("Error while parsing ({0}): ", timeElapsed),
                result);
        }