private static void WriteProductions(Grammar grammar, IndentedTextWriter output)
        {
            output.WriteLine("// Production rules:");
            output.WriteLine();

            foreach (var prod in grammar.Productions)
            {
                output.WriteLine("// {0}:", prod.Index);
                output.Write("{0} :", prod.Outcome.Name);
                if (prod.Pattern.Length == 0)
                {
                    output.Write(" /*empty*/");
                }
                else
                {
                    foreach (var symb in prod.Pattern)
                    {
                        output.Write(' ');
                        output.Write(symb.Name);
                    }
                }

                output.WriteLine(";");
                output.WriteLine();
            }
        }
        public void Test()
        {
            const int tokenCount = 50;
            const int ruleSize = 10;

            int[] tokens = new int[tokenCount];

            this.grammar = new Grammar();

            for (int i = 0; i != tokenCount; ++i)
            {
                tokens[i] = grammar.Symbols.Add(i.ToString()).Index;
            }

            int iterationCount = tokenCount - ruleSize;
            for (int i = 0; i < iterationCount; ++i)
            {
                int outcome = tokens[i];
                int[] pattern = new int[ruleSize];
                Array.Copy(tokens, i + 1, pattern, 0, ruleSize);
                grammar.Productions.Define(outcome, pattern);
            }

            var target = new Lalr1Dfa(new GrammarAnalysis(grammar), LrTableOptimizations.Default);
        }
        private void WriteMatchers(Grammar grammar, IndentedTextWriter output)
        {
            output.WriteLine("// Scanner Conditions: ");
            output.WriteLine();

            foreach (var condition in grammar.Conditions)
            {
                output.WriteLine("condition {0}", condition.Name);
                output.WriteLine("{");
                ++output.Indent;

                foreach (var matcher in condition.Matchers)
                {
                    var transition = matcher.NextCondition == null
                                   ? ""
                                   : string.Format("{{goto {0}}}", matcher.NextCondition.Name);
                    output.WriteLine(
                        "{0} {1}: /{2}/;",
                        Name(matcher.Outcome),
                        transition,
                        matcher.Pattern);
                }

                --output.Indent;
                output.WriteLine("}");
                output.WriteLine();
            }
        }
        public void IsNullableTest()
        {
            var grammar = new Grammar();

            var S = grammar.Symbols.Add("S");
            var a = grammar.Symbols.Add("a");
            var b = grammar.Symbols.Add("b");
            var A = grammar.Symbols.Add("A");
            var B = grammar.Symbols.Add("B");

            grammar.Start = S;
            grammar.Productions.Define(S, new[] { b, A });
            grammar.Productions.Define(A, new[] { a, A, B });
            grammar.Productions.Define(A, new Symbol[0]);
            grammar.Productions.Define(B, new Symbol[0]);

            var target = new RuntimeGrammar(grammar);

            Assert.IsTrue(target.IsNullable(A.Index));
            Assert.IsTrue(target.IsNullable(B.Index));

            Assert.IsFalse(target.IsNullable(a.Index));
            Assert.IsFalse(target.IsNullable(b.Index));
            Assert.IsFalse(target.IsNullable(S.Index));
        }
Пример #5
0
 public SppfGraphWriter(Grammar grammar, IGraphView graph, bool showRules)
 {
     this.grammar   = grammar;
     this.graph     = graph;
     this.visited   = new HashSet<SppfNode>();
     this.front     = new Stack<SppfNode>();
     this.showRules = showRules;
 }
 public static bool GrammarEquals(Grammar x, Grammar y)
 {
     return x == y
         || (x != null
         && y != null
         && Enumerable.SequenceEqual(y.Productions, x.Productions, EqualityComparer<Production>.Default)
         && Enumerable.SequenceEqual(y.Symbols, x.Symbols));
 }
Пример #7
0
 public RuntimeGrammar(Grammar grammar)
 {
     this.grammar = grammar;
     IRuntimeNullableFirstTables tables = new NullableFirstTables(grammar);
     this.isNullable  = tables.TokenToNullable;
     this.MaxRuleSize = tables.MaxRuleSize;
     this.symbolCount = grammar.Symbols.Count;
 }
 public void DefineAmbiguities(Grammar grammar)
 {
     foreach (var prod in stateToTokenProducer.Values)
     {
         var ambSymbol = grammar.Symbols.FindOrAddAmbiguous(prod.MainTokenId, prod.PossibleTokens);
         prod.State.EnvelopeId = ambSymbol.Index;
     #if DEBUG
         Debug.WriteLine("Created ambiguous symbol {0} for state #{1}", ambSymbol, prod.State.Index);
     #endif
     }
 }
        public NullableFirstTables(Grammar grammar)
        {
            this.grammar    = grammar;
            int count       = grammar.Symbols.Count;
            this.tokenSet   = new BitSetType(count);
            this.firsts     = new MutableIntSet[count];
            this.isNullable = new bool[count];

            MaxRuleSize = grammar.Productions.Select(r => r.PatternTokens.Length).Max();
            Build();
        }
        public void Test()
        {
            var lParen = new Symbol("(");
            var rParen = new Symbol(")");
            var num    = new Symbol("NUM");
            var ident  = new Symbol("ID");
            var qStr   = new Symbol("QSTR");

            var grammar = new Grammar
            {
                Symbols =
                {
                    lParen,
                    rParen,
                    num,
                    ident,
                    qStr
                },

                Conditions =
                {
                    new Condition("main")
                    {
                        Matchers =
                        {
                            new Matcher(@"blank+"),
                            new Matcher(@"digit+ ('.' digit+)?  | '.' digit+", num),
                            new Matcher(
                                    @"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*",
                                    ident),
                            new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr),
                            new Matcher(ScanPattern.CreateLiteral("("), lParen),
                            new Matcher(ScanPattern.CreateLiteral(")"), rParen),
                        }
                    }
                }
            };

            var target = new DfaSimulationLexer(
                " (1 (\"bar\" +))",
                ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));

            var collector = new Collector<Msg>();
            target.Accept(collector);
            Assert.AreEqual(
                new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index },
                collector.Select(msg => msg.Id).ToArray());
            Assert.AreEqual(
                new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" },
                collector.Select(msg => msg.Value).ToArray());
        }
 private static void WriteSummaryComment(Grammar grammar, IndentedTextWriter output)
 {
     output.WriteLine("/*");
     ++output.Indent;
     output.WriteLine("symbols       : {0}", grammar.Symbols.Count);
     output.WriteLine("terminals     : {0}", grammar.Symbols.Where(s => s.IsTerminal).Count());
     output.WriteLine("non-terminals : {0}", grammar.Symbols.Where(s => !s.IsTerminal).Count());
     output.WriteLine("productions   : {0}", grammar.Productions.Count);
     output.WriteLine("mergers       : {0}", grammar.Mergers.Count);
     output.WriteLine("matchers      : {0}", grammar.Matchers.Count);
     --output.Indent;
     output.WriteLine("*/");
     output.WriteLine();
 }
        public void Test()
        {
            var originalGrammar = new Grammar();
            var red   = originalGrammar.Symbols.Add("red");
            var green = originalGrammar.Symbols.Add("green", SymbolCategory.ExplicitlyUsed);
            var blue  = originalGrammar.Symbols.Add("blue");
            originalGrammar.Productions.Define(red,  new[] { green, blue });
            originalGrammar.Productions.Define(blue, new[] { red, green });
            originalGrammar.Productions.Define(blue, new Symbol[0]);

            originalGrammar.Start = red;

            GrammarSerializer target = new GrammarSerializer(originalGrammar);
            var factory = new CachedMethod<Func<Grammar>>("GrammarSerializerTest.Assembly0", (emit, args) => { target.Build(emit); return emit.Ret(); }).Delegate;

            var recreated = factory();

            Assert.IsTrue(GrammarEquals(originalGrammar, recreated));
        }
Пример #13
0
 public LrGraph(IReportData data)
 {
     this.grammar  = data.Grammar;
     this.automata = data.ParserAutomata;
 }
 private void WhenGrammarIsInlined()
 {
     IProductionInliner target = new ProductionInliner(originalGrammar);
     this.resultGrammar = target.Inline();
 }
        public void SetUp()
        {
            this.originalGrammar = new Grammar();
            var symbols = originalGrammar.Symbols;
            this.start  = symbols.Add("start");
            this.prefix = symbols.Add("prefix");
            this.suffix = symbols.Add("suffix");
            this.term1  = symbols.Add("term1");
            this.term2  = symbols.Add("term2");
            this.term3  = symbols.Add("term3");
            this.term4  = symbols.Add("term4");
            this.term5  = symbols.Add("term5");
            this.inlinedNonTerm = symbols.Add("inlinedNonTerm");
            this.nestedNonTerm = symbols.Add("nestedNonTerm");

            originalGrammar.Start = start;

            var prod = originalGrammar.Productions.Define(start,  new[] { prefix, inlinedNonTerm, suffix });
            prod.Actions.Add(new ForeignAction(0, 3));
        }
Пример #16
0
 public GrammarAnalysis(Grammar grammar)
 {
     this.grammar = grammar;
     this.tables = new NullableFirstTables(grammar);
 }
Пример #17
0
        public void Test()
        {
            var lParen = new Symbol("(");
            var rParen = new Symbol(")");
            var num    = new Symbol("NUM");
            var ident  = new Symbol("ID");
            var qStr   = new Symbol("QSTR");

            var grammar = new Grammar
            {
                Symbols = {
                    lParen,
                    rParen,
                    num,
                    ident,
                    qStr
                },

                Conditions =
                {
                    new Condition("main")
                    {
                        Matchers =
                        {
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[ \t]+"))
                            {
                                Joint = { new CilMatcher(typeof(void)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"),
                                    num)
                            {
                                Joint = {  new CilMatcher(typeof(Num)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"),
                                    ident)
                            {
                                Joint = { new CilMatcher(typeof(string)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[""](?: \\[""] | [^""])* [""]"),
                                    qStr)
                            {
                                Joint = { new CilMatcher(typeof(QStr)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateLiteral("("),
                                    lParen),
                            new Matcher(
                                    ScanPattern.CreateLiteral(")"),
                                    rParen),
                        }
                    }
                }
            };

            var target = new BootstrapScanner(
                " (1 (\"bar\" +))",
                ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance),
                null,
                ExceptionLogging.Instance);

            var collector = new Collector<Msg>();
            target.Accept(collector);
            Assert.AreEqual(
                new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null },
                collector.Select(msg => msg.Value).ToArray());
        }
Пример #18
0
        private static SymbolBase GetMatcherOutcomeSymbol(
            Grammar                   grammar,
            ICilSymbolResolver        symbolResolver,
            CilSymbolRef              mainOutcome,
            IEnumerable<CilSymbolRef> allOutcomes)
        {
            Symbol main = symbolResolver.GetSymbol(mainOutcome);
            Symbol[] all = (from outcome in allOutcomes
                             select symbolResolver.GetSymbol(outcome))
                             .ToArray();

            switch (all.Length)
            {
                case 0:  return null;
                case 1:  return main;
                default: return new AmbiguousSymbol(main, all);
            }
        }
Пример #19
0
        private static Condition ConditionFromType(Grammar grammar, Type type)
        {
            if (type == null)
            {
                return null;
            }

            foreach (var cond in grammar.Conditions)
            {
                var binding = cond.Joint.The<CilCondition>();
                if (binding.ConditionType == type)
                {
                    return cond;
                }
            }

            throw new InvalidOperationException("Undefined condition: " + type.FullName);
        }
 public TokenIdentitiesSerializer(Grammar grammar)
 {
     this.grammar = grammar;
 }
 public void Write(Grammar grammar, TextWriter output)
 {
     WriteIndented(grammar, new IndentedTextWriter(output, Indent));
 }
Пример #22
0
        private static void InitContextProvider(
            Grammar            grammar,
            CilContextProvider cilProvider,
            ForeignContextProvider    provider)
        {
            provider.Joint.Add(cilProvider);

            foreach (var cilContext in cilProvider.Contexts)
            {
                ForeignContext context;
                if (grammar.Contexts.FindOrAdd(cilContext.UniqueName, out context))
                {
                    context.Joint.Add(cilContext);
                }

                provider.Add(context);
            }
        }
        public void Test()
        {
            var num    = new Symbol("NUM");
            var ident  = new Symbol("ID");
            var qStr   = new Symbol("QSTR");
            var begin  = new Symbol("begin");
            var end    = new Symbol("end");
            var assign = new Symbol(":=");

            var grammar = new Grammar
            {
                Symbols =
                {
                    num,
                    ident,
                    qStr,
                    begin,
                    end,
                    assign
                },

                Conditions =
                {
                    new Condition("main")
                    {
                        Matchers =
                        {
                            new Matcher(
                                    @"digit+ ('.' digit+)?  | '.' digit+",
                                    num),
                            new Matcher(
                                    @"[01234567]+ 'Q'",
                                    num),
                            new Matcher(
                                    @"alpha alnum*",
                                    ident),
                            new Matcher(
                                    @"
                                    quot
                                        ~(quot | esc)*
                                        (esc . ~(quot | esc)* )*
                                    quot
                                    ",
                                    qStr),
                            new Matcher(
                                    ScanPattern.CreateLiteral("begin"),
                                    begin),
                            new Matcher(
                                    ScanPattern.CreateLiteral("end"),
                                    end),
                            new Matcher(
                                    ScanPattern.CreateLiteral(":="),
                                    assign),
                            new Matcher(
                                    @"blank+"),
                        }
                    }
                }
            };

            var target = new TdfaSimulationLexer(
                "b:=10Q \"foo\"",
                ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));

            var collector = new Collector<Msg>();
            target.Accept(collector);

            Assert.AreEqual(
                new int[] { ident.Index, assign.Index, num.Index, qStr.Index },
                collector.Select(msg => msg.Id).ToArray());
            Assert.AreEqual(
                new object[] { "b", ":=", "10Q", "\"foo\"" },
                collector.Select(msg => msg.Value).ToArray());
        }
Пример #24
0
        private static Grammar BuildGrammar(CilGrammar definition)
        {
            var result = new Grammar();

            InitContextProvider(
                result,
                definition.GlobalContextProvider,
                result.GlobalContextProvider);

            var symbolResolver = definition.SymbolResolver;

            // Define grammar tokens
            foreach (var cilSymbol in symbolResolver.Definitions)
            {
                Symbol symbol;
                if (cilSymbol.Type == typeof(Exception))
                {
                    cilSymbol.Symbol = symbol = (Symbol)result.Symbols[PredefinedTokens.Error];
                    symbol.Joint.Add(
                        new CilSymbol
                        {
                            Type       = typeof(Exception),
                            Symbol     = symbol,
                            Categories = SymbolCategory.DoNotDelete | SymbolCategory.DoNotInsert
                        });
                }
                else
                {
                    symbol = new Symbol(cilSymbol.Name)
                    {
                        Categories = cilSymbol.Categories,
                        Joint = { cilSymbol }
                    };
                    result.Symbols.Add(symbol);
                    cilSymbol.Symbol = symbol;
                }
            }

            foreach (CilSymbolFeature<Precedence> feature in definition.Precedence)
            {
                var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
                symbol.Precedence = feature.Value;
            }

            foreach (CilSymbolFeature<CilContextProvider> feature in definition.LocalContextProviders)
            {
                var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
                if (symbol != null)
                {
                    InitContextProvider(result, feature.Value, symbol.LocalContextProvider);
                }
            }

            result.Start = symbolResolver.GetSymbol(definition.Start);

            // Define grammar rules
            foreach (var cilProduction in definition.Productions)
            {
                Symbol outcome = symbolResolver.GetSymbol(cilProduction.Outcome);
                var pattern = Array.ConvertAll(cilProduction.Pattern, symbolResolver.GetSymbol);

                // Try to find existing rules whith same token-signature
                Production production;
                if (result.Productions.FindOrAdd(outcome, pattern, out production))
                {
                    ForeignContextRef contextRef = CreateActionContextRef(cilProduction.Context);
                    production.Actions.Add(new ForeignAction(pattern.Length, contextRef));
                }

                var action = production.Actions[0];
                action.Joint.Add(cilProduction);

                production.ExplicitPrecedence = cilProduction.Precedence;
            }

            // Create conditions to allow referencing them from matchers
            foreach (CilCondition cilCondition in definition.Conditions)
            {
                var cond = CreateCondtion(result, cilCondition);
                result.Conditions.Add(cond);
            }

            // Create matchers
            foreach (CilCondition cilCondition in definition.Conditions)
            {
                var condition = ConditionFromType(result, cilCondition.ConditionType);

                foreach (var cilMatcher in cilCondition.Matchers)
                {
                    SymbolBase outcome = GetMatcherOutcomeSymbol(result, symbolResolver, cilMatcher.MainOutcome, cilMatcher.AllOutcomes);

                    var matcher = new Matcher(
                        cilMatcher.Pattern,
                        outcome,
            #if false
                        context: CreateActionContextRef(cilMatcher.Context),
            #endif
                        nextCondition: ConditionFromType(result, cilMatcher.NextConditionType),
                        disambiguation: cilMatcher.Disambiguation);
                    matcher.Joint.Add(cilMatcher);

                    condition.Matchers.Add(matcher);
                }
            }

            foreach (var cilMerger in definition.Mergers)
            {
                var symbol  = symbolResolver.GetSymbol(cilMerger.Symbol);
                var merger = new Merger(symbol) { Joint = { cilMerger } };
                result.Mergers.Add(merger);
            }

            foreach (var report in definition.Reports)
            {
                result.Reports.Add(report);
            }

            return result;
        }
Пример #25
0
        private static Condition CreateCondtion(Grammar grammar, CilCondition cilCondition)
        {
            var result = new Condition(cilCondition.ConditionType.FullName)
            {
                Joint = { cilCondition }
            };

            InitContextProvider(grammar, cilCondition.ContextProvider, result.ContextProvider);
            return result;
        }
 private void WriteIndented(Grammar grammar, IndentedTextWriter output)
 {
     WriteSummaryComment(grammar, output);
     WriteProductions(grammar, output);
     WriteMatchers(grammar, output);
 }