public void Test()
        {
            var tokens = new []
            {
                CilSymbolRef.Create("foo"),
                CilSymbolRef.Create("bar"),
                CilSymbolRef.Create("other"),
                CilSymbolRef.Create(typeof(string)),
                CilSymbolRef.Create(typeof(string), "foo"),   // links foo and string
                CilSymbolRef.Create(typeof(string), "bar"),   // links bar and string
                CilSymbolRef.Create(typeof(string), "other"), // links other and string
            };

            foreach (var token in tokens)
            {
                target.Link(token);
            }

            Assert.AreEqual(1, target.Definitions.Count());

            var SYM1 = new Symbol("123");
            target.Resolve(tokens[0]).Symbol = SYM1;
            for (int i = 1; i != tokens.Length; ++i)
            {
                var context = "#" + i;
                Assert.AreSame(SYM1, target.GetSymbol(tokens[i]), context);
            }
        }
        public void Test()
        {
            var lParen = new Symbol("(");
            var rParen = new Symbol(")");
            var num    = new Symbol("NUM");
            var ident  = new Symbol("ID");
            var qStr   = new Symbol("QSTR");

            var grammar = new Grammar
            {
                Symbols =
                {
                    lParen,
                    rParen,
                    num,
                    ident,
                    qStr
                },

                Conditions =
                {
                    new Condition("main")
                    {
                        Matchers =
                        {
                            new Matcher(@"blank+"),
                            new Matcher(@"digit+ ('.' digit+)?  | '.' digit+", num),
                            new Matcher(
                                    @"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*",
                                    ident),
                            new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr),
                            new Matcher(ScanPattern.CreateLiteral("("), lParen),
                            new Matcher(ScanPattern.CreateLiteral(")"), rParen),
                        }
                    }
                }
            };

            var target = new DfaSimulationLexer(
                " (1 (\"bar\" +))",
                ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));

            var collector = new Collector<Msg>();
            target.Accept(collector);
            Assert.AreEqual(
                new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index },
                collector.Select(msg => msg.Id).ToArray());
            Assert.AreEqual(
                new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" },
                collector.Select(msg => msg.Value).ToArray());
        }
        public void Test()
        {
            var num    = new Symbol("NUM");
            var ident  = new Symbol("ID");
            var qStr   = new Symbol("QSTR");
            var begin  = new Symbol("begin");
            var end    = new Symbol("end");
            var assign = new Symbol(":=");

            var grammar = new Grammar
            {
                Symbols =
                {
                    num,
                    ident,
                    qStr,
                    begin,
                    end,
                    assign
                },

                Conditions =
                {
                    new Condition("main")
                    {
                        Matchers =
                        {
                            new Matcher(
                                    @"digit+ ('.' digit+)?  | '.' digit+",
                                    num),
                            new Matcher(
                                    @"[01234567]+ 'Q'",
                                    num),
                            new Matcher(
                                    @"alpha alnum*",
                                    ident),
                            new Matcher(
                                    @"
                                    quot
                                        ~(quot | esc)*
                                        (esc . ~(quot | esc)* )*
                                    quot
                                    ",
                                    qStr),
                            new Matcher(
                                    ScanPattern.CreateLiteral("begin"),
                                    begin),
                            new Matcher(
                                    ScanPattern.CreateLiteral("end"),
                                    end),
                            new Matcher(
                                    ScanPattern.CreateLiteral(":="),
                                    assign),
                            new Matcher(
                                    @"blank+"),
                        }
                    }
                }
            };

            var target = new TdfaSimulationLexer(
                "b:=10Q \"foo\"",
                ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));

            var collector = new Collector<Msg>();
            target.Accept(collector);

            Assert.AreEqual(
                new int[] { ident.Index, assign.Index, num.Index, qStr.Index },
                collector.Select(msg => msg.Id).ToArray());
            Assert.AreEqual(
                new object[] { "b", ":=", "10Q", "\"foo\"" },
                collector.Select(msg => msg.Value).ToArray());
        }
        public void SetUp()
        {
            this.originalGrammar = new Grammar();
            var symbols = originalGrammar.Symbols;
            this.start  = symbols.Add("start");
            this.prefix = symbols.Add("prefix");
            this.suffix = symbols.Add("suffix");
            this.term1  = symbols.Add("term1");
            this.term2  = symbols.Add("term2");
            this.term3  = symbols.Add("term3");
            this.term4  = symbols.Add("term4");
            this.term5  = symbols.Add("term5");
            this.inlinedNonTerm = symbols.Add("inlinedNonTerm");
            this.nestedNonTerm = symbols.Add("nestedNonTerm");

            originalGrammar.Start = start;

            var prod = originalGrammar.Productions.Define(start,  new[] { prefix, inlinedNonTerm, suffix });
            prod.Actions.Add(new ForeignAction(0, 3));
        }
        private static Grammar BuildGrammar(CilGrammar definition)
        {
            var result = new Grammar();

            InitContextProvider(
                result,
                definition.GlobalContextProvider,
                result.GlobalContextProvider);

            var symbolResolver = definition.SymbolResolver;

            // Define grammar tokens
            foreach (var cilSymbol in symbolResolver.Definitions)
            {
                Symbol symbol;
                if (cilSymbol.Type == typeof(Exception))
                {
                    cilSymbol.Symbol = symbol = (Symbol)result.Symbols[PredefinedTokens.Error];
                    symbol.Joint.Add(
                        new CilSymbol
                        {
                            Type       = typeof(Exception),
                            Symbol     = symbol,
                            Categories = SymbolCategory.DoNotDelete | SymbolCategory.DoNotInsert
                        });
                }
                else
                {
                    symbol = new Symbol(cilSymbol.Name)
                    {
                        Categories = cilSymbol.Categories,
                        Joint = { cilSymbol }
                    };
                    result.Symbols.Add(symbol);
                    cilSymbol.Symbol = symbol;
                }
            }

            foreach (CilSymbolFeature<Precedence> feature in definition.Precedence)
            {
                var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
                symbol.Precedence = feature.Value;
            }

            foreach (CilSymbolFeature<CilContextProvider> feature in definition.LocalContextProviders)
            {
                var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
                if (symbol != null)
                {
                    InitContextProvider(result, feature.Value, symbol.LocalContextProvider);
                }
            }

            result.Start = symbolResolver.GetSymbol(definition.Start);

            // Define grammar rules
            foreach (var cilProduction in definition.Productions)
            {
                Symbol outcome = symbolResolver.GetSymbol(cilProduction.Outcome);
                var pattern = Array.ConvertAll(cilProduction.Pattern, symbolResolver.GetSymbol);

                // Try to find existing rules whith same token-signature
                Production production;
                if (result.Productions.FindOrAdd(outcome, pattern, out production))
                {
                    ForeignContextRef contextRef = CreateActionContextRef(cilProduction.Context);
                    production.Actions.Add(new ForeignAction(pattern.Length, contextRef));
                }

                var action = production.Actions[0];
                action.Joint.Add(cilProduction);

                production.ExplicitPrecedence = cilProduction.Precedence;
            }

            // Create conditions to allow referencing them from matchers
            foreach (CilCondition cilCondition in definition.Conditions)
            {
                var cond = CreateCondtion(result, cilCondition);
                result.Conditions.Add(cond);
            }

            // Create matchers
            foreach (CilCondition cilCondition in definition.Conditions)
            {
                var condition = ConditionFromType(result, cilCondition.ConditionType);

                foreach (var cilMatcher in cilCondition.Matchers)
                {
                    SymbolBase outcome = GetMatcherOutcomeSymbol(result, symbolResolver, cilMatcher.MainOutcome, cilMatcher.AllOutcomes);

                    var matcher = new Matcher(
                        cilMatcher.Pattern,
                        outcome,
            #if false
                        context: CreateActionContextRef(cilMatcher.Context),
            #endif
                        nextCondition: ConditionFromType(result, cilMatcher.NextConditionType),
                        disambiguation: cilMatcher.Disambiguation);
                    matcher.Joint.Add(cilMatcher);

                    condition.Matchers.Add(matcher);
                }
            }

            foreach (var cilMerger in definition.Mergers)
            {
                var symbol  = symbolResolver.GetSymbol(cilMerger.Symbol);
                var merger = new Merger(symbol) { Joint = { cilMerger } };
                result.Mergers.Add(merger);
            }

            foreach (var report in definition.Reports)
            {
                result.Reports.Add(report);
            }

            return result;
        }
 public AmbiguousSymbol(Symbol main, Symbol[] symbols)
 {
     this.MainToken = main == null ? -1 : main.Index;
     this.Tokens    = new ReadOnlyCollection<int>((from s in symbols select s.Index).ToArray());
 }
        public void Test()
        {
            var lParen = new Symbol("(");
            var rParen = new Symbol(")");
            var num    = new Symbol("NUM");
            var ident  = new Symbol("ID");
            var qStr   = new Symbol("QSTR");

            var grammar = new Grammar
            {
                Symbols = {
                    lParen,
                    rParen,
                    num,
                    ident,
                    qStr
                },

                Conditions =
                {
                    new Condition("main")
                    {
                        Matchers =
                        {
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[ \t]+"))
                            {
                                Joint = { new CilMatcher(typeof(void)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"),
                                    num)
                            {
                                Joint = {  new CilMatcher(typeof(Num)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"),
                                    ident)
                            {
                                Joint = { new CilMatcher(typeof(string)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateRegular(
                                        null,
                                        @"[""](?: \\[""] | [^""])* [""]"),
                                    qStr)
                            {
                                Joint = { new CilMatcher(typeof(QStr)) }
                            },
                            new Matcher(
                                    ScanPattern.CreateLiteral("("),
                                    lParen),
                            new Matcher(
                                    ScanPattern.CreateLiteral(")"),
                                    rParen),
                        }
                    }
                }
            };

            var target = new BootstrapScanner(
                " (1 (\"bar\" +))",
                ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance),
                null,
                ExceptionLogging.Instance);

            var collector = new Collector<Msg>();
            target.Accept(collector);
            Assert.AreEqual(
                new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null },
                collector.Select(msg => msg.Value).ToArray());
        }