예제 #1
0
                public override void Visit(Alts alts)
                {
                    bool stop = true;

                    _reachedInnerAlts = true;
                    if (alts.Mode == LoopMode.None)
                    {
                        stop = false;
                        int startIndex = _index;
                        int length     = -1;

                        foreach (var p in alts.ArmsAndCustomErrorBranch)
                        {
                            _index = startIndex;
                            p.Call(this);

                            int newLen = _index - startIndex;
                            if (length == -1)
                            {
                                length = newLen;
                            }
                            else if (length != newLen)
                            {
                                stop = true;
                            }
                        }
                    }
                    // stop prematching after a variable-length Alts (including any loop)
                    // ...or after a default error branch (we don't know what it consumes)
                    if (stop || alts.HasDefaultErrorBranch(LLPG))
                    {
                        _index = int.MaxValue;
                    }
                }
예제 #2
0
파일: Predicates.cs 프로젝트: dadhi/ecsharp
        public override Pred Clone()
        {
            Alts clone = (Alts)base.Clone();

            clone.Arms = new List <Pred>(Arms.Select(arm => arm.Clone()));
            return(clone);
        }
예제 #3
0
파일: Predicates.cs 프로젝트: dadhi/ecsharp
        private void UpdateSlashDivs(bool slashJoined, int boundary, bool append, Alts bAlts)
        {
            //Debug.Assert(boundary > 0 && boundary < Arms.Count); //not true when joining with error branch
            Division prev  = _divisions.LastOrDefault();
            var      bDivs = bAlts != null ? bAlts._divisions : InternalList <Division> .Empty;

            if (append)
            {
                bDivs = Adjust(bDivs, boundary, true);
                _divisions.AddRange(bDivs);
            }
            else
            {
                prev = bAlts != null?bAlts._divisions.LastOrDefault() : default(Division);

                Adjust(_divisions, boundary, false);
                _divisions.InsertRange(0, bDivs);
            }

            var newDiv = new Division {
                Left = 0, Mid = (short)boundary, Right = (short)Arms.Count, Slash = slashJoined
            };

            _divisions.Add(newDiv);
        }
예제 #4
0
 public virtual void ReplaceChildrenOf(Alts pred, bool includeError)
 {
     VisitAndReplace(pred.Arms);
     if (includeError && pred.ErrorBranch != null && pred.ErrorBranch != DefaultErrorBranch.Value)
     {
         VisitAndReplace(ref pred.ErrorBranch);
     }
 }
예제 #5
0
            /// <summary>
            /// Visit(Alts) is the most important method in this class. It generates
            /// all prediction code, which is the majority of the code in a parser.
            /// </summary>
            public override void Visit(Alts alts)
            {
                PredictionTree tree      = alts.PredictionTree;
                var            timesUsed = new Dictionary <int, int>();

                tree.CountTimesUsed(timesUsed);

                GenerateCodeForAlts(alts, timesUsed, tree);
            }
예제 #6
0
 public void VisitChildrenOf(Alts pred, bool includeError)
 {
     foreach (var p in pred.Arms)
     {
         p.Call(this);
     }
     if (includeError && pred.ErrorBranch != null && pred.ErrorBranch != DefaultErrorBranch.Value)
     {
         pred.ErrorBranch.Call(this);
     }
 }
예제 #7
0
            public override void Visit(Alts alts)
            {
                _currentAlts = alts;
                KthSet[] firstSets = LLPG.ComputeFirstSets(alts);

                if (LLPG.Verbosity > 0)
                {
                    var sb = new StringBuilder();
                    for (int i = 0; i < firstSets.Length; i++)
                    {
                        if (firstSets[i].Alt == -1 || LLPG.Verbosity > 2)
                        {
                            if (sb.Length != 0)
                            {
                                sb.Append('\n');
                            }
                            sb.AppendFormat("First set for {0}: {1}",
                                            firstSets[i].Alt == -1 ? "exit" : "alt #" + (firstSets[i].Alt + 1), firstSets[i]);
                        }
                    }
                    if (sb.Length != 0)
                    {
                        LLPG.Output(Verbose, alts, sb.ToString());
                    }
                }

                try {
                    EzStopwatch timer = new EzStopwatch(true);
                    alts.PredictionTree = ComputePredictionTree(firstSets);
                    if (timer.Millisec > 500)
                    {
                        LLPG.Output(Warning, alts, "Slug? This took a long time to analyze: " + timer.Millisec + "ms");
                    }
                } catch (System.Threading.ThreadAbortException) {
                    LLPG.Output(Error, alts, "ThreadAbortException in rule '" + _currentRule.Name + "'");                     // user diagnostic
                    throw;
                }

                if ((LLPG.Verbosity & 2) != 0)
                {
                    LLPG.Output(Verbose, alts, "(unsimplified) " + alts.PredictionTree.ToString());
                }

                SimplifyPredictionTree(alts.PredictionTree);
                AddElseCases(alts, alts.PredictionTree);
                _currentAlts = null;

                if ((LLPG.Verbosity & 1) != 0)
                {
                    LLPG.Output(Verbose, alts, "(simplified) " + alts.PredictionTree.ToString());
                }

                VisitChildrenOf(alts, true);
            }
예제 #8
0
파일: Predicates.cs 프로젝트: dadhi/ecsharp
        public static Pred Or(Pred a, Pred b, bool slashJoined, LNode basis, BranchMode aMode = BranchMode.None, BranchMode bMode = BranchMode.None, IMessageSink sink = null)
        {
            TerminalPred a_ = a as TerminalPred, b_ = b as TerminalPred;

            if (a_ != null && b_ != null && aMode == BranchMode.None && bMode == BranchMode.None)
            {
                return(new TerminalPred(a_.Basis, a_.Set.Union(b_.Set), true));
            }
            else
            {
                return(Alts.Merge(basis, a, b, slashJoined, aMode, bMode, sink));
            }
        }
예제 #9
0
파일: Predicates.cs 프로젝트: dadhi/ecsharp
        public static Alts Merge(LNode basis, Pred a, Pred b, bool slashJoined, BranchMode aMode, BranchMode bMode, IMessageSink warnings)
        {
            Alts aAlts = a as Alts, bAlts = b as Alts;

            if (aAlts != null && aMode == BranchMode.None)
            {
                return(aAlts.Insert(basis, slashJoined, true, b, bMode, warnings));
            }
            else if (bAlts != null && bMode == BranchMode.None)
            {
                return(bAlts.Insert(basis, slashJoined, false, a, aMode, warnings));
            }
            else
            {
                return(TwoArms(basis, a, b, slashJoined, aMode, bMode, warnings));
            }
        }
예제 #10
0
파일: Predicates.cs 프로젝트: dadhi/ecsharp
        static Alts OneArm(Pred a, BranchMode aMode)
        {
            var alts = new Alts(a.Basis, LoopMode.None);

            if (aMode == BranchMode.ErrorExit || aMode == BranchMode.ErrorContinue)
            {
                alts.ErrorBranch = a;
                alts.ExitOnError = aMode == BranchMode.ErrorExit;
            }
            else
            {
                alts.Arms.Add(a);
                if (aMode == BranchMode.Default)
                {
                    alts.DefaultArm = 0;
                }
            }
            return(alts);
        }
예제 #11
0
파일: Predicates.cs 프로젝트: dadhi/ecsharp
 Alts Insert(LNode newBasis, bool slashJoined, bool append, Pred b, BranchMode bMode, IMessageSink warnings)
 {
     if (Mode == LoopMode.None)
     {
         this.Basis = newBasis ?? this.Basis;
         Alts bAlts = b as Alts;
         int  insertAt = append ? this.Arms.Count : 0, boundary = insertAt;
         if (bAlts != null && bMode == BranchMode.None && bAlts.Mode == LoopMode.None)
         {
             for (int i = 0; i < bAlts.Arms.Count; i++)
             {
                 this.InsertSingle(ref insertAt, bAlts.Arms[i], bAlts.DefaultArm == i ? BranchMode.Default : BranchMode.None, warnings);
             }
             if (bAlts.ErrorBranch != null)
             {
                 this.InsertSingle(ref insertAt, bAlts.ErrorBranch, bAlts.ExitOnError ? BranchMode.ErrorExit : BranchMode.ErrorContinue, warnings);
             }
             Debug.Assert(bAlts.DefaultArm != -1);                     // bAlts has no exit branch
         }
         else
         {
             bAlts = null;
             this.InsertSingle(ref insertAt, b, bMode, warnings);
         }
         if (!append)
         {
             boundary = insertAt;
         }
         UpdateSlashDivs(slashJoined, boundary, append, bAlts);
         return(this);
     }
     else
     {
         var copy = OneArm(this, BranchMode.None);
         copy.Insert(newBasis, slashJoined, append, b, bMode, warnings);
         return(copy);
     }
 }
예제 #12
0
        private Pred TranslateLoopExpr(LNode expr, Context ctx)
        {
            Symbol type   = expr.Name;
            bool?  greedy = null;
            bool   g;

            expr = expr.Args[0];
            if ((g = expr.Calls(_Greedy, 1)) || expr.Calls(_Nongreedy, 1))
            {
                greedy = g;
                expr   = expr.Args[0];
            }
            BranchMode branchMode;
            Pred       subpred = BranchToPred(expr, out branchMode, ctx);

            if (branchMode == BranchMode.ErrorContinue || branchMode == BranchMode.ErrorExit)
            {
                _sink.Error(expr, "'error' only applies when there are multiple arms (a|b, a/b)");
            }

            Pred clone = type == _Plus?subpred.Clone() : null;

            Alts alts = new Alts(expr, type == _Opt ? LoopMode.Opt : LoopMode.Star, clone ?? subpred, greedy);

            if (branchMode == BranchMode.Default)
            {
                alts.DefaultArm = 0;
            }
            if (clone != null)
            {
                return(new Seq(subpred, alts, expr));
            }
            else
            {
                return(alts);
            }
        }
예제 #13
0
 public virtual void Visit(Alts alts)
 {
     VisitOther(alts);
 }
예제 #14
0
            // GENERATED CODE EXAMPLE: The methods in this region generate
            // the for(;;) loop in this example and everything inside it, except
            // the calls to Match() which are generated by Visit(TerminalPred).
            // The generated code uses "goto" and "match" blocks in some cases
            // to avoid code duplication. This occurs when the matching code
            // requires multiple statements AND appears more than once in the
            // prediction tree. Otherwise, matching is done "inline" during
            // prediction. We generate a for(;;) loop for (...)*, and in certain
            // cases, we generates a do...while(false) loop for (...)?.
            //
            // rule Foo @{ (('a'|'A') 'A')* 'a'..'z' 'a'..'z' };
            // public void Foo()
            // {
            //     int la0, la1;
            //     for (;;) {
            //         la0 = LA(0);
            //         if (la0 == 'a') {
            //             la1 = LA(1);
            //             if (la1 == 'A')
            //                 goto match1;
            //             else
            //                 break;
            //         } else if (la0 == 'A')
            //             goto match1;
            //         else
            //             break;
            //         match1:
            //         {
            //             Match('A', 'a');
            //             Match('A');
            //         }
            //     }
            //     MatchRange('a', 'z');
            //     MatchRange('a', 'z');
            // }

            private void GenerateCodeForAlts(Alts alts, Dictionary <int, int> timesUsed, PredictionTree tree)
            {
                bool needError = LLPG.NeedsErrorBranch(tree, alts);

                if (!needError && alts.ErrorBranch != null)
                {
                    LLPG.Output(Warning, alts, "The error branch will not be used because the other alternatives are exhaustive (cover all cases)");
                }
                bool userDefinedError = needError && !_recognizerMode && alts.ErrorBranch != null && alts.ErrorBranch != DefaultErrorBranch.Value;

                // Generate matching code for each arm. the "string" in each pair
                // becomes non-null if the matching code for that branch needs to be
                // split out (separated) from the prediction tree because it appears
                // multiple times in the tree. The string is the goto-label name.
                Pair <LNode, string>[] matchingCode = new Pair <LNode, string> [alts.Arms.Count + (userDefinedError ? 1: 0)];
                MSet <int>             unreachable  = new MSet <int>();
                int separateCount = 0;

                for (int i = 0; i < alts.Arms.Count; i++)
                {
                    if (!timesUsed.ContainsKey(i))
                    {
                        unreachable.Add(i);
                        continue;
                    }

                    var codeForThisArm = new WList <LNode>();
                    VisitWithNewTarget(alts.Arms[i], codeForThisArm);

                    matchingCode[i].A = F.Braces(codeForThisArm.ToVList());
                    if (timesUsed[i] > 1 && !SimpleEnoughToRepeat(matchingCode[i].A))
                    {
                        separateCount++;
                        matchingCode[i].B = alts.Arms[i].ChooseGotoLabel()
                                            ?? "match" + (i + 1).ToString();
                    }
                }

                // Add matching code for the error branch, if present. Note: the
                // default error branch, which is produced by IPGCodeGenHelper.
                // ErrorBranch() is handled differently: default error code can
                // differ at each error point in the prediction tree. Therefore
                // we generate it later, on-demand.
                if (userDefinedError)
                {
                    int i            = alts.Arms.Count;
                    var errorHandler = new WList <LNode>();
                    VisitWithNewTarget(alts.ErrorBranch, errorHandler);
                    matchingCode[i].A = F.Braces(errorHandler.ToVList());
                    if (timesUsed[ErrorAlt] > 1 && !SimpleEnoughToRepeat(matchingCode[i].A))
                    {
                        matchingCode[i].B = "error";
                        separateCount++;
                    }
                }

                // Print unreachability warnings
                if (unreachable.Count == 1)
                {
                    LLPG.Output(Warning, alts, string.Format("Branch {{{0}}} is unreachable.", alts.AltName(unreachable.First())));
                }
                else if (unreachable.Count > 1)
                {
                    LLPG.Output(Warning, alts, string.Format("Branches {{{0}}} are unreachable.", unreachable.Select(i => alts.AltName(i)).Join(", ")));
                }
                if (!timesUsed.ContainsKey(ExitAlt) && alts.Mode != LoopMode.None)
                {
                    LLPG.Output(Warning, alts, "Infinite loop. The exit branch is unreachable.");
                }

                Symbol loopType = null;

                // Choose a loop type for (...)* or (...)?:
                if (alts.Mode == LoopMode.Star)
                {
                    loopType = S.For;
                }
                else if (alts.Mode == LoopMode.Opt)
                {
                    if (alts.HasErrorBranch(LLPG) || alts.NonExitDefaultArmRequested())
                    {
                        loopType = S.DoWhile;
                    }
                }

                // If the code for an arm is nontrivial and appears multiple times
                // in the prediction table, it will have to be split out into a
                // labeled block and reached via "goto". I'd rather just do a goto
                // from inside one "if" statement to inside another, but in C#
                // (unlike in CIL, and unlike in C) that is prohibited :(
                DeduplicateLabels(matchingCode);
                var extraMatching = GenerateExtraMatchingCode(matchingCode, separateCount, ref loopType);

                if (separateCount != 0)
                {
                    loopType = loopType ?? S.DoWhile;
                }

                Symbol breakMode = loopType;                 // used to request a "goto" label in addition to the loop
                LNode  code      = GeneratePredictionTreeCode(tree, matchingCode, ref breakMode);

                // Add break/continue between prediction tree and extra matching code,
                // if necessary.
                if (extraMatching.Count != 0 && CodeGenHelperBase.EndMayBeReachable(code))
                {
                    loopType = loopType ?? S.DoWhile;
                    extraMatching.Insert(0, GetContinueStmt(loopType));
                }

                if (!extraMatching.IsEmpty)
                {
                    code = LNode.MergeLists(code, F.Braces(extraMatching), S.Braces);
                }

                if (loopType == S.For)
                {
                    // (...)* => for (;;) {}
                    code = F.Call(S.For, F.List(), F.Missing, F.List(), code);
                }
                else if (loopType == S.DoWhile)
                {
                    // (...)? becomes "do {...} while(false);" IF the exit branch is NOT the default.
                    // If the exit branch is the default, then no loop and no "break" is needed.
                    code = F.Call(S.DoWhile, code, F.@false);
                }
                if (breakMode != loopType && breakMode != null)
                {
                    // Add "stop:" label (plus extra ";" for C# compatibility, in
                    // case the label ends the block in which it is located.)
                    var stopLabel = F.Call(S.Label, F.Id(breakMode))
                                    .PlusTrailingTrivia(F.Trivia(S.TriviaRawText, ";"));
                    code = LNode.MergeLists(code, stopLabel, S.Braces);
                }

                int oldCount = _target.Count;

                _target.SpliceAdd(code, S.Braces);

                // Add comment before code
                if (LLPG.AddComments)
                {
                    var pos     = alts.Basis.Range.Start;
                    var comment = F.Trivia(S.TriviaSLComment, string.Format(" Line {0}: {1}", pos.Line, alts.ToString()));
                    if (_target.Count > oldCount)
                    {
                        _target[oldCount] = _target[oldCount].PlusAttr(comment);
                    }
                }
            }
예제 #15
0
            /// <summary>Extends each level of the prediction tree so that it has
            /// total coverage. For example, a typicaly prediction tree might have
            /// branches for 'a'..'z' and '0..'9'; this method will add coverage for
            /// all other possible inputs. It does this either by adding an error
            /// branch, or by extending the set handled by the default branch of
            /// each level.</summary>
            private void AddElseCases(Alts alts, PredictionTree tree)
            {
                foreach (var branch in tree.Children)
                {
                    if (branch.Sub.Tree != null)
                    {
                        AddElseCases(alts, branch.Sub.Tree);
                    }
                }

                if (tree.IsAssertionLevel)
                {
                    tree.Children.Last.AndPreds.Clear();
                    tree.Children.Last.AndPreds.Add(Set <AndPred> .Empty);
                }
                else if (!tree.TotalCoverage.ContainsEverything)
                {
                    var rest = tree.TotalCoverage.Inverted();
                    if (alts.HasErrorBranch(LLPG))
                    {
                        tree.Children.Add(new PredictionBranch(rest, new PredictionTreeOrAlt {
                            Alt = ErrorAlt
                        }, tree.TotalCoverage));
                    }
                    else
                    {
                        // No error branch, so use default arm
                                        #if false
                        // First try: this tends to produce less intuitive code. Neither
                        // version is objectively better; sometimes this version gives
                        // faster code and sometimes the other version gives faster code.
                        int defaultArm = alts.DefaultArmInt();
                        foreach (PredictionBranch branch in tree.Children)
                        {
                            if (branch.Sub.Tree == null && branch.Sub.Alt == defaultArm)
                            {
                                branch.Set = branch.Set.Union(rest);
                                goto done;
                            }
                        }
                        if (alts.DefaultArm != null)
                        {
                            tree.Children.Add(new PredictionBranch(rest, defaultArm, tree.TotalCoverage));
                        }
                        else
                        {
                            tree.Children.Last.Set = tree.Children.Last.Set.Union(rest);
                        }
                        done :;
                                        #else
                        PredictionBranch last = tree.Children.Last;
                        if (alts.DefaultArm != null && (last.Sub.Tree != null || last.Sub.Alt != alts.DefaultArm.Value))
                        {
                            tree.Children.Add(new PredictionBranch(rest, alts.DefaultArm.Value, tree.TotalCoverage));
                        }
                        else
                        {
                            last.Set = last.Set.Union(rest);
                        }
                                        #endif
                    }
                }
            }
예제 #16
0
 public override void Visit(Alts alts)
 {
     // Traverse each prediction tree find branches taken and save prematch info
     ScanTree(alts.PredictionTree, alts, new DList <Prematched>());
     VisitChildrenOf(alts, true);
 }
예제 #17
0
 public override void Visit(Alts pred)
 {
     VisitOther(pred); ReplaceChildrenOf(pred, true);
 }
예제 #18
0
            void ScanTree(PredictionTree tree, Alts alts, DList <Prematched> path)
            {
                int oldCount = path.Count;

                while (path.Count <= tree.Lookahead)
                {
                    path.Add(new Prematched {
                        Terminals = Anything
                    });
                }
                Prematched pm = path.Last;

                if (tree.IsAssertionLevel)
                {
                    foreach (PredictionBranch b in tree.Children)
                    {
                        var old      = pm.AndPreds.Clone();
                        var verified = Enumerable.Aggregate(b.AndPreds, (set1, set2) => (set1.Union(set2)));                         // usually empty if more than one
                        pm.AndPreds.UnionWith(verified);

                        if (b.Sub.Tree != null)
                        {
                            ScanTree(b.Sub.Tree, alts, path);
                        }
                        else
                        {
                            Debug.Assert(b.Sub.Alt != ErrorAlt);
                            if (b.Sub.Alt == ExitAlt)
                            {
                                _apply.ApplyPrematchData(alts.Next, path);
                            }
                            else
                            {
                                _apply.ApplyPrematchData(alts.Arms[b.Sub.Alt], path);
                            }
                        }
                        pm.AndPreds = old;
                    }
                }
                else                 // !IsAssertionLevel (terminal-matching level)
                {
                    bool needErrorBranch = LLPG.NeedsErrorBranch(tree, alts);

                    for (int i = 0; i < tree.Children.Count; i++)
                    {
                        PredictionBranch b   = tree.Children[i];
                        IPGTerminalSet   set = b.Set;
                        if (!needErrorBranch && i + 1 == tree.Children.Count)
                        {
                            // Add all the default cases
                            set = set.Union(tree.TotalCoverage.Inverted());
                        }
                        pm.Terminals = set;
                        if (b.Sub.Tree != null)
                        {
                            ScanTree(b.Sub.Tree, alts, path);
                        }
                        else
                        {
                            if (b.Sub.Alt == ExitAlt)
                            {
                                _apply.ApplyPrematchData(alts.Next, path);
                            }
                            else if (b.Sub.Alt != ErrorAlt)
                            {
                                _apply.ApplyPrematchData(alts.Arms[b.Sub.Alt], path);
                            }
                        }
                    }
                    path.PopLast();
                }
                path.Resize(oldCount);
            }
예제 #19
0
 public override void Visit(Alts pred)
 {
     VisitChildrenOf(pred, true);
 }
예제 #20
0
        public LNode GenerateLexerCode()
        {
            _pg = new LLParserGenerator(new IntStreamCodeGenHelper(), MessageSink.Console);

            // Whitespace & comments
            var Newline = Rule("Newline", ((C('\r') + Opt(C('\n'))) | '\n')
                               + Stmt("_lineStartAt = InputPosition")
                               + Stmt("_lineNumber++")
                               + Stmt("_value = WhitespaceTag.Value"), Token);
            var DotIndent = Rule("DotIndent", And("_startPosition == _lineStartAt")
                                 + Stmt("_type = TT.Spaces")
                                 + Plus(C('.') + Plus(C('\t') | ' '))
                                 + Stmt("_indentLevel = MeasureIndent(_indent = Source.Substring(_startPosition, InputPosition - _startPosition))")
                                 + Stmt("_value = WhitespaceTag.Value"), Private);
            var Spaces = Rule("Spaces", Plus(C(' ') | '\t')
                              + Stmt("if (_lineStartAt == _startPosition) "
                                     + "_indentLevel = MeasureIndent(_indent = Source.Substring(_startPosition, InputPosition - _startPosition))")
                              + Stmt("_value = WhitespaceTag.Value"), Token);
            var SLComment    = Rule("SLComment", Seq("//") + Star(Set("[^\r\n]")) + Stmt("_value = WhitespaceTag.Value"), Token);
            var MLCommentRef = new RuleRef(null, null);
            var MLComment    = Rule("MLComment",
                                    Seq("/*") +
                                    Star(MLCommentRef / AnyCh, false) +
                                    Seq("*/") +
                                    Stmt("_value = WhitespaceTag.Value"), Token, 3);

            MLCommentRef.Rule = MLComment;
            _pg.AddRules(Newline, DotIndent, Spaces, SLComment, MLComment);

            // Strings
            var SQString = Rule("SQString", Stmt("_parseNeeded = false") +
                                C('\'')
                                + ((C('\\') + Set("[^\r\n]") + Stmt("_parseNeeded = true"))
                                   / Set("[^'\r\n\\\\]")
                                   / (Seq("") + Expr("_parseNeeded = true")))
                                + Star(Set("[^' \t\n\r]") + Stmt("_parseNeeded = true"))
                                + (C('\'') / (Seq("") + Stmt("_parseNeeded = true")))
                                + Call("ParseSQStringValue"), Token);
            var TQString = Rule("TQString", Stmt("_parseNeeded = true")
                                + (Stmt("_style = NodeStyle.Alternate") +
                                   Seq(@"""""""") + Star(Seq(@"\\""") / AnyCh, false) + Seq(@"""""""")
                                   | Stmt("_style = NodeStyle.Alternate | NodeStyle.Alternate2") +
                                   Seq(@"'''") + Star(Seq(@"\\'") / AnyCh, false) + Seq(@"'''"))
                                + Stmt("ParseStringValue(true)"), Token, 4);
            var DQString = Rule("DQString", Stmt("_parseNeeded = false") +
                                (C('"') + Star(C('\\') + AnyCh + Stmt("_parseNeeded = true") | Set("[^\"\\\\\r\n]")) + '"'
                                 | (Stmt("_style = NodeStyle.Alternate") +
                                    (Seq(@"#""") + Star((Seq(@"""""") + Stmt("_parseNeeded = true")) / Set("[^\"]")) + '"'))
                                ) + Stmt("ParseStringValue(false)"), Token);
            var BQString2 = Rule("BQString2", Stmt("_parseNeeded = false") +
                                 C('`') + Star(C('\\') + AnyCh + Stmt("_parseNeeded = true") | Set("[^`\\\\\r\n]")) + '`', Private);
            var BQString = Rule("BQString", BQString2 + Stmt("ParseBQStringValue()"), Token);

            _pg.AddRules(SQString, DQString, TQString, BQString, BQString2);

            // Identifiers and symbols
            var letterTest    = F.Call(F.Dot("#char", "IsLetter"), F.Call(S.Cast, F.Id("LA0"), F.Id(S.Char)));
            var lettersOrPunc = Set(@"[0-9a-zA-Z_'#~!%^&*\-+=|<>/\\?:.$]");

            Debug.Assert(!((PGIntSet)lettersOrPunc.Set).Contains('`'));
            var IdExtLetter = Rule("IdExtLetter",
                                   And(letterTest) + Set("[\u0080-\uFFFC]"), Private);
            var NormalId = Rule("NormalId", (Set("[#_a-zA-Z]") | IdExtLetter) +
                                Star(Set("[#_a-zA-Z0-9']") | IdExtLetter));
            var CommentStart = Rule("CommentStart", '/' + (C('/') | '*'), Private);
            var FancyId      = Rule("FancyId", BQString2 | Plus(AndNot(CommentStart) + lettersOrPunc | IdExtLetter));
            var Symbol       = Rule("Symbol", Stmt("_parseNeeded = false") +
                                    Seq("@@") + FancyId + Call("ParseSymbolValue"), Token);
            var Id = Rule("Id", Stmt("_parseNeeded = false") +
                          (NormalId | '@' + FancyId + Stmt("_parseNeeded = true")) +
                          Call("ParseIdValue"), Private);

            _pg.AddRules(IdExtLetter, NormalId, CommentStart, FancyId, Symbol, Id);

            // Punctuation
            var Comma       = Rule("Comma", Op(",", "Comma"), Private);
            var Semicolon   = Rule("Semicolon", Op(";", "Semicolon"), Private);
            var At          = Rule("At", C('@') + Stmt("_type = TT.At; _value = GSymbol.Empty"), Private);
            var ops         = Set(@"[~!%^&*\-+=|<>/?:.$]");
            var Operator    = Rule("Operator", Plus(AndNot(CommentStart) + ops) + Stmt("ParseNormalOp()"), Private);
            var BackslashOp = Rule("BackslashOp", '\\' + Opt(FancyId) + Stmt("ParseBackslashOp()"), Private);

            _pg.AddRules(Comma, Semicolon, At, Operator, BackslashOp);

            // Openers & closers
            var LParen = Rule("LParen", C('('), Token);
            var RParen = Rule("RParen", C(')'), Token);
            var LBrack = Rule("LBrack", C('['), Token);
            var RBrack = Rule("RBrack", C(']'), Token);
            var LBrace = Rule("LBrace", C('{'), Token);
            var RBrace = Rule("RBrace", C('}'), Token);

            _pg.AddRules(new[] { LParen, RParen, LBrack, RBrack, LBrace, RBrace });

            Rule Number;

            _pg.AddRules(NumberParts(out Number));

            var  Shebang   = Rule("Shebang", Seq("#!") + Star(Set("[^\r\n]")) + Opt(Newline));
            Alts tokenAlts = (Alts)(
                (And(Expr("InputPosition == 0")) + T(Shebang)) /
                T(Symbol) /
                T(Id) /
                T(Spaces) / T(Newline) / DotIndent /
                T(SLComment) / T(MLComment) /
                T(Number) /
                (Stmt("_type = TT.String") + TQString) /
                (Stmt("_type = TT.String") + DQString) /
                T(SQString) / T(BQString) /
                T(Comma) / T(Semicolon) /
                T(LParen) / T(LBrack) / T(LBrace) /
                T(RParen) / T(RBrack) / T(RBrace) /
                T(At) / BackslashOp / Operator);

            tokenAlts.DefaultArm = 2;             // Id
            var token = Rule("Token", tokenAlts, Token, 3);

            _pg.AddRules(new[] { token, Shebang });
            _pg.FullLLk = true;
            //_pg.Verbosity = 3;

            var members = _pg.Run(F.File);

            members = members.PlusArgs(SymbolsToDeclare.Select(p =>
                                                               F.Var(F.Id("Symbol"), p.Key, F.Call(F.Dot("GSymbol", "Get"), F.Literal(p.Value.Name)))));

            return(F.Attr(F.Public, F.Id(S.Partial),
                          F.Call(S.Class, F.Id(_("LesLexer")), F.Tuple(), members)));
        }