Beispiel #1
0
        //------------------------------------------------------------------------
        //
        //           printRanges        A debugging function.
        //                              dump out all of the range definitions.
        //
        //------------------------------------------------------------------------
        ///CLOVER:OFF
        internal virtual void PrintRanges()
        {
            RangeDescriptor rlRange;
            int             i;

            Console.Out.Write("\n\n Nonoverlapping Ranges ...\n");
            for (rlRange = fRangeList; rlRange != null; rlRange = rlRange.fNext)
            {
                Console.Out.Write(" " + rlRange.fNum + "   " + rlRange.fStartChar + "-" + rlRange.fEndChar);

                for (i = 0; i < rlRange.fIncludesSets.Count; i++)
                {
                    RBBINode usetNode = rlRange.fIncludesSets[i];
                    String   setName  = "anon";
                    RBBINode setRef   = usetNode.fParent;
                    if (setRef != null)
                    {
                        RBBINode varRef = setRef.fParent;
                        if (varRef != null && varRef.fType == RBBINode.varRef)
                        {
                            setName = varRef.fText;
                        }
                    }
                    Console.Out.Write(setName); Console.Out.Write("  ");
                }
                Console.Out.WriteLine("");
            }
        }
Beispiel #2
0
        //-----------------------------------------------------------------------------
        //
        //   calcFollowPos.    Impossible to explain succinctly.  See Aho, section 3.9
        //
        //-----------------------------------------------------------------------------
        internal virtual void CalcFollowPos(RBBINode n)
        {
            if (n == null ||
                n.fType == RBBINode.leafChar ||
                n.fType == RBBINode.endMark)
            {
                return;
            }

            CalcFollowPos(n.fLeftChild);
            CalcFollowPos(n.fRightChild);

            // Aho rule #1
            if (n.fType == RBBINode.opCat)
            {
                foreach (RBBINode i /* is 'i' in Aho's description */ in n.fLeftChild.fLastPosSet)
                {
                    i.fFollowPos.UnionWith(n.fRightChild.fFirstPosSet);
                }
            }

            // Aho rule #2
            if (n.fType == RBBINode.opStar ||
                n.fType == RBBINode.opPlus)
            {
                foreach (RBBINode i /* again, n and i are the names from Aho's description */ in n.fLastPosSet)
                {
                    i.fFollowPos.UnionWith(n.fFirstPosSet);
                }
            }
        }
Beispiel #3
0
        //CLOVER:ON


        //------------------------------------------------------------------------
        //
        //           printRangeGroups     A debugging function.
        //                                dump out all of the range groups.
        //
        //------------------------------------------------------------------------
        ///CLOVER:OFF
        internal virtual void PrintRangeGroups()
        {
            RangeDescriptor rlRange;
            RangeDescriptor tRange;
            int             i;
            int             lastPrintedGroupNum = 0;

            Console.Out.Write("\nRanges grouped by Unicode Set Membership...\n");
            for (rlRange = fRangeList; rlRange != null; rlRange = rlRange.fNext)
            {
                int groupNum = rlRange.fNum & 0xbfff;
                if (groupNum > lastPrintedGroupNum)
                {
                    lastPrintedGroupNum = groupNum;
                    if (groupNum < 10)
                    {
                        Console.Out.Write(" ");
                    }
                    Console.Out.Write(groupNum + " ");

                    if ((rlRange.fNum & 0x4000) != 0)
                    {
                        Console.Out.Write(" <DICT> ");
                    }

                    for (i = 0; i < rlRange.fIncludesSets.Count; i++)
                    {
                        RBBINode usetNode = rlRange.fIncludesSets[i];
                        String   setName  = "anon";
                        RBBINode setRef   = usetNode.fParent;
                        if (setRef != null)
                        {
                            RBBINode varRef = setRef.fParent;
                            if (varRef != null && varRef.fType == RBBINode.varRef)
                            {
                                setName = varRef.fText;
                            }
                        }
                        Console.Out.Write(setName); Console.Out.Write(" ");
                    }

                    i = 0;
                    for (tRange = rlRange; tRange != null; tRange = tRange.fNext)
                    {
                        if (tRange.fNum == rlRange.fNum)
                        {
                            if (i++ % 5 == 0)
                            {
                                Console.Out.Write("\n    ");
                            }
                            RBBINode.PrintHex(tRange.fStartChar, -1);
                            Console.Out.Write("-");
                            RBBINode.PrintHex(tRange.fEndChar, 0);
                        }
                    }
                    Console.Out.Write("\n");
                }
            }
            Console.Out.Write("\n");
        }
Beispiel #4
0
        //-------------------------------------------------------------------------
        //
        //        cloneTree Make a copy of the subtree rooted at this node.
        //                      Discard any variable references encountered along the way,
        //                      and replace with copies of the variable's definitions.
        //                      Used to replicate the expression underneath variable
        //                      references in preparation for generating the DFA tables.
        //
        //-------------------------------------------------------------------------
        internal virtual RBBINode CloneTree()
        {
            RBBINode n;

            if (fType == RBBINode.varRef)
            {
                // If the current node is a variable reference, skip over it
                //   and clone the definition of the variable instead.
                n = fLeftChild.CloneTree();
            }
            else if (fType == RBBINode.uset)
            {
                n = this;
            }
            else
            {
                n = new RBBINode(this);
                if (fLeftChild != null)
                {
                    n.fLeftChild         = fLeftChild.CloneTree();
                    n.fLeftChild.fParent = n;
                }
                if (fRightChild != null)
                {
                    n.fRightChild         = fRightChild.CloneTree();
                    n.fRightChild.fParent = n;
                }
            }
            return(n);
        }
Beispiel #5
0
        //-----------------------------------------------------------------------------
        //
        //   printSet    Debug function.   Print the contents of a set of Nodes
        //
        //-----------------------------------------------------------------------------

        internal virtual void PrintSet(ICollection <RBBINode> s)
        {
            foreach (RBBINode n in s)
            {
                RBBINode.PrintInt32(n.fSerialNum, 8);
            }
            Console.Out.WriteLine();
        }
Beispiel #6
0
        //
        // RBBISymbolTable::lookupNode      Given a key (a variable name), return the
        //                                  corresponding RBBI Node.  If there is no entry
        //                                  in the table for this name, return NULL.
        //
        internal virtual RBBINode LookupNode(String key)
        {
            RBBINode             retNode = null;
            RBBISymbolTableEntry el;

            el = fHashTable.Get(key);
            if (el != null)
            {
                retNode = el.Val;
            }
            return(retNode);
        }
Beispiel #7
0
        //
        //    RBBISymbolTable::addEntry     Add a new entry to the symbol table.
        //                                  Indicate an error if the name already exists -
        //                                    this will only occur in the case of duplicate
        //                                    variable assignments.
        //
        internal virtual void AddEntry(string key, RBBINode val)
        {
            if (fHashTable.TryGetValue(key, out RBBISymbolTableEntry e) && e != null)
            {
                fRuleScanner.Error(RBBIRuleBuilder.U_BRK_VARIABLE_REDFINITION);
                return;
            }

            fHashTable[key] = new RBBISymbolTableEntry {
                Key = key, Val = val
            };
        }
Beispiel #8
0
 //-----------------------------------------------------------------------------
 //
 //           addRuleRootNodes    Recursively walk a parse tree, adding all nodes flagged
 //                               as roots of a rule to a destination vector.
 //
 //-----------------------------------------------------------------------------
 internal virtual void AddRuleRootNodes(IList <RBBINode> dest, RBBINode node)
 {
     if (node == null)
     {
         return;
     }
     if (node.fRuleRoot)
     {
         dest.Add(node);
         // Note: rules cannot nest. If we found a rule start node,
         //       no child node can also be a start node.
         return;
     }
     AddRuleRootNodes(dest, node.fLeftChild);
     AddRuleRootNodes(dest, node.fRightChild);
 }
Beispiel #9
0
        //
        //    RBBISymbolTable::addEntry     Add a new entry to the symbol table.
        //                                  Indicate an error if the name already exists -
        //                                    this will only occur in the case of duplicate
        //                                    variable assignments.
        //
        internal virtual void AddEntry(string key, RBBINode val)
        {
            RBBISymbolTableEntry e;

            e = fHashTable.Get(key);
            if (e != null)
            {
                fRuleScanner.Error(RBBIRuleBuilder.U_BRK_VARIABLE_REDFINITION);
                return;
            }

            e                 = new RBBISymbolTableEntry();
            e.Key             = key;
            e.Val             = val;
            fHashTable[e.Key] = e;
        }
Beispiel #10
0
        //-----------------------------------------------------------------------------
        //
        //   calcNullable.    Impossible to explain succinctly.  See Aho, section 3.9
        //
        //-----------------------------------------------------------------------------
        internal virtual void CalcNullable(RBBINode n)
        {
            if (n == null)
            {
                return;
            }
            if (n.fType == RBBINode.setRef ||
                n.fType == RBBINode.endMark)
            {
                // These are non-empty leaf node types.
                n.fNullable = false;
                return;
            }

            if (n.fType == RBBINode.lookAhead || n.fType == RBBINode.tag)
            {
                // Lookahead marker node.  It's a leaf, so no recursion on children.
                // It's nullable because it does not match any literal text from the input stream.
                n.fNullable = true;
                return;
            }


            // The node is not a leaf.
            //  Calculate nullable on its children.
            CalcNullable(n.fLeftChild);
            CalcNullable(n.fRightChild);

            // Apply functions from table 3.40 in Aho
            if (n.fType == RBBINode.opOr)
            {
                n.fNullable = n.fLeftChild.fNullable || n.fRightChild.fNullable;
            }
            else if (n.fType == RBBINode.opCat)
            {
                n.fNullable = n.fLeftChild.fNullable && n.fRightChild.fNullable;
            }
            else if (n.fType == RBBINode.opStar || n.fType == RBBINode.opQuestion)
            {
                n.fNullable = true;
            }
            else
            {
                n.fNullable = false;
            }
        }
Beispiel #11
0
 internal RBBINode(RBBINode other)
 {
     fSerialNum   = ++gLastSerial;
     fType        = other.fType;
     fInputSet    = other.fInputSet;
     fPrecedence  = other.fPrecedence;
     fText        = other.fText;
     fFirstPos    = other.fFirstPos;
     fLastPos     = other.fLastPos;
     fNullable    = other.fNullable;
     fVal         = other.fVal;
     fRuleRoot    = false;
     fChainIn     = other.fChainIn;
     fFirstPosSet = new HashSet <RBBINode>(other.fFirstPosSet);
     fLastPosSet  = new HashSet <RBBINode>(other.fLastPosSet);
     fFollowPos   = new HashSet <RBBINode>(other.fFollowPos);
 }
Beispiel #12
0
        //-----------------------------------------------------------------------------
        //
        //   bofFixup.    Fixup for state tables that include {bof} beginning of input testing.
        //                Do an swizzle similar to chaining, modifying the followPos set of
        //                the bofNode to include the followPos nodes from other {bot} nodes
        //                scattered through the tree.
        //
        //                This function has much in common with calcChainedFollowPos().
        //
        //-----------------------------------------------------------------------------
        internal virtual void BofFixup()
        {
            //
            //   The parse tree looks like this ...
            //         fTree root  --.       <cat>
            //                               /     \
            //                            <cat>   <#end node>
            //                           /     \
            //                     <bofNode>   rest
            //                               of tree
            //
            //    We will be adding things to the followPos set of the <bofNode>
            //
            RBBINode bofNode = fRB.fTreeRoots[fRootIx].fLeftChild.fLeftChild;

            Assert.Assrt(bofNode.fType == RBBINode.leafChar);
            Assert.Assrt(bofNode.fVal == 2);

            // Get all nodes that can be the start a match of the user-written rules
            //  (excluding the fake bofNode)
            //  We want the nodes that can start a match in the
            //     part labeled "rest of tree"
            //
            ISet <RBBINode> matchStartNodes = fRB.fTreeRoots[fRootIx].fLeftChild.fRightChild.fFirstPosSet;

            foreach (RBBINode startNode in matchStartNodes)
            {
                if (startNode.fType != RBBINode.leafChar)
                {
                    continue;
                }

                if (startNode.fVal == bofNode.fVal)
                {
                    //  We found a leaf node corresponding to a {bof} that was
                    //    explicitly written into a rule.
                    //  Add everything from the followPos set of this node to the
                    //    followPos set of the fake bofNode at the start of the tree.
                    //
                    bofNode.fFollowPos.UnionWith(startNode.fFollowPos);
                }
            }
        }
Beispiel #13
0
        //-----------------------------------------------------------------------------
        //
        //   calcLastPos.    Impossible to explain succinctly.  See Aho, section 3.9
        //
        //-----------------------------------------------------------------------------
        internal virtual void CalcLastPos(RBBINode n)
        {
            if (n == null)
            {
                return;
            }
            if (n.fType == RBBINode.leafChar ||
                n.fType == RBBINode.endMark ||
                n.fType == RBBINode.lookAhead ||
                n.fType == RBBINode.tag)
            {
                // These are non-empty leaf node types.
                n.fLastPosSet.Add(n);
                return;
            }

            // The node is not a leaf.
            //  Calculate lastPos on its children.
            CalcLastPos(n.fLeftChild);
            CalcLastPos(n.fRightChild);

            // Apply functions from table 3.40 in Aho
            if (n.fType == RBBINode.opOr)
            {
                n.fLastPosSet.UnionWith(n.fLeftChild.fLastPosSet);
                n.fLastPosSet.UnionWith(n.fRightChild.fLastPosSet);
            }
            else if (n.fType == RBBINode.opCat)
            {
                n.fLastPosSet.UnionWith(n.fRightChild.fLastPosSet);
                if (n.fRightChild.fNullable)
                {
                    n.fLastPosSet.UnionWith(n.fLeftChild.fLastPosSet);
                }
            }
            else if (n.fType == RBBINode.opStar ||
                     n.fType == RBBINode.opQuestion ||
                     n.fType == RBBINode.opPlus)
            {
                n.fLastPosSet.UnionWith(n.fLeftChild.fLastPosSet);
            }
        }
Beispiel #14
0
        //-----------------------------------------------------------------------------
        //
        //  printPosSets   Debug function.  Dump Nullable, firstpos, lastpos and followpos
        //                 for each node in the tree.
        //
        //-----------------------------------------------------------------------------

        internal virtual void PrintPosSets(RBBINode n)
        {
            if (n == null)
            {
                return;
            }
            RBBINode.PrintNode(n);
            Console.Out.Write("         Nullable:  " + n.fNullable);

            Console.Out.Write("         firstpos:  ");
            PrintSet(n.fFirstPosSet);

            Console.Out.Write("         lastpos:   ");
            PrintSet(n.fLastPosSet);

            Console.Out.Write("         followpos: ");
            PrintSet(n.fFollowPos);

            PrintPosSets(n.fLeftChild);
            PrintPosSets(n.fRightChild);
        }
Beispiel #15
0
        //-------------------------------------------------------------------------
        //
        //       flattenVariables Walk a parse tree, replacing any variable
        //                          references with a copy of the variable's definition.
        //                          Aside from variables, the tree is not changed.
        //
        //                          Return the root of the tree. If the root was not a variable
        //                          reference, it remains unchanged - the root we started with
        //                          is the root we return. If, however, the root was a variable
        //                          reference, the root of the newly cloned replacement tree will
        //                          be returned, and the original tree deleted.
        //
        //                          This function works by recursively walking the tree
        //                          without doing anything until a variable reference is
        //                          found, then calling cloneTree() at that point. Any
        //                          nested references are handled by cloneTree(), not here.
        //
        //-------------------------------------------------------------------------
        internal virtual RBBINode FlattenVariables()
        {
            if (fType == varRef)
            {
                RBBINode retNode = fLeftChild.CloneTree();
                retNode.fRuleRoot = this.fRuleRoot;
                retNode.fChainIn  = this.fChainIn;
                return(retNode);
            }

            if (fLeftChild != null)
            {
                fLeftChild         = fLeftChild.FlattenVariables();
                fLeftChild.fParent = this;
            }
            if (fRightChild != null)
            {
                fRightChild         = fRightChild.FlattenVariables();
                fRightChild.fParent = this;
            }
            return(this);
        }
Beispiel #16
0
        //-------------------------------------------------------------------------
        //
        //        print. Print out a single node, for debugging.
        //
        //-------------------------------------------------------------------------
        ////CLOVER:OFF
        internal static void PrintNode(RBBINode n)
        {
            if (n == null)
            {
                Console.Out.Write(" -- null --\n");
            }
            else
            {
                RBBINode.PrintInt32(n.fSerialNum, 10);
                RBBINode.PrintString(nodeTypeNames[n.fType], 11);
                RBBINode.PrintInt32(n.fParent == null ? 0 : n.fParent.fSerialNum, 11);
                RBBINode.PrintInt32(n.fLeftChild == null ? 0 : n.fLeftChild.fSerialNum, 11);
                RBBINode.PrintInt32(n.fRightChild == null ? 0 : n.fRightChild.fSerialNum, 12);
                RBBINode.PrintInt32(n.fFirstPos, 12);
                RBBINode.PrintInt32(n.fVal, 7);

                if (n.fType == varRef)
                {
                    Console.Out.Write(" " + n.fText);
                }
            }
            Console.Out.WriteLine("");
        }
Beispiel #17
0
        //CLOVER:ON


        //------------------------------------------------------------------------
        //
        //           printSets          A debugging function.
        //                              dump out all of the set definitions.
        //
        //------------------------------------------------------------------------
        ///CLOVER:OFF
        internal virtual void PrintSets()
        {
            int i;

            Console.Out.Write("\n\nUnicode Sets List\n------------------\n");
            for (i = 0; i < fRB.fUSetNodes.Count; i++)
            {
                RBBINode usetNode;
                RBBINode setRef;
                RBBINode varRef;
                String   setName;

                usetNode = fRB.fUSetNodes[i];

                //System.out.print(" " + i + "   ");
                RBBINode.PrintInt32(2, i);
                setName = "anonymous";
                setRef  = usetNode.fParent;
                if (setRef != null)
                {
                    varRef = setRef.fParent;
                    if (varRef != null && varRef.fType == RBBINode.varRef)
                    {
                        setName = varRef.fText;
                    }
                }
                Console.Out.Write("  " + setName);
                Console.Out.Write("   ");
                Console.Out.Write(usetNode.fText);
                Console.Out.Write("\n");
                if (usetNode.fLeftChild != null)
                {
                    usetNode.fLeftChild.PrintTree(true);
                }
            }
            Console.Out.Write("\n");
        }
Beispiel #18
0
        //-------------------------------------------------------------------------
        //
        //      flattenSets Walk the parse tree, replacing any nodes of type setRef
        //                     with a copy of the expression tree for the set. A set's
        //                     equivalent expression tree is precomputed and saved as
        //                     the left child of the uset node.
        //
        //-------------------------------------------------------------------------
        internal virtual void FlattenSets()
        {
            Assert.Assrt(fType != setRef);

            if (fLeftChild != null)
            {
                if (fLeftChild.fType == setRef)
                {
                    RBBINode setRefNode = fLeftChild;
                    RBBINode usetNode   = setRefNode.fLeftChild;
                    RBBINode replTree   = usetNode.fLeftChild;
                    fLeftChild         = replTree.CloneTree();
                    fLeftChild.fParent = this;
                }
                else
                {
                    fLeftChild.FlattenSets();
                }
            }

            if (fRightChild != null)
            {
                if (fRightChild.fType == setRef)
                {
                    RBBINode setRefNode = fRightChild;
                    RBBINode usetNode   = setRefNode.fLeftChild;
                    RBBINode replTree   = usetNode.fLeftChild;
                    fRightChild         = replTree.CloneTree();
                    fRightChild.fParent = this;
                    // delete setRefNode;
                }
                else
                {
                    fRightChild.FlattenSets();
                }
            }
        }
Beispiel #19
0
        //-----------------------------------------------------------------------------
        //
        //   printStates    Debug Function.  Dump the fully constructed state transition table.
        //
        //-----------------------------------------------------------------------------

        internal virtual void PrintStates()
        {
            int c;    // input "character"
            int n;    // state number

            Console.Out.Write("state |           i n p u t     s y m b o l s \n");
            Console.Out.Write("      | Acc  LA    Tag");
            for (c = 0; c < fRB.fSetBuilder.NumCharCategories; c++)
            {
                RBBINode.PrintInt32(c, 3);
            }
            Console.Out.Write("\n");
            Console.Out.Write("      |---------------");
            for (c = 0; c < fRB.fSetBuilder.NumCharCategories; c++)
            {
                Console.Out.Write("---");
            }
            Console.Out.Write("\n");

            for (n = 0; n < fDStates.Count; n++)
            {
                RBBIStateDescriptor sd = fDStates[n];
                RBBINode.PrintInt32(n, 5);
                Console.Out.Write(" | ");

                RBBINode.PrintInt32(sd.fAccepting, 3);
                RBBINode.PrintInt32(sd.fLookAhead, 4);
                RBBINode.PrintInt32(sd.fTagsIdx, 6);
                Console.Out.Write(" ");
                for (c = 0; c < fRB.fSetBuilder.NumCharCategories; c++)
                {
                    RBBINode.PrintInt32(sd.fDtran[c], 3);
                }
                Console.Out.Write("\n");
            }
            Console.Out.Write("\n\n");
        }
Beispiel #20
0
            //-------------------------------------------------------------------------------------
            //
            //          RangeDescriptor::setDictionaryFlag
            //
            //          Character Category Numbers that include characters from
            //          the original Unicode Set named "dictionary" have bit 14
            //          set to 1.  The RBBI runtime engine uses this to trigger
            //          use of the word dictionary.
            //
            //          This function looks through the Unicode Sets that it
            //          (the range) includes, and sets the bit in fNum when
            //          "dictionary" is among them.
            //
            //          TODO:  a faster way would be to find the set node for
            //          "dictionary" just once, rather than looking it
            //          up by name every time.
            //
            // -------------------------------------------------------------------------------------
            internal virtual void SetDictionaryFlag()
            {
                int i;

                for (i = 0; i < this.fIncludesSets.Count; i++)
                {
                    RBBINode usetNode = fIncludesSets[i];
                    string   setName  = "";
                    RBBINode setRef   = usetNode.fParent;
                    if (setRef != null)
                    {
                        RBBINode varRef = setRef.fParent;
                        if (varRef != null && varRef.fType == RBBINode.varRef)
                        {
                            setName = varRef.fText;
                        }
                    }
                    if (setName.Equals("dictionary"))
                    {
                        this.fNum |= 0x4000;
                        break;
                    }
                }
            }
Beispiel #21
0
        internal virtual void AddValToSet(RBBINode usetNode, int val)
        {
            RBBINode leafNode = new RBBINode(RBBINode.leafChar);

            leafNode.fVal = val;
            if (usetNode.fLeftChild == null)
            {
                usetNode.fLeftChild = leafNode;
                leafNode.fParent    = usetNode;
            }
            else
            {
                // There are already input symbols present for this set.
                // Set up an OR node, with the previous stuff as the left child
                //   and the new value as the right child.
                RBBINode orNode = new RBBINode(RBBINode.opOr);
                orNode.fLeftChild          = usetNode.fLeftChild;
                orNode.fRightChild         = leafNode;
                orNode.fLeftChild.fParent  = orNode;
                orNode.fRightChild.fParent = orNode;
                usetNode.fLeftChild        = orNode;
                orNode.fParent             = usetNode;
            }
        }
Beispiel #22
0
        //-----------------------------------------------------------------------------
        //
        //   printRuleStatusTable    Debug Function.  Dump the common rule status table
        //
        //-----------------------------------------------------------------------------

        internal virtual void PrintRuleStatusTable()
        {
            int         thisRecord = 0;
            int         nextRecord = 0;
            int         i;
            IList <int> tbl = fRB.fRuleStatusVals;

            Console.Out.Write("index |  tags \n");
            Console.Out.Write("-------------------\n");

            while (nextRecord < tbl.Count)
            {
                thisRecord = nextRecord;
                nextRecord = thisRecord + tbl[thisRecord] + 1;
                RBBINode.PrintInt32(thisRecord, 7);
                for (i = thisRecord + 1; i < nextRecord; i++)
                {
                    int val = tbl[i];
                    RBBINode.PrintInt32(val, 7);
                }
                Console.Out.Write("\n");
            }
            Console.Out.Write("\n\n");
        }
Beispiel #23
0
        //-----------------------------------------------------------------------------
        //
        //   RBBITableBuilder::build  -  This is the main function for building the DFA state transtion
        //                               table from the RBBI rules parse tree.
        //
        //-----------------------------------------------------------------------------
        internal virtual void Build()
        {
            // If there were no rules, just return.  This situation can easily arise
            //   for the reverse rules.
            if (fRB.fTreeRoots[fRootIx] == null)
            {
                return;
            }

            //
            // Walk through the tree, replacing any references to $variables with a copy of the
            //   parse tree for the substition expression.
            //
            fRB.fTreeRoots[fRootIx] = fRB.fTreeRoots[fRootIx].FlattenVariables();
            if (fRB.fDebugEnv != null && fRB.fDebugEnv.IndexOf("ftree", StringComparison.Ordinal) >= 0)
            {
                Console.Out.WriteLine("Parse tree after flattening variable references.");
                fRB.fTreeRoots[fRootIx].PrintTree(true);
            }

            //
            // If the rules contained any references to {bof}
            //   add a {bof} <cat> <former root of tree> to the
            //   tree.  Means that all matches must start out with the
            //   {bof} fake character.
            //
            if (fRB.fSetBuilder.SawBOF)
            {
                RBBINode bofTop  = new RBBINode(RBBINode.opCat);
                RBBINode bofLeaf = new RBBINode(RBBINode.leafChar);
                bofTop.fLeftChild       = bofLeaf;
                bofTop.fRightChild      = fRB.fTreeRoots[fRootIx];
                bofLeaf.fParent         = bofTop;
                bofLeaf.fVal            = 2; // Reserved value for {bof}.
                fRB.fTreeRoots[fRootIx] = bofTop;
            }

            //
            // Add a unique right-end marker to the expression.
            //   Appears as a cat-node, left child being the original tree,
            //   right child being the end marker.
            //
            RBBINode cn = new RBBINode(RBBINode.opCat);

            cn.fLeftChild = fRB.fTreeRoots[fRootIx];
            fRB.fTreeRoots[fRootIx].fParent = cn;
            cn.fRightChild          = new RBBINode(RBBINode.endMark);
            cn.fRightChild.fParent  = cn;
            fRB.fTreeRoots[fRootIx] = cn;

            //
            //  Replace all references to UnicodeSets with the tree for the equivalent
            //      expression.
            //
            fRB.fTreeRoots[fRootIx].FlattenSets();
            if (fRB.fDebugEnv != null && fRB.fDebugEnv.IndexOf("stree", StringComparison.Ordinal) >= 0)
            {
                Console.Out.WriteLine("Parse tree after flattening Unicode Set references.");
                fRB.fTreeRoots[fRootIx].PrintTree(true);
            }


            //
            // calculate the functions nullable, firstpos, lastpos and followpos on
            // nodes in the parse tree.
            //    See the alogrithm description in Aho.
            //    Understanding how this works by looking at the code alone will be
            //       nearly impossible.
            //
            CalcNullable(fRB.fTreeRoots[fRootIx]);
            CalcFirstPos(fRB.fTreeRoots[fRootIx]);
            CalcLastPos(fRB.fTreeRoots[fRootIx]);
            CalcFollowPos(fRB.fTreeRoots[fRootIx]);
            if (fRB.fDebugEnv != null && fRB.fDebugEnv.IndexOf("pos", StringComparison.Ordinal) >= 0)
            {
                Console.Out.Write("\n");
                PrintPosSets(fRB.fTreeRoots[fRootIx]);
            }

            //
            //  For "chained" rules, modify the followPos sets
            //
            if (fRB.fChainRules)
            {
                CalcChainedFollowPos(fRB.fTreeRoots[fRootIx]);
            }

            //
            //  BOF (start of input) test fixup.
            //
            if (fRB.fSetBuilder.SawBOF)
            {
                BofFixup();
            }

            //
            // Build the DFA state transition tables.
            //
            BuildStateTable();
            FlagAcceptingStates();
            FlagLookAheadStates();
            FlagTaggedStates();

            //
            // Update the global table of rule status {tag} values
            // The rule builder has a global vector of status values that are common
            //    for all tables.  Merge the ones from this table into the global set.
            //
            MergeRuleStatusVals();

            if (fRB.fDebugEnv != null && fRB.fDebugEnv.IndexOf("states", StringComparison.Ordinal) >= 0)
            {
                PrintStates();
            }
        }
Beispiel #24
0
        //-----------------------------------------------------------------------------
        //
        //   calcChainedFollowPos.    Modify the previously calculated followPos sets
        //                            to implement rule chaining.  NOT described by Aho
        //
        //-----------------------------------------------------------------------------
        internal virtual void CalcChainedFollowPos(RBBINode tree)
        {
            IList <RBBINode> endMarkerNodes = new JCG.List <RBBINode>();
            IList <RBBINode> leafNodes      = new JCG.List <RBBINode>();

            // get a list of all endmarker nodes.
            tree.FindNodes(endMarkerNodes, RBBINode.endMark);

            // get a list all leaf nodes
            tree.FindNodes(leafNodes, RBBINode.leafChar);

            // Collect all leaf nodes that can start matches for rules
            // with inbound chaining enabled, which is the union of the
            // firstPosition sets from each of the rule root nodes.

            IList <RBBINode> ruleRootNodes = new JCG.List <RBBINode>();

            AddRuleRootNodes(ruleRootNodes, tree);

            ISet <RBBINode> matchStartNodes = new JCG.HashSet <RBBINode>();

            foreach (RBBINode node in ruleRootNodes)
            {
                if (node.fChainIn)
                {
                    matchStartNodes.UnionWith(node.fFirstPosSet);
                }
            }

            // Iterate over all leaf nodes,
            //
            foreach (RBBINode tNode in leafNodes)
            {
                RBBINode endNode = null;

                // Identify leaf nodes that correspond to overall rule match positions.
                //   These include an endMarkerNode in their followPos sets.
                foreach (RBBINode endMarkerNode in endMarkerNodes)
                {
                    if (tNode.fFollowPos.Contains(endMarkerNode))
                    {
                        endNode = tNode;
                        break;
                    }
                }
                if (endNode == null)
                {
                    // node wasn't an end node.  Try again with the next.
                    continue;
                }

                // We've got a node that can end a match.

                // Line Break Specific hack:  If this node's val correspond to the $CM char class,
                //                            don't chain from it.
                // TODO:  Add rule syntax for this behavior, get specifics out of here and
                //        into the rule file.
                if (fRB.fLBCMNoChain)
                {
                    int c = this.fRB.fSetBuilder.GetFirstChar(endNode.fVal);
                    if (c != -1)
                    {
                        // c == -1 occurs with sets containing only the {eof} marker string.
                        int cLBProp = UChar.GetIntPropertyValue(c, UProperty.Line_Break);
                        if (cLBProp == LineBreak.CombiningMark)
                        {
                            continue;
                        }
                    }
                }


                // Now iterate over the nodes that can start a match, looking for ones
                //   with the same char class as our ending node.
                foreach (RBBINode startNode in matchStartNodes)
                {
                    if (startNode.fType != RBBINode.leafChar)
                    {
                        continue;
                    }

                    if (endNode.fVal == startNode.fVal)
                    {
                        // The end val (character class) of one possible match is the
                        //   same as the start of another.

                        // Add all nodes from the followPos of the start node to the
                        //  followPos set of the end node, which will have the effect of
                        //  letting matches transition from a match state at endNode
                        //  to the second char of a match starting with startNode.
                        endNode.fFollowPos.UnionWith(startNode.fFollowPos);
                    }
                }
            }
        }