예제 #1
0
        public MatchSet(OutputModelFactory factory, GrammarAST ast)
            : base(factory, ast)
        {
            SetTransition st       = (SetTransition)ast.atnState.Transition(0);
            int           wordSize = factory.GetGenerator().GetTarget().GetInlineTestSetWordSize();

            expr = new TestSetInline(factory, null, st.set, wordSize);
            Decl.Decl d = new TokenTypeDecl(factory, expr.varName);
            factory.GetCurrentRuleFunction().AddLocalDecl(d);
            capture = new CaptureNextTokenType(factory, expr.varName);
        }
예제 #2
0
        public void Equals_DifferentTypes_ReturnsFalse()
        {
            var comparer = this.GetComparer();

            var transitionA = new SimpleTransition();
            var transitionB = new SetTransition();

            var result = comparer.Equals(transitionA, transitionB);

            Assert.IsFalse(result);
        }
예제 #3
0
        private void HandleSetTransition(SetTransition setTransition, int currentTokenListIndex, TokenizationResult sentenceTokens)
        {
            IToken nextToken = sentenceTokens.Tokens.ElementAt(currentTokenListIndex);

            foreach (int transitionTokenType in setTransition.Label.ToList())
            {
                if (transitionTokenType == nextToken.TokenIndex)
                {
                    GetSuggestionsFromParser(setTransition.target, currentTokenListIndex + 1, sentenceTokens);
                }
            }
        }
예제 #4
0
        public void Equals_EqualSetTransitions_ReturnsTrue()
        {
            var comparer = this.GetComparer();

            var transitionA = new SetTransition {
                GlyphIdSet = new HashSet <ushort> {
                    1, 2
                }
            };
            var transitionB = new SetTransition {
                GlyphIdSet = new HashSet <ushort> {
                    1, 2
                }
            };

            var result = comparer.Equals(transitionA, transitionB);

            Assert.IsTrue(result);
        }
예제 #5
0
        private static Transition updateTransition(Transition t, char openDelimiter, char closeDelimiter)
        {
            Transition updated = null;

            if (t is RuleTransition)
            {
                return(null);
            }
            else if (t is AtomTransition)
            {
                AtomTransition atomTransition = (AtomTransition)t;
                int            newLabel;
                if (atomTransition.label == OpenDelimiterPlaceholder)
                {
                    newLabel = openDelimiter;
                }
                else if (atomTransition.label == CloseDelimiterPlaceholder)
                {
                    newLabel = closeDelimiter;
                }
                else
                {
                    return(null);
                }

                updated = new AtomTransition(t.target, newLabel);
            }
            else if (t is NotSetTransition)
            {
                NotSetTransition notSetTransition = (NotSetTransition)t;
                int removeLabel;
                int addLabel;
                if (notSetTransition.set.Contains(OpenDelimiterPlaceholder))
                {
                    removeLabel = OpenDelimiterPlaceholder;
                    addLabel    = openDelimiter;
                }
                else if (notSetTransition.set.Contains(CloseDelimiterPlaceholder))
                {
                    removeLabel = CloseDelimiterPlaceholder;
                    addLabel    = closeDelimiter;
                }
                else
                {
                    return(null);
                }

                IntervalSet set = new IntervalSet(notSetTransition.set);
                set.Remove(removeLabel);
                set.Add(addLabel);
                set.SetReadonly(true);

                updated = new NotSetTransition(t.target, set);
            }
            else if (t is SetTransition)
            {
                SetTransition setTransition = (SetTransition)t;
                int           removeLabel;
                int           addLabel;
                if (setTransition.set.Contains(OpenDelimiterPlaceholder))
                {
                    removeLabel = OpenDelimiterPlaceholder;
                    addLabel    = openDelimiter;
                }
                else if (setTransition.set.Contains(CloseDelimiterPlaceholder))
                {
                    removeLabel = CloseDelimiterPlaceholder;
                    addLabel    = closeDelimiter;
                }
                else
                {
                    return(null);
                }

                IntervalSet set = new IntervalSet(setTransition.set);
                set.Remove(removeLabel);
                set.Add(addLabel);
                set.SetReadonly(true);

                updated = createSetTransition(t.target, set);
            }
            else if (t is RangeTransition)
            {
                RangeTransition rangeTransition = (RangeTransition)t;
                int             removeLabel;
                int             addLabel;
                if (rangeTransition.from <= OpenDelimiterPlaceholder && rangeTransition.to >= OpenDelimiterPlaceholder)
                {
                    removeLabel = OpenDelimiterPlaceholder;
                    addLabel    = openDelimiter;
                }
                else if (rangeTransition.from <= CloseDelimiterPlaceholder && rangeTransition.to >= CloseDelimiterPlaceholder)
                {
                    removeLabel = CloseDelimiterPlaceholder;
                    addLabel    = closeDelimiter;
                }
                else
                {
                    return(null);
                }

                IntervalSet set = IntervalSet.Of(rangeTransition.from, rangeTransition.to);
                set.Remove(removeLabel);
                set.Add(addLabel);
                set.SetReadonly(true);

                updated = createSetTransition(t.target, set);
            }

            return(updated);
        }
예제 #6
0
        private static void OptimizeSets(Grammar g, ATN atn)
        {
            if (g.IsParser())
            {
                // parser codegen doesn't currently support SetTransition
                return;
            }

            int removedStates = 0;
            IList <DecisionState> decisions = atn.decisionToState;

            foreach (DecisionState decision in decisions)
            {
                if (decision.ruleIndex >= 0)
                {
                    Rule rule = g.GetRule(decision.ruleIndex);
                    if (char.IsLower(rule.name[0]))
                    {
                        // parser codegen doesn't currently support SetTransition
                        continue;
                    }
                }

                IntervalSet setTransitions = new IntervalSet();
                for (int i = 0; i < decision.NumberOfTransitions; i++)
                {
                    Transition epsTransition = decision.Transition(i);
                    if (!(epsTransition is EpsilonTransition))
                    {
                        continue;
                    }

                    if (epsTransition.target.NumberOfTransitions != 1)
                    {
                        continue;
                    }

                    Transition transition = epsTransition.target.Transition(0);
                    if (!(transition.target is BlockEndState))
                    {
                        continue;
                    }

                    if (transition is NotSetTransition)
                    {
                        // TODO: not yet implemented
                        continue;
                    }

                    if (transition is AtomTransition ||
                        transition is RangeTransition ||
                        transition is SetTransition)
                    {
                        setTransitions.Add(i);
                    }
                }

                // due to min alt resolution policies, can only collapse sequential alts
                for (int i = setTransitions.GetIntervals().Count - 1; i >= 0; i--)
                {
                    Interval interval = setTransitions.GetIntervals()[i];
                    if (interval.Length <= 1)
                    {
                        continue;
                    }

                    ATNState    blockEndState = decision.Transition(interval.a).target.Transition(0).target;
                    IntervalSet matchSet      = new IntervalSet();
                    for (int j = interval.a; j <= interval.b; j++)
                    {
                        Transition matchTransition = decision.Transition(j).target.Transition(0);
                        if (matchTransition is NotSetTransition)
                        {
                            throw new NotImplementedException();
                        }

                        IntervalSet set     = matchTransition.Label;
                        int         minElem = set.MinElement;
                        int         maxElem = set.MaxElement;
                        for (int k = minElem; k <= maxElem; k++)
                        {
                            if (matchSet.Contains(k))
                            {
                                char setMin = (char)set.MinElement;
                                char setMax = (char)set.MaxElement;
                                // TODO: Token is missing (i.e. position in source will not be displayed).
                                g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName,
                                                           null, (char)minElem + "-" + (char)maxElem, "[" + setMin + "-" + setMax + "]");
                                break;
                            }
                        }

                        matchSet.AddAll(set);
                    }

                    Transition newTransition;
                    if (matchSet.GetIntervals().Count == 1)
                    {
                        if (matchSet.Count == 1)
                        {
                            newTransition = new AtomTransition(blockEndState, matchSet.MinElement);
                        }
                        else
                        {
                            Interval matchInterval = matchSet.GetIntervals()[0];
                            newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
                        }
                    }
                    else
                    {
                        newTransition = new SetTransition(blockEndState, matchSet);
                    }

                    decision.Transition(interval.a).target.SetTransition(0, newTransition);
                    for (int j = interval.a + 1; j <= interval.b; j++)
                    {
                        Transition removed = decision.Transition(interval.a + 1);
                        decision.RemoveTransition(interval.a + 1);
                        atn.RemoveState(removed.target);
                        removedStates++;
                    }
                }
            }

            //System.Console.WriteLine("ATN optimizer removed " + removedStates + " states by collapsing sets.");
        }
예제 #7
0
        /** Return a String containing a DOT description that, when displayed,
         *  will show the incoming state machine visually.  All nodes reachable
         *  from startState will be included.
         */
        public virtual string GetDOT(ATNState startState, string[] ruleNames, bool isLexer)
        {
            if (startState == null)
            {
                return(null);
            }

            // The output DOT graph for visualization
            ISet <ATNState> markedStates = new HashSet <ATNState>();
            Template        dot          = stlib.GetInstanceOf("atn");

            dot.Add("startState", startState.stateNumber);
            dot.Add("rankdir", rankdir);

            Queue <ATNState> work = new Queue <ATNState>();

            work.Enqueue(startState);
            while (work.Count > 0)
            {
                ATNState s = work.Peek();
                if (markedStates.Contains(s))
                {
                    work.Dequeue();
                    continue;
                }
                markedStates.Add(s);

                // don't go past end of rule node to the follow states
                if (s is RuleStopState)
                {
                    continue;
                }

                // special case: if decision point, then line up the alt start states
                // unless it's an end of block
                //			if ( s instanceof BlockStartState ) {
                //				ST rankST = stlib.getInstanceOf("decision-rank");
                //				DecisionState alt = (DecisionState)s;
                //				for (int i=0; i<alt.getNumberOfTransitions(); i++) {
                //					ATNState target = alt.transition(i).target;
                //					if ( target!=null ) {
                //						rankST.add("states", target.stateNumber);
                //					}
                //				}
                //				dot.add("decisionRanks", rankST);
                //			}

                // make a DOT edge for each transition
                Template edgeST;
                for (int i = 0; i < s.NumberOfTransitions; i++)
                {
                    Transition edge = s.Transition(i);
                    if (edge is RuleTransition)
                    {
                        RuleTransition rr = ((RuleTransition)edge);
                        // don't jump to other rules, but display edge to follow node
                        edgeST = stlib.GetInstanceOf("edge");

                        string label = "<" + ruleNames[rr.ruleIndex];
                        if (((RuleStartState)rr.target).isPrecedenceRule)
                        {
                            label += "[" + rr.precedence + "]";
                        }
                        label += ">";

                        edgeST.Add("label", label);
                        edgeST.Add("src", "s" + s.stateNumber);
                        edgeST.Add("target", "s" + rr.followState.stateNumber);
                        edgeST.Add("arrowhead", arrowhead);
                        dot.Add("edges", edgeST);
                        work.Enqueue(rr.followState);
                        continue;
                    }
                    if (edge is ActionTransition)
                    {
                        edgeST = stlib.GetInstanceOf("action-edge");
                        edgeST.Add("label", GetEdgeLabel(edge.ToString()));
                    }
                    else if (edge is AbstractPredicateTransition)
                    {
                        edgeST = stlib.GetInstanceOf("edge");
                        edgeST.Add("label", GetEdgeLabel(edge.ToString()));
                    }
                    else if (edge.IsEpsilon)
                    {
                        edgeST = stlib.GetInstanceOf("epsilon-edge");
                        edgeST.Add("label", GetEdgeLabel(edge.ToString()));
                        bool loopback = false;
                        if (edge.target is PlusBlockStartState)
                        {
                            loopback = s.Equals(((PlusBlockStartState)edge.target).loopBackState);
                        }
                        else if (edge.target is StarLoopEntryState)
                        {
                            loopback = s.Equals(((StarLoopEntryState)edge.target).loopBackState);
                        }
                        edgeST.Add("loopback", loopback);
                    }
                    else if (edge is AtomTransition)
                    {
                        edgeST = stlib.GetInstanceOf("edge");
                        AtomTransition atom  = (AtomTransition)edge;
                        string         label = atom.label.ToString();
                        if (isLexer)
                        {
                            label = "'" + GetEdgeLabel(((char)atom.label).ToString()) + "'";
                        }
                        else if (grammar != null)
                        {
                            label = grammar.GetTokenDisplayName(atom.label);
                        }
                        edgeST.Add("label", GetEdgeLabel(label));
                    }
                    else if (edge is SetTransition)
                    {
                        edgeST = stlib.GetInstanceOf("edge");
                        SetTransition set   = (SetTransition)edge;
                        string        label = set.Label.ToString();
                        if (isLexer)
                        {
                            label = set.Label.ToString(true);
                        }
                        else if (grammar != null)
                        {
                            label = set.Label.ToString(grammar.GetVocabulary());
                        }
                        if (edge is NotSetTransition)
                        {
                            label = "~" + label;
                        }
                        edgeST.Add("label", GetEdgeLabel(label));
                    }
                    else if (edge is RangeTransition)
                    {
                        edgeST = stlib.GetInstanceOf("edge");
                        RangeTransition range = (RangeTransition)edge;
                        string          label = range.Label.ToString();
                        if (isLexer)
                        {
                            label = range.ToString();
                        }
                        else if (grammar != null)
                        {
                            label = range.Label.ToString(grammar.GetVocabulary());
                        }
                        edgeST.Add("label", GetEdgeLabel(label));
                    }
                    else
                    {
                        edgeST = stlib.GetInstanceOf("edge");
                        edgeST.Add("label", GetEdgeLabel(edge.ToString()));
                    }
                    edgeST.Add("src", "s" + s.stateNumber);
                    edgeST.Add("target", "s" + edge.target.stateNumber);
                    edgeST.Add("arrowhead", arrowhead);
                    if (s.NumberOfTransitions > 1)
                    {
                        edgeST.Add("transitionIndex", i);
                    }
                    else
                    {
                        edgeST.Add("transitionIndex", false);
                    }
                    dot.Add("edges", edgeST);
                    work.Enqueue(edge.target);
                }
            }

            // define nodes we visited (they will appear first in DOT output)
            // this is an example of ST's lazy eval :)
            // define stop state first; seems to be a bug in DOT where doublecircle
            // shape only works if we define them first. weird.
            //		ATNState stopState = startState.atn.ruleToStopState.get(startState.rule);
            //		if ( stopState!=null ) {
            //			ST st = stlib.getInstanceOf("stopstate");
            //			st.add("name", "s"+stopState.stateNumber);
            //			st.add("label", getStateLabel(stopState));
            //			dot.add("states", st);
            //		}
            foreach (ATNState s in markedStates)
            {
                if (!(s is RuleStopState))
                {
                    continue;
                }
                Template st = stlib.GetInstanceOf("stopstate");
                st.Add("name", "s" + s.stateNumber);
                st.Add("label", GetStateLabel(s));
                dot.Add("states", st);
            }

            foreach (ATNState s in markedStates)
            {
                if (s is RuleStopState)
                {
                    continue;
                }
                Template st = stlib.GetInstanceOf("state");
                st.Add("name", "s" + s.stateNumber);
                st.Add("label", GetStateLabel(s));
                st.Add("transitions", s.Transitions);
                dot.Add("states", st);
            }

            return(dot.Render());
        }
예제 #8
0
        protected void CompilePositioningAppendixFromMachine(GlyphTypeface typeface, uint featureId, PositioningAppendix positioning, GlyphClassesAppendix glyphClasses, double emSize, StateMachine machine)
        {
            Debug.Assert(machine.States[0] == machine.EntryState, "First state is not the entry state.");

            PositioningAppendix.Feature feature = new PositioningAppendix.Feature();

            checked
            {
                for (ushort requiredState = 0; requiredState < machine.States.Count; requiredState++)
                {
                    var state = machine.States[requiredState];

                    foreach (var transition in state.Transitions)
                    {
                        PositioningAppendix.Rule rule = new PositioningAppendix.Rule();
                        rule.RequiredState = requiredState;
                        rule.NewState      = (ushort)machine.States.IndexOf(transition.TargetState);

                        if (transition is AlwaysTransition)
                        {
                            rule.Condition = PositioningAppendix.RuleCondition.Unconditional;
                        }
                        else if (transition is SimpleTransition)
                        {
                            rule.Condition          = PositioningAppendix.RuleCondition.Glyph;
                            rule.ConditionParameter = ((SimpleTransition)transition).GlyphId;
                        }
                        else if (transition is SetTransition)
                        {
                            SetTransition setTransition = (SetTransition)transition;
                            int[]         glyphs        = setTransition.GlyphIdSet.Select(id => (int)id).ToArray();

                            GlyphClassesAppendix.Coverage coverage = glyphClasses.FindCoverage(glyphs);
                            if (coverage == null)
                            {
                                coverage = glyphClasses.AppendCoverage(glyphs);
                            }

                            if (!glyphClasses.Coverages.Contains(coverage))
                            {
                                glyphClasses.Coverages.Add(coverage);
                            }

                            rule.Condition          = SubstitutionAppendix.RuleCondition.GlyphClass;
                            rule.ConditionParameter = (ushort)glyphClasses.Coverages.IndexOf(coverage);
                        }
                        else
                        {
                            Debug.Assert(false, "Unknown condition: " + transition.GetType());
                            continue;
                        }

                        if (transition.Action == null)
                        {
                            rule.Action = PositioningAppendix.RuleAction.Nothing;
                        }
                        else
                        {
                            if (transition.Action is AnchorPointToAnchorPointAction)
                            {
                                AnchorPointToAnchorPointAction anchorAction = transition.Action as AnchorPointToAnchorPointAction;

                                sbyte x = ToPixels(anchorAction.PreviousGlyphAnchorPoint.X - anchorAction.CurrentGlyphAnchorPoint.X, emSize, typeface);
                                sbyte y = ToPixels(anchorAction.PreviousGlyphAnchorPoint.Y - anchorAction.CurrentGlyphAnchorPoint.Y, emSize, typeface);

                                rule.Action          = PositioningAppendix.RuleAction.PositionOffset;
                                rule.ActionParameter = Pack(x, y);
                                rule.ActionOffset    = 0;
                            }
                            else if (transition.Action is PositioningAdjustmentAction)
                            {
                                PositioningAdjustmentAction positioningAction = transition.Action as PositioningAdjustmentAction;
                                List <GlyphPositionChange>  changes           = positioningAction.PositionChanges.ToList();

                                rule.ActionOffset = (sbyte)(1 - changes.Count);

                                rule.TapeMovement = (sbyte)TrimEnd(changes);
                                if (changes.Count == 0)
                                {
                                    rule.Action       = SubstitutionAppendix.RuleAction.Nothing;
                                    rule.ActionOffset = 0;
                                }
                                else
                                {
                                    if (changes.Count == 1)
                                    {
                                        GlyphPositionChange position = changes[0];

                                        if ((position.AdvanceX != 0 || position.AdvanceY != 0) && (position.OffsetX == 0 && position.OffsetY == 0))
                                        {
                                            sbyte x = ToPixels(position.AdvanceX, emSize, typeface);
                                            sbyte y = ToPixels(position.AdvanceY, emSize, typeface);

                                            rule.Action          = PositioningAppendix.RuleAction.PositionAdvance;
                                            rule.ActionParameter = Pack(x, y);
                                        }
                                        else if ((position.OffsetX != 0 || position.OffsetY != 0) && (position.AdvanceX == 0 && position.AdvanceY == 0))
                                        {
                                            sbyte x = ToPixels(position.OffsetX, emSize, typeface);
                                            sbyte y = ToPixels(position.OffsetY, emSize, typeface);

                                            rule.Action          = PositioningAppendix.RuleAction.PositionOffset;
                                            rule.ActionParameter = Pack(x, y);
                                        }
                                    }

                                    if (rule.Action == StateMachineAppendix.RuleAction.Nothing)
                                    {
                                        PositioningAppendix.PositionChangesParameters parameters = new StateMachineAppendix.PositionChangesParameters();

                                        foreach (GlyphPositionChange position in changes)
                                        {
                                            PositioningAppendix.PositionChange change = new StateMachineAppendix.PositionChange(ToPixels(position.OffsetX, emSize, typeface), ToPixels(position.OffsetY, emSize, typeface), ToPixels(position.AdvanceX, emSize, typeface), ToPixels(position.AdvanceY, emSize, typeface));
                                            parameters.PositionChanges.Add(change);
                                        }

                                        rule.Action          = PositioningAppendix.RuleAction.PositionComplex;
                                        rule.ActionParameter = positioning.AppendParameters(parameters);
                                    }
                                }
                            }
                            else
                            {
                                Debug.Assert(false, "Unknown transition action: " + transition.Action.GetType());
                                continue;
                            }
                        }

                        rule.TapeMovement += (sbyte)(transition.HeadShift);
                        feature.Rules.Add(rule);
                    }
                }
            }

            positioning.Features.Add(feature);
            positioning.FeatureOffsets.Add(new SubstitutionAppendix.FeatureOffset {
                Tag = featureId
            });
        }
예제 #9
0
        protected void CompileSubstitutionAppendingFromMachine(uint featureId, SubstitutionAppendix substitution, GlyphClassesAppendix glyphClasses, StateMachine machine)
        {
            Debug.Assert(machine.States[0] == machine.EntryState, "First state is not the entry state.");

            SubstitutionAppendix.Feature feature = new SubstitutionAppendix.Feature();

            checked
            {
                for (ushort requiredState = 0; requiredState < machine.States.Count; requiredState++)
                {
                    var state = machine.States[requiredState];

                    foreach (var transition in state.Transitions)
                    {
                        SubstitutionAppendix.Rule rule = new SubstitutionAppendix.Rule();
                        rule.RequiredState = requiredState;
                        rule.NewState      = (ushort)machine.States.IndexOf(transition.TargetState);

                        if (transition is AlwaysTransition)
                        {
                            rule.Condition = SubstitutionAppendix.RuleCondition.Unconditional;
                        }
                        else if (transition is SimpleTransition)
                        {
                            rule.Condition          = SubstitutionAppendix.RuleCondition.Glyph;
                            rule.ConditionParameter = ((SimpleTransition)transition).GlyphId;
                        }
                        else if (transition is SetTransition)
                        {
                            SetTransition setTransition = (SetTransition)transition;
                            int[]         glyphs        = setTransition.GlyphIdSet.Select(id => (int)id).ToArray();

                            GlyphClassesAppendix.Coverage coverage = glyphClasses.FindCoverage(glyphs);
                            if (coverage == null)
                            {
                                coverage = glyphClasses.AppendCoverage(glyphs);
                            }

                            if (!glyphClasses.Coverages.Contains(coverage))
                            {
                                glyphClasses.Coverages.Add(coverage);
                            }

                            rule.Condition          = SubstitutionAppendix.RuleCondition.GlyphClass;
                            rule.ConditionParameter = (ushort)glyphClasses.Coverages.IndexOf(coverage);
                        }
                        else
                        {
                            Debug.Assert(false, "Unknown condition: " + transition.GetType());
                            continue;
                        }

                        if (transition.Action == null)
                        {
                            rule.Action = SubstitutionAppendix.RuleAction.Nothing;
                        }
                        else
                        {
                            SubstitutionAction action = transition.Action as SubstitutionAction;
                            if (action == null)
                            {
                                Debug.Assert(false, "Unknown action: " + transition.Action.GetType());
                                continue;
                            }

                            int replacementGlyphCount = action.ReplacementGlyphIds.Count();
                            if (replacementGlyphCount == 1 && action.ReplacedGlyphCount <= 3)
                            {
                                if (action.ReplacedGlyphCount == 0)
                                {
                                    rule.Action = SubstitutionAppendix.RuleAction.GlyphInsertion;
                                }
                                else if (action.ReplacedGlyphCount == 1)
                                {
                                    rule.Action = SubstitutionAppendix.RuleAction.GlyphOverwrite;
                                }
                                else if (action.ReplacedGlyphCount == 2)
                                {
                                    rule.Action = SubstitutionAppendix.RuleAction.GlyphRewrite_2_1;
                                }
                                else if (action.ReplacedGlyphCount == 3)
                                {
                                    rule.Action = SubstitutionAppendix.RuleAction.GlyphRewrite_3_1;
                                }
                                else
                                {
                                    Debug.Assert(false, "Unknown action: " + action);
                                    continue;
                                }

                                rule.ActionParameter = action.ReplacementGlyphIds.First();
                            }
                            else if (replacementGlyphCount == 0)
                            {
                                rule.Action          = SubstitutionAppendix.RuleAction.GlyphDeletion;
                                rule.ActionParameter = (ushort)action.ReplacedGlyphCount;
                            }
                            else
                            {
                                rule.Action          = SubstitutionAppendix.RuleAction.GlyphRewrite_N_M;
                                rule.ActionParameter = substitution.AppendParameters(new SubstitutionAppendix.GlyphRewriteParameters((byte)action.ReplacedGlyphCount, action.ReplacementGlyphIds.Select(g => (int)g).ToArray()));
                            }

                            rule.ActionOffset = (sbyte)(1 - action.SkippedGlyphCount - action.ReplacedGlyphCount);
                            rule.TapeMovement = (sbyte)action.SkippedGlyphCount;
                        }

                        rule.TapeMovement += (sbyte)(transition.HeadShift);
                        feature.Rules.Add(rule);
                    }
                }
            }

            substitution.Features.Add(feature);
            substitution.FeatureOffsets.Add(new SubstitutionAppendix.FeatureOffset {
                Tag = featureId
            });
        }
예제 #10
0
        public override Handle Set(GrammarAST associatedAST, IList <GrammarAST> alts, bool invert)
        {
            ATNState    left  = NewState(associatedAST);
            ATNState    right = NewState(associatedAST);
            IntervalSet set   = new IntervalSet();

            foreach (GrammarAST t in alts)
            {
                if (t.Type == ANTLRParser.RANGE)
                {
                    int a = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(0).Text);
                    int b = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(1).Text);
                    if (CheckRange((GrammarAST)t.GetChild(0), (GrammarAST)t.GetChild(1), a, b))
                    {
                        CheckSetCollision(associatedAST, set, a, b);
                        set.Add(a, b);
                    }
                }
                else if (t.Type == ANTLRParser.LEXER_CHAR_SET)
                {
                    set.AddAll(GetSetFromCharSetLiteral(t));
                }
                else if (t.Type == ANTLRParser.STRING_LITERAL)
                {
                    int c = CharSupport.GetCharValueFromGrammarCharLiteral(t.Text);
                    if (c != -1)
                    {
                        CheckSetCollision(associatedAST, set, c);
                        set.Add(c);
                    }
                    else
                    {
                        g.tool.errMgr.GrammarError(ErrorType.INVALID_LITERAL_IN_LEXER_SET,
                                                   g.fileName, t.Token, t.Text);
                    }
                }
                else if (t.Type == ANTLRParser.TOKEN_REF)
                {
                    g.tool.errMgr.GrammarError(ErrorType.UNSUPPORTED_REFERENCE_IN_LEXER_SET,
                                               g.fileName, t.Token, t.Text);
                }
            }
            if (invert)
            {
                left.AddTransition(new NotSetTransition(right, set));
            }
            else
            {
                Transition transition;
                if (set.GetIntervals().Count == 1)
                {
                    Interval interval = set.GetIntervals()[0];
                    transition = new RangeTransition(right, interval.a, interval.b);
                }
                else
                {
                    transition = new SetTransition(right, set);
                }

                left.AddTransition(transition);
            }
            associatedAST.atnState = left;
            return(new Handle(left, right));
        }
예제 #11
0
        public virtual string AsString()
        {
            if (start == null)
            {
                return(null);
            }
            marked = new HashSet <ATNState>();

            work = new List <ATNState>();
            work.Add(start);

            StringBuilder buf = new StringBuilder();
            ATNState      s;

            while (work.Count > 0)
            {
                s = work[0];
                work.RemoveAt(0);
                if (marked.Contains(s))
                {
                    continue;
                }
                int n = s.NumberOfTransitions;
                //System.Console.WriteLine("visit " + s + "; edges=" + n);
                marked.Add(s);
                for (int i = 0; i < n; i++)
                {
                    Transition t = s.Transition(i);
                    if (!(s is RuleStopState))
                    { // don't add follow states to work
                        if (t is RuleTransition)
                        {
                            work.Add(((RuleTransition)t).followState);
                        }
                        else
                        {
                            work.Add(t.target);
                        }
                    }
                    buf.Append(GetStateString(s));
                    if (t is EpsilonTransition)
                    {
                        buf.Append("->").Append(GetStateString(t.target)).Append('\n');
                    }
                    else if (t is RuleTransition)
                    {
                        buf.Append("-").Append(g.GetRule(((RuleTransition)t).ruleIndex).name).Append("->").Append(GetStateString(t.target)).Append('\n');
                    }
                    else if (t is ActionTransition)
                    {
                        ActionTransition a = (ActionTransition)t;
                        buf.Append("-").Append(a.ToString()).Append("->").Append(GetStateString(t.target)).Append('\n');
                    }
                    else if (t is SetTransition)
                    {
                        SetTransition st  = (SetTransition)t;
                        bool          not = st is NotSetTransition;
                        if (g.IsLexer())
                        {
                            buf.Append("-").Append(not ? "~" : "").Append(st.ToString()).Append("->").Append(GetStateString(t.target)).Append('\n');
                        }
                        else
                        {
                            buf.Append("-").Append(not ? "~" : "").Append(st.Label.ToString(g.GetVocabulary())).Append("->").Append(GetStateString(t.target)).Append('\n');
                        }
                    }
                    else if (t is AtomTransition)
                    {
                        AtomTransition a     = (AtomTransition)t;
                        string         label = g.GetTokenDisplayName(a.label);
                        buf.Append("-").Append(label).Append("->").Append(GetStateString(t.target)).Append('\n');
                    }
                    else
                    {
                        buf.Append("-").Append(t.ToString()).Append("->").Append(GetStateString(t.target)).Append('\n');
                    }
                }
            }
            return(buf.ToString());
        }
예제 #12
0
        protected override ATNState GetReachableTarget(ATNConfig source, Transition trans, int ttype)
        {
            if (ttype == AntlrV4.CaretToken.CaretTokenType)
            {
                ATNState       target         = null;
                AtomTransition atomTransition = trans as AtomTransition;
                if (atomTransition != null)
                {
                    if (GetWordlikeTokenTypes().Contains(atomTransition.label))
                    {
                        target = atomTransition.target;
                    }
                }
                else
                {
                    SetTransition setTransition = trans as SetTransition;
                    if (setTransition != null)
                    {
                        bool not = trans is NotSetTransition;
                        foreach (int t in GetWordlikeTokenTypes().ToArray())
                        {
                            if (!not && setTransition.set.Contains(t) || not && !setTransition.set.Contains(t))
                            {
                                target = setTransition.target;
                                break;
                            }
                        }
                    }
                    else
                    {
                        RangeTransition rangeTransition = trans as RangeTransition;
                        if (rangeTransition != null)
                        {
                            // TODO: there must be a better algorithm here
                            int[] wordlikeTokenTypes = GetWordlikeTokenTypes().ToArray();
                            int   lowerBound         = Array.BinarySearch(wordlikeTokenTypes, rangeTransition.from);
                            int   upperBound         = Array.BinarySearch(wordlikeTokenTypes, rangeTransition.to);
                            if (lowerBound >= 0 || upperBound >= 0 || lowerBound != upperBound)
                            {
                                target = rangeTransition.target;
                            }
                        }
                        else
                        {
                            WildcardTransition wildcardTransition = trans as WildcardTransition;
                            if (wildcardTransition != null)
                            {
                                target = trans.target;
                            }
                        }
                    }
                }

                if (_caretTransitions == null)
                {
                    _caretTransitions = new Dictionary <ATNConfig, IList <Transition> >();
                }

                IList <Transition> configTransitions;
                if (!_caretTransitions.TryGetValue(source, out configTransitions))
                {
                    configTransitions         = new List <Transition>();
                    _caretTransitions[source] = configTransitions;
                }

                configTransitions.Add(trans);
                return(target);
            }

            return(base.GetReachableTarget(source, trans, ttype));
        }