protected internal virtual DFAState AddDFAState(ATNConfigSet configs) { System.Diagnostics.Debug.Assert(!configs.HasSemanticContext); DFAState proposed = new DFAState(configs, 0, MaxDfaEdge); DFAState existing; if (atn.modeToDFA[mode].states.TryGetValue(proposed, out existing)) { return(existing); } configs.OptimizeConfigs(this); DFAState newState = new DFAState(configs.Clone(true), 0, MaxDfaEdge); ATNConfig firstConfigWithRuleStopState = null; foreach (ATNConfig c in configs) { if (c.State is RuleStopState) { firstConfigWithRuleStopState = c; break; } } if (firstConfigWithRuleStopState != null) { newState.isAcceptState = true; newState.lexerActionExecutor = firstConfigWithRuleStopState.ActionExecutor; newState.prediction = atn.ruleToTokenType[firstConfigWithRuleStopState.State.ruleIndex]; } return(atn.modeToDFA[mode].AddState(newState)); }
public virtual int Match([NotNull] ICharStream input, int mode) { match_calls++; this.mode = mode; int mark = input.Mark(); try { this.startIndex = input.Index; this.prevAccept.Reset(); DFAState s0 = atn.modeToDFA[mode].s0.Get(); if (s0 == null) { return(MatchATN(input)); } else { return(ExecATN(input, s0)); } } finally { input.Release(mark); } }
protected int MatchATN(ICharStream input) { ATNState startState = atn.modeToStartState[mode]; if (debug) { Console.WriteLine("matchATN mode " + mode + " start: " + startState); } int old_mode = mode; ATNConfigSet s0_closure = ComputeStartState(input, startState); bool suppressEdge = s0_closure.hasSemanticContext; s0_closure.hasSemanticContext = false; DFAState next = AddDFAState(s0_closure); if (!suppressEdge) { decisionToDFA[mode].s0 = next; } int predict = ExecATN(input, next); if (debug) { Console.WriteLine("DFA after matchATN: " + decisionToDFA[old_mode].ToString()); } return(predict); }
public void Reset() { index = -1; line = 0; charPos = -1; dfaState = null; }
protected internal virtual DFAState AddDFAEdge([NotNull] DFAState from, int t, [NotNull] ATNConfigSet q) { /* leading to this call, ATNConfigSet.hasSemanticContext is used as a * marker indicating dynamic predicate evaluation makes this edge * dependent on the specific input sequence, so the static edge in the * DFA should be omitted. The target DFAState is still created since * execATN has the ability to resynchronize with the DFA state cache * following the predicate evaluation step. * * TJP notes: next time through the DFA, we see a pred again and eval. * If that gets us to a previously created (but dangling) DFA * state, we can continue in pure DFA mode from there. */ bool suppressEdge = q.HasSemanticContext; if (suppressEdge) { q.ClearExplicitSemanticContext(); } DFAState to = AddDFAState(q); if (suppressEdge) { return(to); } AddDFAEdge(from, t, to); return(to); }
public SimulatorState(ParserRuleContext outerContext, DFAState s0, bool useContext, ParserRuleContext remainingOuterContext) { this.outerContext = outerContext != null ? outerContext : ParserRuleContext.EmptyContext; this.s0 = s0; this.useContext = useContext; this.remainingOuterContext = remainingOuterContext; }
protected internal virtual DFAState AddDFAState([NotNull] ATNConfigSet configs) { /* the lexer evaluates predicates on-the-fly; by this point configs * should not contain any configurations with unevaluated predicates. */ System.Diagnostics.Debug.Assert(!configs.HasSemanticContext); DFAState proposed = new DFAState(atn.modeToDFA[mode], configs); DFAState existing; if (atn.modeToDFA[mode].states.TryGetValue(proposed, out existing)) { return(existing); } configs.OptimizeConfigs(this); DFAState newState = new DFAState(atn.modeToDFA[mode], configs.Clone(true)); ATNConfig firstConfigWithRuleStopState = null; foreach (ATNConfig c in configs) { if (c.State is RuleStopState) { firstConfigWithRuleStopState = c; break; } } if (firstConfigWithRuleStopState != null) { int prediction = atn.ruleToTokenType[firstConfigWithRuleStopState.State.ruleIndex]; LexerActionExecutor lexerActionExecutor = firstConfigWithRuleStopState.ActionExecutor; newState.AcceptStateInfo = new AcceptStateInfo(prediction, lexerActionExecutor); } return(atn.modeToDFA[mode].AddState(newState)); }
protected internal virtual void AddDFAEdge([NotNull] DFAState p, int t, [NotNull] DFAState q) { if (p != null) { p.SetTarget(t, q); } }
// forces unicode to stay in ATN protected internal virtual void Reset() { index = -1; line = 0; charPos = -1; dfaState = null; }
protected internal override DFAState GetExistingTargetState(DFAState previousD, int t) { // this method is called after each time the input position advances if (currentState.useContext) { _llStopIndex = _input.Index; } else { _sllStopIndex = _input.Index; } DFAState existingTargetState = base.GetExistingTargetState(previousD, t); if (existingTargetState != null) { // this method is directly called by execDFA; must construct a SimulatorState // to represent the current state for this case currentState = new SimulatorState(currentState.outerContext, existingTargetState, currentState.useContext, currentState.remainingOuterContext); if (currentState.useContext) { decisions[currentDecision].LL_DFATransitions++; } else { decisions[currentDecision].SLL_DFATransitions++; } // count only if we transition over a DFA state if (existingTargetState == Error) { SimulatorState state = new SimulatorState(currentState.outerContext, previousD, currentState.useContext, currentState.remainingOuterContext); decisions[currentDecision].errors.Add(new ErrorInfo(currentDecision, state, _input, _startIndex, _input.Index)); } } return(existingTargetState); }
protected virtual string GetStateLabel(DFAState s) { if (s == null) { return("null"); } StringBuilder buf = new StringBuilder(250); buf.Append('s'); buf.Append(s.stateNumber); if (s.IsAcceptState) { buf.Append("=>").Append(s.Prediction); } if (grammar != null) { Runtime.Sharpen.BitSet alts = s.configs.RepresentedAlternatives; buf.Append("\\n"); IEnumerable <ATNConfig> configurations = s.configs; for (int alt = alts.NextSetBit(0); alt >= 0; alt = alts.NextSetBit(alt + 1)) { if (alt > alts.NextSetBit(0)) { buf.Append("\\n"); } buf.Append("alt"); buf.Append(alt); buf.Append(':'); // get a list of configs for just this alt // it will help us print better later IList <ATNConfig> configsInAlt = new List <ATNConfig>(); foreach (ATNConfig c in configurations) { if (c.Alt != alt) { continue; } configsInAlt.Add(c); } int n = 0; for (int cIndex = 0; cIndex < configsInAlt.Count; cIndex++) { ATNConfig c = configsInAlt[cIndex]; n++; buf.Append(c.ToString(null, false)); if ((cIndex + 1) < configsInAlt.Count) { buf.Append(", "); } if (n % 5 == 0 && (configsInAlt.Count - cIndex) > 3) { buf.Append("\\n"); } } } } string stateLabel = buf.ToString(); return(stateLabel); }
//This Method Compare 2 DFAStates public bool isEqualto(DFAState otherState) { if (states.Count == otherState.states.Count) { bool ans = true; foreach (State x in otherState.states) { ans = false; foreach (State y in states) { if (x.Name == y.Name) { ans = true; break; } } if (!ans) { break; } } return(ans); } else { return(false); } }
protected override void ReportAmbiguity(DFA dfa, DFAState D, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configSet) { int prediction; if (ambigAlts != null) { prediction = ambigAlts.NextSetBit(0); } else { prediction = configSet.GetAlts().NextSetBit(0); } if (configSet.fullCtx && prediction != conflictingAltResolvedBySLL) { // Even though this is an ambiguity we are reporting, we can // still detect some context sensitivities. Both SLL and LL // are showing a conflict, hence an ambiguity, but if they resolve // to different minimum alternatives we have also identified a // context sensitivity. decisions[currentDecision].contextSensitivities.Add(new ContextSensitivityInfo(currentDecision, null /*configs*/, input, startIndex, stopIndex)); } decisions[currentDecision].ambiguities.Add( new AmbiguityInfo(currentDecision, null /*configs, ambigAlts*/, input, startIndex, stopIndex /*, configs.IsFullContext*/) ); base.ReportAmbiguity(dfa, D, startIndex, stopIndex, exact, ambigAlts, configSet); }
protected override DFAState ComputeTargetState(DFA dfa, DFAState previousD, int t) { DFAState state = base.ComputeTargetState(dfa, previousD, t); currentState = state; return(state); }
static DFAState InitERROR() { DFAState state = new DFAState(new ATNConfigSet()); state.stateNumber = Int32.MaxValue; return(state); }
private static void InitNumericStates() { DFAState <int, LexicalFunction> s5 = AddState(new LexicalState(5, false)); //Numeric begin state; DFAState <int, LexicalFunction> s6 = AddState(new LexicalState(6, false)); //Number dot state; DFAState <int, LexicalFunction> s7 = AddState(new LexicalState(7, true, LexicalFunction.OutputNumeric)); //Number quit state; //s0 [0-9] s5 s0.AddNextState('0', '9', s5.Id); s0.AddNextState('0', '9', s5.Id); //s5 [0-9] s5 s5.AddNextState('0', '9', s5.Id); s5.AddNextState('0', '9', s5.Id); //s5 [\.] s6 s5.AddNextState('.', s6.Id); //s5 else s7 (integer) s5.AddElseState(s7.Id); //s6 [0-9] s6 s6.AddNextState('0', '9', s6.Id); s6.AddNextState('0', '9', s6.Id); //s6 else s7 (float) s6.AddElseState(s7.Id); }
public static void Init() { var percent_terminal = new DFAState() { IsTerminal = true, Type = NumType.percent }; var dec_terminal = new DFAState() { IsTerminal = true, NextDict = new Dictionary <StateType, DFAState>(), Type = NumType.dec }; var cdot_terminal = new DFAState() { IsTerminal = true, NextDict = new Dictionary <StateType, DFAState>(), Type = NumType.cdot }; var frac = new DFAState() { NextDict = new Dictionary <StateType, DFAState>(), Type = NumType.frac }; var colon = new DFAState() { NextDict = new Dictionary <StateType, DFAState>(), Type = NumType.colon }; var num_terminal = new DFAState() { IsTerminal = true, NextDict = new Dictionary <StateType, DFAState>(), Type = NumType.num }; num_terminal.NextDict.Add(StateType.percent, percent_terminal); num_terminal.NextDict.Add(StateType.num, num_terminal); num_terminal.NextDict.Add(StateType.dec, dec_terminal); num_terminal.NextDict.Add(StateType.cdot, cdot_terminal); num_terminal.NextDict.Add(StateType.frac, frac); num_terminal.NextDict.Add(StateType.colon, colon); dec_terminal.NextDict.Add(StateType.num, num_terminal); frac.NextDict.Add(StateType.num, num_terminal); cdot_terminal.NextDict.Add(StateType.num, num_terminal); colon.NextDict.Add(StateType.num, num_terminal); var sign = new DFAState() { NextDict = new Dictionary <StateType, DFAState>(), Type = NumType.signed }; sign.NextDict.Add(StateType.num, num_terminal); _start = new DFAState() { NextDict = new Dictionary <StateType, DFAState>() }; _start.NextDict.Add(StateType.num, num_terminal); _start.NextDict.Add(StateType.signed, sign); }
public void AddDfaState(TransitionState stateDfa, Hashset setEclosure) { DFAState stateRecord = new DFAState(); stateRecord.SetEclosure = setEclosure; _stateHashTable[stateDfa] = stateRecord; }
protected internal virtual void CaptureSimState(LexerATNSimulator.SimState settings , ICharStream input, DFAState dfaState) { settings.index = input.Index; settings.line = line; settings.charPos = charPositionInLine; settings.dfaState = dfaState; }
protected void CaptureSimState(SimState settings, ICharStream input, DFAState dfaState) { settings.index = input.Index; settings.line = thisLine; settings.charPos = charPositionInLine; settings.dfaState = dfaState; }
public GrammarInsufficientPredicatesMessage(DecisionProbe probe, DFAState problemState, IDictionary <int, ICollection <IToken> > altToLocations) : base(ErrorManager.MSG_INSUFFICIENT_PREDICATES) { this.probe = probe; this.problemState = problemState; this.altToLocations = altToLocations; }
private void WalkCreatingDfaDgml(DFAState dfaState) { if (!_markedStates.Add(dfaState.StateNumber)) { return; } // first add this node string nodeCategory; if (dfaState.IsAcceptState) { nodeCategory = Categories.StopState; } else { nodeCategory = Categories.State; } XElement node = new XElement(Elements.Node, new XAttribute(Attributes.Id, "state_" + dfaState.StateNumber), new XAttribute(Attributes.Label, GetStateLabel(dfaState)), new XAttribute(Attributes.Category, nodeCategory)); _nodes.Add(dfaState, node); if (GroupNodes) { _extraLinks.Add(CreateContainmentLink(_groupId, "state_" + dfaState.StateNumber)); } // make an edge for each transition for (int i = 0; i < dfaState.NumberOfTransitions; i++) { Transition edge = dfaState.GetTransition(i); if (StripNonreducedStates) { DFAState target = edge.Target as DFAState; // don't generate nodes for terminal states if (target != null && target.AcceptStateReachable != Reachable.Yes) { continue; } } string edgeCategory = Categories.Edge; XElement edgeElement = new XElement(Elements.Link, new XAttribute(Attributes.Source, "state_" + dfaState.StateNumber), new XAttribute(Attributes.Target, "state_" + edge.Target.StateNumber), new XAttribute(Attributes.Category, edgeCategory), new XAttribute(Attributes.Label, GetEdgeLabel(edge))); _links.Add(new KeyValuePair <State, Transition>(dfaState, edge), edgeElement); WalkCreatingDfaDgml((DFAState)edge.Target); } }
public string GenerateGraph(State state) { _nodes.Clear(); _links.Clear(); _extraNodes.Clear(); _extraLinks.Clear(); _markedStates.Clear(); _canSkipStates.Clear(); DFAState dfaState = state as DFAState; if (GroupNodes) { if (dfaState != null) { _groupId = "decision_"; _extraNodes.Add(new XElement(Elements.Node, new XAttribute(Attributes.Id, _groupId), new XAttribute(Attributes.Label, dfaState.StateNumber.ToString()), new XAttribute(Attributes.Group, "Collapsed"))); } else { NFAState nfaState = (NFAState)state; _groupId = "rule_" + nfaState.enclosingRule.Name; _extraNodes.Add(new XElement(Elements.Node, new XAttribute(Attributes.Id, _groupId), new XAttribute(Attributes.Label, nfaState.enclosingRule.Name), new XAttribute(Attributes.Group, "Collapsed"))); } } if (dfaState != null) { WalkCreatingDfaDgml(dfaState); } else { WalkRuleNfaCreatingDgml(state); LocateVerboseStates(state); } XDocument document = new XDocument( new XDeclaration("1.0", "utf-8", "yes"), new XElement(Elements.DirectedGraph, new XAttribute(Attributes.GraphDirection, GraphDirection.TopToBottom), new XAttribute(Attributes.Layout, Layout.Sugiyama), GetNodes(), GetLinks(), GetCategories(), GetProperties(), GetStyles())); return(document.ToString()); }
protected internal virtual int ExecATN(ICharStream input, DFAState ds0) { //System.out.println("enter exec index "+input.index()+" from "+ds0.configs); int t = input.La(1); DFAState s = ds0; // s is current/from DFA state while (true) { // while more work // As we move src->trg, src->trg, we keep track of the previous trg to // avoid looking up the DFA state again, which is expensive. // If the previous target was already part of the DFA, we might // be able to avoid doing a reach operation upon t. If s!=null, // it means that semantic predicates didn't prevent us from // creating a DFA state. Once we know s!=null, we check to see if // the DFA state has an edge already for t. If so, we can just reuse // it's configuration set; there's no point in re-computing it. // This is kind of like doing DFA simulation within the ATN // simulation because DFA simulation is really just a way to avoid // computing reach/closure sets. Technically, once we know that // we have a previously added DFA state, we could jump over to // the DFA simulator. But, that would mean popping back and forth // a lot and making things more complicated algorithmically. // This optimization makes a lot of sense for loops within DFA. // A character will take us back to an existing DFA state // that already has lots of edges out of it. e.g., .* in comments. DFAState target = GetExistingTargetState(s, t); if (target == null) { target = ComputeTargetState(input, s, t); } if (target == Error) { break; } if (target.isAcceptState) { CaptureSimState(prevAccept, input, target); if (t == IntStreamConstants.Eof) { break; } } if (t != IntStreamConstants.Eof) { Consume(input); t = input.La(1); } s = target; } // flip; current DFA target becomes new src/from state return(FailOrAccept(prevAccept, input, s.configs, t)); }
protected internal virtual DFAState GetExistingTargetState(DFAState s, int t) { DFAState target = s.GetTarget(t); #if !PORTABLE if (debug && target != null) { System.Console.Out.WriteLine("reuse state " + s.stateNumber + " edge to " + target.stateNumber); } #endif return(target); }
private static void InitChineseStates() { DFAState <int, LexicalFunction> s8 = AddState(new LexicalState(8, false)); //Numeric begin state; DFAState <int, LexicalFunction> s9 = AddState(new LexicalState(9, true, LexicalFunction.OutputChinese)); //Number quit state; //s0 [4e00-9fa5] s5 s0.AddNextState(0x4e00, 0x9fa5, s8.Id); s8.AddNextState(0x4e00, 0x9fa5, s8.Id); s8.AddElseState(s9.Id); }
public static NumMatch Match(string input, int start) { var nm = new NumMatch(); var typeCache = NumType.unknown; if (string.IsNullOrEmpty(input) || start < 0) { return(nm); } DFAState cur = _start; DFAState pre = null; for (int i = start; i < input.Length; i++) { pre = cur; cur = cur.Process(input[i]); if (cur == null) { if (!pre.IsTerminal) // 上一个状态非终止状态 { nm.Length--; } return(nm); } if (cur.Type == NumType.signed) { nm.IsSigned = true; } nm.Length++; if (cur.IsTerminal) { if (typeCache > nm.Type) { nm.Type = typeCache; } } if (cur.Type > nm.Type) { if (cur.IsTerminal) { nm.Type = cur.Type; } else { typeCache = cur.Type; } } } return(nm); }
public RecursionOverflowMessage(DecisionProbe probe, DFAState sampleBadState, int alt, ICollection <string> targetRules, ICollection <ICollection <NFAState> > callSiteStates) : base(ErrorManager.MSG_RECURSION_OVERLOW) { this.probe = probe; this.sampleBadState = sampleBadState; this.alt = alt; this.targetRules = targetRules; this.callSiteStates = callSiteStates; }
private static void InitCommentAndArithmeticStates() { ArithmeticTbl.Add("+", SyntaxType.Plus); ArithmeticTbl.Add("-", SyntaxType.Subtract); ArithmeticTbl.Add("*", SyntaxType.Multiply); ArithmeticTbl.Add("/", SyntaxType.Divide); DFAState <int, LexicalFunction> s8 = AddState(new LexicalState(8, false)); //Comment / state; DFAState <int, LexicalFunction> s9 = AddState(new LexicalState(9, false)); //Comment * start state; DFAState <int, LexicalFunction> s10 = AddState(new LexicalState(10, false)); //Comment * end state; DFAState <int, LexicalFunction> s11 = AddState(new LexicalState(11, false)); //Comment - first state; DFAState <int, LexicalFunction> s12 = AddState(new LexicalState(12, false)); //Comment - second state; DFAState <int, LexicalFunction> s13 = AddState(new LexicalState(13, true, LexicalFunction.OutputComment)); //Comment quit state; DFAState <int, LexicalFunction> s14 = AddState(new LexicalState(14, false)); //Arithmetic begin state; DFAState <int, LexicalFunction> s15 = AddState(new LexicalState(15, true, LexicalFunction.OutputArithmeticOpr)); //Arithmetic quit state; //s0 [\/] s8 s0.AddNextState('/', s8.Id); //s8 [\*] s9 s8.AddNextState('*', s9.Id); s8.AddElseState(s15.Id); //s9 [\*] s10 s9.AddNextState('*', s10.Id); //s9 else s9 s9.AddElseState(s9.Id); //s10 [\/] s13 s10.AddNextState('/', s13.Id); //s10 else s9 s10.AddElseState(s9.Id); //s0 [-] s11 s0.AddNextState('-', s11.Id); //s11 [-] s12 s11.AddNextState('-', s12.Id); //For comment s11.AddNextState('0', '9', 5); //For Negative numeric s11.AddElseState(s15.Id); //s12 [\n] s13 s12.AddNextState(new int[] { 0, '\n' }, s13.Id); s12.AddElseState(s12.Id); //s0 [\+\*] s14 s0.AddNextState(new int[] { '+', '*' }, s14.Id); s14.AddElseState(s15.Id); }
private static void InitSpaceStates() { DFAState <int, LexicalFunction> s3 = AddState(new LexicalState(3, false, LexicalFunction.DoSpace)); //Space begin state; DFAState <int, LexicalFunction> s4 = AddState(new LexicalState(4, true, LexicalFunction.OutputSpace)); //Space quit state; //s0 [ \t\r\n] s3 s0.AddNextState(new int[] { ' ', '\t', '\r', '\n' }, s3.Id); //s3 [ \t\r\n] s3 s3.AddNextState(new int[] { ' ', '\t', '\r', '\n' }, s3.Id); //s3 ^[ \t\r\n] s4 s3.AddElseState(s4.Id); }
public DFAGraph(NFAGraph nfa) { Alphabet = nfa.Alphabet; FinalStates = new HashSet<DFAState>(); AdjList = new Dictionary<DFAState, IDictionary<CharacterClassElement, DFAState>>(); var stateVisitQueue = new Queue<DFAState>(); var marked = new HashSet<DFAState>(); //kgo subset construction var startEClosure = nfa.EClosure(nfa.StartState); StartState = new DFAState(startEClosure); stateVisitQueue.Enqueue(StartState); if (StartState.Tags.Contains(FAStateTags.Final)) FinalStates.Add(StartState); AdjList[StartState] = new Dictionary<CharacterClassElement, DFAState>(); while (stateVisitQueue.Count > 0) { var thisState = stateVisitQueue.Dequeue(); marked.Add(thisState); foreach (var symbol in Alphabet) { var u = nfa.EClosure(nfa.Move(thisState.Constituents.ToList(), symbol)); if (u.Count > 0) { var dfaU = new DFAState(u); if (!AdjList.Keys.Contains(dfaU)) { if (dfaU.Tags.Contains(FAStateTags.Final)) FinalStates.Add(dfaU); stateVisitQueue.Enqueue(dfaU); AdjList[dfaU] = new Dictionary<CharacterClassElement, DFAState>(); } LinkStates(thisState, dfaU, symbol); } } } }
private bool SimulateMatch(DFAState node, string input) { if (input.Length == 0) return FinalStates.Contains(node); var thisC = input[0]; var inputRest = input.Substring(1); if (!Alphabet.Any(r => r.Contains(thisC))) //not in alphabet return false; var inputSymbol = Alphabet.Single(r => r.Contains(thisC)); try { return SimulateMatch(AdjList[node][inputSymbol], inputRest); } catch (KeyNotFoundException) { return false; } }
private void LinkStates(DFAState src, DFAState dst, CharacterClassElement el) { AdjList[src][el] = dst; }
private void SimulateSubmatch(DFAState node, string input, int pointer, Stack<int> submatchPointers, ref List<string> submatches) { if (pointer == input.Length) { if (!node.Tags.Contains(FAStateTags.Final)) submatches = null; return; } var thisC = input[pointer]; if (!Alphabet.Any(r => r.Contains(thisC))) { submatches = null; return; } var inputSymbol = Alphabet.Single(r => r.Contains(thisC)); //submatch stuff? foreach (var tag in node.Tags) { switch (tag) { case FAStateTags.PushSubmatch: submatchPointers.Push(pointer); break; case FAStateTags.PopSubmatch: int mStart = submatchPointers.Pop(); submatches.Add(input.Substring(mStart, pointer - mStart)); break; } } try { var nextNode = AdjList[node][inputSymbol]; SimulateSubmatch(nextNode, input, pointer + 1, submatchPointers, ref submatches); } catch (KeyNotFoundException) { submatches = null; return; } }
public DFAEdge(int symbol, DFAState targetState) : this() { Symbol = symbol; TargetState = targetState; }