internal void AddMove(NfaState newState) { if (!epsilonMoves.Contains(newState)) InsertInOrder(epsilonMoves, newState); }
private static void DumpCompositeStatesAsciiMoves(TextWriter ostr, string key, int byteNum, bool[] dumped) { int i; int[] nameSet = allNextStates[key]; if (nameSet.Length == 1 || dumped[StateNameForComposite(key)]) return; NfaState toBePrinted = null; int neededStates = 0; NfaState tmp; NfaState stateForCase = null; String toPrint = ""; bool stateBlock = stateBlockTable.ContainsKey(key); for (i = 0; i < nameSet.Length; i++) { tmp = allStates[nameSet[i]]; if (tmp.asciiMoves[byteNum] != 0L) { if (neededStates++ == 1) break; else toBePrinted = tmp; } else dumped[tmp.stateName] = true; if (tmp.stateForCase != null) { if (stateForCase != null) throw new InvalidOperationException("CSharpCC Bug: Please send a report: "); stateForCase = tmp.stateForCase; } } if (stateForCase != null) toPrint = stateForCase.PrintNoBreak(ostr, byteNum, dumped); if (neededStates == 0) { if (stateForCase != null && toPrint.Equals("")) ostr.WriteLine(" break;"); return; } if (neededStates == 1) { if (!toPrint.Equals("")) ostr.Write(toPrint); ostr.WriteLine(" case " + StateNameForComposite(key) + ":"); if (!dumped[toBePrinted.stateName] && !stateBlock && toBePrinted.inNextOf > 1) ostr.WriteLine(" case " + toBePrinted.stateName + ":"); dumped[toBePrinted.stateName] = true; toBePrinted.DumpAsciiMove(ostr, byteNum, dumped); return; } var partition = PartitionStatesSetForAscii(nameSet, byteNum); if (!toPrint.Equals("")) ostr.Write(toPrint); int keyState = StateNameForComposite(key); ostr.WriteLine(" case " + keyState + ":"); if (keyState < generatedStates) dumped[keyState] = true; for (i = 0; i < partition.Count; i++) { var subSet = partition[i]; for (int j = 0; j < subSet.Count; j++) { tmp = subSet[j]; if (stateBlock) dumped[tmp.stateName] = true; tmp.DumpAsciiMoveForCompositeState(ostr, byteNum, j != 0); } } if (stateBlock) ostr.WriteLine(" break;"); else ostr.WriteLine(" break;"); }
private void MergeMoves(NfaState other) { // Warning : This function does not merge epsilon moves if (asciiMoves == other.asciiMoves) { CSharpCCErrors.SemanticError("Bug in CSharpCC : Please send " + "a report along with the input that caused this. Thank you."); throw new InvalidOperationException(); } asciiMoves[0] = asciiMoves[0] | other.asciiMoves[0]; asciiMoves[1] = asciiMoves[1] | other.asciiMoves[1]; if (other.charMoves != null) { if (charMoves == null) charMoves = other.charMoves; else { char[] tmpCharMoves = new char[charMoves.Length + other.charMoves.Length]; Array.Copy(charMoves, 0, tmpCharMoves, 0, charMoves.Length); charMoves = tmpCharMoves; for (int i = 0; i < other.charMoves.Length; i++) AddChar(other.charMoves[i]); } } if (other.rangeMoves != null) { if (rangeMoves == null) rangeMoves = other.rangeMoves; else { char[] tmpRangeMoves = new char[rangeMoves.Length + other.rangeMoves.Length]; Array.Copy(rangeMoves, 0, tmpRangeMoves, 0, rangeMoves.Length); rangeMoves = tmpRangeMoves; for (int i = 0; i < other.rangeMoves.Length; i += 2) AddRange(other.rangeMoves[i], other.rangeMoves[i + 1]); } } if (other.kind < kind) kind = other.kind; if (other.kindToPrint < kindToPrint) kindToPrint = other.kindToPrint; isFinal |= other.isFinal; }
public static int moveFromSetForRegEx(char c, NfaState[] states, NfaState[] newStates, int round) { int start = 0; int sz = states.Length; for (int i = 0; i < sz; i++) { NfaState tmp1, tmp2; if ((tmp1 = states[i]) == null) break; if (tmp1.CanMoveUsingChar(c)) { if (tmp1.kindToPrint != Int32.MaxValue) { newStates[start] = null; return 1; } NfaState[] v = tmp1.next.epsilonMoveArray; for (int j = v.Length; j-- > 0;) { if ((tmp2 = v[j]).round != round) { tmp2.round = round; newStates[start++] = tmp2; } } } } newStates[start] = null; return Int32.MaxValue; }
private static void InsertInOrder(IList<NfaState> v, NfaState s) { int j; for (j = 0; j < v.Count; j++) if (v[j].id > s.id) break; else if (v[j].id == s.id) return; v.Insert(j, s); }
private NfaState CreateClone() { NfaState retVal = new NfaState(); retVal.isFinal = isFinal; retVal.kind = kind; retVal.lookingFor = lookingFor; retVal.lexState = lexState; retVal.inNextOf = inNextOf; retVal.MergeMoves(this); return retVal; }
public Nfa() { Start = new NfaState(); End = new NfaState(); }
private static void FindStatesWithNoBreak() { Dictionary<string, string> printed = new Dictionary<string, string>(); bool[] put = new bool[generatedStates]; int cnt = 0; int i, j, foundAt = 0; for (j = 0; j < allStates.Count; j++) { NfaState stateForCase = null; NfaState tmpState = allStates[j]; if (tmpState.stateName == -1 || tmpState.dummy || !tmpState.UsefulState() || tmpState.next == null || tmpState.next.usefulEpsilonMoves < 1) continue; String s = tmpState.next.epsilonMovesString; if (compositeStateTable.ContainsKey(s) || printed.ContainsKey(s)) continue; printed[s] = s; int[] nexts = allNextStates[s]; if (nexts.Length == 1) continue; int state = cnt; //System.out.println("State " + tmpState.stateName + " : " + s); for (i = 0; i < nexts.Length; i++) { if ((state = nexts[i]) == -1) continue; NfaState tmp = allStates[state]; if (!tmp.isComposite && tmp.inNextOf == 1) { if (put[state]) throw new InvalidOperationException("CSharpCC Bug: Please send a report"); foundAt = i; cnt++; stateForCase = tmp; put[state] = true; //System.out.print(state + " : " + tmp.inNextOf + ", "); break; } } //System.out.println(""); if (stateForCase == null) continue; for (i = 0; i < nexts.Length; i++) { if ((state = nexts[i]) == -1) continue; NfaState tmp = allStates[state]; if (!put[state] && tmp.inNextOf > 1 && !tmp.isComposite && tmp.stateForCase == null) { cnt++; nexts[i] = -1; put[state] = true; int toSwap = nexts[0]; nexts[0] = nexts[foundAt]; nexts[foundAt] = toSwap; tmp.stateForCase = stateForCase; stateForCase.stateForCase = tmp; stateSetsToFix[s] = nexts; //System.out.println("For : " + s + "; " + stateForCase.stateName + //" and " + tmp.stateName); goto Outer; } } for (i = 0; i < nexts.Length; i++) { if ((state = nexts[i]) == -1) continue; NfaState tmp = (NfaState) allStates[state]; if (tmp.inNextOf <= 1) put[state] = false; } Outer: ; } }
internal Nfa(NfaState start, NfaState end) { Start = start; End = end; }
internal static void GenerateNfaStartStates(TextWriter ostr, NfaState initialState) { bool[] seen = new bool[NfaState.generatedStates]; IDictionary<string, string> stateSets = new Dictionary<string, string>(); String stateSetString = ""; int i, j, kind, jjmatchedPos = 0; int maxKindsReqd = maxStrKind/64 + 1; long[] actives; IList<NfaState> newStates = new List<NfaState>(); IList<NfaState> oldStates = null, jjtmpStates; statesForPos = new IDictionary<string, long[]>[maxLen]; intermediateKinds = new int[maxStrKind + 1][]; intermediateMatchedPos = new int[maxStrKind + 1][]; for (i = 0; i < maxStrKind; i++) { if (LexGen.lexStates[i] != LexGen.lexStateIndex) continue; String image = allImages[i]; if (image == null || image.Length < 1) continue; try { if ((oldStates = new List<NfaState>(initialState.epsilonMoves)).Count == 0) { DumpNfaStartStatesCode(statesForPos, ostr); return; } } catch (Exception e) { CSharpCCErrors.SemanticError("Error cloning state vector"); } intermediateKinds[i] = new int[image.Length]; intermediateMatchedPos[i] = new int[image.Length]; jjmatchedPos = 0; kind = Int32.MaxValue; for (j = 0; j < image.Length; j++) { if (oldStates == null || oldStates.Count <= 0) { // Here, j > 0 kind = intermediateKinds[i][j] = intermediateKinds[i][j - 1]; jjmatchedPos = intermediateMatchedPos[i][j] = intermediateMatchedPos[i][j - 1]; } else { kind = NfaState.MoveFromSet(image[j], oldStates, newStates); oldStates.Clear(); if (j == 0 && kind != Int32.MaxValue && LexGen.canMatchAnyChar[LexGen.lexStateIndex] != -1 && kind > LexGen.canMatchAnyChar[LexGen.lexStateIndex]) kind = LexGen.canMatchAnyChar[LexGen.lexStateIndex]; if (GetStrKind(image.Substring(0, j + 1)) < kind) { intermediateKinds[i][j] = kind = Int32.MaxValue; jjmatchedPos = 0; } else if (kind != Int32.MaxValue) { intermediateKinds[i][j] = kind; jjmatchedPos = intermediateMatchedPos[i][j] = j; } else if (j == 0) kind = intermediateKinds[i][j] = Int32.MaxValue; else { kind = intermediateKinds[i][j] = intermediateKinds[i][j - 1]; jjmatchedPos = intermediateMatchedPos[i][j] = intermediateMatchedPos[i][j - 1]; } stateSetString = NfaState.GetStateSetString(newStates); } if (kind == Int32.MaxValue && (newStates == null || newStates.Count == 0)) continue; int p; if (stateSets.ContainsKey(stateSetString)) { stateSets[stateSetString] = stateSetString; for (p = 0; p < newStates.Count; p++) { if (seen[newStates[p].stateName]) newStates[p].inNextOf++; else seen[newStates[p].stateName] = true; } } else { for (p = 0; p < newStates.Count; p++) seen[newStates[p].stateName] = true; } jjtmpStates = oldStates; oldStates = newStates; (newStates = jjtmpStates).Clear(); if (statesForPos[j] == null) statesForPos[j] = new Dictionary<string, long[]>(); if (!(statesForPos[j].TryGetValue(kind + ", " + jjmatchedPos + ", " + stateSetString, out actives))) { actives = new long[maxKindsReqd]; statesForPos[j][kind + ", " + jjmatchedPos + ", " + stateSetString] = actives; } actives[i/64] |= 1L << (i%64); //String name = NfaState.StoreStateSet(stateSetString); } } DumpNfaStartStatesCode(statesForPos, ostr); }
public override Nfa GenerateNfa(bool ignoreCase) { if (Image.Length == 1) { RCharacterList temp = new RCharacterList(Image[0]); return temp.GenerateNfa(ignoreCase); } NfaState startState = new NfaState(); NfaState theStartState = startState; NfaState finalState = null; if (Image.Length == 0) return new Nfa(theStartState, theStartState); int i; for (i = 0; i < Image.Length; i++) { finalState = new NfaState(); startState.charMoves = new char[1]; startState.AddChar(Image[i]); if (Options.getIgnoreCase() || ignoreCase) { startState.AddChar(Char.ToLower(Image[i])); startState.AddChar(Char.ToUpper(Image[i])); } startState.next = finalState; startState = finalState; } return new Nfa(theStartState, finalState); }
public static void start() { if (!Options.getBuildTokenManager() || Options.getUserTokenManager() || CSharpCCErrors.ErrorCount > 0) return; keepLineCol = Options.getKeepLineColumn(); List<RegularExpression> choices = new List<RegularExpression>(); IEnumerator e; TokenProduction tp; int i, j; staticString = (Options.getStatic() ? "static " : ""); tokMgrClassName = CSharpCCGlobals.cu_name + "TokenManager"; PrintClassHead(); BuildLexStatesTable(); e = allTpsForState.Keys.GetEnumerator(); bool ignoring = false; while (e.MoveNext()) { NfaState.ReInit(); RStringLiteral.ReInit(); String key = (String) e.Current; lexStateIndex = GetIndex(key); lexStateSuffix = "_" + lexStateIndex; IList<TokenProduction> allTps = allTpsForState[key]; initStates[key] = initialState = new NfaState(); ignoring = false; singlesToSkip[lexStateIndex] = new NfaState(); singlesToSkip[lexStateIndex].dummy = true; if (key.Equals("DEFAULT")) defaultLexState = lexStateIndex; for (i = 0; i < allTps.Count; i++) { tp = allTps[i]; int kind = tp.Kind; bool ignore = tp.IgnoreCase; IList<RegExprSpec> rexps = tp.RegexSpecs; if (i == 0) ignoring = ignore; for (j = 0; j < rexps.Count; j++) { RegExprSpec respec = rexps[j]; curRE = respec.RegularExpression; rexprs[curKind = curRE.Ordinal] = curRE; lexStates[curRE.Ordinal] = lexStateIndex; ignoreCase[curRE.Ordinal] = ignore; if (curRE.IsPrivate) { kinds[curRE.Ordinal] = -1; continue; } if (curRE is RStringLiteral && !((RStringLiteral) curRE).Image.Equals("")) { ((RStringLiteral) curRE).GenerateDfa(ostr, curRE.Ordinal); if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore) mixed[lexStateIndex] = true; } else if (curRE.CanMatchAnyChar) { if (canMatchAnyChar[lexStateIndex] == -1 || canMatchAnyChar[lexStateIndex] > curRE.Ordinal) canMatchAnyChar[lexStateIndex] = curRE.Ordinal; } else { Nfa temp; if (curRE is RChoice) choices.Add(curRE); temp = curRE.GenerateNfa(ignore); temp.End.isFinal = true; temp.End.kind = curRE.Ordinal; initialState.AddMove(temp.Start); } if (kinds.Length < curRE.Ordinal) { int[] tmp = new int[curRE.Ordinal + 1]; Array.Copy(kinds, 0, tmp, 0, kinds.Length); kinds = tmp; } //System.out.println(" ordina : " + curRE.ordinal); kinds[curRE.Ordinal] = kind; if (respec.NextState != null && !respec.NextState.Equals(lexStateName[lexStateIndex])) newLexState[curRE.Ordinal] = respec.NextState; if (respec.Action != null && respec.Action.ActionTokens != null && respec.Action.ActionTokens.Count > 0) actions[curRE.Ordinal] = respec.Action; switch (kind) { case TokenProduction.SPECIAL: hasSkipActions |= (actions[curRE.Ordinal] != null) || (newLexState[curRE.Ordinal] != null); hasSpecial = true; toSpecial[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64); toSkip[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64); break; case TokenProduction.SKIP: hasSkipActions |= (actions[curRE.Ordinal] != null); hasSkip = true; toSkip[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64); break; case TokenProduction.MORE: hasMoreActions |= (actions[curRE.Ordinal] != null); hasMore = true; toMore[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64); if (newLexState[curRE.Ordinal] != null) canReachOnMore[GetIndex(newLexState[curRE.Ordinal])] = true; else canReachOnMore[lexStateIndex] = true; break; case TokenProduction.TOKEN: hasTokenActions |= (actions[curRE.Ordinal] != null); toToken[curRE.Ordinal/64] |= 1L << (curRE.Ordinal%64); break; } } } // Generate a static block for initializing the nfa transitions NfaState.ComputeClosures(); for (i = 0; i < initialState.epsilonMoves.Count; i++) initialState.epsilonMoves[i].GenerateCode(); if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0)) { initialState.GenerateCode(); initialState.GenerateInitMoves(ostr); } if (initialState.kind != Int32.MaxValue && initialState.kind != 0) { if ((toSkip[initialState.kind/64] & (1L << initialState.kind)) != 0L || (toSpecial[initialState.kind/64] & (1L << initialState.kind)) != 0L) hasSkipActions = true; else if ((toMore[initialState.kind/64] & (1L << initialState.kind)) != 0L) hasMoreActions = true; else hasTokenActions = true; if (initMatch[lexStateIndex] == 0 || initMatch[lexStateIndex] > initialState.kind) { initMatch[lexStateIndex] = initialState.kind; hasEmptyMatch = true; } } else if (initMatch[lexStateIndex] == 0) initMatch[lexStateIndex] = Int32.MaxValue; RStringLiteral.FillSubString(); if (hasNfa[lexStateIndex] && !mixed[lexStateIndex]) RStringLiteral.GenerateNfaStartStates(ostr, initialState); RStringLiteral.DumpDfaCode(ostr); if (hasNfa[lexStateIndex]) NfaState.DumpMoveNfa(ostr); if (stateSetSize < NfaState.generatedStates) stateSetSize = NfaState.generatedStates; } for (i = 0; i < choices.Count; i++) ((RChoice) choices[i]).CheckUnmatchability(); NfaState.DumpStateSets(ostr); CheckEmptyStringMatch(); NfaState.DumpNonAsciiMoveMethods(ostr); RStringLiteral.DumpStrLiteralImages(ostr); DumpStaticVarDeclarations(); DumpFillToken(); DumpGetNextToken(); if (Options.getDebugTokenManager()) { NfaState.DumpStatesForKind(ostr); DumpDebugMethods(); } if (hasLoop) { ostr.WriteLine("{0}int[] ccEmptyLineNo = new int[{1}];", staticString, maxLexStates); ostr.WriteLine("{0}int[] ccEmptyColNo = new int[{1}];", staticString, maxLexStates); ostr.WriteLine("{0}bool[] ccBeenHere = new bool[{1}];", staticString, maxLexStates); } if (hasSkipActions) DumpSkipActions(); if (hasMoreActions) DumpMoreActions(); if (hasTokenActions) DumpTokenActions(); NfaState.PrintBoilerPlate(ostr); ostr.WriteLine( /*{*/ "}"); if (namespaceInserted) ostr.WriteLine("}"); ostr.Close(); }
public static void reInit() { ostr = null; staticString = null; tokMgrClassName = null; allTpsForState = new Dictionary<string, IList<TokenProduction>>(); lexStateIndex = 0; kinds = null; maxOrdinal = 1; lexStateSuffix = null; newLexState = null; lexStates = null; ignoreCase = null; actions = null; initStates = new Dictionary<string, NfaState>(); stateSetSize = 0; maxLexStates = 0; lexStateName = null; singlesToSkip = null; toSkip = null; toSpecial = null; toMore = null; toToken = null; defaultLexState = 0; rexprs = null; maxLongsReqd = null; initMatch = null; canMatchAnyChar = null; hasEmptyMatch = false; canLoop = null; stateHasActions = null; hasLoop = false; canReachOnMore = null; hasNfa = null; mixed = null; initialState = null; curKind = 0; hasSkipActions = false; hasMoreActions = false; hasTokenActions = false; hasSpecial = false; hasSkip = false; hasMore = false; curRE = null; }