private static Surf.Set Split(Surf.Set S, char character, Surf.Set transitions, Surf.Set P2) { //System.Console.WriteLine("START Split2 " + System.DateTime.Now.ToString()); Surf.Set f_nP = PartitionMap(P2); Surf.Set partition_map = new Surf.Set(); foreach (int fromState in S) { object key = new Surf.Tuple(new object[] { fromState, character }); if (transitions.IsDefined(key)) { object value = transitions.Apply(key); int toState = (int)value; int partition = (int)f_nP.Apply(toState); partition_map.Add(new Surf.Tuple(new object[] { partition, fromState })); } else { partition_map.Add(new Surf.Tuple(new object[] { -1, fromState })); } } //System.Console.WriteLine("END Split2 " + System.DateTime.Now.ToString()); return(partition_map.Nest().Range()); }
private Surf.Set FOLLOW(string symbol) { if (!follow.IsDefined(symbol)) { return(new Surf.Set()); } return((Surf.Set)follow.Apply(symbol)); }
private static FiniteAutomaton Reorder(Bamboo.Parsing.FiniteAutomata.FiniteAutomaton fa) { System.Console.WriteLine("START Reorder " + System.DateTime.Now.ToString()); Surf.Set stateMap = new Surf.Set(); GetToStates(fa.StartState, fa.Transitions, stateMap, new Counter()); Surf.Set states = new Surf.Set(); Surf.Set alphabet = new Surf.Set(); Surf.Set transitions = new Surf.Set(); int startState; Surf.Set finalStates = new Surf.Set(); Surf.Set tokens = new Surf.Set(); for (int i = 0; i < fa.States.Length; i++) { states.Add(stateMap.Apply(fa.States[i])); } alphabet = new Surf.Set(fa.Alphabet); for (int i = 0; i < fa.Transitions.Length; i++) { if (fa.Transitions[i].Epsilon) { transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { stateMap.Apply(fa.Transitions[i].FromState) }), stateMap.Apply(fa.Transitions[i].ToState) })); } else { transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { stateMap.Apply(fa.Transitions[i].FromState), fa.Transitions[i].Character }), stateMap.Apply(fa.Transitions[i].ToState) })); } } startState = (int)stateMap.Apply(fa.StartState); for (int i = 0; i < fa.FinalStates.Length; i++) { finalStates.Add(stateMap.Apply(fa.FinalStates[i])); tokens.Add(new Surf.Tuple(new object[] { stateMap.Apply(fa.FinalStates[i]), Lookup(fa.Tokens, fa.FinalStates[i]) })); } System.Console.WriteLine("END Reorder " + System.DateTime.Now.ToString()); return(new FiniteAutomaton(states, alphabet, transitions, startState, finalStates, tokens)); }
public static Surf.Set Predict(Bamboo.Parsing.Grammars.Grammar grammar, Surf.Set FIRST, Surf.Set FOLLOW) { Surf.Set PREDICT = new Surf.Set(); int i = 0; foreach (Bamboo.Parsing.Grammars.Production production in grammar.Productions) { Surf.Set PREDICT_i = ((Surf.Set)FIRST.Apply(GetFirstSymbol(production.Expression))).Difference(EPSILON); if (((Surf.Set)FIRST.Apply(GetFirstSymbol(production.Expression))).Contains("EPSILON")) { Union(PREDICT_i, (Surf.Set)FOLLOW.Apply(production.Nonterminal)); } PREDICT.Add(new Surf.Tuple(new object[] { i, production, PREDICT_i })); i++; } AssertDisjoint(PREDICT, grammar.Nonterminals); return(PREDICT); }
private static void First(string A, Bamboo.Parsing.Grammars.Expression expression, Surf.Set FIRST) { List <Bamboo.Parsing.Grammars.Expression> B = ToList(expression); Surf.Set FIRST_A = (Surf.Set)FIRST.Apply(A); Union(FIRST_A, ((Surf.Set)FIRST.Apply(((Bamboo.Parsing.Grammars.Symbol)B[0]).Token)).Difference(EPSILON)); int i = 0; while (((Surf.Set)FIRST.Apply(((Bamboo.Parsing.Grammars.Symbol)B[0]).Token)).Contains("EPSILON") && i < (B.Count - 1)) { Union(FIRST_A, ((Surf.Set)FIRST.Apply(((Bamboo.Parsing.Grammars.Symbol)B[i + 1]).Token)).Difference(EPSILON)); i++; } if (i == (B.Count - 1) && ((Surf.Set)FIRST.Apply(((Bamboo.Parsing.Grammars.Symbol)B[B.Count - 1]).Token)).Contains("EPSILON")) { Union(FIRST_A, EPSILON); } }
private static void DependencyGraph(Surf.Set dependencies, Surf.Set recalculationList, Surf.Set dependencyGraph) { foreach (string identifier in recalculationList) { if (dependencies.IsDefined(identifier)) { Surf.Set dependencyList = ((Surf.Set)dependencies.Apply(identifier)).Intersection(recalculationList); if (dependencyList.Count > 0) { dependencyGraph.Define(identifier, dependencyList); } } } }
private static void RecalculationList(string name, Surf.Set dependents, Surf.Set recalculationList) { if (recalculationList.Contains(name)) { throw new System.Exception("Cycle detected."); } recalculationList.Add(name); if (dependents.IsDefined(name)) { foreach (string dependent in (Surf.Set)dependents.Apply(name)) { RecalculationList(dependent, dependents, recalculationList); } } }
private static Surf.Set EpsilonClosure(Surf.Set states, Surf.Set transitions, Surf.Set epsilonCache) { Surf.Tuple key = Tuple(states); if (epsilonCache.IsDefined(key)) { return((Surf.Set)epsilonCache.Apply(key)); } //System.Console.WriteLine("START EpsilonClosure " + System.DateTime.Now.ToString()); Surf.Set epsilonClosure = new Surf.Set(); Stack <int> worklist = new Stack <int>(); foreach (int state in states) { epsilonClosure.Add(state); worklist.Push(state); } while (worklist.Count > 0) { int state = worklist.Pop(); foreach (Surf.Tuple transition in transitions) { Surf.Tuple input = (Surf.Tuple)transition[0]; if (input.Count == 1 && (int)input[0] == state) { int toState = (int)transition[1]; if (!epsilonClosure.Contains(toState)) { epsilonClosure.Add(toState); worklist.Push(toState); } } } } epsilonCache.Add(new Surf.Tuple(new object[] { key, epsilonClosure })); //System.Console.WriteLine("END EpsilonClosure " + System.DateTime.Now.ToString()); return(epsilonClosure); }
// Hopcroft's Algorithm public static FiniteAutomaton Minimize(FiniteAutomaton dfa) { System.Console.WriteLine("START Minimize " + System.DateTime.Now.ToString()); Surf.Set dfa_transitions = Set(dfa.Transitions); // Split final states and non-final states. Surf.Set worklist = new Surf.Set(); Surf.Set P = new Surf.Set(); Surf.Set f = SplitFinalStates(dfa.Tokens); foreach (Surf.Set f2 in f) { worklist.Add(f2); P.Add(f2); } Surf.Set nonFinalStates = new Surf.Set(dfa.States).Difference(new Surf.Set(dfa.FinalStates)); worklist.Add(nonFinalStates); P.Add(nonFinalStates); // While there are more states to split. while (worklist.Count > 0) { Surf.Set p = (Surf.Set)worklist[0]; worklist.Remove(0); Surf.Set t = Split(p, dfa_transitions, P); if (t.Count > 1) { int i = 0; foreach (Surf.Set p2 in P) { if (p2.Equals(p)) { P.Remove(i); break; } i++; } foreach (Surf.Set t2 in t) { worklist.Add(t2); P.Add(t2); } } } /* * // Split final states and non-final states. * Surf.Set P = new Surf.Set(); * Surf.Set f = SplitFinalStates(dfa.Tokens); * foreach (Surf.Set f2 in f) * { * P.Add(f2); * } * P.Add(new Surf.Set(dfa.States).Difference(new Surf.Set(dfa.FinalStates))); * * // While there are more states to split. * bool isChanging = true; * while (isChanging) * { * isChanging = false; * * Surf.Set T = new Surf.Set(); * foreach (Surf.Set p in P) * { * Surf.Set t = Split(p, dfa_transitions, P); * * if (t.Count > 1) * { * isChanging = true; * } * foreach (Surf.Set t2 in t) * { * T.Add(t2); * } * } * P = T; * } */ Surf.Set states = new Surf.Set(); Surf.Set alphabet = new Surf.Set(); Surf.Set transitions = new Surf.Set(); int startState; Surf.Set finalStates = new Surf.Set(); Surf.Set tokens = new Surf.Set(); Surf.Set P_map = PartitionMap(P); foreach (int state in dfa.States) { states.Add(P_map.Apply(state)); } alphabet = new Surf.Set(dfa.Alphabet); foreach (Surf.Tuple transition in dfa_transitions) { Surf.Tuple input = (Surf.Tuple)transition[0]; int fromState = (int)P_map.Apply(input[0]); char character = (char)input[1]; int toState = (int)P_map.Apply(transition[1]); transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { fromState, character }), toState })); } startState = (int)P_map.Apply(dfa.StartState); foreach (int state in dfa.FinalStates) { finalStates.Add(P_map.Apply(state)); if (!tokens.IsDefined(P_map.Apply(state))) { tokens.Add(new Surf.Tuple(new object[] { P_map.Apply(state), Lookup(dfa.Tokens, state) })); } } System.Console.WriteLine("END Minimize " + System.DateTime.Now.ToString()); return(Reorder(new FiniteAutomaton(states, alphabet, transitions, startState, finalStates, tokens))); }
public static FiniteAutomaton DFA(Bamboo.Parsing.FiniteAutomata.FiniteAutomaton nfa, Counter counter) { System.Console.WriteLine("START DFA " + System.DateTime.Now.ToString()); Surf.Set epsilonCache = new Surf.Set(); Surf.Set moveCache = new Surf.Set(); Surf.Set nfa_transitions = Set(nfa.Transitions); Dictionary <string, int> nfa_transitions_lookup = new Dictionary <string, int>(); foreach (Surf.Tuple transition in nfa_transitions) { Surf.Tuple input = (Surf.Tuple)transition[0]; if (input.Count == 2) { int state = (int)input[0]; char ch = (char)input[1]; nfa_transitions_lookup.Add("" + state + "::" + ch, (int)transition[1]); } } Surf.Set q0 = EpsilonClosure(new Surf.Set(new object[] { nfa.StartState }), nfa_transitions, epsilonCache); Surf.Set Q = new Surf.Set(); // A set whose element are sets of states that are subsets of N. Q.Add(q0); Surf.Set T = new Surf.Set(); Stack <Surf.Set> worklist = new Stack <Surf.Set>(); worklist.Push(q0); while (worklist.Count > 0) { Surf.Set q = worklist.Pop(); foreach (char character in nfa.Alphabet) { Surf.Set t = EpsilonClosure(Move(q, character, nfa_transitions_lookup), nfa_transitions, epsilonCache); if (t.Count > 0) { T.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { q, character }), t })); if (!Q.Contains(t)) { Q.Add(t); worklist.Push(t); } } } } Surf.Set states = new Surf.Set(); Surf.Set alphabet = new Surf.Set(); Surf.Set transitions = new Surf.Set(); int startState; Surf.Set finalStates = new Surf.Set(); Surf.Set tokens = new Surf.Set(); Surf.Set D = new Surf.Set(); foreach (Surf.Set q in Q) { D.Add(new Surf.Tuple(new object[] { q, counter.Next() })); } foreach (Surf.Set q in Q) { states.Add(D.Apply(q)); } alphabet = new Surf.Set(nfa.Alphabet); foreach (Surf.Tuple t in T) { Surf.Tuple input = (Surf.Tuple)t[0]; int fromState = (int)D.Apply(input[0]); char character = (char)input[1]; int toState = (int)D.Apply(t[1]); transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { fromState, character }), toState })); } startState = (int)states[0]; foreach (int state in nfa.FinalStates) { foreach (Surf.Tuple d in D) { Surf.Set q = (Surf.Set)d[0]; if (q.Contains(state)) { finalStates.Add(d[1]); foreach (int q_state in q) { string token = Lookup(nfa.Tokens, (int)q_state); if (token.Length > 0) { tokens.Add(new Surf.Tuple(new object[] { d[1], token })); break; } } } } } System.Console.WriteLine("END DFA " + System.DateTime.Now.ToString()); //foreach (Token token in nfa.Tokens) //{ // System.Console.WriteLine(token.Name + " " + token.Number); //} //System.Console.WriteLine(); //Surf.Printer.Print(tokens, System.Console.Out); //System.Console.WriteLine(); //System.Console.WriteLine(); return(Minimize(Reorder(new FiniteAutomaton(states, alphabet, transitions, startState, finalStates, tokens)))); }
private Surf.Set FIRST(Bamboo.Parsing.Grammars.Expression expression) { return((Surf.Set)first.Apply(((Bamboo.Parsing.Grammars.Symbol)expression).Token)); }
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer) { writer.WriteLine("//"); writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + ""); writer.WriteLine("//"); writer.WriteLine("using System;"); writer.WriteLine(""); writer.WriteLine("namespace " + nspace + ""); writer.WriteLine("{"); writer.WriteLine(" public class "+ name + "Tokenizer"); writer.WriteLine(" {"); writer.WriteLine(" private static readonly int[,] TABLE = new int[,] {"); Surf.Set transitions = new Surf.Set(); foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions) { transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { transition.FromState, transition.Character }), transition.ToState })); } foreach (int state in finiteAutomaton.States) { writer.Write(" { "); foreach (object character in finiteAutomaton.Alphabet) { Surf.Tuple key = new Surf.Tuple(new object[] { state, character }); if (transitions.IsDefined(key)) { int toState = (int)transitions.Apply(key); writer.Write("" + toState + ", "); } else { writer.Write("-1, "); } } writer.WriteLine("},"); } writer.WriteLine(" };"); writer.WriteLine(""); writer.WriteLine(" private System.Text.StringBuilder _stringBuilder = new System.Text.StringBuilder();"); writer.WriteLine(""); writer.WriteLine(" public "+ name + "Tokenizer()"); writer.WriteLine(" {"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" public "+ name + "Token Tokenize(" + name + "TextReader reader)"); writer.WriteLine(" {"); writer.WriteLine(" int n;"); writer.WriteLine(" char ch;"); writer.WriteLine(""); writer.WriteLine(" _stringBuilder.Length = 0;"); writer.WriteLine(""); /* * // * // Trim whitespace * // * while ((n = reader.Peek()) != -1) * { * ch = (char)n; * switch (ch) * { * case ' ': * case '\t': * case '\r': * case '\n': * { * break; * } * default: * { * goto s0; * } * } * } */ writer.WriteLine(" n = reader.Peek();"); writer.WriteLine(" ch = (char)0;"); writer.WriteLine(" int state = 0;"); writer.WriteLine(""); writer.WriteLine(" if (n == -1)"); writer.WriteLine(" {"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._EOF_);"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" while(n != -1 && state != -1)"); writer.WriteLine(" {"); writer.WriteLine(" ch = (char)n;"); writer.WriteLine(""); writer.WriteLine(" switch(ch)"); writer.WriteLine(" {"); int i = 0; foreach (char character in finiteAutomaton.Alphabet) { writer.WriteLine(" case '"+ Escape(character) + "':"); writer.WriteLine(" {"); writer.WriteLine(" int state2 = TABLE[state, "+ i + "];"); writer.WriteLine(" if(state2 == -1)"); writer.WriteLine(" {"); writer.WriteLine(" goto EXIT;"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" state = state2;"); writer.WriteLine(" _stringBuilder.Append(ch);"); writer.WriteLine(" reader.Read();"); writer.WriteLine(" n = reader.Peek();"); writer.WriteLine(" break;"); writer.WriteLine(" }"); i++; } writer.WriteLine(" default:"); writer.WriteLine(" {"); writer.WriteLine(" goto EXIT;"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" EXIT:"); writer.WriteLine(" switch(state)"); writer.WriteLine(" {"); foreach (int finalState in finiteAutomaton.FinalStates) { Bamboo.Parsing.FiniteAutomata.Token token = GetToken(finalState, finiteAutomaton.Tokens); writer.WriteLine(" case "+ finalState + ":"); writer.WriteLine(" {"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());"); writer.WriteLine(" }"); } writer.WriteLine(" default:"); writer.WriteLine(" {"); writer.WriteLine(" if (n > -1)"); writer.WriteLine(" {"); writer.WriteLine(" _stringBuilder.Append(ch);"); writer.WriteLine(" }"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_, _stringBuilder.ToString());"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" }"); writer.WriteLine("}"); }