public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer) { writer.WriteLine("//"); writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + ""); writer.WriteLine("//"); writer.WriteLine("using System;"); writer.WriteLine(""); writer.WriteLine("namespace " + nspace + ""); writer.WriteLine("{"); writer.WriteLine(" public class "+ name + "Tokenizer"); writer.WriteLine(" {"); writer.WriteLine(" private System.Text.StringBuilder _stringBuilder = new System.Text.StringBuilder();"); writer.WriteLine(""); writer.WriteLine(" public "+ name + "Tokenizer()"); writer.WriteLine(" {"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" public "+ name + "Token Tokenize(" + name + "TextReader reader)"); writer.WriteLine(" {"); writer.WriteLine(" int n;"); writer.WriteLine(" char ch;"); writer.WriteLine(""); writer.WriteLine(" _stringBuilder.Length = 0;"); writer.WriteLine(""); writer.WriteLine(" //"); writer.WriteLine(" // Trim whitespace"); writer.WriteLine(" //"); writer.WriteLine(" while ((n = reader.Peek()) != -1)"); writer.WriteLine(" {"); writer.WriteLine(" ch = (char)n;"); writer.WriteLine(" switch (ch)"); writer.WriteLine(" {"); writer.WriteLine(" case ' ':"); writer.WriteLine(" case '\\t':"); if (!HasNewline(finiteAutomaton.Tokens)) { writer.WriteLine(" case '\\r':"); writer.WriteLine(" case '\\n':"); } writer.WriteLine(" {"); writer.WriteLine(" reader.Read();"); writer.WriteLine(" break;"); writer.WriteLine(" }"); writer.WriteLine(" default:"); writer.WriteLine(" {"); writer.WriteLine(" goto s0;"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(""); foreach (int state in finiteAutomaton.States) { writer.WriteLine(" s"+ state + ":"); if (Operators.IsFinal(state, finiteAutomaton.FinalStates) && !Operators.HasTransitions(state, finiteAutomaton.Transitions)) { Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());"); } else { writer.WriteLine(" n = reader.Peek();"); writer.WriteLine(" if (n == -1)"); writer.WriteLine(" {"); writer.WriteLine(" if (_stringBuilder.Length == 0)"); writer.WriteLine(" {"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._EOF_);"); writer.WriteLine(" }"); writer.WriteLine(" else"); writer.WriteLine(" {"); if (Operators.IsFinal(state, finiteAutomaton.FinalStates)) { Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());"); } else { writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_);"); } writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" ch = (char)n;"); writer.WriteLine(" switch (ch)"); writer.WriteLine(" {"); Surf.Set transitions = new Surf.Set(); foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions) { if (transition.FromState == state) { transitions.Add(new Surf.Tuple(new object[] { transition.ToState, transition.Character })); } } transitions = transitions.Nest(); foreach (Surf.Tuple transition in transitions) { foreach (char character in (Surf.Set)transition[1]) { writer.WriteLine(" case '"+ Operators.Escape(character) + "':"); } writer.WriteLine(" {"); writer.WriteLine(" reader.Read();"); writer.WriteLine(" _stringBuilder.Append(ch);"); writer.WriteLine(" goto s"+ transition[0] + ";"); writer.WriteLine(" }"); } writer.WriteLine(" default:"); writer.WriteLine(" {"); if (Operators.IsFinal(state, finiteAutomaton.FinalStates)) { Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());"); } else { writer.WriteLine(" _stringBuilder.Append(ch);"); writer.WriteLine(" goto se;"); } writer.WriteLine(" }"); writer.WriteLine(" }"); } writer.WriteLine(""); } writer.WriteLine(" se:"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_, _stringBuilder.ToString());"); writer.WriteLine(""); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" }"); writer.WriteLine("}"); }
public static void GenerateClass(string name, string multinamespace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer) { writer.WriteLine("//"); writer.WriteLine("// " + name + "Tokenizer.cpp"); writer.WriteLine("//"); writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + ""); writer.WriteLine("//"); writer.WriteLine(""); writer.WriteLine("#include <istream>"); writer.WriteLine("#include \"" + name + "Parser.h\""); writer.WriteLine("#include \"" + name + "Token.h\""); writer.WriteLine("#include \"" + name + "Tokenizer.h\""); writer.WriteLine("#include \"" + name + "TokenType.h\""); writer.WriteLine(""); string[] namespaces = multinamespace.Split(new string[] { "::" }, StringSplitOptions.None); foreach (string nspace in namespaces) { writer.WriteLine("namespace " + nspace + ""); writer.WriteLine("{"); } writer.WriteLine(" int "+ name + "Tokenizer::_counter = 0;"); writer.WriteLine(""); writer.WriteLine(" void "+ name + "Tokenizer::next(char ch)"); writer.WriteLine(" {"); writer.WriteLine(" switch(_state)"); writer.WriteLine(" {"); //TODO if ch == -1, purge remaining token. foreach (int state in finiteAutomaton.States) { bool isFinal = Operators.IsFinal(state, finiteAutomaton.FinalStates); bool hasTransitions = Operators.HasTransitions(state, finiteAutomaton.Transitions); if (!isFinal || hasTransitions) { writer.WriteLine(" case "+ state + ":"); writer.WriteLine(" {"); writer.WriteLine(" switch (ch)"); writer.WriteLine(" {"); if (state == 0) { writer.WriteLine(" case ' ':"); writer.WriteLine(" case '\\t':"); writer.WriteLine(" case '\\r':"); writer.WriteLine(" case '\\n':"); writer.WriteLine(" {"); writer.WriteLine(" // Trim whitespace"); writer.WriteLine(" break;"); writer.WriteLine(" }"); } Surf.Set transitions = new Surf.Set(); foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions) { if (transition.FromState == state) { transitions.Add(new Surf.Tuple(new object[] { transition.ToState, transition.Character })); } } transitions = transitions.Nest(); foreach (Surf.Tuple transition in transitions) { foreach (char character in (Surf.Set)transition[1]) { writer.WriteLine(" case '"+ Operators.Escape(character) + "':"); } writer.WriteLine(" {"); bool isFinal2 = Operators.IsFinal((int)transition[0], finiteAutomaton.FinalStates); bool hasTransitions2 = Operators.HasTransitions((int)transition[0], finiteAutomaton.Transitions); writer.WriteLine(" _buffer += ch;"); if (isFinal2 && !hasTransitions2) { Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken((int)transition[0], finiteAutomaton.Tokens); writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::" + token.Name + ", _buffer);"); writer.WriteLine(" _parser->next(token);"); writer.WriteLine(" _state = 0;"); writer.WriteLine(" _buffer.clear();"); } else { writer.WriteLine(" _state = "+ transition[0] + ";"); } writer.WriteLine(" break;"); writer.WriteLine(" }"); } writer.WriteLine(" default:"); writer.WriteLine(" {"); if (isFinal) { Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens); writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::" + token.Name + ", _buffer);"); writer.WriteLine(" _parser->next(token);"); writer.WriteLine(" _state = 0;"); writer.WriteLine(" _buffer.clear();"); writer.WriteLine(" next(ch);"); } else { writer.WriteLine(" _buffer += ch;"); writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::_ERROR_, _buffer);"); writer.WriteLine(" _parser->next(token);"); } writer.WriteLine(" break;"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" break;"); writer.WriteLine(" }"); } } writer.WriteLine(" default:"); writer.WriteLine(" {"); writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::_ERROR_, _buffer);"); writer.WriteLine(" _parser->next(token);"); writer.WriteLine(" break;"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" }"); foreach (string nspace in namespaces) { writer.WriteLine("}"); } }
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer) { writer.WriteLine("//"); writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + ""); writer.WriteLine("//"); writer.WriteLine("using System;"); writer.WriteLine(""); writer.WriteLine("namespace " + nspace + ""); writer.WriteLine("{"); writer.WriteLine(" public class "+ name + "Tokenizer"); writer.WriteLine(" {"); writer.WriteLine(" private static readonly int[,] TABLE = new int[,] {"); Surf.Set transitions = new Surf.Set(); foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions) { transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { transition.FromState, transition.Character }), transition.ToState })); } foreach (int state in finiteAutomaton.States) { writer.Write(" { "); foreach (object character in finiteAutomaton.Alphabet) { Surf.Tuple key = new Surf.Tuple(new object[] { state, character }); if (transitions.IsDefined(key)) { int toState = (int)transitions.Apply(key); writer.Write("" + toState + ", "); } else { writer.Write("-1, "); } } writer.WriteLine("},"); } writer.WriteLine(" };"); writer.WriteLine(""); writer.WriteLine(" private System.Text.StringBuilder _stringBuilder = new System.Text.StringBuilder();"); writer.WriteLine(""); writer.WriteLine(" public "+ name + "Tokenizer()"); writer.WriteLine(" {"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" public "+ name + "Token Tokenize(" + name + "TextReader reader)"); writer.WriteLine(" {"); writer.WriteLine(" int n;"); writer.WriteLine(" char ch;"); writer.WriteLine(""); writer.WriteLine(" _stringBuilder.Length = 0;"); writer.WriteLine(""); /* * // * // Trim whitespace * // * while ((n = reader.Peek()) != -1) * { * ch = (char)n; * switch (ch) * { * case ' ': * case '\t': * case '\r': * case '\n': * { * break; * } * default: * { * goto s0; * } * } * } */ writer.WriteLine(" n = reader.Peek();"); writer.WriteLine(" ch = (char)0;"); writer.WriteLine(" int state = 0;"); writer.WriteLine(""); writer.WriteLine(" if (n == -1)"); writer.WriteLine(" {"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._EOF_);"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" while(n != -1 && state != -1)"); writer.WriteLine(" {"); writer.WriteLine(" ch = (char)n;"); writer.WriteLine(""); writer.WriteLine(" switch(ch)"); writer.WriteLine(" {"); int i = 0; foreach (char character in finiteAutomaton.Alphabet) { writer.WriteLine(" case '"+ Escape(character) + "':"); writer.WriteLine(" {"); writer.WriteLine(" int state2 = TABLE[state, "+ i + "];"); writer.WriteLine(" if(state2 == -1)"); writer.WriteLine(" {"); writer.WriteLine(" goto EXIT;"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" state = state2;"); writer.WriteLine(" _stringBuilder.Append(ch);"); writer.WriteLine(" reader.Read();"); writer.WriteLine(" n = reader.Peek();"); writer.WriteLine(" break;"); writer.WriteLine(" }"); i++; } writer.WriteLine(" default:"); writer.WriteLine(" {"); writer.WriteLine(" goto EXIT;"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" EXIT:"); writer.WriteLine(" switch(state)"); writer.WriteLine(" {"); foreach (int finalState in finiteAutomaton.FinalStates) { Bamboo.Parsing.FiniteAutomata.Token token = GetToken(finalState, finiteAutomaton.Tokens); writer.WriteLine(" case "+ finalState + ":"); writer.WriteLine(" {"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());"); writer.WriteLine(" }"); } writer.WriteLine(" default:"); writer.WriteLine(" {"); writer.WriteLine(" if (n > -1)"); writer.WriteLine(" {"); writer.WriteLine(" _stringBuilder.Append(ch);"); writer.WriteLine(" }"); writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_, _stringBuilder.ToString());"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(" }"); writer.WriteLine(""); writer.WriteLine(" }"); writer.WriteLine("}"); }