public static void GenerateClass(string name, string multinamespace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// " + name + "Token.cpp");
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("");
writer.WriteLine("#include \"" + name + "Token.h\"");
writer.WriteLine("");
string[] namespaces = multinamespace.Split(new string[] { "::" }, StringSplitOptions.None);
foreach (string nspace in namespaces)
{
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
}
writer.WriteLine(" int "+ name + "Token::_counter = 0;");
writer.WriteLine("");
foreach (string nspace in namespaces)
{
writer.WriteLine("}");
}
}
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("using System;");
writer.WriteLine("");
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
writer.WriteLine(" public class "+ name + "Token");
writer.WriteLine(" {");
writer.WriteLine(" public int Type;");
writer.WriteLine(" public string Value;");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Token(int type)");
writer.WriteLine(" {");
writer.WriteLine(" Type = type;");
writer.WriteLine(" Value = null;");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Token(int type, string value)");
writer.WriteLine(" {");
writer.WriteLine(" Type = type;");
writer.WriteLine(" Value = value;");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" }");
writer.WriteLine("}");
}
public static void GenerateHeader(string name, string multinamespace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// " + name + "Token.h");
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("");
writer.WriteLine("#ifndef " + name.ToUpper() + "TOKEN_H");
writer.WriteLine("#define " + name.ToUpper() + "TOKEN_H");
writer.WriteLine("");
writer.WriteLine("#include <iostream>");
writer.WriteLine("#include <string>");
writer.WriteLine("#include \"" + name + "TokenType.h\"");
writer.WriteLine("");
string[] namespaces = multinamespace.Split(new string[] { "::" }, StringSplitOptions.None);
foreach (string nspace in namespaces)
{
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
}
writer.WriteLine(" class "+ name + "Token");
writer.WriteLine(" {");
writer.WriteLine(" private:");
writer.WriteLine(" static int _counter;");
writer.WriteLine("");
writer.WriteLine(" int _type;");
writer.WriteLine(" std::string _value;");
writer.WriteLine(" public:");
writer.WriteLine(" "+ name + "Token(int type) : _type(type) { _counter++; }");
writer.WriteLine(" "+ name + "Token(int type, std::string value) : _type(type), _value(value) { _counter++; }");
writer.WriteLine(" ~"+ name + "Token() { _counter--; }");
writer.WriteLine(" static int counter() { return _counter; }");
writer.WriteLine("");
writer.WriteLine(" int type() { return _type; }");
writer.WriteLine(" std::string value() { return _value; }");
writer.WriteLine("");
writer.WriteLine(" friend std::ostream& operator<<(std::ostream& stream, Token& token) { ");
writer.WriteLine(" stream << \"TOKEN \" << token._type;");
writer.WriteLine(" if(!token._value.empty())");
writer.WriteLine(" stream << \" \" << token._value;");
writer.WriteLine(" return stream << std::endl; }");
writer.WriteLine(" friend std::ostream& operator<<(std::ostream& stream, Token* token) { ");
writer.WriteLine(" stream << \"TOKEN \" << token->_type;");
writer.WriteLine(" if(!token->_value.empty())");
writer.WriteLine(" stream << \" \" << token->_value;");
writer.WriteLine(" return stream << std::endl; }");
writer.WriteLine(" };");
foreach (string nspace in namespaces)
{
writer.WriteLine("}");
}
writer.WriteLine("#endif");
}
public static void GenerateHeader(string name, string multinamespace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// " + name + "Tokenizer.h");
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("");
writer.WriteLine("#ifndef " + name.ToUpper() + "TOKENIZER_H");
writer.WriteLine("#define " + name.ToUpper() + "TOKENIZER_H");
writer.WriteLine("");
writer.WriteLine("#include <string>");
writer.WriteLine("#include \"" + name + "Parser.h\"");
writer.WriteLine("#include \"" + name + "Token.h\"");
writer.WriteLine("");
string[] namespaces = multinamespace.Split(new string[] { "::" }, StringSplitOptions.None);
foreach (string nspace in namespaces)
{
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
}
writer.WriteLine(" class "+ name + "Tokenizer");
writer.WriteLine(" {");
writer.WriteLine(" private:");
writer.WriteLine(" "+ name + "Tokenizer(const " + name + "Tokenizer&);");
writer.WriteLine(" "+ name + "Tokenizer operator=(const " + name + "Tokenizer&);");
writer.WriteLine(" static int _counter;");
writer.WriteLine("");
writer.WriteLine(" "+ name + "Parser* _parser;");
writer.WriteLine(" int _state;");
writer.WriteLine(" std::string _buffer;");
writer.WriteLine(" public:");
writer.WriteLine(" "+ name + "Tokenizer(" + name + "Parser* parser) : _parser(parser), _state(0) { _counter++; }");
writer.WriteLine(" ~"+ name + "Tokenizer() { _counter--; }");
writer.WriteLine(" static int counter() { return _counter; }");
writer.WriteLine("");
writer.WriteLine(" void next(char ch);");
writer.WriteLine(" };");
foreach (string nspace in namespaces)
{
writer.WriteLine("}");
}
writer.WriteLine("#endif");
}
public static void GenerateHeader(string name, string multinamespace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// " + name + "TokenType.h");
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("");
writer.WriteLine("#pragma once");
writer.WriteLine("");
string[] namespaces = multinamespace.Split(new string[] { "::" }, StringSplitOptions.None);
foreach (string nspace in namespaces)
{
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
}
writer.WriteLine("namespace " + name + "TokenType");
writer.WriteLine("{");
Dictionary <string, string> tokens = new Dictionary <string, string>();
foreach (Bamboo.Parsing.FiniteAutomata.Token token in finiteAutomaton.Tokens)
{
if (!tokens.ContainsKey(token.Name))
{
tokens.Add(token.Name, token.Name);
}
}
int i = 0;
foreach (string tokenName in tokens.Keys)
{
writer.WriteLine(" const int "+ tokenName + " = " + i++ + ";");
}
writer.WriteLine("");
writer.WriteLine(" const int _ERROR_ = "+ i++ + ";");
writer.WriteLine(" const int _EOF_ = "+ i++ + ";");
writer.WriteLine("}");
foreach (string nspace in namespaces)
{
writer.WriteLine("}");
}
}
private static FiniteAutomaton Reorder(Bamboo.Parsing.FiniteAutomata.FiniteAutomaton fa)
{
System.Console.WriteLine("START Reorder " + System.DateTime.Now.ToString());
Surf.Set stateMap = new Surf.Set();
GetToStates(fa.StartState, fa.Transitions, stateMap, new Counter());
Surf.Set states = new Surf.Set();
Surf.Set alphabet = new Surf.Set();
Surf.Set transitions = new Surf.Set();
int startState;
Surf.Set finalStates = new Surf.Set();
Surf.Set tokens = new Surf.Set();
for (int i = 0; i < fa.States.Length; i++)
{
states.Add(stateMap.Apply(fa.States[i]));
}
alphabet = new Surf.Set(fa.Alphabet);
for (int i = 0; i < fa.Transitions.Length; i++)
{
if (fa.Transitions[i].Epsilon)
{
transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { stateMap.Apply(fa.Transitions[i].FromState) }), stateMap.Apply(fa.Transitions[i].ToState) }));
}
else
{
transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { stateMap.Apply(fa.Transitions[i].FromState), fa.Transitions[i].Character }), stateMap.Apply(fa.Transitions[i].ToState) }));
}
}
startState = (int)stateMap.Apply(fa.StartState);
for (int i = 0; i < fa.FinalStates.Length; i++)
{
finalStates.Add(stateMap.Apply(fa.FinalStates[i]));
tokens.Add(new Surf.Tuple(new object[] { stateMap.Apply(fa.FinalStates[i]), Lookup(fa.Tokens, fa.FinalStates[i]) }));
}
System.Console.WriteLine("END Reorder " + System.DateTime.Now.ToString());
return(new FiniteAutomaton(states, alphabet, transitions, startState, finalStates, tokens));
}
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("using System;");
writer.WriteLine("");
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
writer.WriteLine(" public class "+ name + "TextReader");
writer.WriteLine(" {");
writer.WriteLine(" private System.IO.TextReader _reader;");
writer.WriteLine("");
writer.WriteLine(" public int Line = 1;");
writer.WriteLine(" public int Column = 1;");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "TextReader(System.IO.TextReader reader)");
writer.WriteLine(" {");
writer.WriteLine(" this._reader = reader;");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" public int Peek()");
writer.WriteLine(" {");
writer.WriteLine(" return this._reader.Peek();");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" public int Read()");
writer.WriteLine(" {");
writer.WriteLine(" char ch = (char)this._reader.Read();");
writer.WriteLine(" if(ch == '\\n')");
writer.WriteLine(" {");
writer.WriteLine(" Line++;");
writer.WriteLine(" Column = 1;");
writer.WriteLine(" }");
writer.WriteLine(" return ch;");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" }");
writer.WriteLine("}");
}
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("using System;");
writer.WriteLine("");
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
writer.WriteLine(" public class "+ name + "TokenType");
writer.WriteLine(" {");
Dictionary <string, string> tokens = new Dictionary <string, string>();
foreach (Bamboo.Parsing.FiniteAutomata.Token token in finiteAutomaton.Tokens)
{
if (!tokens.ContainsKey(token.Name))
{
tokens.Add(token.Name, token.Name);
}
}
int i = 0;
foreach (string tokenName in tokens.Keys)
{
writer.WriteLine(" public const int "+ tokenName + " = " + i + ";");
i++;
}
writer.WriteLine("");
writer.WriteLine(" public const int _ERROR_ = "+ i + ";");
i++;
writer.WriteLine(" public const int _EOF_ = "+ i + ";");
writer.WriteLine(" }");
writer.WriteLine("}");
}
public static void GenerateClass(string name, string multinamespace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// " + name + "Tokenizer.cpp");
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("");
writer.WriteLine("#include <istream>");
writer.WriteLine("#include \"" + name + "Parser.h\"");
writer.WriteLine("#include \"" + name + "Token.h\"");
writer.WriteLine("#include \"" + name + "Tokenizer.h\"");
writer.WriteLine("#include \"" + name + "TokenType.h\"");
writer.WriteLine("");
string[] namespaces = multinamespace.Split(new string[] { "::" }, StringSplitOptions.None);
foreach (string nspace in namespaces)
{
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
}
writer.WriteLine(" int "+ name + "Tokenizer::_counter = 0;");
writer.WriteLine("");
writer.WriteLine(" void "+ name + "Tokenizer::next(char ch)");
writer.WriteLine(" {");
writer.WriteLine(" switch(_state)");
writer.WriteLine(" {");
//TODO if ch == -1, purge remaining token.
foreach (int state in finiteAutomaton.States)
{
bool isFinal = Operators.IsFinal(state, finiteAutomaton.FinalStates);
bool hasTransitions = Operators.HasTransitions(state, finiteAutomaton.Transitions);
if (!isFinal || hasTransitions)
{
writer.WriteLine(" case "+ state + ":");
writer.WriteLine(" {");
writer.WriteLine(" switch (ch)");
writer.WriteLine(" {");
if (state == 0)
{
writer.WriteLine(" case ' ':");
writer.WriteLine(" case '\\t':");
writer.WriteLine(" case '\\r':");
writer.WriteLine(" case '\\n':");
writer.WriteLine(" {");
writer.WriteLine(" // Trim whitespace");
writer.WriteLine(" break;");
writer.WriteLine(" }");
}
Surf.Set transitions = new Surf.Set();
foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions)
{
if (transition.FromState == state)
{
transitions.Add(new Surf.Tuple(new object[] { transition.ToState, transition.Character }));
}
}
transitions = transitions.Nest();
foreach (Surf.Tuple transition in transitions)
{
foreach (char character in (Surf.Set)transition[1])
{
writer.WriteLine(" case '"+ Operators.Escape(character) + "':");
}
writer.WriteLine(" {");
bool isFinal2 = Operators.IsFinal((int)transition[0], finiteAutomaton.FinalStates);
bool hasTransitions2 = Operators.HasTransitions((int)transition[0], finiteAutomaton.Transitions);
writer.WriteLine(" _buffer += ch;");
if (isFinal2 && !hasTransitions2)
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken((int)transition[0], finiteAutomaton.Tokens);
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::" + token.Name + ", _buffer);");
writer.WriteLine(" _parser->next(token);");
writer.WriteLine(" _state = 0;");
writer.WriteLine(" _buffer.clear();");
}
else
{
writer.WriteLine(" _state = "+ transition[0] + ";");
}
writer.WriteLine(" break;");
writer.WriteLine(" }");
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
if (isFinal)
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::" + token.Name + ", _buffer);");
writer.WriteLine(" _parser->next(token);");
writer.WriteLine(" _state = 0;");
writer.WriteLine(" _buffer.clear();");
writer.WriteLine(" next(ch);");
}
else
{
writer.WriteLine(" _buffer += ch;");
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::_ERROR_, _buffer);");
writer.WriteLine(" _parser->next(token);");
}
writer.WriteLine(" break;");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" break;");
writer.WriteLine(" }");
}
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::_ERROR_, _buffer);");
writer.WriteLine(" _parser->next(token);");
writer.WriteLine(" break;");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" }");
foreach (string nspace in namespaces)
{
writer.WriteLine("}");
}
}
public static FiniteAutomaton DFA(Bamboo.Parsing.FiniteAutomata.FiniteAutomaton nfa, Counter counter)
{
System.Console.WriteLine("START DFA " + System.DateTime.Now.ToString());
Surf.Set epsilonCache = new Surf.Set();
Surf.Set moveCache = new Surf.Set();
Surf.Set nfa_transitions = Set(nfa.Transitions);
Dictionary <string, int> nfa_transitions_lookup = new Dictionary <string, int>();
foreach (Surf.Tuple transition in nfa_transitions)
{
Surf.Tuple input = (Surf.Tuple)transition[0];
if (input.Count == 2)
{
int state = (int)input[0];
char ch = (char)input[1];
nfa_transitions_lookup.Add("" + state + "::" + ch, (int)transition[1]);
}
}
Surf.Set q0 = EpsilonClosure(new Surf.Set(new object[] { nfa.StartState }), nfa_transitions, epsilonCache);
Surf.Set Q = new Surf.Set(); // A set whose element are sets of states that are subsets of N.
Q.Add(q0);
Surf.Set T = new Surf.Set();
Stack <Surf.Set> worklist = new Stack <Surf.Set>();
worklist.Push(q0);
while (worklist.Count > 0)
{
Surf.Set q = worklist.Pop();
foreach (char character in nfa.Alphabet)
{
Surf.Set t = EpsilonClosure(Move(q, character, nfa_transitions_lookup), nfa_transitions, epsilonCache);
if (t.Count > 0)
{
T.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { q, character }), t }));
if (!Q.Contains(t))
{
Q.Add(t);
worklist.Push(t);
}
}
}
}
Surf.Set states = new Surf.Set();
Surf.Set alphabet = new Surf.Set();
Surf.Set transitions = new Surf.Set();
int startState;
Surf.Set finalStates = new Surf.Set();
Surf.Set tokens = new Surf.Set();
Surf.Set D = new Surf.Set();
foreach (Surf.Set q in Q)
{
D.Add(new Surf.Tuple(new object[] { q, counter.Next() }));
}
foreach (Surf.Set q in Q)
{
states.Add(D.Apply(q));
}
alphabet = new Surf.Set(nfa.Alphabet);
foreach (Surf.Tuple t in T)
{
Surf.Tuple input = (Surf.Tuple)t[0];
int fromState = (int)D.Apply(input[0]);
char character = (char)input[1];
int toState = (int)D.Apply(t[1]);
transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { fromState, character }), toState }));
}
startState = (int)states[0];
foreach (int state in nfa.FinalStates)
{
foreach (Surf.Tuple d in D)
{
Surf.Set q = (Surf.Set)d[0];
if (q.Contains(state))
{
finalStates.Add(d[1]);
foreach (int q_state in q)
{
string token = Lookup(nfa.Tokens, (int)q_state);
if (token.Length > 0)
{
tokens.Add(new Surf.Tuple(new object[] { d[1], token }));
break;
}
}
}
}
}
System.Console.WriteLine("END DFA " + System.DateTime.Now.ToString());
//foreach (Token token in nfa.Tokens)
//{
// System.Console.WriteLine(token.Name + " " + token.Number);
//}
//System.Console.WriteLine();
//Surf.Printer.Print(tokens, System.Console.Out);
//System.Console.WriteLine();
//System.Console.WriteLine();
return(Minimize(Reorder(new FiniteAutomaton(states, alphabet, transitions, startState, finalStates, tokens))));
}
//
// Subset Construction
//
public static FiniteAutomaton DFA(Bamboo.Parsing.FiniteAutomata.FiniteAutomaton nfa)
{
return(DFA(nfa, new Counter()));
}
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("using System;");
writer.WriteLine("");
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
writer.WriteLine(" public class "+ name + "Tokenizer");
writer.WriteLine(" {");
writer.WriteLine(" private System.Text.StringBuilder _stringBuilder = new System.Text.StringBuilder();");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Tokenizer()");
writer.WriteLine(" {");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Token Tokenize(" + name + "TextReader reader)");
writer.WriteLine(" {");
writer.WriteLine(" int n;");
writer.WriteLine(" char ch;");
writer.WriteLine("");
writer.WriteLine(" _stringBuilder.Length = 0;");
writer.WriteLine("");
writer.WriteLine(" //");
writer.WriteLine(" // Trim whitespace");
writer.WriteLine(" //");
writer.WriteLine(" while ((n = reader.Peek()) != -1)");
writer.WriteLine(" {");
writer.WriteLine(" ch = (char)n;");
writer.WriteLine(" switch (ch)");
writer.WriteLine(" {");
writer.WriteLine(" case ' ':");
writer.WriteLine(" case '\\t':");
if (!HasNewline(finiteAutomaton.Tokens))
{
writer.WriteLine(" case '\\r':");
writer.WriteLine(" case '\\n':");
}
writer.WriteLine(" {");
writer.WriteLine(" reader.Read();");
writer.WriteLine(" break;");
writer.WriteLine(" }");
writer.WriteLine(" default:");
writer.WriteLine(" {");
writer.WriteLine(" goto s0;");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine("");
foreach (int state in finiteAutomaton.States)
{
writer.WriteLine(" s"+ state + ":");
if (Operators.IsFinal(state, finiteAutomaton.FinalStates) && !Operators.HasTransitions(state, finiteAutomaton.Transitions))
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());");
}
else
{
writer.WriteLine(" n = reader.Peek();");
writer.WriteLine(" if (n == -1)");
writer.WriteLine(" {");
writer.WriteLine(" if (_stringBuilder.Length == 0)");
writer.WriteLine(" {");
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._EOF_);");
writer.WriteLine(" }");
writer.WriteLine(" else");
writer.WriteLine(" {");
if (Operators.IsFinal(state, finiteAutomaton.FinalStates))
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());");
}
else
{
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_);");
}
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" ch = (char)n;");
writer.WriteLine(" switch (ch)");
writer.WriteLine(" {");
Surf.Set transitions = new Surf.Set();
foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions)
{
if (transition.FromState == state)
{
transitions.Add(new Surf.Tuple(new object[] { transition.ToState, transition.Character }));
}
}
transitions = transitions.Nest();
foreach (Surf.Tuple transition in transitions)
{
foreach (char character in (Surf.Set)transition[1])
{
writer.WriteLine(" case '"+ Operators.Escape(character) + "':");
}
writer.WriteLine(" {");
writer.WriteLine(" reader.Read();");
writer.WriteLine(" _stringBuilder.Append(ch);");
writer.WriteLine(" goto s"+ transition[0] + ";");
writer.WriteLine(" }");
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
if (Operators.IsFinal(state, finiteAutomaton.FinalStates))
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());");
}
else
{
writer.WriteLine(" _stringBuilder.Append(ch);");
writer.WriteLine(" goto se;");
}
writer.WriteLine(" }");
writer.WriteLine(" }");
}
writer.WriteLine("");
}
writer.WriteLine(" se:");
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_, _stringBuilder.ToString());");
writer.WriteLine("");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" }");
writer.WriteLine("}");
}
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("using System;");
writer.WriteLine("");
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
writer.WriteLine(" public class "+ name + "Tokenizer");
writer.WriteLine(" {");
writer.WriteLine(" private static readonly int[,] TABLE = new int[,] {");
Surf.Set transitions = new Surf.Set();
foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions)
{
transitions.Add(new Surf.Tuple(new object[] { new Surf.Tuple(new object[] { transition.FromState, transition.Character }), transition.ToState }));
}
foreach (int state in finiteAutomaton.States)
{
writer.Write(" { ");
foreach (object character in finiteAutomaton.Alphabet)
{
Surf.Tuple key = new Surf.Tuple(new object[] { state, character });
if (transitions.IsDefined(key))
{
int toState = (int)transitions.Apply(key);
writer.Write("" + toState + ", ");
}
else
{
writer.Write("-1, ");
}
}
writer.WriteLine("},");
}
writer.WriteLine(" };");
writer.WriteLine("");
writer.WriteLine(" private System.Text.StringBuilder _stringBuilder = new System.Text.StringBuilder();");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Tokenizer()");
writer.WriteLine(" {");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Token Tokenize(" + name + "TextReader reader)");
writer.WriteLine(" {");
writer.WriteLine(" int n;");
writer.WriteLine(" char ch;");
writer.WriteLine("");
writer.WriteLine(" _stringBuilder.Length = 0;");
writer.WriteLine("");
/*
* //
* // Trim whitespace
* //
* while ((n = reader.Peek()) != -1)
* {
* ch = (char)n;
* switch (ch)
* {
* case ' ':
* case '\t':
* case '\r':
* case '\n':
* {
* break;
* }
* default:
* {
* goto s0;
* }
* }
* }
*/
writer.WriteLine(" n = reader.Peek();");
writer.WriteLine(" ch = (char)0;");
writer.WriteLine(" int state = 0;");
writer.WriteLine("");
writer.WriteLine(" if (n == -1)");
writer.WriteLine(" {");
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._EOF_);");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" while(n != -1 && state != -1)");
writer.WriteLine(" {");
writer.WriteLine(" ch = (char)n;");
writer.WriteLine("");
writer.WriteLine(" switch(ch)");
writer.WriteLine(" {");
int i = 0;
foreach (char character in finiteAutomaton.Alphabet)
{
writer.WriteLine(" case '"+ Escape(character) + "':");
writer.WriteLine(" {");
writer.WriteLine(" int state2 = TABLE[state, "+ i + "];");
writer.WriteLine(" if(state2 == -1)");
writer.WriteLine(" {");
writer.WriteLine(" goto EXIT;");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" state = state2;");
writer.WriteLine(" _stringBuilder.Append(ch);");
writer.WriteLine(" reader.Read();");
writer.WriteLine(" n = reader.Peek();");
writer.WriteLine(" break;");
writer.WriteLine(" }");
i++;
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
writer.WriteLine(" goto EXIT;");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" EXIT:");
writer.WriteLine(" switch(state)");
writer.WriteLine(" {");
foreach (int finalState in finiteAutomaton.FinalStates)
{
Bamboo.Parsing.FiniteAutomata.Token token = GetToken(finalState, finiteAutomaton.Tokens);
writer.WriteLine(" case "+ finalState + ":");
writer.WriteLine(" {");
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());");
writer.WriteLine(" }");
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
writer.WriteLine(" if (n > -1)");
writer.WriteLine(" {");
writer.WriteLine(" _stringBuilder.Append(ch);");
writer.WriteLine(" }");
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_, _stringBuilder.ToString());");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" }");
writer.WriteLine("}");
}
