private static Surf.Set Split(Surf.Set S, char character, Surf.Set transitions, Surf.Set P2)
{
//System.Console.WriteLine("START Split2 " + System.DateTime.Now.ToString());
Surf.Set f_nP = PartitionMap(P2);
Surf.Set partition_map = new Surf.Set();
foreach (int fromState in S)
{
object key = new Surf.Tuple(new object[] { fromState, character });
if (transitions.IsDefined(key))
{
object value = transitions.Apply(key);
int toState = (int)value;
int partition = (int)f_nP.Apply(toState);
partition_map.Add(new Surf.Tuple(new object[] { partition, fromState }));
}
else
{
partition_map.Add(new Surf.Tuple(new object[] { -1, fromState }));
}
}
//System.Console.WriteLine("END Split2 " + System.DateTime.Now.ToString());
return(partition_map.Nest().Range());
}
private static Surf.Set SplitFinalStates(Token[] tokens)
{
Surf.Set fn = new Surf.Set();
foreach (Token token in tokens)
{
fn.Add(new Surf.Tuple(new object[] { token.Name, token.Number }));
}
return(fn.Nest().Range());
}
public static void GenerateClass(string name, string multinamespace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// " + name + "Tokenizer.cpp");
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("");
writer.WriteLine("#include <istream>");
writer.WriteLine("#include \"" + name + "Parser.h\"");
writer.WriteLine("#include \"" + name + "Token.h\"");
writer.WriteLine("#include \"" + name + "Tokenizer.h\"");
writer.WriteLine("#include \"" + name + "TokenType.h\"");
writer.WriteLine("");
string[] namespaces = multinamespace.Split(new string[] { "::" }, StringSplitOptions.None);
foreach (string nspace in namespaces)
{
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
}
writer.WriteLine(" int "+ name + "Tokenizer::_counter = 0;");
writer.WriteLine("");
writer.WriteLine(" void "+ name + "Tokenizer::next(char ch)");
writer.WriteLine(" {");
writer.WriteLine(" switch(_state)");
writer.WriteLine(" {");
//TODO if ch == -1, purge remaining token.
foreach (int state in finiteAutomaton.States)
{
bool isFinal = Operators.IsFinal(state, finiteAutomaton.FinalStates);
bool hasTransitions = Operators.HasTransitions(state, finiteAutomaton.Transitions);
if (!isFinal || hasTransitions)
{
writer.WriteLine(" case "+ state + ":");
writer.WriteLine(" {");
writer.WriteLine(" switch (ch)");
writer.WriteLine(" {");
if (state == 0)
{
writer.WriteLine(" case ' ':");
writer.WriteLine(" case '\\t':");
writer.WriteLine(" case '\\r':");
writer.WriteLine(" case '\\n':");
writer.WriteLine(" {");
writer.WriteLine(" // Trim whitespace");
writer.WriteLine(" break;");
writer.WriteLine(" }");
}
Surf.Set transitions = new Surf.Set();
foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions)
{
if (transition.FromState == state)
{
transitions.Add(new Surf.Tuple(new object[] { transition.ToState, transition.Character }));
}
}
transitions = transitions.Nest();
foreach (Surf.Tuple transition in transitions)
{
foreach (char character in (Surf.Set)transition[1])
{
writer.WriteLine(" case '"+ Operators.Escape(character) + "':");
}
writer.WriteLine(" {");
bool isFinal2 = Operators.IsFinal((int)transition[0], finiteAutomaton.FinalStates);
bool hasTransitions2 = Operators.HasTransitions((int)transition[0], finiteAutomaton.Transitions);
writer.WriteLine(" _buffer += ch;");
if (isFinal2 && !hasTransitions2)
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken((int)transition[0], finiteAutomaton.Tokens);
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::" + token.Name + ", _buffer);");
writer.WriteLine(" _parser->next(token);");
writer.WriteLine(" _state = 0;");
writer.WriteLine(" _buffer.clear();");
}
else
{
writer.WriteLine(" _state = "+ transition[0] + ";");
}
writer.WriteLine(" break;");
writer.WriteLine(" }");
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
if (isFinal)
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::" + token.Name + ", _buffer);");
writer.WriteLine(" _parser->next(token);");
writer.WriteLine(" _state = 0;");
writer.WriteLine(" _buffer.clear();");
writer.WriteLine(" next(ch);");
}
else
{
writer.WriteLine(" _buffer += ch;");
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::_ERROR_, _buffer);");
writer.WriteLine(" _parser->next(token);");
}
writer.WriteLine(" break;");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" break;");
writer.WriteLine(" }");
}
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
writer.WriteLine(" "+ name + "Token token(" + name + "TokenType::_ERROR_, _buffer);");
writer.WriteLine(" _parser->next(token);");
writer.WriteLine(" break;");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" }");
foreach (string nspace in namespaces)
{
writer.WriteLine("}");
}
}
public static void Generate(string name, string nspace, Bamboo.Parsing.FiniteAutomata.FiniteAutomaton finiteAutomaton, System.IO.TextWriter writer)
{
writer.WriteLine("//");
writer.WriteLine("// AUTOGENERATED " + System.DateTime.Now + "");
writer.WriteLine("//");
writer.WriteLine("using System;");
writer.WriteLine("");
writer.WriteLine("namespace " + nspace + "");
writer.WriteLine("{");
writer.WriteLine(" public class "+ name + "Tokenizer");
writer.WriteLine(" {");
writer.WriteLine(" private System.Text.StringBuilder _stringBuilder = new System.Text.StringBuilder();");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Tokenizer()");
writer.WriteLine(" {");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" public "+ name + "Token Tokenize(" + name + "TextReader reader)");
writer.WriteLine(" {");
writer.WriteLine(" int n;");
writer.WriteLine(" char ch;");
writer.WriteLine("");
writer.WriteLine(" _stringBuilder.Length = 0;");
writer.WriteLine("");
writer.WriteLine(" //");
writer.WriteLine(" // Trim whitespace");
writer.WriteLine(" //");
writer.WriteLine(" while ((n = reader.Peek()) != -1)");
writer.WriteLine(" {");
writer.WriteLine(" ch = (char)n;");
writer.WriteLine(" switch (ch)");
writer.WriteLine(" {");
writer.WriteLine(" case ' ':");
writer.WriteLine(" case '\\t':");
if (!HasNewline(finiteAutomaton.Tokens))
{
writer.WriteLine(" case '\\r':");
writer.WriteLine(" case '\\n':");
}
writer.WriteLine(" {");
writer.WriteLine(" reader.Read();");
writer.WriteLine(" break;");
writer.WriteLine(" }");
writer.WriteLine(" default:");
writer.WriteLine(" {");
writer.WriteLine(" goto s0;");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine("");
foreach (int state in finiteAutomaton.States)
{
writer.WriteLine(" s"+ state + ":");
if (Operators.IsFinal(state, finiteAutomaton.FinalStates) && !Operators.HasTransitions(state, finiteAutomaton.Transitions))
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());");
}
else
{
writer.WriteLine(" n = reader.Peek();");
writer.WriteLine(" if (n == -1)");
writer.WriteLine(" {");
writer.WriteLine(" if (_stringBuilder.Length == 0)");
writer.WriteLine(" {");
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._EOF_);");
writer.WriteLine(" }");
writer.WriteLine(" else");
writer.WriteLine(" {");
if (Operators.IsFinal(state, finiteAutomaton.FinalStates))
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());");
}
else
{
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_);");
}
writer.WriteLine(" }");
writer.WriteLine(" }");
writer.WriteLine(" ch = (char)n;");
writer.WriteLine(" switch (ch)");
writer.WriteLine(" {");
Surf.Set transitions = new Surf.Set();
foreach (Bamboo.Parsing.FiniteAutomata.Transition transition in finiteAutomaton.Transitions)
{
if (transition.FromState == state)
{
transitions.Add(new Surf.Tuple(new object[] { transition.ToState, transition.Character }));
}
}
transitions = transitions.Nest();
foreach (Surf.Tuple transition in transitions)
{
foreach (char character in (Surf.Set)transition[1])
{
writer.WriteLine(" case '"+ Operators.Escape(character) + "':");
}
writer.WriteLine(" {");
writer.WriteLine(" reader.Read();");
writer.WriteLine(" _stringBuilder.Append(ch);");
writer.WriteLine(" goto s"+ transition[0] + ";");
writer.WriteLine(" }");
}
writer.WriteLine(" default:");
writer.WriteLine(" {");
if (Operators.IsFinal(state, finiteAutomaton.FinalStates))
{
Bamboo.Parsing.FiniteAutomata.Token token = Operators.GetToken(state, finiteAutomaton.Tokens);
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType." + token.Name + ", _stringBuilder.ToString());");
}
else
{
writer.WriteLine(" _stringBuilder.Append(ch);");
writer.WriteLine(" goto se;");
}
writer.WriteLine(" }");
writer.WriteLine(" }");
}
writer.WriteLine("");
}
writer.WriteLine(" se:");
writer.WriteLine(" return new "+ name + "Token(" + name + "TokenType._ERROR_, _stringBuilder.ToString());");
writer.WriteLine("");
writer.WriteLine(" }");
writer.WriteLine("");
writer.WriteLine(" }");
writer.WriteLine("}");
}
