private ail.net.parser.FsaPair GetPairFromStates(ail.net.parser.FsaState xi_p_state, ail.net.parser.FsaState xi_q_state, Hashtable xi_table)
{
ail.net.framework.Assert.NonNullReference(xi_p_state, "xi_p_state");
ail.net.framework.Assert.NonNullReference(xi_q_state, "xi_q_state");
ail.net.framework.Assert.NonNullReference(xi_table, "xi_table");
ail.net.parser.FsaPair result = null;
ail.net.parser.FsaPairKey key;
key.PState = xi_p_state;
key.QState = xi_q_state;
result = (ail.net.parser.FsaPair)xi_table[key];
if (result == (object)null)
{
key.PState = xi_q_state;
key.QState = xi_p_state;
result = (ail.net.parser.FsaPair)xi_table[key];
}
ail.net.framework.Assert.NonNullReference(result, "result");
return(result);
}
public ail.net.parser.FsaTransition AddTransition(ail.net.parser.FsaState xi_start, ail.net.parser.FsaState xi_end, string xi_predicate, string xi_context, int xi_rank)
{
ail.net.framework.Assert.NonNullReference(xi_start, "xi_start");
ail.net.framework.Assert.NonNullReference(xi_end, "xi_end");
ail.net.framework.Assert.NonEmptyString(xi_predicate, "xi_predicate");
return(AddTransition(xi_start.Id, xi_end.Id, xi_predicate, (char)0, xi_context, xi_rank));
}
public ail.net.parser.FsaTransition AddTransition(ail.net.parser.FsaState xi_start, ail.net.parser.FsaState xi_end, string xi_predicate, char xi_switch_char)
{
ail.net.framework.Assert.NonNullReference(xi_start, "xi_start");
ail.net.framework.Assert.NonNullReference(xi_end, "xi_end");
ail.net.framework.Assert.NonEmptyString(xi_predicate, "xi_predicate");
return(AddTransition(xi_start.Id, xi_end.Id, xi_predicate, xi_switch_char, "", 0));
}
public ail.net.parser.Fsa Minimize(ail.net.parser.Fsa.EMinimizationMode xi_mode)
{
ail.net.parser.Fsa result = new ail.net.parser.Fsa();
// phase I (clean)
RemoveUselessStates();
// phase II (divide fsa into equivalent groups, each group will be new fsa state)
Hashtable partition = new Hashtable();
BuildPartition(ref partition, xi_mode);
// phase III (compose fsa)
foreach (ail.net.parser.FsaStateSet group in partition.Values)
{
ail.net.parser.FsaState new_state = result.AddState(group.Id);
ail.net.parser.FsaState old_state = (ail.net.parser.FsaState)ail.net.framework.Functor.FirstElementOfCollection(group.States.Values);
// check if group has start state
foreach (ail.net.parser.FsaState state in group.States.Values)
{
if (IsStartState(state))
{
result.StartState = new_state;
break;
}
}
// check if group final state
foreach (ail.net.parser.FsaState state in group.States.Values)
{
if (IsFinalState(state))
{
result.AddFinalState(new_state, state.Token);
break;
}
}
// add transitions
foreach (ail.net.parser.FsaTransition transition in old_state.Transitions.Values)
{
ail.net.parser.FsaStateSet transition_group = GetGroupFromState(transition.End, partition);
ail.net.framework.Assert.NonNullReference(transition_group, "transition_group");
result.AddTransition(new_state.Id,
transition_group.Id,
transition.Predicate.Text,
transition.Predicate.SwitchChar,
transition.Predicate.Context,
transition.Predicate.Rank);
}
}
// phase IV (clean)
result.RemoveUselessStates();
return(result);
}
public void RemoveTransition(ail.net.parser.FsaState xi_state, ail.net.parser.FsaTransition xi_transition)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
ail.net.framework.Assert.NonNullReference(xi_transition, "xi_transition");
ail.net.framework.Assert.Condition(States.Contains(xi_state.Id), "xi_state.Id");
ail.net.framework.Assert.Condition(xi_state.Transitions.Contains(xi_transition.Id), "xi_state.Transitions.Contains(xi_transition.Id)");
xi_state.Transitions.Remove(xi_transition.Id);
}
public void RemoveState(ail.net.parser.FsaState xi_state)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
ail.net.framework.Assert.Condition(States.Contains(xi_state.Id), "States.Contains(xi_state.Id)");
FinalStates.Remove(xi_state.Id);
States.Remove(xi_state.Id);
}
public ail.net.parser.FsaStateSet CalculateStateEclosure(ail.net.parser.FsaState xi_state)
{
// Calculate e-closure:
// Push all states in T onto stack;
// Initialize e-closure(T) to T;
// While stack <> empty do
// Pop t;
// For each transition (t, u) in e-transition do
// If u is not in e-closure(T) then
// Add u to e-closure(T);
// Push u;
// End;
// End;
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
ail.net.parser.FsaStateSet result = new ail.net.parser.FsaStateSet();
ail.net.parser.FsaState state = xi_state;
result.States.Add(state.Id, state);
Stack stack = new Stack();
for (;;)
{
int count = stack.Count;
foreach (ail.net.parser.FsaTransition transition in state.Transitions.Values)
{
if (transition.IsEpsilon())
{
if (!result.States.Contains(transition.End))
{
ail.net.parser.FsaState end_state = (ail.net.parser.FsaState)States[transition.End];
ail.net.framework.Assert.NonNullReference(end_state, "end_state");
result.States.Add(end_state.Id, end_state);
stack.Push(end_state);
}
}
}
if (stack.Count != count && !result.States.Contains(state.Id))
{
result.States.Add(state.Id, state);
}
if (stack.Count == 0)
{
break;
}
state = (ail.net.parser.FsaState)stack.Pop();
}
return(result);
}
public void AddFsa(ail.net.parser.Fsa xi_fsa)
{
ail.net.framework.Assert.NonNullReference(xi_fsa, "xi_fsa");
bool empty = IsEmpty();
Hashtable map = new Hashtable(ail.net.framework.Functor.Align(xi_fsa.States.Count, ail.net.framework.Functor.kAlignValue));
for (int i = 0; i < xi_fsa.StateCounter.Current; i++)
{
ail.net.parser.FsaState old_state = (ail.net.parser.FsaState)xi_fsa.States[i];
ail.net.framework.Assert.NonNullReference(old_state, "old_state");
ail.net.parser.FsaState new_state = AddState();
ail.net.framework.Assert.NonNullReference(new_state, "new_state");
new_state.Label = old_state.Label;
new_state.Token = old_state.Token;
if (xi_fsa.IsFinalState(old_state))
{
AddFinalState(new_state, new_state.Token);
}
map[old_state.Id] = new_state.Id;
}
foreach (ail.net.parser.FsaState state in xi_fsa.States.Values)
{
foreach (ail.net.parser.FsaTransition transition in state.Transitions.Values)
{
ail.net.parser.FsaState start_state = (ail.net.parser.FsaState)States[map[transition.Start]];
ail.net.parser.FsaState end_state = (ail.net.parser.FsaState)States[map[transition.End]];
ail.net.framework.Assert.NonNullReference(start_state, "start_state");
ail.net.framework.Assert.NonNullReference(end_state, "end_state");
AddTransition(start_state.Id,
end_state.Id,
transition.Predicate.Text,
transition.Predicate.SwitchChar,
transition.Predicate.Context,
transition.Predicate.Rank);
}
}
if (!empty)
{
if (xi_fsa.StartState != (object)null)
{
AddTransition(StartState, (ail.net.parser.FsaState)States[map[xi_fsa.StartState.Id]], ail.net.parser.FsaTransition.kEpsilonPredicate);
}
}
}
public void AddFinalState(ail.net.parser.FsaState xi_state, ail.net.parser.Token xi_token)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
ail.net.framework.Assert.NonNullReference(xi_token, "xi_token");
if (!FinalStates.Contains(xi_state.Id))
{
xi_state.Token = xi_token;
FinalStates.Add(xi_state.Id, xi_state);
}
}
private bool AreEqual(ail.net.parser.FsaState xi_state, ail.net.parser.FsaState xi_curr_state, Hashtable xi_partition)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
ail.net.framework.Assert.NonNullReference(xi_curr_state, "xi_curr_state");
ail.net.framework.Assert.NonNullReference(xi_partition, "xi_partition");
bool result = false;
// loop on all predicates relevant for these two states only,
// as automaton is deterministic every state has at most one transition with given predicate
#if USE_STATE_PREDICATE_SET
Hashtable predicates = CombinePredicates(xi_state, xi_curr_state);
foreach (ail.net.parser.FsaTransitionPredicate predicate in predicates.Values)
#else
foreach (ail.net.parser.FsaTransitionPredicate predicate in Predicates.Values)
#endif
{
ail.net.parser.FsaTransition t1 = xi_state.GetTransitionByPredicate(predicate);
ail.net.parser.FsaTransition t2 = xi_curr_state.GetTransitionByPredicate(predicate);
#if !USE_STATE_PREDICATE_SET
if (t1 == (object)null && t2 == (object)null) // skip not relevant
{
continue;
}
#endif
result = false; // reset
if (t1 == (object)null || t2 == (object)null)
{
break;
}
// attempt to find group with these two states
foreach (ail.net.parser.FsaStateSet group in xi_partition.Values)
{
if (group.States.Contains(t1.End) && group.States.Contains(t2.End))
{
result = true;
break;
}
}
if (!result)
{
break;
}
}
return(result);
}
private Hashtable SplitGroup(ail.net.parser.FsaStateSet xi_group, Hashtable xi_partition)
{
ail.net.framework.Assert.NonNullReference(xi_group, "xi_group");
ail.net.framework.Assert.Condition(xi_group.States.Count > 0, "xi_group.States.Count > 0");
ail.net.framework.Assert.NonNullReference(xi_partition, "xi_partition");
Hashtable result = new Hashtable();
// first state introduces new subgroup
ail.net.parser.FsaStateSet group = new ail.net.parser.FsaStateSet(GroupCounter.Next());
result.Add(group.Id, group);
group.States.Add(((ail.net.parser.FsaState)ail.net.framework.Functor.FirstElementOfCollection(xi_group.States.Values)).Id,
(ail.net.parser.FsaState)ail.net.framework.Functor.FirstElementOfCollection(xi_group.States.Values));
// go over states in the group
foreach (ail.net.parser.FsaState state in xi_group.States.Values)
{
bool added = false;
// check if state is not ortogonal to any subgroup
foreach (ail.net.parser.FsaStateSet subgroup in result.Values)
{
if (subgroup.States.Contains(state.Id)) // state == state
{
added = true;
break;
}
ail.net.parser.FsaState group_state = (ail.net.parser.FsaState)ail.net.framework.Functor.FirstElementOfCollection(subgroup.States.Values);
if (AreEqual(state, group_state, xi_partition))
{
subgroup.States.Add(state.Id, state);
added = true;
break;
}
}
// if state is ortogonal introduce a new group and add state to this new group
if (!added)
{
group = new ail.net.parser.FsaStateSet(GroupCounter.Next());
result.Add(group.Id, group);
group.States.Add(state.Id, state);
}
}
return(result);
}
private static string GenerateStartStateCode(ail.net.parser.FsaState xi_state)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
StringBuilder result = new StringBuilder();
for (int rank = 0; ; rank++) // fix point
{
int length = result.Length;
foreach (ail.net.parser.FsaTransition transition in xi_state.Transitions.Values)
{
ail.net.framework.Assert.NonNullReference(transition, "transition");
if (transition.Predicate.Rank == rank)
{
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"else if(");
if (transition.Predicate.Context.Length != 0)
{
result.Append("xi_context == ");
result.Append(transition.Predicate.Context);
result.Append(" && ");
}
result.Append(transition.Predicate.Text);
result.Append(@"(Current))");
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"{");
result.Append(Environment.NewLine);
result.Append(kTab4);
result.Append(@"goto _q");
result.Append(transition.End.ToString());
result.Append(@";");
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"}");
}
}
if (result.Length == length && rank > ail.net.parser.FsaTransition.kMaxRankValue)
{
break;
}
}
return(result.ToString());
}
public ail.net.parser.FsaState AddState(int xi_id)
{
ail.net.parser.FsaState result = null;
if (!States.Contains(xi_id))
{
result = new ail.net.parser.FsaState(xi_id, this);
States.Add(xi_id, result);
if (StartState == (object)null) // avoid calling operator == ()
{
StartState = result;
}
}
return(result);
}
private void RemoveUselessStates()
{
// removes only states which are not accessible from start state
// does not touch states which are not on the path for any input string
ResetMarkedStates();
Queue queue = new Queue();
queue.Enqueue(StartState);
while (queue.Count > 0)
{
ail.net.parser.FsaState state = (ail.net.parser.FsaState)queue.Dequeue();
if (!state.Marked)
{
state.Marked = true;
foreach (ail.net.parser.FsaTransition transition in state.Transitions.Values)
{
queue.Enqueue(States[transition.End]);
}
}
}
ArrayList keys = new ArrayList();
foreach (ail.net.parser.FsaState state in States.Values)
{
if (!state.Marked)
{
keys.Add(state);
}
}
foreach (ail.net.parser.FsaState key in keys)
{
RemoveState(key);
}
ResetMarkedStates();
}
public ail.net.parser.FsaTransition AddTransition(int xi_start, int xi_end, string xi_predicate, char xi_switch_char, string xi_context, int xi_rank)
{
ail.net.framework.Assert.NonEmptyString(xi_predicate, "xi_predicate");
ail.net.parser.FsaTransition result = null;
ail.net.parser.FsaState state = (ail.net.parser.FsaState)States[xi_start];
ail.net.framework.Assert.NonNullReference(state, "state");
int transition_id = TransitionCounter.Next();
result = new ail.net.parser.FsaTransition(transition_id, xi_start, xi_end, xi_predicate, xi_switch_char, xi_context, xi_rank);
state.Transitions.Add(transition_id, result);
if (!Predicates.Contains(xi_predicate))
{
Predicates.Add(xi_predicate, result.Predicate);
}
return(result);
}
private Hashtable CombinePredicates(ail.net.parser.FsaState xi_state1, ail.net.parser.FsaState xi_state2)
{
ail.net.framework.Assert.NonNullReference(xi_state1, "xi_state1");
ail.net.framework.Assert.NonNullReference(xi_state2, "xi_state2");
Hashtable result = new Hashtable();
foreach (ail.net.parser.FsaTransition transition in xi_state1.Transitions.Values)
{
result.Add(transition.Predicate.Text, transition.Predicate);
}
foreach (ail.net.parser.FsaTransition transition in xi_state2.Transitions.Values)
{
if (!result.Contains(transition.Predicate.Text))
{
result.Add(transition.Predicate.Text, transition.Predicate);
}
}
return(result);
}
public ail.net.parser.FsaStateSet CalculateMove(ail.net.parser.FsaStateSet xi_state_set, string xi_predicate)
{
// Algorithm to compute T=Move(S,c)
// T = 0;
// for each state s in S
// {
// for each edge e from s to some s’
// {
// if(e is labelled with c)
// {
// T = T U closure({s’});
// }
// }
// }
ail.net.framework.Assert.NonNullReference(xi_state_set, "xi_state_set");
ail.net.framework.Assert.NonEmptyString(xi_predicate, "xi_predicate");
ail.net.parser.FsaStateSet result = new ail.net.parser.FsaStateSet();
foreach (ail.net.parser.FsaState state in xi_state_set.States.Values)
{
foreach (ail.net.parser.FsaTransition transition in state.Transitions.Values)
{
if (transition.Predicate.Text == xi_predicate)
{
if (!result.States.Contains(transition.End))
{
ail.net.parser.FsaState end_state = (ail.net.parser.FsaState)States[transition.End];
ail.net.framework.Assert.NonNullReference(end_state, "end_state");
result.States.Add(end_state.Id, end_state);
}
}
}
}
return(result);
}
public static bool operator ==(FsaStateSet xi_lhs, FsaStateSet xi_rhs)
{
bool result = (object)xi_lhs != (object)null && (object)xi_rhs != (object)null && xi_lhs.States.Count == xi_rhs.States.Count;
if (result)
{
IEnumerator lhs_it = xi_lhs.States.GetEnumerator();
IEnumerator rhs_it = xi_rhs.States.GetEnumerator();
while (lhs_it.MoveNext() && rhs_it.MoveNext())
{
ail.net.parser.FsaState lhs_state = (ail.net.parser.FsaState)((DictionaryEntry)lhs_it.Current).Value;
ail.net.parser.FsaState rhs_state = (ail.net.parser.FsaState)((DictionaryEntry)rhs_it.Current).Value;
if (lhs_state.Id != rhs_state.Id)
{
result = false;
break;
}
}
}
return(result);
}
public void AddToken(ail.net.parser.Fsa xi_fsa, string xi_token, int xi_token_type, int xi_escape_token_type, string xi_escape_predicate)
{
ail.net.framework.Assert.NonNullReference(xi_fsa, "xi_fsa");
ail.net.framework.Assert.NonEmptyString(xi_token, "xi_token");
ail.net.parser.FsaState q1 = null;
if (xi_fsa.StartState != (object)null)
{
q1 = xi_fsa.AddState();
xi_fsa.AddTransition(xi_fsa.StartState, q1, ail.net.parser.FsaTransition.kEpsilonPredicate);
}
else
{
q1 = xi_fsa.AddState();
}
ail.net.parser.FsaState eq = null; // escape state
if (xi_escape_token_type != ail.net.parser.Token.kUndefinedEscapeTokenType)
{
eq = xi_fsa.AddState();
xi_fsa.AddTransition(eq, eq, xi_escape_predicate, ail.net.parser.FsaTransition.kMaxRankValue);
ail.net.parser.Token token = (ail.net.parser.Token)Factory.Create(Token.GetType());
token.Type = xi_escape_token_type;
token.Priority = ail.net.parser.LexAnalyzer.kEscapeTokenPriority;
xi_fsa.AddFinalState(eq, token);
}
ail.net.parser.FsaState q_prev = q1;
ail.net.parser.FsaState q_curr = q1;
for (int i = 0; i < xi_token.Length; i++)
{
q_curr = xi_fsa.AddState();
ail.net.framework.Assert.NonNullReference(q_curr, "q_curr");
xi_fsa.AddTransition(q_prev,
q_curr,
ail.net.framework.CharPredicate.BuildAsciiCharPredicate(xi_token[i]),
(char)xi_token[i]);
if (i == xi_token.Length - 1)
{
ail.net.parser.Token token = (ail.net.parser.Token)Factory.Create(Token.GetType());
token.Type = xi_token_type;
token.Priority = ail.net.parser.LexAnalyzer.kDefaultTokenPriority;
xi_fsa.AddFinalState(q_curr, token);
}
else if (xi_escape_token_type != ail.net.parser.Token.kUndefinedEscapeTokenType)
{
ail.net.parser.Token token = (ail.net.parser.Token)Factory.Create(Token.GetType());
token.Type = xi_escape_token_type;
token.Priority = ail.net.parser.LexAnalyzer.kEscapeTokenPriority;
xi_fsa.AddFinalState(q_curr, token);
}
if (xi_escape_token_type != ail.net.parser.Token.kUndefinedEscapeTokenType)
{
xi_fsa.AddTransition(q_curr, eq, xi_escape_predicate, ail.net.parser.FsaTransition.kMaxRankValue);
}
q_prev = q_curr;
}
}
private string GenerateStateCode(ail.net.parser.FsaState xi_state)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
StringBuilder result = new StringBuilder();
result.Append(Environment.NewLine);
result.Append(@"_q");
result.Append(xi_state.Id.ToString());
result.Append(@":");
bool finalstate = xi_state.Papa.FinalStates.Contains(xi_state.Id);
if (finalstate)
{
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"PushState((int)");
result.Append(Token.GetClassName());
result.Append(@".EType.");
result.Append(Token.GetTokenName(xi_state.Token.Type));
result.Append(@"); // save accepted state for backtracking");
result.Append(Environment.NewLine);
}
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"Next();");
result.Append(Environment.NewLine);
for (int rank = 0, i = 0; ; rank++) // fix point
{
int length = result.Length;
foreach (ail.net.parser.FsaTransition transition in xi_state.Transitions.Values)
{
ail.net.framework.Assert.NonNullReference(transition, "transition");
if (transition.Predicate.Rank == rank)
{
result.Append(Environment.NewLine);
result.Append(kTab3);
if (i > 0)
{
result.Append(@"else ");
}
else
{
i++;
}
result.Append(@"if(");
if (transition.Predicate.Context.Length != 0)
{
result.Append("xi_context == ");
result.Append(transition.Predicate.Context);
result.Append(" && ");
}
// result.Append(@"ail.net.framework.CharPredicate.");
result.Append(transition.Predicate.Text);
result.Append(@"(Current))");
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"{");
result.Append(Environment.NewLine);
result.Append(kTab4);
result.Append(@"goto _q");
result.Append(transition.End.ToString());
result.Append(@";");
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"}");
}
}
if (result.Length == length && rank > ail.net.parser.FsaTransition.kMaxRankValue)
{
break;
}
}
if (HasBacktracking && !finalstate)
{
if (xi_state.Transitions.Values.Count > 0)
{
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"else");
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"{");
result.Append(Environment.NewLine);
result.Append(kTab4);
}
else
{
result.Append(kTab3);
}
result.Append(@"PopState(); // restore the last accepted state, backtracking");
if (xi_state.Transitions.Values.Count > 0)
{
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"}");
}
}
result.Append(Environment.NewLine);
result.Append(kTab3);
result.Append(@"goto _exit;");
return(result.ToString());
}
public ail.net.parser.Fsa Nfa2Dfa()
{
// Conversion of an NFA into a DFA (subset construction)
// .....................................................
// Input: An NFA N.
// Output: A DFA D accepting the same language.
// Operations:
// e-closure(s) is the set of NFA states reachable from s on e-transitions alone.
// e-closure(T) is the union of e-closure(r) for all r in T.
// move(T, a) is the set of NFA states to which there is a transition on input a from some NFA state in T.
//
// set the start state to e-closure(s0) and unmark it.
// While there is an unmarked state T in Dstates do
// Mark T
// For each input symbol a do
// If U := e-closure(move(T, a));
// If U is not in Dstates then
// Add U as an unmarked state to Dstates;
// Dtran(T, a) := U;
// End;
// End;
ail.net.parser.Fsa result = new ail.net.parser.Fsa();
ArrayList dfa_states = new ArrayList();
// build pseudo dfa
ail.net.parser.FsaStateSet start_dfa_state = CalculateStateEclosure(StartState);
start_dfa_state.Id = dfa_states.Count;
start_dfa_state.Marked = false;
dfa_states.Add(start_dfa_state);
bool proceed = false;
for (;;)
{
IEnumerator dfa_state_enum = dfa_states.GetEnumerator();
while (dfa_state_enum.MoveNext())
{
ail.net.parser.FsaStateSet dfa_state = (ail.net.parser.FsaStateSet)dfa_state_enum.Current;
if (!dfa_state.Marked)
{
dfa_state.Marked = true;
foreach (ail.net.parser.FsaTransitionPredicate predicate in Predicates.Values)
{
if (predicate.Text != ail.net.parser.FsaTransition.kEpsilonPredicate)
{
ail.net.parser.FsaStateSet move_set = CalculateMove(dfa_state, predicate.Text);
if (move_set != (object)null)
{
ail.net.parser.FsaStateSet pseudo_dfa_state = CalculateEClosureFromMove(move_set);
if (pseudo_dfa_state != (object)null && pseudo_dfa_state.States.Count > 0)
{
ail.net.parser.FsaStateSet new_dfa_state = HasDfaCompoundState(dfa_states, pseudo_dfa_state);
if (new_dfa_state == (object)null)
{
new_dfa_state = pseudo_dfa_state;
new_dfa_state.Id = dfa_states.Count;
new_dfa_state.Marked = false;
dfa_states.Add(new_dfa_state);
dfa_state_enum = dfa_states.GetEnumerator(); // reset iterator
}
if (!dfa_state.Transitions.Contains(dfa_state.Transitions.Count))
{
ail.net.parser.FsaTransition transition = new ail.net.parser.FsaTransition(dfa_state.Transitions.Count,
dfa_state.Id,
new_dfa_state.Id,
predicate.Text,
predicate.SwitchChar,
predicate.Context,
predicate.Rank);
dfa_state.Transitions.Add(transition.Id, transition);
}
}
}
}
}
proceed = true;
}
}
if (!proceed)
{
break;
}
proceed = false;
}
// populate states and final states
foreach (ail.net.parser.FsaStateSet dfa_state in dfa_states)
{
ail.net.parser.FsaState state = result.AddState(dfa_state.Id);
ail.net.framework.Assert.NonNullReference(state, "state");
foreach (ail.net.parser.FsaTransition transition in dfa_state.Transitions.Values)
{
result.AddTransition(transition.Start,
transition.End,
transition.Predicate.Text,
transition.Predicate.SwitchChar,
transition.Predicate.Context,
transition.Predicate.Rank);
}
ail.net.parser.FsaState final_state = null;
foreach (ail.net.parser.FsaState tmp_state in dfa_state.States.Values)
{
if (FinalStates.Contains(tmp_state.Id))
{
ail.net.parser.FsaState org_state = (ail.net.parser.FsaState)States[tmp_state.Id];
ail.net.framework.Assert.NonNullReference(org_state, "org_state");
if (final_state == (object)null || org_state.Token.Priority > final_state.Token.Priority)
{
final_state = org_state;
}
}
}
if (final_state != (object)null)
{
result.AddFinalState(state, final_state.Token);
}
}
result.StateCounter.Reset(States.Count);
return(result);
}
private void GenerateStateCodeWalker(ail.net.parser.Fsa xi_fsa, ail.net.parser.FsaState xi_state, int xi_token_type, StringBuilder xio_code)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
if (xi_state.Id != xi_fsa.StartState.Id) // start state was generated earley with switch/case code
{
xi_state.Marked = true;
xio_code.Append(Environment.NewLine);
xio_code.Append(@"_q");
xio_code.Append(xi_state.Id.ToString());
xio_code.Append(@":");
bool finalstate = xi_state.Papa.FinalStates.Contains(xi_state.Id);
if (finalstate && xi_state.Token != (object)null)
{
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"PushState((int)");
xio_code.Append(Token.GetClassName());
xio_code.Append(@".EType.");
xio_code.Append(Token.GetTokenName(xi_state.Token.Type));
xio_code.Append(@"); // save accepted state for backtracking");
xio_code.Append(Environment.NewLine);
}
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"Next();");
xio_code.Append(Environment.NewLine);
for (int rank = 0, i = 0; ; rank++) // fix point
{
int length = xio_code.Length;
foreach (ail.net.parser.FsaTransition transition in xi_state.Transitions.Values)
{
ail.net.framework.Assert.NonNullReference(transition, "transition");
if (transition.Predicate.Rank == rank)
{
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
if (i > 0)
{
xio_code.Append(@"else ");
}
else
{
i++;
}
xio_code.Append(@"if(");
if (transition.Predicate.Context.Length != 0)
{
xio_code.Append("xi_context == ");
xio_code.Append(transition.Predicate.Context);
xio_code.Append(" && ");
}
xio_code.Append(transition.Predicate.Text);
xio_code.Append(@"(Current))");
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"{");
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab4);
xio_code.Append(@"goto _q");
xio_code.Append(transition.End.ToString());
xio_code.Append(@";");
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"}");
}
}
if (xio_code.Length == length && rank > ail.net.parser.FsaTransition.kMaxRankValue)
{
break;
}
}
if (!finalstate && HasBacktracking && !NoBacktrackingTokens.Contains(xi_token_type))
{
if (xi_state.Transitions.Values.Count > 0)
{
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"else");
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"{");
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab4);
}
else
{
xio_code.Append(kTab3);
}
xio_code.Append(@"PopState(); // restore the last accepted state, backtracking");
if (xi_state.Transitions.Values.Count > 0)
{
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"}");
}
}
xio_code.Append(Environment.NewLine);
xio_code.Append(kTab3);
xio_code.Append(@"goto _exit;");
}
int token_type = xi_token_type;
if (xi_state.Token != (object)null)
{
token_type = xi_state.Token.Type;
}
foreach (ail.net.parser.FsaTransition transition in xi_state.Transitions.Values)
{
ail.net.framework.Assert.NonNullReference(transition, "transition");
ail.net.parser.FsaState state = (ail.net.parser.FsaState)xi_fsa.States[transition.End];
ail.net.framework.Assert.NonNullReference(state, "state");
if (!state.Marked)
{
GenerateStateCodeWalker(xi_fsa, state, token_type, xio_code);
}
}
}
public static void TestLexAnalyzer()
{
try
{
ail.net.parser.ReLexAnalyzer lexer = new ail.net.parser.ReLexAnalyzer(new ail.net.parser.ReToken(), new ArrayList());
// string ss = lexer.Token.GetTokenName(10);
// string sc = lexer.Token.GetClassName();
ail.net.parser.Fsa fsa = new ail.net.parser.Fsa();
ail.net.parser.FsaState q0 = fsa.AddState();
ail.net.parser.FsaState q1 = fsa.AddState();
ail.net.parser.FsaState q2 = fsa.AddState();
ail.net.parser.FsaState q3 = fsa.AddState();
ail.net.parser.FsaState q4 = fsa.AddState();
ail.net.parser.FsaState q5 = fsa.AddState();
ail.net.parser.FsaState q6 = fsa.AddState();
fsa.AddTransition(q0, q1, "IsCharLowerCaseF");
fsa.AddTransition(q1, q2, "IsCharLowerCaseO");
fsa.AddTransition(q2, q3, "IsCharLowerCaseR");
fsa.AddTransition(q2, q4, "IsCharLowerCaseO");
fsa.AddTransition(q4, q5, "IsCharLowerCaseL");
fsa.AddTransition(q2, q6, "IsIdentifierChar", 1);
fsa.AddTransition(q6, q6, "IsIdentifierChar", 1);
/*
* fsa.AddFinalState(q1, (int)ail.net.parser.Token.EType.eWhiteSpace);//.eIdentifier);
* fsa.AddFinalState(q2, (int)ail.net.parser.Token.EType.eWhiteSpace);//.eIdentifier);
* fsa.AddFinalState(q3, (int)ail.net.parser.Token.EType.eWhiteSpace);//.eFor);
* fsa.AddFinalState(q4, (int)ail.net.parser.Token.EType.eWhiteSpace);//.eIdentifier);
* fsa.AddFinalState(q5, (int)ail.net.parser.Token.EType.eWhiteSpace);//.eFool);
* fsa.AddFinalState(q6, (int)ail.net.parser.Token.EType.eWhiteSpace);//.eIdentifier);
*/
// lexer.AddToken(fsa, "while", (int)ail.net.parser.Token.EType.eWhiteSpace/*.eWhile*/, (int)ail.net.parser.Token.EType.eWhiteSpace);//.eIdentifier);
// fsa.Print();
ail.net.parser.Fsa cloned_fsa = (ail.net.parser.Fsa)fsa.Clone();
// fsa.Print();
lexer.Load("forfool", ail.net.parser.Context.ELoadMedia.eMediaString);
lexer.NextLexeme();
// Console.WriteLine(lexer.Lexeme);
// lexer.NextLexeme();
// Console.WriteLine(lexer.Lexeme);
// string code = lexer.GenerateCode(fsa);
// Console.WriteLine(code);
lexer.GenerateCode("c:\\tmp\\fsa.txt", fsa);
}
catch (Exception ex)
{
string m = ex.Message;
}
}
public static void TestFsa()
{
try
{
/*
* ail.net.parser.Fsa fsa = new ail.net.parser.Fsa();
*
* ail.net.parser.FsaState q0 = fsa.AddState();
* q0.Label = q0.Id.ToString();
* ail.net.parser.FsaState q1 = fsa.AddState();
* q1.Label = q1.Id.ToString();
* ail.net.parser.FsaState q2 = fsa.AddState();
* q2.Label = q2.Id.ToString();
* ail.net.parser.FsaState q3 = fsa.AddState();
* q3.Label = q3.Id.ToString();
* ail.net.parser.FsaState q4 = fsa.AddState();
* q4.Label = q4.Id.ToString();
* ail.net.parser.FsaState q5 = fsa.AddState();
* q5.Label = q5.Id.ToString();
* ail.net.parser.FsaState q6 = fsa.AddState();
* q6.Label = q6.Id.ToString();
* ail.net.parser.FsaState q7 = fsa.AddState();
* q7.Label = q7.Id.ToString();
* ail.net.parser.FsaState q8 = fsa.AddState();
* q8.Label = q8.Id.ToString();
* ail.net.parser.FsaState q9 = fsa.AddState();
* q9.Label = q9.Id.ToString();
* ail.net.parser.FsaState q10 = fsa.AddState();
* q10.Label = q10.Id.ToString();
* ail.net.parser.FsaState q11 = fsa.AddState();
* q11.Label = q11.Id.ToString();
* ail.net.parser.FsaState q12 = fsa.AddState();
* q12.Label = q12.Id.ToString();
* ail.net.parser.FsaState q13 = fsa.AddState();
* q13.Label = q13.Id.ToString();
* ail.net.parser.FsaState q14 = fsa.AddState();
* q14.Label = q14.Id.ToString();
*
* fsa.AddTransition(q0, q1, "IsCharLowerCaseI");
* fsa.AddTransition(q1, q2, "IsCharLowerCaseF");
*
* fsa.AddTransition(q0, q3, ail.net.parser.FsaTransition.kEpsilonPredicate);
* fsa.AddTransition(q3, q4, "IsCharLowerCaseA");
* fsa.AddTransition(q4, q7, ail.net.parser.FsaTransition.kEpsilonPredicate);
* fsa.AddTransition(q7, q5, ail.net.parser.FsaTransition.kEpsilonPredicate);
* fsa.AddTransition(q5, q6, "IsCharLowerCaseA");
* fsa.AddTransition(q5, q6, "IsCharDecimalZero");
* fsa.AddTransition(q6, q7, ail.net.parser.FsaTransition.kEpsilonPredicate);
*
* fsa.AddTransition(q0, q8, ail.net.parser.FsaTransition.kEpsilonPredicate);
* fsa.AddTransition(q8, q9, "IsCharDecimalZero");
* fsa.AddTransition(q9, q12, ail.net.parser.FsaTransition.kEpsilonPredicate);
* fsa.AddTransition(q12, q10, ail.net.parser.FsaTransition.kEpsilonPredicate);
* fsa.AddTransition(q10, q11, "IsCharDecimalZero");
* fsa.AddTransition(q11, q12, ail.net.parser.FsaTransition.kEpsilonPredicate);
*
* fsa.AddTransition(q0, q13, ail.net.parser.FsaTransition.kEpsilonPredicate);
* fsa.AddTransition(q13, q14, "IsWhiteSpaceChar");
*
* // fsa.AddFinalState(q2, (int)ail.net.parser.ReToken.EType.eBar);
* // fsa.AddFinalState(q7, (int)ail.net.parser.ReToken.EType.eChar);
* // fsa.AddFinalState(q12, (int)ail.net.parser.ReToken.EType.eInteger);
* // fsa.AddFinalState(q14, (int)ail.net.parser.ReToken.EType.eHexChar);
*
* // fsa.Print();
*/
/*
* foreach(ail.net.parser.FsaState state in fsa.States.Values)
* {
* ail.net.parser.FsaStateSet eclosure = fsa.CalculateStateEclosure(state);
*
* if(eclosure.States.Count > 0)
* {
* Console.WriteLine("state: {0}", state.Id);
* eclosure.Print();
* Console.WriteLine("");
*
* ail.net.parser.FsaStateSet move = fsa.CalculateMove(eclosure, "IsCharLowerCaseA");
*
* if(move.States.Count > 0)
* {
* Console.WriteLine("state: {0}", state.Id);
* move.Print();
* Console.WriteLine("");
* }
* }
* }
*/
// ail.net.parser.Fsa nfa = fsa.Nfa2Dfa();
// nfa.Print();
{
// aho
ail.net.parser.Fsa dfa = new ail.net.parser.Fsa();
ail.net.parser.FsaState q0 = dfa.AddState();
q0.Label = "A";
ail.net.parser.FsaState q1 = dfa.AddState();
q1.Label = "B";
ail.net.parser.FsaState q2 = dfa.AddState();
q2.Label = "C";
ail.net.parser.FsaState q3 = dfa.AddState();
q3.Label = "D";
ail.net.parser.FsaState q4 = dfa.AddState();
q4.Label = "E";
dfa.AddTransition(q0, q1, "a");
dfa.AddTransition(q0, q2, "b");
dfa.AddTransition(q1, q1, "a");
dfa.AddTransition(q1, q3, "b");
dfa.AddTransition(q2, q2, "b");
dfa.AddTransition(q2, q1, "a");
dfa.AddTransition(q3, q1, "a");
dfa.AddTransition(q3, q4, "b");
dfa.AddTransition(q4, q1, "a");
dfa.AddTransition(q4, q2, "b");
dfa.AddFinalState(q4, new ail.net.parser.CppToken((int)ail.net.parser.CppToken.EType.eIdentifier, 0));
// ail.net.parser.Fsa mfa = dfa.Minimize(ail.net.parser.Fsa.EMinimizationMode.eTable);
}
{
// minimize_dfa.pdf
ail.net.parser.Fsa dfa = new ail.net.parser.Fsa();
ail.net.parser.FsaState q0 = dfa.AddState();
q0.Label = "A";
ail.net.parser.FsaState q1 = dfa.AddState();
q1.Label = "B";
ail.net.parser.FsaState q2 = dfa.AddState();
q2.Label = "C";
// ail.net.parser.FsaState q3 = dfa.AddState();
// q3.Label = "D";
ail.net.parser.FsaState q4 = dfa.AddState();
q4.Label = "E";
ail.net.parser.FsaState q5 = dfa.AddState();
q5.Label = "F";
ail.net.parser.FsaState q6 = dfa.AddState();
q6.Label = "G";
ail.net.parser.FsaState q7 = dfa.AddState();
q7.Label = "H";
dfa.AddTransition(q0, q1, "0");
dfa.AddTransition(q0, q5, "1");
dfa.AddTransition(q1, q6, "0");
dfa.AddTransition(q1, q2, "1");
dfa.AddTransition(q2, q0, "0");
dfa.AddTransition(q2, q2, "1");
// dfa.AddTransition(q3, q2, "0");
// dfa.AddTransition(q3, q6, "1");
dfa.AddTransition(q4, q7, "0");
dfa.AddTransition(q4, q5, "1");
dfa.AddTransition(q5, q2, "0");
dfa.AddTransition(q5, q6, "1");
dfa.AddTransition(q6, q6, "0");
dfa.AddTransition(q6, q4, "1");
dfa.AddTransition(q7, q6, "0");
dfa.AddTransition(q7, q2, "1");
dfa.AddFinalState(q2, new ail.net.parser.CppToken((int)ail.net.parser.CppToken.EType.eIdentifier, 0));
ail.net.parser.Fsa mfa = dfa.Minimize(ail.net.parser.Fsa.EMinimizationMode.ePartition);
Console.WriteLine(mfa.States.Count);
}
}
catch (Exception ex)
{
string m = ex.Message;
}
}
private void BuildPrimaryPartition(ref Hashtable xio_partition)
{
// builds primary partition, which consist of two groups: {F} anf {Q-F}
ail.net.framework.Assert.NonNullReference(xio_partition, "xio_partition");
xio_partition.Clear();
ail.net.parser.FsaStateSet nf_states = new ail.net.parser.FsaStateSet(GroupCounter.Next()); // non-final states
ail.net.parser.FsaStateSet fn_states = new ail.net.parser.FsaStateSet(GroupCounter.Next()); // final states
xio_partition.Add(nf_states.Id, nf_states); // final states will be split and add later
foreach (ail.net.parser.FsaState state in States.Values)
{
if (IsFinalState(state))
{
fn_states.States.Add(state.Id, state);
}
else
{
nf_states.States.Add(state.Id, state);
}
}
// also, for lexical analyzer we should separate final states by tokens into different groups
Hashtable subgroups = new Hashtable();
ail.net.parser.FsaStateSet group = new ail.net.parser.FsaStateSet(GroupCounter.Next());
ail.net.parser.FsaState pin_state = (ail.net.parser.FsaState)ail.net.framework.Functor.FirstElementOfCollection(fn_states.States.Values);
group.States.Add(pin_state.Id, pin_state);
subgroups.Add(group.Id, group);
_start:
foreach (ail.net.parser.FsaState state in fn_states.States.Values)
{
if (state.Marked)
{
continue;
}
ail.net.parser.FsaStateSet subgroup_to_add = null;
// find subgroup it may belong to
foreach (ail.net.parser.FsaStateSet subgroup in subgroups.Values)
{
foreach (ail.net.parser.FsaState subgroup_state in subgroup.States.Values)
{
if (state.Token.Type == subgroup_state.Token.Type)
{
subgroup_to_add = subgroup;
break;
}
}
if (subgroup_to_add != (object)null)
{
break;
}
}
if (subgroup_to_add == (object)null) // not found - split
{
ail.net.parser.FsaStateSet new_group = new ail.net.parser.FsaStateSet(GroupCounter.Next());
new_group.States.Add(state.Id, state);
subgroups.Add(new_group.Id, new_group);
state.Marked = true;
goto _start; // fortunately not so many final states :((
}
else
{
if (!subgroup_to_add.States.Contains(state.Id))
{
subgroup_to_add.States.Add(state.Id, state);
}
}
}
// clean up
_reset:
foreach (ail.net.parser.FsaState state in fn_states.States.Values)
{
if (state.Marked)
{
state.Marked = false;
fn_states.States.Remove(state.Id);
goto _reset;
}
}
// insert subgroups
foreach (ail.net.parser.FsaStateSet subgroup in subgroups.Values)
{
xio_partition.Add(subgroup.Id, subgroup);
}
}
public bool IsFinalState(ail.net.parser.FsaState xi_state)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
return(FinalStates.Contains(xi_state.Id));
}
private void BuildTable(ref Hashtable xio_partition)
{
//!! not implemented due to huge tables being built, postponed :(
// Mark the distinguishable pairs of states.
// To achieve this task, we first mark all pairs (p,q), where p belongs to F and
// q does not belong to F as distinguishable. Then, we proceed as follows:
// repeat
// for all non-marked pairs {p,q} do
// for each letter 'a' do
// if the pair {(s)igma(p,a), (s)igma(q,a)} is marked
// then mark {p,q}
// until no new pairs are marked
ail.net.framework.Assert.NonNullReference(xio_partition, "xio_partition");
// initialization, this algorithm is sensitive to states organization
ResetMarkedStates();
xio_partition.Clear();
// table for 'A B C D E' states, cells in use marked with '*'
// 0 A
// 1 B *
// 2 C * *
// 3 D * * *
// 4 E * * * *
// A B C D E
// 0 1 2 3 4
// access by (p, q) = table[ArithmeticProgressionSum(Math.Max(0, p_id-1)+q_id]
// map[index][state]
ArrayList map = new ArrayList(States.Values);
Hashtable table = new Hashtable(CalculateTableSize());
// create table of pairs, also mark all pairs of accept and non-accept states as non-equivalent
// keep only one of pairs (p, q) or (q, p)
ArrayList states = new ArrayList(States.Values);
states.Sort(); // states must be numbered in sequence
for (int i = 0; i < states.Count - 1; i++)
{
for (int j = i + 1; j < states.Count; j++)
{
ail.net.parser.FsaState p_state = (ail.net.parser.FsaState)states[i];
ail.net.parser.FsaState q_state = (ail.net.parser.FsaState)states[j];
ail.net.parser.FsaPair pair = new ail.net.parser.FsaPair(p_state, q_state);
table.Add(new ail.net.parser.FsaPairKey(p_state, q_state), pair);
bool p_final = IsFinalState(p_state);
bool q_final = IsFinalState(q_state);
bool mark = ((p_final && !q_final) || (!p_final && q_final));
if (!mark && p_final && q_final)
{
mark = p_state.Token.Type != q_state.Token.Type;
}
if (mark)
{
pair.Marked = true;
}
}
}
// populate table
for (;;)
{
bool process = false;
foreach (ail.net.parser.FsaPair pair in table.Values)
{
if (!pair.Marked)
{
Hashtable predicates = CombinePredicates(pair.PState, pair.QState);
foreach (ail.net.parser.FsaTransitionPredicate predicate in predicates.Values)
{
ail.net.parser.FsaTransition p_state_t = pair.PState.GetTransitionByPredicate(predicate);
ail.net.parser.FsaTransition q_state_t = pair.QState.GetTransitionByPredicate(predicate);
if ((p_state_t == (object)null || q_state_t == (object)null))
{
continue;
}
ail.net.parser.FsaState p_state = (ail.net.parser.FsaState)States[p_state_t.End];
ail.net.parser.FsaState q_state = (ail.net.parser.FsaState)States[q_state_t.End];
if ((object)p_state == (object)q_state) // pairs with same states are assumped unmarked
{
continue;
}
ail.net.parser.FsaPair sigma_pair = GetPairFromStates(p_state, q_state, table);
if (sigma_pair.Marked)
{
pair.Marked = true;
process = true;
}
}
}
}
if (!process)
{
break;
}
}
ResetMarkedStates();
// build partition
foreach (ail.net.parser.FsaState state in states)
{
if (!state.Marked)
{
// add equivalent groups, optimistic assumption
ail.net.parser.FsaStateSet group = new ail.net.parser.FsaStateSet(0);
for (int i = 0; i < state.Id; i++)
{
ail.net.parser.FsaPair pair = GetPairFromStates((ail.net.parser.FsaState)States[i], state, table);
if (!pair.Marked)
{
if ((object)pair.QState != (object)state)
{
if (!pair.QState.Marked)
{
group.States.Add(pair.QState.Id, pair.QState);
}
}
else
{
if (!pair.PState.Marked)
{
group.States.Add(pair.PState.Id, pair.PState);
}
}
}
}
if (group.States.Count > 0)
{
group.States.Add(state.Id, state);
group.Id = GroupCounter.Next(); // correct id
xio_partition.Add(group.Id, group);
foreach (ail.net.parser.FsaState group_state in group.States.Values)
{
group_state.Marked = true;
}
}
}
}
foreach (ail.net.parser.FsaState state in states)
{
if (!state.Marked)
{
// add non-equivalent groups
ail.net.parser.FsaStateSet group = new ail.net.parser.FsaStateSet(GroupCounter.Next());
group.States.Add(state.Id, state);
xio_partition.Add(group.Id, group);
}
}
ResetMarkedStates();
}
public bool IsStartState(ail.net.parser.FsaState xi_state)
{
ail.net.framework.Assert.NonNullReference(xi_state, "xi_state");
return(xi_state == StartState);
}
public FsaPairKey(ail.net.parser.FsaState xi_p_state, ail.net.parser.FsaState xi_q_state)
{
PState = xi_p_state;
QState = xi_q_state;
}
public FsaPair(ail.net.parser.FsaState xi_p_state, ail.net.parser.FsaState xi_q_state)
{
MarkedAttr = false;
PStateAttr = xi_p_state;
QStateAttr = xi_q_state;
}
