public Tuple <int, string> ProcessWord(string word)
{
int State = StartState;
int HeadPosition = 0;
char[] Tape = word.ToCharArray();
while (!EndStates.Contains(State))
{
char currentSymbol = GetSymbol(Tape, HeadPosition);
Transition t = TransitionFunction(State, currentSymbol);
Debug.WriteLine($"({State}, {currentSymbol}) -> ({t.NextState}, {t.Write ?? currentSymbol}, {t.MovementDirection}); {new String(Tape).Insert(HeadPosition, $"(q{State})")}");
State = t.NextState;
if (t.Write.HasValue)
{
Tape[HeadPosition] = t.Write.Value;
}
if (t.MovementDirection == Direction.Left)
{
HeadPosition--;
}
else
{
HeadPosition++;
}
}
return(new Tuple <int, string>(State, new string(Tape)));
}
private DFA <Tuple <T, T>, U> andor(DFA <T, U> other, bool and)
{
if (!Alphabet.SetEquals(other.Alphabet))
{
return(null);
}
var curr = new HashSet <Tuple <T, T> >(Transitions.Count * other.Transitions.Count());
curr.Add(Tuple.Create(StartState, other.StartState));
var result = new DFA <Tuple <T, T>, U>();
result.addStartState(curr.First());
while (result.Transitions.Count() < Transitions.Count * other.Transitions.Count())
{
var next = new HashSet <Tuple <T, T> >(Transitions.Count * other.Transitions.Count());
foreach (var currState in curr)
{
if (and ? (EndStates.Contains(currState.Item1) && other.EndStates.Contains(currState.Item2))
: (EndStates.Contains(currState.Item1) || other.EndStates.Contains(currState.Item2)))
{
result.addEndState(currState);
}
foreach (var terminal in Alphabet)
{
var nextState = Tuple.Create(Transitions[currState.Item1][terminal], other.Transitions[currState.Item2][terminal]);
next.Add(nextState);
result.addTransition(terminal, currState, nextState);
}
}
curr = next;
}
return(result);
}
public DFA <T, U> negative()
{
DFA <T, U> result = new DFA <T, U>();
result.addStartState(StartState);
result.Transitions = Transitions;
result.States = States;
foreach (var state in States)
{
if (!EndStates.Contains(state))
{
result.addEndState(state);
}
}
return(result);
}
public bool accept(U[] input)
{
if (!isValid())
{
throw new AutomatonInvalidException();
}
var currState = StartState;
foreach (U u in input)
{
if (!Alphabet.Contains(u))
{
throw new System.ArgumentException($"{u} is not part of alphabet");
}
currState = Transitions[currState][u];
}
return(EndStates.Contains(currState));
}
public void RunTraining()
{
QMethod.Validate(this);
/*
* For each episode: Select random initial state
* Do while not reach goal state
* Select one among all possible actions for the current state
* Using this possible action, consider to go to the next state
* Get maximum Q value of this next state based on all possible actions
* Set the next state as the current state
*/
// For each episode
var rand = new Random();
long maxloopEventCount = 0;
// Train episodes
for (long i = 0; i < Episodes; i++)
{
long maxloop = 0;
// Select random initial state
int stateIndex = rand.Next(States.Count);
QState state = States[stateIndex];
QAction action = null;
do
{
if (++maxloop > MaxExploreStepsWithinOneEpisode)
{
if (ShowWarning)
{
string msg = string.Format(
"{0} !! MAXLOOP state: {1} action: {2}, {3} endstate is to difficult to reach?",
++maxloopEventCount, state, action, "maybe your path setup is wrong or the ");
QMethod.Log(msg);
}
break;
}
// no actions, skip this state
if (state.Actions.Count == 0)
{
break;
}
// Selection strategy is random based on probability
int index = rand.Next(state.Actions.Count);
action = state.Actions[index];
// Using this possible action, consider to go to the next state
// Pick random Action outcome
QActionResult nextStateResult = action.PickActionByProbability();
string nextStateName = nextStateResult.StateName;
double q = nextStateResult.QEstimated;
double r = nextStateResult.Reward;
double maxQ = MaxQ(nextStateName);
// Q(s,a)= Q(s,a) + alpha * (R(s,a) + gamma * Max(next state, all actions) - Q(s,a))
double value = q + Alpha * (r + Gamma * maxQ - q); // q-learning
nextStateResult.QValue = value; // update
// is end state go to next episode
if (EndStates.Contains(nextStateResult.StateName))
{
break;
}
// Set the next state as the current state
state = StateLookup[nextStateResult.StateName];
} while (true);
}
}
