public Dictionary<Cell<Double>, Double> evaluate(Dictionary<Cell<Double>, CS8803AGA.PsychSim.State.Action> pi_i, Dictionary<Cell<Double>, Double> U,
MDP mdp)
{
Dictionary<Cell<Double>, Double> U_i = new Dictionary<Cell<Double>, Double>(U);
Dictionary<Cell<Double>, Double> U_ip1 = new Dictionary<Cell<Double>, Double>(U);
for (int i = 0; i < k; i++)
{
foreach (var s in U.Keys)
{
double aSum = 0;
if (pi_i.ContainsKey(s))
{
CS8803AGA.PsychSim.State.Action ap_i = pi_i[s];
foreach (var sDelta in U.Keys)
{
aSum += mdp.transitionProbability(sDelta, s, ap_i)
* U_i[sDelta];
}
}
U_ip1[s] = (mdp.reward(s) + gamma * aSum);
}
Util.Merge(U_i, U_ip1);
}
return U_ip1;
}
public LookupPolicy policyIteration(MDP mdp, MDP mdp1, MDP mdp2)
{
Dictionary<Cell<Double>, Double> U = Util.create(mdp.GetStates(), new Double());
Dictionary<Cell<Double>, CS8803AGA.PsychSim.State.Action> pi = initialPolicyVector(mdp);
bool unchanged;
do {
U = policyEvaluation.evaluate(pi, U, mdp, mdp1, mdp2);
unchanged = true;
foreach (var s in mdp.GetStates())
{
if (pi.ContainsKey(s))
{
double aMax = Double.MinValue, piVal = 0;
CS8803AGA.PsychSim.State.Action aArgmax = pi[s];
foreach (var a in mdp.actions(s))
{
double aSum = 0;
foreach (var sDelta in mdp.GetStates())
{
aSum += mdp.transitionProbability(sDelta, s, a)
* U[sDelta];
}
if (aSum > aMax)
{
aMax = aSum;
aArgmax = a;
}
if (a.Equals(pi[s]))
{
piVal = aSum;
}
}
if (aMax > piVal)
{
pi[s] = aArgmax;
unchanged = false;
}
}
}
} while (!unchanged);
return new LookupPolicy(pi);
}
public Dictionary<Cell<Double>, Double> valueIteration(MDP mdp, double epsilon)
{
Dictionary<Cell<Double>, Double> U = Util.create(mdp.GetStates(), 0.0);
Dictionary<Cell<Double>, Double> Udelta = Util.create(mdp.GetStates(), 0.0);
double delta = 0;
double minDelta = epsilon * (1 - gamma) / gamma;
do
{
Util.Merge(U, Udelta);
delta = 0;
foreach (Cell<Double> s in mdp.GetStates())
{
HashSet<CS8803AGA.PsychSim.State.Action> actions = mdp.actions(s);
double aMax = 0;
if (actions.Count > 0)
{
aMax = Double.MinValue;
}
foreach (CS8803AGA.PsychSim.State.Action a in actions)
{
double aSum = 0;
foreach (Cell<Double> sDelta in mdp.GetStates())
{
aSum += mdp.transitionProbability(sDelta, s, a)
* U[sDelta];
}
if (aSum > aMax)
{
aMax = aSum;
}
}
var val = mdp.reward(s) + gamma * aMax;
Udelta[s] = val;
double aDiff = Math.Abs(Udelta[s] - U[s]);
if (aDiff > delta)
{
delta = aDiff;
}
}
} while (delta > minDelta);
return U;
}
