// function VALUE-ITERATION(mdp, ε) returns a utility function
/**
* The value iteration algorithm for calculating the utility of states.
*
* @param mdp
* an MDP with states S, actions A(s), <br>
* transition model P(s' | s, a), rewards R(s)
* @param epsilon
* the maximum error allowed in the utility of any state
* @return a vector of utilities for states in S
*/
public IMap <S, double> valueIteration(IMarkovDecisionProcess <S, A> mdp, double epsilon)
{
//
// local variables: U, U', vectors of utilities for states in S,
// initially zero
IMap <S, double> U = Util.create(mdp.states(), 0D);
IMap <S, double> Udelta = Util.create(mdp.states(), 0D);
// δ the maximum change in the utility of any state in an
// iteration
double delta = 0;
// Note: Just calculate this once for efficiency purposes:
// ε(1 - γ)/γ
double minDelta = epsilon * (1 - gamma) / gamma;
// repeat
do
{
// U <- U'; δ <- 0
U.PutAll(Udelta);
delta = 0;
// for each state s in S do
foreach (S s in mdp.states())
{
// max<sub>a ∈ A(s)</sub>
ISet <A> actions = mdp.actions(s);
// Handle terminal states (i.e. no actions).
double aMax = 0;
if (actions.Size() > 0)
{
aMax = double.NegativeInfinity;
}
foreach (A a in actions)
{
// Σ<sub>s'</sub>P(s' | s, a) U[s']
double aSum = 0;
foreach (S sDelta in mdp.states())
{
aSum += mdp.transitionProbability(sDelta, s, a) * U.Get(sDelta);
}
if (aSum > aMax)
{
aMax = aSum;
}
}
// U'[s] <- R(s) + γ
// max<sub>a ∈ A(s)</sub>
Udelta.Put(s, mdp.reward(s) + gamma * aMax);
// if |U'[s] - U[s]| > δ then δ <- |U'[s] - U[s]|
double aDiff = System.Math.Abs(Udelta.Get(s) - U.Get(s));
if (aDiff > delta)
{
delta = aDiff;
}
}
// until δ < ε(1 - γ)/γ
} while (delta > minDelta);
// return U
return(U);
}
public IMap <S, double> evaluate(IMap <S, A> pi_i, IMap <S, double> U, IMarkovDecisionProcess <S, A> mdp)
{
IMap <S, double> U_i = CollectionFactory.CreateMap <S, double>(U);
IMap <S, double> U_ip1 = CollectionFactory.CreateMap <S, double>(U);
// repeat k times to produce the next utility estimate
for (int i = 0; i < k; ++i)
{
// U<sub>i+1</sub>(s) <- R(s) +
// γΣ<sub>s'</sub>P(s'|s,π<sub>i</sub>(s))U<sub>i</sub>(s')
foreach (S s in U.GetKeys())
{
A ap_i = pi_i.Get(s);
double aSum = 0;
// Handle terminal states (i.e. no actions)
if (null != ap_i)
{
foreach (S sDelta in U.GetKeys())
{
aSum += mdp.transitionProbability(sDelta, s, ap_i) * U_i.Get(sDelta);
}
}
U_ip1.Put(s, mdp.reward(s) + gamma * aSum);
}
U_i.PutAll(U_ip1);
}
return(U_ip1);
}
public void testRewardFunction()
{
// Ensure all actions can be performed in each cell.
foreach (Cell <double> s in cw.GetCells())
{
if (4 == s.getX() && 3 == s.getY())
{
Assert.AreEqual(1.0, mdp.reward(s));
}
else if (4 == s.getX() && 2 == s.getY())
{
Assert.AreEqual(-1.0, mdp.reward(s));
}
else
{
Assert.AreEqual(-0.04, mdp.reward(s));
}
}
}
