public IMap <S, double> evaluate(IMap <S, A> pi_i, IMap <S, double> U, IMarkovDecisionProcess <S, A> mdp)
{
IMap <S, double> U_i = CollectionFactory.CreateMap <S, double>(U);
IMap <S, double> U_ip1 = CollectionFactory.CreateMap <S, double>(U);
// repeat k times to produce the next utility estimate
for (int i = 0; i < k; ++i)
{
// U<sub>i+1</sub>(s) <- R(s) +
// γΣ<sub>s'</sub>P(s'|s,π<sub>i</sub>(s))U<sub>i</sub>(s')
foreach (S s in U.GetKeys())
{
A ap_i = pi_i.Get(s);
double aSum = 0;
// Handle terminal states (i.e. no actions)
if (null != ap_i)
{
foreach (S sDelta in U.GetKeys())
{
aSum += mdp.transitionProbability(sDelta, s, ap_i) * U_i.Get(sDelta);
}
}
U_ip1.Put(s, mdp.reward(s) + gamma * aSum);
}
U_i.PutAll(U_ip1);
}
return(U_ip1);
}
static void passiveTDAgentDemo()
{
CellWorld <double> cw = CellWorldFactory.CreateCellWorldForFig17_1();
CellWorldEnvironment cwe = new CellWorldEnvironment(
cw.GetCellAt(1, 1),
cw.GetCells(),
MDPFactory.createTransitionProbabilityFunctionForFigure17_1(cw),
CommonFactory.CreateRandom());
IMap <Cell <double>, CellWorldAction> fixedPolicy = CollectionFactory.CreateMap <Cell <double>, CellWorldAction>();
fixedPolicy.Put(cw.GetCellAt(1, 1), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(1, 2), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(1, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(2, 1), CellWorldAction.Left);
fixedPolicy.Put(cw.GetCellAt(2, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(3, 1), CellWorldAction.Left);
fixedPolicy.Put(cw.GetCellAt(3, 2), CellWorldAction.Up);
fixedPolicy.Put(cw.GetCellAt(3, 3), CellWorldAction.Right);
fixedPolicy.Put(cw.GetCellAt(4, 1), CellWorldAction.Left);
PassiveTDAgent <Cell <double>, CellWorldAction> ptda
= new PassiveTDAgent <Cell <double>, CellWorldAction>(fixedPolicy, 0.2, 1.0);
cwe.AddAgent(ptda);
output_utility_learning_rates(ptda, 20, 500, 100, 1);
}
public Assignment <VAR, VAL> Clone()
{
Assignment <VAR, VAL> result;
result = new Assignment <VAR, VAL>();
result.variableToValueMap = CollectionFactory.CreateMap <VAR, VAL>(variableToValueMap);
return(result);
}
public VacuumEnvironmentState Clone()
{
VacuumEnvironmentState result = null;
result = new VacuumEnvironmentState();
result.state = CollectionFactory.CreateMap <string, VacuumEnvironment.LocationState>(state);
agentLocations = CollectionFactory.CreateMap <IAgent, string>(agentLocations);
return(result);
}
// END-BayesianNetwork
//
//
// PRIVATE METHODS
//
private void checkIsDAGAndCollectVariablesInTopologicalOrder()
{
// Topological sort based on logic described at:
// http://en.wikipedia.org/wiki/Topoligical_sorting
ISet <INode> seenAlready = CollectionFactory.CreateSet <INode>();
IMap <INode, ICollection <INode> > incomingEdges = CollectionFactory.CreateMap <INode, ICollection <INode> >();
ICollection <INode> s = CollectionFactory.CreateFifoQueueNoDuplicates <INode>();
foreach (INode n in this.rootNodes)
{
walkNode(n, seenAlready, incomingEdges, s);
}
while (!s.IsEmpty())
{
INode n = s.Pop();
variables.Add(n.GetRandomVariable());
varToNodeMap.Put(n.GetRandomVariable(), n);
foreach (INode m in n.GetChildren())
{
ICollection <INode> edges = incomingEdges.Get(m);
edges.Remove(n);
if (edges.IsEmpty())
{
s.Add(m);
}
}
}
foreach (ICollection <INode> edges in incomingEdges.GetValues())
{
if (!edges.IsEmpty())
{
throw new IllegalArgumentException("Network contains at least one cycle in it, must be a DAG.");
}
}
}
public override IMap <S, double> getUtility()
{
return(CollectionFactory.CreateMap <S, double>(U));
}
public void checkForPossibleAnswers(ISet <Clause> resolvents)
{
// If no bindings being looked for, then
// is just a true false query.
foreach (Clause aClause in resolvents)
{
if (answerClause.isEmpty())
{
if (aClause.isEmpty())
{
proofs.Add(new ProofFinal(aClause.getProofStep(), CollectionFactory.CreateMap <Variable, Term>()));
complete = true;
}
}
else
{
if (aClause.isEmpty())
{
// This should not happen
// as added an answer literal, which
// implies the database (i.e. premises) are
// unsatisfiable to begin with.
throw new IllegalStateException("Generated an empty clause while looking for an answer, implies original KB is unsatisfiable");
}
if (aClause.isUnitClause() &&
aClause.isDefiniteClause() &&
aClause
.getPositiveLiterals()
.Get(0)
.getAtomicSentence()
.getSymbolicName()
.Equals(answerLiteral.getAtomicSentence()
.getSymbolicName()))
{
IMap <Variable, Term> answerBindings = CollectionFactory.CreateMap <Variable, Term>();
ICollection <Term> answerTerms = aClause.getPositiveLiterals().Get(0).getAtomicSentence().getArgs();
int idx = 0;
foreach (Variable v in answerLiteralVariables)
{
answerBindings.Put(v, answerTerms.Get(idx));
idx++;
}
bool addNewAnswer = true;
foreach (Proof p in proofs)
{
if (p.getAnswerBindings().SequenceEqual(answerBindings))
{
addNewAnswer = false;
break;
}
}
if (addNewAnswer)
{
proofs.Add(new ProofFinal(aClause.getProofStep(), answerBindings));
}
}
}
if (CommonFactory.Now().BiggerThan(finishTime))
{
complete = true;
// Indicate that I have run out of query time
timedOut = true;
}
}
}
public bool isAnswer(Chain nearParent)
{
bool isAns = false;
if (answerChain.isEmpty())
{
if (nearParent.isEmpty())
{
proofs.Add(new ProofFinal(nearParent.getProofStep(), CollectionFactory.CreateMap <Variable, Term>()));
complete = true;
isAns = true;
}
}
else
{
if (nearParent.isEmpty())
{
// This should not happen
// as added an answer literal to sos, which
// implies the database (i.e. premises) are
// unsatisfiable to begin with.
throw new IllegalStateException(
"Generated an empty chain while looking for an answer, implies original KB is unsatisfiable");
}
if (1 == nearParent.getNumberLiterals() &&
nearParent
.getHead()
.getAtomicSentence()
.getSymbolicName()
.Equals(answerChain.getHead()
.getAtomicSentence().getSymbolicName()))
{
IMap <Variable, Term> answerBindings = CollectionFactory.CreateMap <Variable, Term>();
ICollection <Term> answerTerms = nearParent.getHead()
.getAtomicSentence().getArgs();
int idx = 0;
foreach (Variable v in answerLiteralVariables)
{
answerBindings.Put(v, answerTerms.Get(idx));
idx++;
}
bool addNewAnswer = true;
foreach (Proof p in proofs)
{
if (p.getAnswerBindings().SequenceEqual(answerBindings))
{
addNewAnswer = false;
break;
}
}
if (addNewAnswer)
{
proofs.Add(new ProofFinal(nearParent.getProofStep(), answerBindings));
}
isAns = true;
}
}
if (CommonFactory.Now().BiggerThan(finishTime))
{
complete = true;
// Indicate that I have run out of query time
timedOut = true;
}
return(isAns);
}
