public IMap <S, double> evaluate(IMap <S, A> pi_i, IMap <S, double> U, IMarkovDecisionProcess <S, A> mdp)
        {
            IMap <S, double> U_i   = CollectionFactory.CreateMap <S, double>(U);
            IMap <S, double> U_ip1 = CollectionFactory.CreateMap <S, double>(U);

            // repeat k times to produce the next utility estimate
            for (int i = 0; i < k; ++i)
            {
                // U<sub>i+1</sub>(s) <- R(s) +
                // &gamma;&Sigma;<sub>s'</sub>P(s'|s,&pi;<sub>i</sub>(s))U<sub>i</sub>(s')
                foreach (S s in U.GetKeys())
                {
                    A      ap_i = pi_i.Get(s);
                    double aSum = 0;
                    // Handle terminal states (i.e. no actions)
                    if (null != ap_i)
                    {
                        foreach (S sDelta in U.GetKeys())
                        {
                            aSum += mdp.transitionProbability(sDelta, s, ap_i) * U_i.Get(sDelta);
                        }
                    }
                    U_ip1.Put(s, mdp.reward(s) + gamma * aSum);
                }

                U_i.PutAll(U_ip1);
            }
            return(U_ip1);
        }
예제 #2
0
        static void passiveTDAgentDemo()
        {
            CellWorld <double>   cw  = CellWorldFactory.CreateCellWorldForFig17_1();
            CellWorldEnvironment cwe = new CellWorldEnvironment(
                cw.GetCellAt(1, 1),
                cw.GetCells(),
                MDPFactory.createTransitionProbabilityFunctionForFigure17_1(cw),
                CommonFactory.CreateRandom());

            IMap <Cell <double>, CellWorldAction> fixedPolicy = CollectionFactory.CreateMap <Cell <double>, CellWorldAction>();

            fixedPolicy.Put(cw.GetCellAt(1, 1), CellWorldAction.Up);
            fixedPolicy.Put(cw.GetCellAt(1, 2), CellWorldAction.Up);
            fixedPolicy.Put(cw.GetCellAt(1, 3), CellWorldAction.Right);
            fixedPolicy.Put(cw.GetCellAt(2, 1), CellWorldAction.Left);
            fixedPolicy.Put(cw.GetCellAt(2, 3), CellWorldAction.Right);
            fixedPolicy.Put(cw.GetCellAt(3, 1), CellWorldAction.Left);
            fixedPolicy.Put(cw.GetCellAt(3, 2), CellWorldAction.Up);
            fixedPolicy.Put(cw.GetCellAt(3, 3), CellWorldAction.Right);
            fixedPolicy.Put(cw.GetCellAt(4, 1), CellWorldAction.Left);

            PassiveTDAgent <Cell <double>, CellWorldAction> ptda
                = new PassiveTDAgent <Cell <double>, CellWorldAction>(fixedPolicy, 0.2, 1.0);

            cwe.AddAgent(ptda);

            output_utility_learning_rates(ptda, 20, 500, 100, 1);
        }
예제 #3
0
        public Assignment <VAR, VAL> Clone()
        {
            Assignment <VAR, VAL> result;

            result = new Assignment <VAR, VAL>();
            result.variableToValueMap = CollectionFactory.CreateMap <VAR, VAL>(variableToValueMap);

            return(result);
        }
예제 #4
0
        public VacuumEnvironmentState Clone()
        {
            VacuumEnvironmentState result = null;

            result         = new VacuumEnvironmentState();
            result.state   = CollectionFactory.CreateMap <string, VacuumEnvironment.LocationState>(state);
            agentLocations = CollectionFactory.CreateMap <IAgent, string>(agentLocations);

            return(result);
        }
예제 #5
0
        // END-BayesianNetwork
        //

        //
        // PRIVATE METHODS
        //
        private void checkIsDAGAndCollectVariablesInTopologicalOrder()
        {
            // Topological sort based on logic described at:
            // http://en.wikipedia.org/wiki/Topoligical_sorting
            ISet <INode> seenAlready = CollectionFactory.CreateSet <INode>();
            IMap <INode, ICollection <INode> > incomingEdges = CollectionFactory.CreateMap <INode, ICollection <INode> >();
            ICollection <INode> s = CollectionFactory.CreateFifoQueueNoDuplicates <INode>();

            foreach (INode n in this.rootNodes)
            {
                walkNode(n, seenAlready, incomingEdges, s);
            }
            while (!s.IsEmpty())
            {
                INode n = s.Pop();
                variables.Add(n.GetRandomVariable());
                varToNodeMap.Put(n.GetRandomVariable(), n);
                foreach (INode m in n.GetChildren())
                {
                    ICollection <INode> edges = incomingEdges.Get(m);
                    edges.Remove(n);
                    if (edges.IsEmpty())
                    {
                        s.Add(m);
                    }
                }
            }

            foreach (ICollection <INode> edges in incomingEdges.GetValues())
            {
                if (!edges.IsEmpty())
                {
                    throw new IllegalArgumentException("Network contains at least one cycle in it, must be a DAG.");
                }
            }
        }
예제 #6
0
 public override IMap <S, double> getUtility()
 {
     return(CollectionFactory.CreateMap <S, double>(U));
 }
예제 #7
0
            public void checkForPossibleAnswers(ISet <Clause> resolvents)
            {
                // If no bindings being looked for, then
                // is just a true false query.
                foreach (Clause aClause in resolvents)
                {
                    if (answerClause.isEmpty())
                    {
                        if (aClause.isEmpty())
                        {
                            proofs.Add(new ProofFinal(aClause.getProofStep(), CollectionFactory.CreateMap <Variable, Term>()));
                            complete = true;
                        }
                    }
                    else
                    {
                        if (aClause.isEmpty())
                        {
                            // This should not happen
                            // as added an answer literal, which
                            // implies the database (i.e. premises) are
                            // unsatisfiable to begin with.
                            throw new IllegalStateException("Generated an empty clause while looking for an answer, implies original KB is unsatisfiable");
                        }

                        if (aClause.isUnitClause() &&
                            aClause.isDefiniteClause() &&
                            aClause
                            .getPositiveLiterals()
                            .Get(0)
                            .getAtomicSentence()
                            .getSymbolicName()
                            .Equals(answerLiteral.getAtomicSentence()
                                    .getSymbolicName()))
                        {
                            IMap <Variable, Term> answerBindings = CollectionFactory.CreateMap <Variable, Term>();
                            ICollection <Term>    answerTerms    = aClause.getPositiveLiterals().Get(0).getAtomicSentence().getArgs();
                            int idx = 0;
                            foreach (Variable v in answerLiteralVariables)
                            {
                                answerBindings.Put(v, answerTerms.Get(idx));
                                idx++;
                            }
                            bool addNewAnswer = true;
                            foreach (Proof p in proofs)
                            {
                                if (p.getAnswerBindings().SequenceEqual(answerBindings))
                                {
                                    addNewAnswer = false;
                                    break;
                                }
                            }
                            if (addNewAnswer)
                            {
                                proofs.Add(new ProofFinal(aClause.getProofStep(), answerBindings));
                            }
                        }
                    }

                    if (CommonFactory.Now().BiggerThan(finishTime))
                    {
                        complete = true;
                        // Indicate that I have run out of query time
                        timedOut = true;
                    }
                }
            }
예제 #8
0
            public bool isAnswer(Chain nearParent)
            {
                bool isAns = false;

                if (answerChain.isEmpty())
                {
                    if (nearParent.isEmpty())
                    {
                        proofs.Add(new ProofFinal(nearParent.getProofStep(), CollectionFactory.CreateMap <Variable, Term>()));
                        complete = true;
                        isAns    = true;
                    }
                }
                else
                {
                    if (nearParent.isEmpty())
                    {
                        // This should not happen
                        // as added an answer literal to sos, which
                        // implies the database (i.e. premises) are
                        // unsatisfiable to begin with.
                        throw new IllegalStateException(
                                  "Generated an empty chain while looking for an answer, implies original KB is unsatisfiable");
                    }
                    if (1 == nearParent.getNumberLiterals() &&
                        nearParent
                        .getHead()
                        .getAtomicSentence()
                        .getSymbolicName()
                        .Equals(answerChain.getHead()
                                .getAtomicSentence().getSymbolicName()))
                    {
                        IMap <Variable, Term> answerBindings = CollectionFactory.CreateMap <Variable, Term>();
                        ICollection <Term>    answerTerms    = nearParent.getHead()
                                                               .getAtomicSentence().getArgs();
                        int idx = 0;
                        foreach (Variable v in answerLiteralVariables)
                        {
                            answerBindings.Put(v, answerTerms.Get(idx));
                            idx++;
                        }
                        bool addNewAnswer = true;
                        foreach (Proof p in proofs)
                        {
                            if (p.getAnswerBindings().SequenceEqual(answerBindings))
                            {
                                addNewAnswer = false;
                                break;
                            }
                        }
                        if (addNewAnswer)
                        {
                            proofs.Add(new ProofFinal(nearParent.getProofStep(), answerBindings));
                        }
                        isAns = true;
                    }
                }

                if (CommonFactory.Now().BiggerThan(finishTime))
                {
                    complete = true;
                    // Indicate that I have run out of query time
                    timedOut = true;
                }

                return(isAns);
            }