Пример #1
0
 /// <summary>
 /// Initializes a new MDPSuccessorState.
 /// </summary>
 /// <param name="action">Action to the state.</param>
 /// <param name="cost">Cost of the transition state.</param>
 /// <param name="state">Transition state.</param>
 /// <param name="reward">Reward value.</param>
 public MDPSuccessorState(IAction action, double cost, IMDPState state, double reward)
 {
     this.Action = action;
     this.Cost   = cost;
     this.State  = state;
     this.Reward = reward;
 }
Пример #2
0
 /// <summary>
 /// Initializes a new MDPSuccessorState.
 /// </summary>
 /// <param name="action">Action to the state.</param>
 /// <param name="cost">Cost of the transition state.</param>
 /// <param name="state">Transition state.</param>
 /// <param name="reward">Reward value.</param>
 public MDPSuccessorState(IAction action, double cost, IMDPState state, double reward)
 {
     this.Action = action;
     this.Cost = cost;
     this.State = state;
     this.Reward = reward;
 }
Пример #3
0
 /// <summary>
 ///   Initializes a new MDPSuccessorState.
 /// </summary>
 /// <param name="action">Action to the state.</param>
 /// <param name="cost">Cost of the transition state.</param>
 /// <param name="state">Transition state.</param>
 /// <param name="reward">Reward value.</param>
 public MDPSuccessorState(IAction action, double cost, IMDPState state, double reward)
 {
     Action = action;
     Cost   = cost;
     State  = state;
     Reward = reward;
 }
Пример #4
0
        public IMDPDecisionSpace GenDecisionSpace(IMDPState s)
        {
            IMDPDecisionSpace res = new DecisionSpace();

            foreach (Decision _d in _ds)
            {
                if ((s as State).CanSupportDecision(_d))
                {
                    res.Add(_d);
                }
            }
            return(res);
        }
Пример #5
0
        /// <summary>
        ///   Converts a Matrix of states into an array of State objects.
        /// </summary>
        /// <param name="states">State matrix.</param>
        /// <param name="properties">(Optional) Feature summary.</param>
        /// <param name="discretizer">Disretization function to apply for reducing states.</param>
        /// <returns></returns>
        public static IEnumerable <IMDPState> GetStates(Matrix states, Summary properties, IDiscretizer discretizer)
        {
            var summary = properties ?? Summary.Summarize(states);

            var slist = new IMDPState[states.Rows];

            for (var i = 0; i < states.Rows; i++)
            {
                slist[i] = GetState(states[i], summary, discretizer);
            }

            return(slist);
        }
Пример #6
0
        /// <summary>
        /// Converts a Matrix of states into an array of State objects.
        /// </summary>
        /// <param name="states">State matrix.</param>
        /// <param name="properties">(Optional) Feature summary.</param>
        /// <param name="discretizer">Disretization function to apply for reducing states.</param>
        /// <returns></returns>
        public static IEnumerable <IMDPState> GetStates(Matrix states, Math.Summary properties, IDiscretizer discretizer)
        {
            Math.Summary summary = (properties != null ? properties : Math.Summary.Summarize(states));

            var slist = new IMDPState[states.Rows];

            for (int i = 0; i < states.Rows; i++)
            {
                slist[i] = MDPConverter.GetState(states[i], summary, discretizer);
            }

            return(slist);
        }
Пример #7
0
        public IMDPState CreateOrFind(IMDPState _s)
        {
            IMDPState res = _ss.FirstOrDefault(s => s.Equals(_s));

            if (res == null)
            {
                _ss.Add(_s);
                return(_s);
            }
            else
            {
                return(res);
            }
        }
Пример #8
0
 public bool Equals(IMDPState other)
 {
     if (this.Keys.Count != (other as State).Keys.Count)
     {
         return(false);
     }
     foreach (IALPResource ts in this.Keys)
     {
         if (!((other as State).Keys.Contains(ts) && (other as State)[ts] == this[ts]))
         {
             return(false);
         }
     }
     return(true);
 }
Пример #9
0
        public static State MinusOneUnit(this StateSpace ss, IMDPState s, Route route)
        {
            IMDPState _s = s.Clone();

            foreach (Product p in route)
            {
                foreach (Resource r in p)
                {
                    if ((_s as State).Keys.Contains(r) && (_s as State)[r] > 0)
                    {
                        (_s as State)[r] -= 1;
                    }
                }
            }
            return(ss.FirstOrDefault(i => i.Equals(_s)) as State);
        }
Пример #10
0
        /// <summary>
        ///   Converts the experience pair into their equivalent math forms.
        /// </summary>
        /// <param name="state">IMDPState instance.</param>
        /// <param name="nodes">List of nodes added to the result set.</param>
        /// <param name="states">Matrix to store contained successor state vectors.</param>
        /// <param name="actions">Vector to store the contained action values.</param>
        /// <param name="statesP">Matrix to store all contained successor transition state vectors.</param>
        /// <param name="rewards">Vector to store all contained reward values.</param>
        /// <returns>HashSet&lt;string&gt;</returns>
        private static void Convert(
            this IMDPState state,
            ref List <string> nodes,
            ref Matrix states,
            ref Vector actions,
            ref Matrix statesP,
            ref Vector rewards)
        {
            if (state != null)
            {
                foreach (IMDPSuccessor successor in state.GetSuccessors())
                {
                    if (state.Features.Length != states.Cols)
                    {
                        states = Matrix.Reshape(states, states.Rows, state.Features.Length);
                    }
                    if (state.Features.Length != statesP.Cols)
                    {
                        statesP = Matrix.Reshape(statesP, statesP.Rows, ((IMDPState)successor.State).Features.Length);
                    }

                    string id = $"{state.Id}:{successor.State.Id}";
                    if (!nodes.Contains(id))
                    {
                        states  = states.Insert(state.ToVector(), states.Rows - 1, VectorType.Row);
                        actions = actions.Insert(actions.Length - 1, successor.Action.Id);
                        statesP = statesP.Insert(((IMDPState)successor.State).ToVector(), statesP.Rows - 1, VectorType.Row);
                        rewards = rewards.Insert(rewards.Length - 1, successor.Reward);
                        nodes.Add(id);
                    }

                    if (!successor.State.IsTerminal)
                    {
                        var successorState = (IMDPState)successor.State;
                        if (successorState.Id != state.Id)
                        {
                            successorState.Convert(ref nodes, ref states, ref actions, ref statesP, ref rewards);
                        }
                    }
                }
            }
        }
Пример #11
0
        public IALPState CreateOrFind(MetaResouceState RecDic)
        {
            State state = new State();

            foreach (Resource ts in this.ResSpace)
            {
                state.Add(ts, RecDic.GetRemainNum(ts));//这里没有管安全性
            }
            IMDPState res = _ss.FirstOrDefault(i => i.Equals(state));

            if (res == null)
            {
                _ss.Add(state);
                return(state);
            }
            else
            {
                return(res as IALPState);
            }
        }
Пример #12
0
        public IALPState CreateOrFind(IDictionary <IALPResource, int> RecDic)
        {
            State state = new State();

            foreach (Resource ts in this.ResSpace)
            {
                state.Add(ts, RecDic[ts]);//这里没有管安全性
            }
            IMDPState res = _ss.FirstOrDefault(i => i.Equals(state));

            if (res == null)
            {
                _ss.Add(state);
                return(state);
            }
            else
            {
                return(res as IALPState);
            }
        }
Пример #13
0
 public double Prob(int time, IMDPState s1, IMDPState s2, IMDPDecision a)
 {
     if (s1.Equals(s2))
     {
         return(1 - suppRoute((a as Decision).OpenProductSet).Sum(p => Ro(time) * P(time, p, a as Decision)));
     }
     else
     {
         //Find a route via which s1 transits to s2.
         Route r = suppRoute((a as Decision).OpenProductSet).
                   FirstOrDefault(i => (GenStateSpace(s1, a) as StateSpace).MinusOneUnit(s1, i).Equals(s2));
         if (r != null)
         {
             return(Ro(time) * P(time, r, a as Decision));
         }
         else
         {
             return(0);
         }
     }
 }
Пример #14
0
        public IMDPStateSpace GenStateSpace(IMDPState s, IMDPDecision a)
        {
            //目前状态减去开放产品集中的产品
            IMDPStateSpace subss = new StateSpace();

            if ((s as State).CanSupportDecision(a as Decision))
            {
                foreach (Product p in (a as Decision).OpenProductSet)
                {
                    IMDPState _s = (_ss as StateSpace).MinusOneUnit(s, p);
                    if (!subss.Contains(_s))
                    {
                        subss.Add(_s);
                    }
                }
                //再加入本身
                subss.Add(s);
            }

            return(subss);
        }
Пример #15
0
        /// <summary>
        /// Converts a Matrix of states into an array of State objects.
        /// </summary>
        /// <param name="states">State matrix.</param>
        /// <param name="properties">(Optional) Feature summary.</param>
        /// <param name="discretizer">Disretization function to apply for reducing states.</param>
        /// <returns></returns>
        public static IEnumerable<IMDPState> GetStates(Matrix states, Math.Summary properties, IDiscretizer discretizer)
        {
            Math.Summary summary = (properties != null ? properties : Math.Summary.Summarize(states));

            var slist = new IMDPState[states.Rows];

            for (int i = 0; i < states.Rows; i++)
            {
                slist[i] = MDPConverter.GetState(states[i], summary, discretizer);
            }

            return slist;
        }
Пример #16
0
        public void Test_QLearning_Path_Finder()
        {
            // start
            var master  = new MDPState(2);
            var kitchen = new MDPState(3);

            master.Successors.Add(new MDPSuccessorState(new AI.Action(1, "Goto Kitchen"), 0.1, kitchen, 0));

            var entrance = new MDPState(1);
            var lounge   = new MDPState(4);

            kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(2, "Goto Lounge"), 0.1, lounge, -15));
            kitchen.Successors.Add(new MDPSuccessorState(new AI.Action(3, "Goto Entrance Hall"), 0, entrance, -30));

            var spare = new MDPState(0);

            lounge.Successors.Add(new MDPSuccessorState(new AI.Action(4, "Goto Spare Room"), 0.1, spare, -10));

            var outside = new MDPState(5);

            lounge.Successors.Add(new MDPSuccessorState(new AI.Action(5, "Go Outside"), 0.1, outside, 30));
            entrance.Successors.Add(new MDPSuccessorState(new AI.Action(6, "Go Outside"), 0.1, outside, 50));
            outside.Successors.Add(new MDPSuccessorState(new AI.Action(7, "Stay Outside"), 0.2, outside, 50));

            var examples = MDPConverter.ToExamples(master);

            Assert.Equal(7, examples.Item1.Rows);
            Assert.Equal(7, examples.Item2.Length);
            Assert.Equal(7, examples.Item3.Rows);
            Assert.Equal(7, examples.Item4.Length);

            var generator = new Reinforcement.QLearning.QLearnerGenerator()
            {
                Lambda = 0.9
            };

            Reinforcement.QLearning.QLearnerModel model = (Reinforcement.QLearning.QLearnerModel)generator.Generate(examples.Item1, examples.Item2, examples.Item3, examples.Item4);

            Assert.Equal(3, (int)model.Predict(kitchen.ToVector()) /*, "Expected to move from kitchen to entrance hall"*/);
            Assert.Equal(5, (int)model.Predict(lounge.ToVector()) /*, "Expected to move from lounge to outside"*/);
            Assert.Equal(7, (int)model.Predict(outside.ToVector()) /*, "Expected to stay outside"*/);

            string path = "Start: " + master.Id; IMDPState current = master;
            int    counter = 0;

            while (current.Id != outside.Id)
            {
                if (counter > 20)
                {
                    break;
                }

                double v    = model.Predict(current.ToVector());
                var    next = current.GetSuccessors().Where(w => w.Action.Id == (int)v).FirstOrDefault() as IMDPSuccessor;
                if (next == null)
                {
                    break;
                }

                current = next.State as IMDPState;

                counter++;

                path += $"\n next: { current.Id } ({ next.Reward.ToString("N2") })";
            }

            Console.Write(path);
        }
Пример #17
0
 /// <summary>
 ///   Converts the MDP State into a vector form.
 /// </summary>
 /// <param name="state">MDP State.</param>
 /// <returns>Vector.</returns>
 public static Vector ToVector(this IMDPState state)
 {
     return(state.Features);
 }
Пример #18
0
 public bool Equals(IMDPState other)
 {
     throw new NotImplementedException();
 }
Пример #19
0
 /// <summary>
 ///   Converts an MDP State (recursively) into it's equivalent math form, including all successor states.
 /// </summary>
 /// <param name="state">The starting state.</param>
 /// <returns>Tuple&lt;Matrix, Vector, Matrix&gt;</returns>
 public static Tuple <Matrix, Vector, Matrix, Vector> ToExamples(this IMDPState state)
 {
     return(new[] { state }.ToExamples());
 }
Пример #20
0
 public double Reward(int t, IMDPState s, IMDPDecision a)
 {
     return(suppRoute((a as Decision).OpenProductSet).
            Where(i => (s as State).CanSupportRoute(i)).
            Sum(i => Ro(t) * P(t, i, a as Decision) * f(i)));
 }