/// <summary> /// Generates a <see cref="QLearnerModel"/> based on states/actions with transitions and rewards. /// </summary> /// <param name="X1">Initial State matrix.</param> /// <param name="y">Action label vector.</param> /// <param name="X2">Transition State matrix.</param> /// <param name="r">Reward values.</param> /// <returns>QLearnerModel.</returns> public override IReinforcementModel Generate(Matrix X1, Vector y, Matrix X2, Vector r) { this.Preprocess(X1, y, X2, r); var examples = MDPConverter.GetStates(X1, y, X2, this.FeatureProperties, this.FeatureDiscretizer); var states = examples.Item1; var actions = examples.Item2; var statesP = examples.Item3; QTable Q = new QTable(); // construct Q table for (int i = 0; i < states.Count(); i++) { var state = states.ElementAt(i); var action = actions.ElementAt(i); var stateP = statesP.ElementAt(i); Q.AddOrUpdate(state, action, r[i]); if (!Q.ContainsKey(stateP)) { Q.AddKey(stateP); } } double count = states.Select(s => s.Id).Distinct().Count(); double change = 0; for (int pass = 0; pass < this.MaxIterations; pass++) { change = 0; for (int i = 0; i < states.Count(); i++) { IState state = states.ElementAt(i); IAction action = actions.ElementAt(i); IState stateP = statesP.ElementAt(i); double reward = r[i]; double q = (1.0 - this.LearningRate) * Q[state, action] + this.LearningRate * (reward + this.Lambda * Q[stateP, Q.GetMaxAction(stateP)]); change += (1.0 / count) * System.Math.Abs((Q[state, action] - q)); Q[state, action] = q; } if (change <= this.Epsilon) { break; } } return(new QLearnerModel() { Descriptor = this.Descriptor, TransitionDescriptor = this.TransitionDescriptor, NormalizeFeatures = this.NormalizeFeatures, FeatureNormalizer = this.FeatureNormalizer, FeatureProperties = this.FeatureProperties, FeatureDiscretizer = this.FeatureDiscretizer, LearningRate = this.LearningRate, Lambda = this.Lambda, Q = Q }); }
/// <summary> /// Generates a <see cref="QLearnerModel"/> based on states/actions with transitions and rewards. /// </summary> /// <param name="X1">Initial State matrix.</param> /// <param name="y">Action label vector.</param> /// <param name="X2">Transition State matrix.</param> /// <param name="r">Reward values.</param> /// <returns>QLearnerModel.</returns> public override IReinforcementModel Generate(Matrix X1, Vector y, Matrix X2, Vector r) { this.Preprocess(X1, y, X2, r); var examples = MDPConverter.GetStates(X1, y, X2, this.FeatureProperties, this.FeatureDiscretizer); var states = examples.Item1; var actions = examples.Item2; var statesP = examples.Item3; QTable Q = new QTable(); // construct Q table for (int i = 0; i < states.Count(); i++) { var state = states.ElementAt(i); var action = actions.ElementAt(i); var stateP = statesP.ElementAt(i); Q.AddOrUpdate(state, action, r[i]); if (!Q.ContainsKey(stateP)) Q.AddKey(stateP); } double count = states.Select(s => s.Id).Distinct().Count(); double change = 0; for (int pass = 0; pass < this.MaxIterations; pass++) { change = 0; for (int i = 0; i < states.Count(); i++) { IState state = states.ElementAt(i); IAction action = actions.ElementAt(i); IState stateP = statesP.ElementAt(i); double reward = r[i]; double q = (1.0 - this.LearningRate) * Q[state, action] + this.LearningRate * (reward + this.Lambda * Q[stateP, Q.GetMaxAction(stateP)]); change += (1.0 / count) * System.Math.Abs((Q[state, action] - q)); Q[state, action] = q; } if (change <= this.Epsilon) break; } return new QLearnerModel() { Descriptor = this.Descriptor, TransitionDescriptor = this.TransitionDescriptor, NormalizeFeatures = this.NormalizeFeatures, FeatureNormalizer = this.FeatureNormalizer, FeatureProperties = this.FeatureProperties, FeatureDiscretizer = this.FeatureDiscretizer, LearningRate = this.LearningRate, Lambda = this.Lambda, Q = Q }; }
/// <summary> /// Generates a <see cref="QLearnerModel" /> based on states/actions with transitions and rewards. /// </summary> /// <param name="X1">Initial State matrix.</param> /// <param name="y">Action label vector.</param> /// <param name="X2">Transition State matrix.</param> /// <param name="r">Reward values.</param> /// <returns>QLearnerModel.</returns> public override IReinforcementModel Generate(Matrix X1, Vector y, Matrix X2, Vector r) { Preprocess(X1, y, X2, r); var examples = MDPConverter.GetStates(X1, y, X2, FeatureProperties, FeatureDiscretizer); var states = examples.Item1; var actions = examples.Item2; var statesP = examples.Item3; var Q = new QTable(); // construct Q table for (var i = 0; i < states.Count(); i++) { var state = states.ElementAt(i); var action = actions.ElementAt(i); var stateP = statesP.ElementAt(i); Q.AddOrUpdate(state, action, r[i]); if (!Q.ContainsKey(stateP)) { Q.AddKey(stateP); } } double count = states.Select(s => s.Id).Distinct().Count(); for (var pass = 0; pass < MaxIterations; pass++) { double change = 0; for (var i = 0; i < states.Count(); i++) { var state = states.ElementAt(i); var action = actions.ElementAt(i); var stateP = statesP.ElementAt(i); var reward = r[i]; var q = (1.0 - LearningRate) * Q[state, action] + LearningRate * (reward + Lambda * Q[stateP, Q.GetMaxAction(stateP)]); change += 1.0 / count * System.Math.Abs(Q[state, action] - q); Q[state, action] = q; } if (change <= Epsilon) { break; } } return(new QLearnerModel { Descriptor = Descriptor, TransitionDescriptor = TransitionDescriptor, NormalizeFeatures = NormalizeFeatures, FeatureNormalizer = FeatureNormalizer, FeatureProperties = FeatureProperties, FeatureDiscretizer = FeatureDiscretizer, LearningRate = LearningRate, Lambda = Lambda, Q = Q }); }