public static void AddScaledSubVector2Matrix(ref Matrix m, Vector v, int v_position, double scalar) { if (v.Dimension - v_position < m.Height * m.Width) throw new Exception("Computing.MatrixToolbox.AddScaledSubVector2Matrix()"); for (int i = 0; i < m.Height; i++) for (int j = 0; j < m.Width; j++) m[i, j] += v[v_position++] * scalar; }
public static void AddSquaredVector(ref Matrix m, Vector a) { int dim = a.Dimension; if (dim!=m.Height || dim!=m.Width) throw (new System.Exception("Adding squared vector is impossible due to wrong dimensions")); for (int i=0; i<dim; i++) for (int j=0; j<dim; j++) m[i,j] += a[i]*a[j]; }
public void AddToWeights(Vector vect, double scalar) { if (vect == null || vect.Dimension != WeightsCount) throw new Exception(this.GetType().ToString() + ".AddToWeights()"); int k = 0; for (int l = 0; l < Layer.Length; l++) { Matrix mat = Layer[l].Weights; int msize = mat.Height * mat.Width; MatrixToolbox.AddScaledSubVector2Matrix(ref mat, vect, k, scalar); k += msize; } }
public AVisit(int time_index, Vector state, Vector[] actions, Vector[] next_states) { TimeIndex = time_index; State = state.Clone(); Actions = new Vector[actions.Length]; for (int i = 0; i < actions.Length; i++) Actions[i] = actions[i].Clone(); NextStates = new Vector[next_states.Length]; //for (int i = 0; i < next_states.Length; i++) // NextStates[i] = next_states[i].Clone(); int j = -1; while (++j < next_states.Length && next_states[j] != null) NextStates[j] = next_states[j].Clone(); }
public static void Copy(Matrix pattern, ref Vector v) { int dim = Math.Max(pattern.Height, pattern.Width); Vector.AssureDimension(ref v, dim); if (pattern.Width==1) { for (int i=0; i<dim; i++) v[i] = pattern[i,0]; } else if (pattern.Height==1) { for (int j=0; j<dim; j++) v[j] = pattern[0,j]; } else throw new Exception("The matrix is not a vector"); }
public void BackPropagateGradient(Vector out_gradient, ref Vector weight_gradient) { int out_dim = this.OutDimension; if (out_gradient.Dimension != out_dim) throw new Exception(this.GetType().ToString() + ".BackPropagateGradient"); if (dL_dOutput != out_gradient) dL_dOutput.InsertPart(0, out_dim, out_gradient, 0); BackPropagateGradient(); Vector.AssureDimension(ref weight_gradient, WeightsCount); int k = 0; for (int l = 0; l < Layer.Length; l++) { int size = Layer[l].dL_dWeights.Height * Layer[l].dL_dWeights.Width; MatrixToolbox.InsertMatrix2Vector(ref weight_gradient, k, Layer[l].dL_dWeights); k += size; } }
public void SaveForwardState(ref Vector state) { int offset = 0; Vector.AssureDimension(ref state, InternalStateDimension); for (int l = 0; l < Layer.Length; l++) { state.InsertPart(offset, Layer[l].X.Dimension, Layer[l].X, 0); offset += Layer[l].X.Dimension; state.InsertPart(offset, Layer[l].Output.Dimension, Layer[l].Output, 0); offset += Layer[l].Output.Dimension; } }
protected void PrepareHorizon(int iterationLimit, Vector state, ref Vector[] currentActions, ref Vector[] currentNextStates, ref double stateValue) { Phi = 0; IterationsNrOpti = 0; var sigma = Sigma; while (IterationsNrOpti < iterationLimit || sigma < SigmaMin) { ++IterationsNrOpti; AdjustActions(state, ref sigma, ref currentActions, ref currentNextStates, ref stateValue); } }
protected Vector ModelNextState(Vector state, Vector action) { return new Vector(_model.StateFunction(state.Table.ToList(), action.Table.ToList()).ToArray()); }
protected double CalculateStateValue(Vector state, Vector[] actions, ref Vector[] next_states) { // TODO h? Actions.length? HorizonSize? int h = actions.Length; next_states = new Vector[h]; var stateTmp = state; double value = 0; double gammai = 1; // Get the last state with the current actions for (int i = 0; i < h; i++) { next_states[i] = stateTmp = ModelNextState(stateTmp, actions[i]); value += gammai * ModelReward(stateTmp); if (!ModelIsStateAcceptable(stateTmp)) { for (int j = i+1; j < h; ++j) { next_states[j] = new Vector(MODEL_STATE_VAR_COUNT, 0.0); } return value; } gammai *= Gamma; } V.Approximate(new Vector(_model.TurnStateToNNAcceptable(stateTmp.Table.ToList()).ToArray()), ref Vval); value += gammai * Vval[0]; return value; }
// sterowanie zostało wykonane, // system przeszedł do następnego stanu (należy zignorować reward) public void ThisHappened(double[] nextState) { //Thread.Sleep(100); var reset = false; if (ModelIsStateAcceptable(new Vector(nextState))) { State = new Vector(nextState); for (int i = 0; i < HORIZON_SIZE - 1; i++) { Actions[i] = Actions[i + 1]; NextStates[i] = NextStates[i + 1]; } } else { reset = true; } VestSum += ModelReward(State); // dokonujemy poprawek historycznych sterowań i funkcji V for (int k = 0; k < TimesToTeach; k++) { // Get random state & calculate its value Visit = (AVisit)AllVisits[Sampler.Next(AllVisits.Count - 1)]; Vest = CalculateStateValue(Visit.State, Visit.Actions, ref Visit.NextStates); // Try to Adjust its actions //for (int i = 0; i < TimesToAdjust; i++) // AdjustActions(Visit.State, Sigma, ref Visit.Actions, ref Visit.NextStates, ref Vest); PrepareHorizon(TimesToAdjustPastActions, Visit.State, ref Visit.Actions, ref Visit.NextStates, ref Vest); // Get the state value from the approximator & add calculated discrepancy to its weights(NN). V.Approximate(new Vector(_model.TurnStateToNNAcceptable(Visit.State.Table.ToList()).ToArray()), ref Vval); Vgrad[0] = Vval[0] - Vest; V.BackPropagateGradient(Vgrad, ref VparamGrad); V.AddToWeights(VparamGrad, -BetaV); } if (reset) NextEpisode(); }
/// <summary>Prepares the model for the next epizode. The Approximator stays unchanged.</summary> public void NextEpisode() { VestSum /= TimeIndex; UpdateVestList(VestSum); VestSum = 0; TimeIndex = 0; ++_episodeNr; StartInState(_model.MeddleWithGoalAndStartingState().ToArray()); Actions = new Vector[HORIZON_SIZE]; NextStates = new Vector[HORIZON_SIZE]; for (int i = 0; i < HORIZON_SIZE; i++) { Actions[i] = new Vector(MODEL_ACTION_VAR_COUNT, _model.GenerateControlVariables()[0]); NextStates[i] = new Vector(MODEL_STATE_VAR_COUNT, 0.0); } }
public static void DActivIdentity(Vector x, Vector v, ref Vector derivatives) { if (v == null || derivatives == null || v.Dimension < x.Dimension || derivatives.Dimension < x.Dimension) throw new Exception("Neural.MLPerceptron2.ALayer.DActivIdentity()"); derivatives.FillWith(1); }
public static void ActivLogit(Vector x, ref Vector v) { if (v == null || v.Dimension < x.Dimension) throw new Exception("Neural.MLPerceptron2.ALayer.ActivLogit()"); int dim = x.Dimension; for (int i = 0; i < dim; i++) v[i] = 1.0 / (1.0 + Math.Exp(-x[i])); }
public static void ActivIdentity(Vector x, ref Vector v) { if (v == null || v.Dimension < x.Dimension) throw new Exception("Neural.MLPerceptron2.ALayer.ActivIdentity()"); int dim = x.Dimension; for (int i = 0; i < dim; i++) v[i] = x[i]; }
protected void SetInput(Vector arguments) { if (arguments == null || arguments.Dimension != Input.Dimension) throw new Exception(this.GetType().ToString() + ".SetInput"); Input = arguments - InputAverage; // w C++ szybciej bedzie: Input.SetDifference(arguments, InputAverage); Input = Input & InputInvStddev; // w C++ szybciej bedzie: Input.SetProducts(Input, InputInvStddev); }
public void SetWeights(Vector weights) { if (weights == null || weights.Dimension != WeightsCount) throw new Exception(this.GetType().ToString() + ".SetWeights()"); int k = 0; for (int l = 0; l < Layer.Length; l++) { Matrix mat = Layer[l].Weights; int msize = mat.Height * mat.Width; mat.FillWith(0); MatrixToolbox.AddScaledSubVector2Matrix(ref mat, weights, k, 1); k += msize; } }
public void SetInputDescription(Vector in_av, Vector in_stddev) { int i, in_dim = Input.Dimension; if (in_av.Dimension != in_dim || in_stddev.Dimension != in_dim) throw new Exception(this.GetType().ToString() + ".SetInputDescription(...)"); Vector.Copy(in_av, ref InputAverage); for (i = 0; i < in_dim; i++) { if (in_stddev[i] <= 0) throw new Exception(this.GetType().ToString() + ".SetInputDescription(...): stddev nonpositive"); InputInvStddev[i] = 1.0 / in_stddev[i]; } }
public void Init(double[] stateAv, double[] stateStddev, double[] actionMin, int vsize) { int stateDim = stateAv.GetLength(0); int actionDim = actionMin.GetLength(0); #region budowa sieci neuronowej V = new MLPerceptron2(); V.Build(stateDim, CellType.Arcustangent, new int[] { vsize, 1 }); V.SetInputDescription(new Vector(stateAv), new Vector(stateStddev)); V.InitWeights(1.0 / Math.Sqrt(vsize + 1)); Vval = new Vector(1); Vgrad = new Vector(1); #endregion koniec budowy sieci neuronowej Actions = new Vector[HORIZON_SIZE]; NextStates = new Vector[HORIZON_SIZE]; for (int i = 0; i < HORIZON_SIZE; i++) { Actions[i] = new Vector(actionDim, _model.GenerateControlVariables()[0]); NextStates[i] = new Vector(stateDim, 0.0); } }
public static void DActivLogit(Vector x, Vector v, ref Vector derivatives) { if (v == null || derivatives == null || v.Dimension < x.Dimension || derivatives.Dimension < x.Dimension) throw new Exception("Neural.MLPerceptron2.ALayer.DActivLogit()"); int dim = x.Dimension; for (int i = 0; i < dim; i++) derivatives[i] = v[i] * (1.0 - v[i]); }
public void StartInState(double[] state) { State = new Vector(state); }
public void Build(int in_dim, CellType act_type, int out_dim) { Input = new Vector(in_dim + 1); Input[in_dim] = 1; dL_dInput = new Vector(in_dim + 1); Weights = new Matrix(out_dim, in_dim + 1); dL_dWeights = new Matrix(out_dim, in_dim + 1); X = new Vector(out_dim); dL_dX = new Vector(out_dim); switch (act_type) { case CellType.Linear: Activation = new Delegate11(ActivIdentity); DActivation = new Delegate12(DActivIdentity); break; case CellType.Expotential: Activation = new Delegate11(ActivLogit); DActivation = new Delegate12(DActivLogit); break; case CellType.Arcustangent: Activation = new Delegate11(ActivAtan); DActivation = new Delegate12(DActivAtan); break; default: throw new System.Exception(this.GetType().ToString() + ".Build(): unknown cell type"); } Output = new Vector(out_dim); dOutput_dX = new Vector(out_dim); dL_dOutput = new Vector(out_dim); }
/// <summary>Iteracja algorytmu (1+1) </summary> /// <returns>Zwraca, czy nastąpiła poprawa </returns> protected void AdjustActions(Vector state, ref double sigma, ref Vector[] currentActions, ref Vector[] currentNextStates, ref double stateValue) { var modifiedActions = new Vector[HORIZON_SIZE]; var newNextStates = new Vector[HORIZON_SIZE]; double sigmaDiscount; #region Modify the Action vector for (int i = 0; i < HORIZON_SIZE; i++) { modifiedActions[i] = currentActions[i].Clone(); sigmaDiscount = (double)(i + 1)/HORIZON_SIZE; // < MinAction ; _action + Rand ; MaxAction > for (int j = 0; j < modifiedActions[i].Dimension; j++) { modifiedActions[i][j] = Math.Min( Math.Max(MinAction[j], modifiedActions[i][j] + Sampler.SampleFromNormal(0, sigma) * sigmaDiscount), MaxAction[j]); } } #endregion // If the value of the state with new action vector is bigger, replace the previous action vector #region if bigger, replace the previous action vector double stateNewValue = CalculateStateValue(state, modifiedActions, ref newNextStates); if (stateNewValue > stateValue) { stateValue = stateNewValue; currentActions = modifiedActions; currentNextStates = newNextStates; ++Phi; } #endregion UpdateSigma(ref sigma); }
public void CalculateBackward() { DActivation(X, Output, ref dOutput_dX); dL_dX = dOutput_dX & dL_dOutput; // w C++ szybciej bedzie: dL_dX.SetProducts(dOutput_dX, dL_dOutput); dL_dWeights = dL_dX | Input; // w C++ szybciej bedzie: dL_dWeights.SetProduct(dL_dX, Input); dL_dInput = dL_dX * Weights; // w C++ szybciej bedzie: dL_dInput.SetProduct(dL_dX, Weights); }
protected bool ModelIsStateAcceptable(Vector state) { return _model.IsStateAcceptable(state.Table.ToList()); }
public void CalculateForwad() { X = Weights * Input; // w C++ szybciej bedzie: X.SetProduct(Weights, Input); Activation(X, ref Output); }
protected double ModelReward(Vector nextState) { return ModelIsStateAcceptable(nextState) ? _model.GetReward(nextState.Table.ToList()) : _model.Penalty(); }
public void Approximate(Vector input, ref Vector output) { SetInput(input); CalculateAhead(); Vector.AssureDimension(ref output, OutDimension); output.InsertPart(0, OutDimension, Layer[Layer.Length - 1].Output, 0); }
public Advisor(IModel model, List<Property> properties, List<LoggedValue> loggedValues) { _model = model; MODEL_ACTION_VAR_COUNT = _model.GenerateControlVariables().Count; MODEL_STATE_VAR_COUNT = _model.GetInitialState().Count; HORIZON_SIZE = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("Horizon")).Value); Vsize = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("Neurons Number")).Value); BetaV = Convert.ToDouble(properties.Find(p => p.Name.Equals("BetaV")).Value); Gamma = Convert.ToDouble(properties.Find(p => p.Name.Equals("Discount")).Value);; Sigma = Convert.ToDouble(properties.Find(p => p.Name.Equals("Sigma")).Value); SigmaMin = Convert.ToDouble(properties.Find(p => p.Name.Equals("SigmaMin")).Value); TimesToAdjust = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("TimesToAdjust")).Value); var externalDiscretization = Convert.ToDouble(properties.Find(p => p.Name.Equals("ExternalDiscretization")).Value); var internalDiscretization = Convert.ToDouble(properties.Find(p => p.Name.Equals("InternalDiscretization")).Value); var TimeLimit = Convert.ToDouble(properties.Find(p => p.Name.Equals("TimeLimit")).Value); TimesToTeach = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("TimesToTeach")).Value); TimesToAdjustPastActions = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("TimesToAdjustPastActions")).Value); _logger = new LogIt("", loggedValues); _model.SetDiscretizations(externalDiscretization, internalDiscretization); TimeIndex = 0; _episodeNr = 0; AllVisits = new ArrayList(); Sampler = new ASampler(); IterationsLimit = (int) (TimeLimit / externalDiscretization); VestSum = 0; StartInState(_model.GetInitialState().ToArray()); MinAction = new Vector(_model.MinActionValues()); MaxAction = new Vector(_model.MaxActionValues()); double[] stateAverage = _model.GetStateValuesAverageNN().ToArray(); double[] stateStandardDeviation = _model.GetStateValuesStandardDeviationNN().ToArray(); Init(stateAverage, stateStandardDeviation, MinAction.Table, Vsize); }
public void RestoreForwardState(Vector state) { int offset = 0; if (state.Dimension != InternalStateDimension) throw new Exception(GetType().ToString() + ".RestoreForwardState"); for (int l = 0; l < Layer.Length; l++) { Layer[l].X.InsertPart(0, Layer[l].X.Dimension, state, offset); offset += Layer[l].X.Dimension; Layer[l].Output.InsertPart(0, Layer[l].Output.Dimension, state, offset); offset += Layer[l].Output.Dimension; } }