public static void AddScaledSubVector2Matrix(ref Matrix m, Vector v, int v_position, double scalar)
{
if (v.Dimension - v_position < m.Height * m.Width)
throw new Exception("Computing.MatrixToolbox.AddScaledSubVector2Matrix()");
for (int i = 0; i < m.Height; i++)
for (int j = 0; j < m.Width; j++)
m[i, j] += v[v_position++] * scalar;
}
public static void AddSquaredVector(ref Matrix m, Vector a)
{
int dim = a.Dimension;
if (dim!=m.Height || dim!=m.Width)
throw (new System.Exception("Adding squared vector is impossible due to wrong dimensions"));
for (int i=0; i<dim; i++)
for (int j=0; j<dim; j++)
m[i,j] += a[i]*a[j];
}
public void AddToWeights(Vector vect, double scalar)
{
if (vect == null || vect.Dimension != WeightsCount)
throw new Exception(this.GetType().ToString() + ".AddToWeights()");
int k = 0;
for (int l = 0; l < Layer.Length; l++)
{
Matrix mat = Layer[l].Weights;
int msize = mat.Height * mat.Width;
MatrixToolbox.AddScaledSubVector2Matrix(ref mat, vect, k, scalar);
k += msize;
}
}
public AVisit(int time_index, Vector state, Vector[] actions, Vector[] next_states)
{
TimeIndex = time_index;
State = state.Clone();
Actions = new Vector[actions.Length];
for (int i = 0; i < actions.Length; i++)
Actions[i] = actions[i].Clone();
NextStates = new Vector[next_states.Length];
//for (int i = 0; i < next_states.Length; i++)
// NextStates[i] = next_states[i].Clone();
int j = -1;
while (++j < next_states.Length && next_states[j] != null)
NextStates[j] = next_states[j].Clone();
}
public static void Copy(Matrix pattern, ref Vector v)
{
int dim = Math.Max(pattern.Height, pattern.Width);
Vector.AssureDimension(ref v, dim);
if (pattern.Width==1)
{
for (int i=0; i<dim; i++)
v[i] = pattern[i,0];
}
else
if (pattern.Height==1)
{
for (int j=0; j<dim; j++)
v[j] = pattern[0,j];
}
else
throw new Exception("The matrix is not a vector");
}
public void BackPropagateGradient(Vector out_gradient, ref Vector weight_gradient)
{
int out_dim = this.OutDimension;
if (out_gradient.Dimension != out_dim)
throw new Exception(this.GetType().ToString() + ".BackPropagateGradient");
if (dL_dOutput != out_gradient)
dL_dOutput.InsertPart(0, out_dim, out_gradient, 0);
BackPropagateGradient();
Vector.AssureDimension(ref weight_gradient, WeightsCount);
int k = 0;
for (int l = 0; l < Layer.Length; l++)
{
int size = Layer[l].dL_dWeights.Height * Layer[l].dL_dWeights.Width;
MatrixToolbox.InsertMatrix2Vector(ref weight_gradient, k, Layer[l].dL_dWeights);
k += size;
}
}
public void SaveForwardState(ref Vector state)
{
int offset = 0;
Vector.AssureDimension(ref state, InternalStateDimension);
for (int l = 0; l < Layer.Length; l++)
{
state.InsertPart(offset, Layer[l].X.Dimension, Layer[l].X, 0);
offset += Layer[l].X.Dimension;
state.InsertPart(offset, Layer[l].Output.Dimension, Layer[l].Output, 0);
offset += Layer[l].Output.Dimension;
}
}
protected void PrepareHorizon(int iterationLimit, Vector state, ref Vector[] currentActions, ref Vector[] currentNextStates, ref double stateValue)
{
Phi = 0;
IterationsNrOpti = 0;
var sigma = Sigma;
while (IterationsNrOpti < iterationLimit || sigma < SigmaMin)
{
++IterationsNrOpti;
AdjustActions(state, ref sigma, ref currentActions, ref currentNextStates, ref stateValue);
}
}
protected Vector ModelNextState(Vector state, Vector action)
{
return new Vector(_model.StateFunction(state.Table.ToList(), action.Table.ToList()).ToArray());
}
protected double CalculateStateValue(Vector state, Vector[] actions, ref Vector[] next_states)
{
// TODO h? Actions.length? HorizonSize?
int h = actions.Length;
next_states = new Vector[h];
var stateTmp = state;
double value = 0;
double gammai = 1;
// Get the last state with the current actions
for (int i = 0; i < h; i++)
{
next_states[i] = stateTmp = ModelNextState(stateTmp, actions[i]);
value += gammai * ModelReward(stateTmp);
if (!ModelIsStateAcceptable(stateTmp))
{
for (int j = i+1; j < h; ++j)
{
next_states[j] = new Vector(MODEL_STATE_VAR_COUNT, 0.0);
}
return value;
}
gammai *= Gamma;
}
V.Approximate(new Vector(_model.TurnStateToNNAcceptable(stateTmp.Table.ToList()).ToArray()), ref Vval);
value += gammai * Vval[0];
return value;
}
// sterowanie zostało wykonane,
// system przeszedł do następnego stanu (należy zignorować reward)
public void ThisHappened(double[] nextState)
{
//Thread.Sleep(100);
var reset = false;
if (ModelIsStateAcceptable(new Vector(nextState)))
{
State = new Vector(nextState);
for (int i = 0; i < HORIZON_SIZE - 1; i++)
{
Actions[i] = Actions[i + 1];
NextStates[i] = NextStates[i + 1];
}
}
else
{
reset = true;
}
VestSum += ModelReward(State);
// dokonujemy poprawek historycznych sterowań i funkcji V
for (int k = 0; k < TimesToTeach; k++)
{
// Get random state & calculate its value
Visit = (AVisit)AllVisits[Sampler.Next(AllVisits.Count - 1)];
Vest = CalculateStateValue(Visit.State, Visit.Actions, ref Visit.NextStates);
// Try to Adjust its actions
//for (int i = 0; i < TimesToAdjust; i++)
// AdjustActions(Visit.State, Sigma, ref Visit.Actions, ref Visit.NextStates, ref Vest);
PrepareHorizon(TimesToAdjustPastActions, Visit.State, ref Visit.Actions, ref Visit.NextStates, ref Vest);
// Get the state value from the approximator & add calculated discrepancy to its weights(NN).
V.Approximate(new Vector(_model.TurnStateToNNAcceptable(Visit.State.Table.ToList()).ToArray()), ref Vval);
Vgrad[0] = Vval[0] - Vest;
V.BackPropagateGradient(Vgrad, ref VparamGrad);
V.AddToWeights(VparamGrad, -BetaV);
}
if (reset) NextEpisode();
}
/// <summary>Prepares the model for the next epizode. The Approximator stays unchanged.</summary>
public void NextEpisode()
{
VestSum /= TimeIndex;
UpdateVestList(VestSum);
VestSum = 0;
TimeIndex = 0;
++_episodeNr;
StartInState(_model.MeddleWithGoalAndStartingState().ToArray());
Actions = new Vector[HORIZON_SIZE];
NextStates = new Vector[HORIZON_SIZE];
for (int i = 0; i < HORIZON_SIZE; i++)
{
Actions[i] = new Vector(MODEL_ACTION_VAR_COUNT, _model.GenerateControlVariables()[0]);
NextStates[i] = new Vector(MODEL_STATE_VAR_COUNT, 0.0);
}
}
public static void DActivIdentity(Vector x, Vector v, ref Vector derivatives)
{
if (v == null || derivatives == null || v.Dimension < x.Dimension || derivatives.Dimension < x.Dimension)
throw new Exception("Neural.MLPerceptron2.ALayer.DActivIdentity()");
derivatives.FillWith(1);
}
public static void ActivLogit(Vector x, ref Vector v)
{
if (v == null || v.Dimension < x.Dimension)
throw new Exception("Neural.MLPerceptron2.ALayer.ActivLogit()");
int dim = x.Dimension;
for (int i = 0; i < dim; i++)
v[i] = 1.0 / (1.0 + Math.Exp(-x[i]));
}
public static void ActivIdentity(Vector x, ref Vector v)
{
if (v == null || v.Dimension < x.Dimension)
throw new Exception("Neural.MLPerceptron2.ALayer.ActivIdentity()");
int dim = x.Dimension;
for (int i = 0; i < dim; i++)
v[i] = x[i];
}
protected void SetInput(Vector arguments)
{
if (arguments == null || arguments.Dimension != Input.Dimension)
throw new Exception(this.GetType().ToString() + ".SetInput");
Input = arguments - InputAverage; // w C++ szybciej bedzie: Input.SetDifference(arguments, InputAverage);
Input = Input & InputInvStddev; // w C++ szybciej bedzie: Input.SetProducts(Input, InputInvStddev);
}
public void SetWeights(Vector weights)
{
if (weights == null || weights.Dimension != WeightsCount)
throw new Exception(this.GetType().ToString() + ".SetWeights()");
int k = 0;
for (int l = 0; l < Layer.Length; l++)
{
Matrix mat = Layer[l].Weights;
int msize = mat.Height * mat.Width;
mat.FillWith(0);
MatrixToolbox.AddScaledSubVector2Matrix(ref mat, weights, k, 1);
k += msize;
}
}
public void SetInputDescription(Vector in_av, Vector in_stddev)
{
int i, in_dim = Input.Dimension;
if (in_av.Dimension != in_dim || in_stddev.Dimension != in_dim)
throw new Exception(this.GetType().ToString() + ".SetInputDescription(...)");
Vector.Copy(in_av, ref InputAverage);
for (i = 0; i < in_dim; i++)
{
if (in_stddev[i] <= 0)
throw new Exception(this.GetType().ToString() + ".SetInputDescription(...): stddev nonpositive");
InputInvStddev[i] = 1.0 / in_stddev[i];
}
}
public void Init(double[] stateAv, double[] stateStddev, double[] actionMin, int vsize)
{
int stateDim = stateAv.GetLength(0);
int actionDim = actionMin.GetLength(0);
#region budowa sieci neuronowej
V = new MLPerceptron2();
V.Build(stateDim, CellType.Arcustangent, new int[] { vsize, 1 });
V.SetInputDescription(new Vector(stateAv), new Vector(stateStddev));
V.InitWeights(1.0 / Math.Sqrt(vsize + 1));
Vval = new Vector(1);
Vgrad = new Vector(1);
#endregion koniec budowy sieci neuronowej
Actions = new Vector[HORIZON_SIZE];
NextStates = new Vector[HORIZON_SIZE];
for (int i = 0; i < HORIZON_SIZE; i++)
{
Actions[i] = new Vector(actionDim, _model.GenerateControlVariables()[0]);
NextStates[i] = new Vector(stateDim, 0.0);
}
}
public static void DActivLogit(Vector x, Vector v, ref Vector derivatives)
{
if (v == null || derivatives == null || v.Dimension < x.Dimension || derivatives.Dimension < x.Dimension)
throw new Exception("Neural.MLPerceptron2.ALayer.DActivLogit()");
int dim = x.Dimension;
for (int i = 0; i < dim; i++)
derivatives[i] = v[i] * (1.0 - v[i]);
}
public void StartInState(double[] state)
{
State = new Vector(state);
}
public void Build(int in_dim, CellType act_type, int out_dim)
{
Input = new Vector(in_dim + 1);
Input[in_dim] = 1;
dL_dInput = new Vector(in_dim + 1);
Weights = new Matrix(out_dim, in_dim + 1);
dL_dWeights = new Matrix(out_dim, in_dim + 1);
X = new Vector(out_dim);
dL_dX = new Vector(out_dim);
switch (act_type)
{
case CellType.Linear:
Activation = new Delegate11(ActivIdentity);
DActivation = new Delegate12(DActivIdentity);
break;
case CellType.Expotential:
Activation = new Delegate11(ActivLogit);
DActivation = new Delegate12(DActivLogit);
break;
case CellType.Arcustangent:
Activation = new Delegate11(ActivAtan);
DActivation = new Delegate12(DActivAtan);
break;
default:
throw new System.Exception(this.GetType().ToString() + ".Build(): unknown cell type");
}
Output = new Vector(out_dim);
dOutput_dX = new Vector(out_dim);
dL_dOutput = new Vector(out_dim);
}
/// <summary>Iteracja algorytmu (1+1) </summary>
/// <returns>Zwraca, czy nastąpiła poprawa </returns>
protected void AdjustActions(Vector state, ref double sigma, ref Vector[] currentActions, ref Vector[] currentNextStates, ref double stateValue)
{
var modifiedActions = new Vector[HORIZON_SIZE];
var newNextStates = new Vector[HORIZON_SIZE];
double sigmaDiscount;
#region Modify the Action vector
for (int i = 0; i < HORIZON_SIZE; i++)
{
modifiedActions[i] = currentActions[i].Clone();
sigmaDiscount = (double)(i + 1)/HORIZON_SIZE;
// < MinAction ; _action + Rand ; MaxAction >
for (int j = 0; j < modifiedActions[i].Dimension; j++)
{
modifiedActions[i][j] = Math.Min(
Math.Max(MinAction[j], modifiedActions[i][j] + Sampler.SampleFromNormal(0, sigma) * sigmaDiscount),
MaxAction[j]);
}
}
#endregion
// If the value of the state with new action vector is bigger, replace the previous action vector
#region if bigger, replace the previous action vector
double stateNewValue = CalculateStateValue(state, modifiedActions, ref newNextStates);
if (stateNewValue > stateValue)
{
stateValue = stateNewValue;
currentActions = modifiedActions;
currentNextStates = newNextStates;
++Phi;
}
#endregion
UpdateSigma(ref sigma);
}
public void CalculateBackward()
{
DActivation(X, Output, ref dOutput_dX);
dL_dX = dOutput_dX & dL_dOutput; // w C++ szybciej bedzie: dL_dX.SetProducts(dOutput_dX, dL_dOutput);
dL_dWeights = dL_dX | Input; // w C++ szybciej bedzie: dL_dWeights.SetProduct(dL_dX, Input);
dL_dInput = dL_dX * Weights; // w C++ szybciej bedzie: dL_dInput.SetProduct(dL_dX, Weights);
}
protected bool ModelIsStateAcceptable(Vector state)
{
return _model.IsStateAcceptable(state.Table.ToList());
}
public void CalculateForwad()
{
X = Weights * Input; // w C++ szybciej bedzie: X.SetProduct(Weights, Input);
Activation(X, ref Output);
}
protected double ModelReward(Vector nextState)
{
return ModelIsStateAcceptable(nextState) ? _model.GetReward(nextState.Table.ToList()) : _model.Penalty();
}
public void Approximate(Vector input, ref Vector output)
{
SetInput(input);
CalculateAhead();
Vector.AssureDimension(ref output, OutDimension);
output.InsertPart(0, OutDimension, Layer[Layer.Length - 1].Output, 0);
}
public Advisor(IModel model, List<Property> properties, List<LoggedValue> loggedValues)
{
_model = model;
MODEL_ACTION_VAR_COUNT = _model.GenerateControlVariables().Count;
MODEL_STATE_VAR_COUNT = _model.GetInitialState().Count;
HORIZON_SIZE = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("Horizon")).Value);
Vsize = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("Neurons Number")).Value);
BetaV = Convert.ToDouble(properties.Find(p => p.Name.Equals("BetaV")).Value);
Gamma = Convert.ToDouble(properties.Find(p => p.Name.Equals("Discount")).Value);;
Sigma = Convert.ToDouble(properties.Find(p => p.Name.Equals("Sigma")).Value);
SigmaMin = Convert.ToDouble(properties.Find(p => p.Name.Equals("SigmaMin")).Value);
TimesToAdjust = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("TimesToAdjust")).Value);
var externalDiscretization =
Convert.ToDouble(properties.Find(p => p.Name.Equals("ExternalDiscretization")).Value);
var internalDiscretization =
Convert.ToDouble(properties.Find(p => p.Name.Equals("InternalDiscretization")).Value);
var TimeLimit = Convert.ToDouble(properties.Find(p => p.Name.Equals("TimeLimit")).Value);
TimesToTeach = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("TimesToTeach")).Value);
TimesToAdjustPastActions = (int)Convert.ToDouble(properties.Find(p => p.Name.Equals("TimesToAdjustPastActions")).Value);
_logger = new LogIt("", loggedValues);
_model.SetDiscretizations(externalDiscretization, internalDiscretization);
TimeIndex = 0;
_episodeNr = 0;
AllVisits = new ArrayList();
Sampler = new ASampler();
IterationsLimit = (int) (TimeLimit / externalDiscretization);
VestSum = 0;
StartInState(_model.GetInitialState().ToArray());
MinAction = new Vector(_model.MinActionValues());
MaxAction = new Vector(_model.MaxActionValues());
double[] stateAverage = _model.GetStateValuesAverageNN().ToArray();
double[] stateStandardDeviation = _model.GetStateValuesStandardDeviationNN().ToArray();
Init(stateAverage, stateStandardDeviation, MinAction.Table, Vsize);
}
public void RestoreForwardState(Vector state)
{
int offset = 0;
if (state.Dimension != InternalStateDimension)
throw new Exception(GetType().ToString() + ".RestoreForwardState");
for (int l = 0; l < Layer.Length; l++)
{
Layer[l].X.InsertPart(0, Layer[l].X.Dimension, state, offset);
offset += Layer[l].X.Dimension;
Layer[l].Output.InsertPart(0, Layer[l].Output.Dimension, state, offset);
offset += Layer[l].Output.Dimension;
}
}
