public ReinforcementAgent(TemporalNetwork network)
{
inputs = new List <double[]>();
outputs = new List <double[]>();
rewards = new List <double>();
startingvs = new List <List <NeuronValues> >();
n = network;
reward = 0;
}
public static EvolutionaryLine <Game> FromString(string str, Game startingState, MutationInfo mutationInfo)
{
string[] networksString = str.Split(new string[] { "\nnetwork:\n" }, StringSplitOptions.RemoveEmptyEntries);
List <FittedNetwork <Game> > networks = new List <FittedNetwork <Game> >();
for (int i = 0; i < networksString.Length; i++)
{
networks.Add(new FittedNetwork <Game>(TemporalNetwork.FromString(networksString[i]), startingState));
}
return(new EvolutionaryLine <Game>(networks));
}
public FittedNetwork(TemporalNetwork network, Game game)
{
this.game = game;
if (network != null)
{
this.network = network;
}
else
{
throw new ArgumentException("Use constructor for random networks");
}
fitnessScore = 0;
}
/// <summary>
/// reproduce all networks mostly with the fittest 3 networks
/// </summary>
/// <param name="numberOfCrossOvers">number of times to crossover each network</param>
/// <param name="reproduceWithHighestFit"></param>
public void CrossoverNetworks(int numberOfCrossOvers, Game startingState)
{
int miliS = DateTime.Now.Millisecond;
Random r = new Random(miliS + numberOfCrossOvers);
List <FittedNetwork <Game> > sons = new List <FittedNetwork <Game> >();
for (int i = 0; i < networks.Count; i++)
{
for (int crossI = 0; crossI < numberOfCrossOvers; crossI++)
{
int crossoverPick = r.Next(0, 4);
if (10 * r.NextDouble() <= 1)
{
crossoverPick += (int)(Math.Min(r.Next(0, 28), networks.Count - crossoverPick) * r.NextDouble());
}
sons.Add(new FittedNetwork <Game>(TemporalNetwork.CrossOver(networks[i].network, networks[crossoverPick].network, -1), startingState));
}
}
networks.AddRange(sons);
}
/// <summary>
/// if expected has NaN it has not expected output
/// </summary>
public void SupervisedBackProp(List <double[]> input, List <double[]> expectedOutput, CostFunctions costFunction, List <List <NeuronValues> > startingStates, out double cost, out List <List <List <NeuronValues> > > grads, out List <double[]> biasGrads)
{
cost = 0;
if (input.Count != expectedOutput.Count)
{
throw new ArgumentOutOfRangeException();
}
var copy = new TemporalNetwork(ActivationFunction, TemporalLayers);
copy.SetTemporalStates(startingStates);
List <double[]> costs = new List <double[]>();
for (int t = 0; t < input.Count; t++)
{
double[] outputT = copy.ExecuteNetwork(input[t]);
costs.Add(new double[expectedOutput[t].Length]);
cost += Cost.GetCostOf(outputT, expectedOutput[t], costFunction);//REPAIR
for (int i = 0; i < expectedOutput[i].Length; i++)
{
costs[t][i] = Derivatives.DerivativeOf(outputT[i], expectedOutput[t][i], costFunction);
}
}
GetGradients(costs, input, startingStates, out grads, out biasGrads);
}
/// <summary>
///
/// </summary>
/// <param name="costs"></param>
/// <param name="input"></param>
/// <param name="startingRecState">use getStates() or getEmptyStates()</param>
/// <param name="gradients">layers at each time gradients</param>
/// <param name="biasGrads"></param>
public void GetGradients(List <double[]> costs, List <double[]> input, List <List <NeuronValues> > startingRecState, out List <List <List <NeuronValues> > > gradients, out List <double[]> biasGrads /*, out double cost*/)//TODO: document code
{
//cost = 0;
//for (int i = 0; i < costs.Count; i++)
//{
// double currentCost = 0;
// for (int l = 0; l < costs[i].Length; l++)
// {
// currentCost += costs[i][l];
// }
// currentCost /= costs[i].Length;
// cost += currentCost;
//}
//cost /= costs.Count;
///<summary>State of the neurons at each execution step</summary>
List <List <List <NeuronValues> > > executionStates = new List <List <List <NeuronValues> > >();//TODO: order to be i before t
///<summary>Layer output at each execution step</summary>
List <List <List <double> > > layerOuts = new List <List <List <double> > >();//TODO: order to be i before t
TemporalNetwork n = this;
n.SetTemporalStates(startingRecState);
for (int t = 0; t < input.Count; t++)
{
executionStates.Add(new List <List <NeuronValues> >());
layerOuts.Add(new List <List <double> >());
for (int i = 0; i < Lenght; i++)
{
executionStates[t].Add(TemporalLayers[i].GetStates());
layerOuts[t].Add(new List <double>(TemporalLayers[i].ExecuteLayer(i == 0 ? input[t] : layerOuts[t][i - 1].ToArray(), ActivationFunction, out _)));
}
}
//From time containing networks of execution to networks containing each time step
ListReorder <NeuronValues> reorderedExecutionStates = new ListReorder <NeuronValues>(executionStates);
ListReorder <double> reorderedLayerOuts = new ListReorder <double>(layerOuts);
reorderedExecutionStates.Reorder();
reorderedLayerOuts.Reorder();
var layerOutsArrayd = reorderedLayerOuts.ToLowerLevelArray();
biasGrads = new List <double[]>();
gradients = new List <List <List <NeuronValues> > >();
for (int t = input.Count - 1; t >= 0; t--)
{
var networkBiasGrads = new double[Lenght];
gradients.Add(new List <List <NeuronValues> >());
List <double[]> prevActGrads = new List <double[]>();
for (int i = Lenght - 1; i >= 0; i--)
{
TemporalLayers[i].GetGrads(i == 0? costs: prevActGrads
, i == 0? input: layerOutsArrayd[i - 1] /*reordered layerouts i==0? input: layerout[i-1]*/
, reorderedExecutionStates.list[i] /*reorder executionStates[i]*/, ActivationFunction,
out prevActGrads, out double biasGrad, out List <NeuronValues> grads
);
gradients[t][i] = grads;
networkBiasGrads[i] = biasGrad;
}
biasGrads.Add(networkBiasGrads);
}
/*for (int t = costs.Count - 1; t >= 0; t--)
* {
* gradients.Add(new List<List<NeuronValues>>());
* List<double[]> prevActivationsGrads = costs;
* for (int i = Lenght - 1; i >= 0; i--)
* {
* TemporalLayers[i].GetGrads(prevActivationsGrads, i == 0 ? input[t].ToArray() : layerOuts[t][i - 1].ToArray(), executionResults[t], ActivationFunction
* , out prevActivationsGrads, out double tempBias, out List<NeuronValues> currentGrads);
* biasGrads[i] += tempBias;
* gradients[t].Add(currentGrads);
* }
* }*/
}
/// <summary>
/// Activation functions must be the same for both parents to properly evolve
/// </summary>
/// <param name="flipsCount">Number of time the network parent will change, -1 for the half of the min lenght of the parentLayers</param>
/// <param name="parentToFixEnds">f = a, t = b</param>
public static TemporalNetwork CrossOver(TemporalNetwork a, TemporalNetwork b, int flipsCount = -1, bool fixInput_OutputParent = false, bool parentToFixEnds = true)
{
if (flipsCount < 0)
{
flipsCount = Math.Max(1, Math.Min(a.Lenght, b.Lenght) / 2);
flipsCount += 1 * Convert.ToInt32(flipsCount == 0);
}
Random random = new Random(DateTime.Now.Millisecond);
List <bool> layerParents = new List <bool>();
List <int> parentChangeIndex = new List <int>();
int maxLayers = Math.Max(a.Lenght, b.Lenght), minLayers = Math.Min(a.Lenght, b.InputLength);
for (int i = 0; i < flipsCount; i++)
{
layerParents.Add(Convert.ToBoolean(Math.Round(random.NextDouble())));
parentChangeIndex.Add(i);
}
int fillingIndex = 0;
while (layerParents.Count < minLayers)
{
layerParents.Insert(parentChangeIndex[fillingIndex], layerParents[parentChangeIndex[fillingIndex]]);
for (int i = fillingIndex; i < parentChangeIndex.Count; i++)
{
parentChangeIndex[i] += 1;
}
}
bool maxLayersParent = Convert.ToBoolean(maxLayers == a.Lenght ? 0 : 1);
while (layerParents.Count < maxLayers - minLayers)
{
layerParents.Insert(layerParents.Count - 1, maxLayersParent);
parentChangeIndex[parentChangeIndex.Count - 1] += 1;
}
if (fixInput_OutputParent)
{
layerParents[0] = layerParents[layerParents.Count - 1] = parentToFixEnds;
}
List <List <NeuronTypes> > neuronTypes = new List <List <NeuronTypes> >();
List <List <Interconnection> > interconnections = new List <List <Interconnection> >();
for (int i = 0; i < maxLayers; i++)
{
TemporalLayer currentLayer = layerParents[i] ? a[i] : b[i];
for (int neuronIndex = 0; neuronIndex < maxLayers; neuronIndex++)
{
neuronTypes.Add(currentLayer.GetNeuronTypes());
}
List <Interconnection> compatibleInterconnections = new List <Interconnection>();
for (int connectionIndex = 0; connectionIndex < currentLayer.Connections.Count; connectionIndex++)// Check for connections of the same layerParent
{
if (layerParents[connectionIndex] == layerParents[currentLayer.Connections[connectionIndex].connectedIndex])
{
compatibleInterconnections.Add(currentLayer.Connections[connectionIndex]);
}
}
interconnections.Add(currentLayer.Connections);
}
TemporalNetwork output = new TemporalNetwork
(a.ActivationFunction, neuronTypes[0].Count, neuronTypes, interconnections);
return(output);
}
public FittedNetwork(Game game, MutationInfo mutationInfo, TemporalNetwork.ActivationFunctions activationFunction, int inputLenght, int outputLenght, int layersCount, int minNeurons, int maxNeurons)
{
this.game = game;
network = new TemporalNetwork(activationFunction, inputLenght, outputLenght, layersCount, minNeurons, maxNeurons, mutationInfo);
fitnessScore = 0;
}
