/// <summary>
/// Probe the given black box, and record the responses in <paramref name="responseArr"/>
/// </summary>
/// <param name="box">The black box to probe.</param>
/// <param name="responseArr">Response array.</param>
public void Probe(IBlackBox <double> box, double[] responseArr)
{
Debug.Assert(responseArr.Length == _paramSamplingInfo.SampleResolution);
double[] xArr = _paramSamplingInfo.XArrNetwork;
for (int i = 0; i < xArr.Length; i++)
{
// Reset black box internal state.
box.ResetState();
// Set bias input, and function input value.
box.InputVector[0] = 1.0;
box.InputVector[1] = xArr[i];
// Activate the black box.
box.Activate();
// Get the black box's output value.
// TODO: Review this scheme. This replicates the behaviour in SharpNEAT 2.x but not sure if it's ideal,
// for one it depends on the output range of the neural net activation function in use.
double output = box.OutputVector[0];
Clip(ref output);
responseArr[i] = ((output - 0.5) * _scale) + _offset;
}
}
private static void TwoInputs_WeightHalf_Inner(
IBlackBox <double> net,
IActivationFunction <double> actFn)
{
double x;
// Activate and test.
net.InputVector[0] = 0.0;
net.InputVector[1] = 0.0;
net.Activate();
Assert.Equal(0.5, net.OutputVector[0]);
// Activate and test.
net.InputVector[0] = 1.0;
net.InputVector[1] = 2.0;
net.Activate();
x = 1.5; actFn.Fn(ref x);
Assert.Equal(x, net.OutputVector[0]);
// Activate and test.
net.InputVector[0] = 10.0;
net.InputVector[1] = 20.0;
net.Activate();
x = 15.0; actFn.Fn(ref x);
Assert.Equal(x, net.OutputVector[0]);
}
/// <summary>
/// Probe the given black box, and record the responses in <paramref name="responseArr"/>.
/// </summary>
/// <param name="box">The black box to probe.</param>
/// <param name="responseArr">Response array.</param>
public void Probe(IBlackBox <double> box, double[] responseArr)
{
Debug.Assert(responseArr.Length == _sampleCount);
// Reset black box internal state.
box.Reset();
// Get the blackbox input and output spans.
var inputs = box.Inputs.Span;
var outputs = box.Outputs.Span;
// Take the required number of samples.
for (int i = 0; i < _sampleCount; i++)
{
// Set bias input.
inputs[0] = 1.0;
// Activate the black box.
box.Activate();
// Get the black box's output value.
// TODO: Review this scheme. This replicates the behaviour in SharpNEAT 2.x but not sure if it's ideal,
// for one it depends on the output range of the neural net activation function in use.
double output = outputs[0];
Clip(ref output);
responseArr[i] = ((output - 0.5) * _scale) + _offset;
}
}
private double[] activate(IBlackBox box, IEnumerable <double> inputs, double[] outputs)
{
box.ResetState();
// Give inputs to the network
var nodeId = 0;
foreach (var input in inputs)
{
box.InputSignalArray[nodeId++] = input;
}
// Activate the network and get outputs back
box.Activate();
box.OutputSignalArray.CopyTo(outputs, 0);
// Normalize outputs when using NEAT
if (phenotype == Phenotype.Neat)
{
for (var i = 0; i < outputs.Count(); i++)
{
outputs[i] = (outputs[i] + 1.0) / 2.0;
}
}
return(outputs);
}
private void RemoveExistingExperiment()
{
if (_ea != null)
{
//_ea.Stop();
_ea.Dispose();
}
if (_experiment != null)
{
//_experiment.
}
_experiment = null;
_ea = null;
_experimentArgs = null;
_hyperneatArgs = null;
_harness = null;
_harnessArgs = null;
_evalArgs = null;
_winningBrain = null;
}
/// <summary>
/// Evaluate the provided black box against the function regression task,
/// and return its fitness score.
/// </summary>
/// <param name="box">The black box to evaluate.</param>
/// <returns>A new instance of <see cref="FitnessInfo"/>.</returns>
public FitnessInfo Evaluate(IBlackBox <double> box)
{
// Probe the black box over the full range of the input parameter.
_blackBoxProbe.Probe(box, _yArr);
// Calc gradients.
FuncRegressionUtils.CalcGradients(_paramSamplingInfo, _yArr, _gradientArr);
// Calc y position mean squared error (MSE), and apply weighting.
double yMse = MathSpanUtils.MeanSquaredDelta(_yArr, _yArrTarget);
yMse *= _yMseWeight;
// Calc gradient mean squared error.
double gradientMse = MathSpanUtils.MeanSquaredDelta(_gradientArr, _gradientArrTarget);
gradientMse *= _gradientMseWeight;
// Calc fitness as the inverse of MSE (higher value is fitter).
// Add a constant to avoid divide by zero, and to constrain the fitness range between bad and good solutions;
// this allows the selection strategy to select solutions that are mediocre and therefore helps preserve diversity.
double fitness = 20.0 / (yMse + gradientMse + 0.02);
return(new FitnessInfo(fitness));
}
private ForagingAgent getAgent(int i, IBlackBox phenome)
{
switch (AgentType)
{
case AgentTypes.Neural:
return(new NeuralAgent(i, _genomeList[i].SpecieIdx, phenome, _agentsNavigate, _agentsHide));
case AgentTypes.Social:
var a = new SocialAgent(i, _genomeList[i].SpecieIdx, phenome, _agentsNavigate, _agentsHide,
sar => sar.Last.Value.Reward > 0)
{
MemorySize = CurrentMemorySize
};
var network = (FastCyclicNetwork)phenome;
network.Momentum = ((SocialAgent)a).Momentum;
network.BackpropLearningRate = ((SocialAgent)a).LearningRate;
return(a);
case AgentTypes.QLearning:
return(new QLearningAgent(i, _genomeList[i].SpecieIdx, phenome, _agentsNavigate, _agentsHide,
8, 4, _world));
case AgentTypes.Spinning:
return(new SpinningAgent(i));
default:
return(null);
}
}
/// <summary>
/// Evaluate the provided black box against the cart and double pole balancing task,
/// and return its fitness score.
/// </summary>
/// <param name="box">The black box to evaluate.</param>
/// <returns>A new instance of <see cref="FitnessInfo"/>.</returns>
public FitnessInfo Evaluate(IBlackBox <double> box)
{
// The evaluation consists of four separate trials, each with their own fitness score.
// The final overall fitness is given by the root mean squared (RMS) fitness. Using an RMS
// score ensures that improvements in the worst scoring trial are prioritised (by evolution) over
// a similar level of improvement in a better scoring trial. RMS also has the nice quality of giving
// a maximum overall fitness that is equal to the max fitness for a single trial.
// Keep a running sum of the squared fitness scores.
float fitnessSqrSum = 0f;
// Trial 1.
float fitness = RunTrial(box, 0.0f, DegreesToRadians(1f), 0f);
fitnessSqrSum += fitness * fitness;
// Trial 2.
fitness = RunTrial(box, 0.0f, DegreesToRadians(4f), 0f);
fitnessSqrSum += fitness * fitness;
// Trial 3.
fitness = RunTrial(box, 0.0f, DegreesToRadians(-2f), 0f);
fitnessSqrSum += fitness * fitness;
// Trial 4.
fitness = RunTrial(box, 0.0f, DegreesToRadians(-3f), 0f);
fitnessSqrSum += fitness * fitness;
// Calculate the final overall fitness score.
// Take the mean of the sum of squared fitnesses, and then take the square root.
fitness = MathF.Sqrt(fitnessSqrSum / 4f);
return(new FitnessInfo(fitness));
}
public double[] GetChampOutput(int nodeDepth, int nodeSiblingNum, int maxDepth, int maxBranching)
{
IGenomeDecoder <NeatGenome, IBlackBox> decoder = experiment.CreateGenomeDecoder();
IBlackBox box = decoder.Decode(contentEA.CurrentChampGenome);
// Normalize nodeDepth and nodeSiblingNum to range of 0-1. This may affect outputs?
double normDepth = (double)nodeDepth / (double)maxDepth;
double normSib;
if (nodeDepth == 0)
{
normSib = 0; // Only one node at depth 0, prevent a divide by 0 error
}
else
{
normSib = (double)nodeSiblingNum / (double)(Mathf.Pow(maxBranching, nodeDepth) - 1);
}
box.InputSignalArray[0] = normDepth;
box.InputSignalArray[1] = normSib;
box.ResetState();
box.Activate();
//Debug.Log("(" + normDepth + "," + normSib + ") -> (" + box.OutputSignalArray[0] + "," + box.OutputSignalArray[1] + "," + box.OutputSignalArray[2] + ","
//+ box.OutputSignalArray[3] + "," + box.OutputSignalArray[4] + "," + box.OutputSignalArray[5] + ")");
double[] outputs = new double[box.OutputCount];
for (int i = 0; i < box.OutputCount; i++)
{
outputs[i] = box.OutputSignalArray[i];
}
return(outputs);
}
public BrainToMalmoController(IBlackBox givenBrain, ProgramMalmo givenMalmo)
{
programMalmo = givenMalmo;
SetDotAsDecimalSeparator();
brain = givenBrain;
programMalmo.ObservationsEvent += new EventHandler <ObservationEventArgs>(WhenObservationsEvent);
}
public void Evaluate(IBlackBox box)
{
// Random starting point in radius 20
Vector3 dir = new Vector3(UnityEngine.Random.Range(-1f, 1f), 0, UnityEngine.Random.Range(-1f, 1f));
Vector3 pos = dir.normalized * 20;
pos.y = 1;
GameObject obj = Instantiate(Robot, pos, Quaternion.identity) as GameObject;
TrainingController robo = obj.GetComponent <TrainingController>();
Target.transform.position = new Vector3(0, 1, 0);
Target.transform.localScale = new Vector3(Settings.Brain.TargetSize, 2, Settings.Brain.TargetSize);
dict.Add(box, robo);
if (Settings.Brain.MultipleTargets)
{
GameObject t = Instantiate(Target, new Vector3(0, 1, 0), Quaternion.identity) as GameObject;
t.transform.localScale = new Vector3(1, 1, 1);
TargetController tc = t.AddComponent <TargetController>();
tc.Activate(obj.transform);
targetDict.Add(robo, tc);
robo.Activate(box, t);
}
else
{
robo.Activate(box, Target);
}
evaluationStartTime = 0;
}
public void SetBrains(IBlackBox _brain1, IBlackBox _brain2, IBlackBox _brain3, IBlackBox _brain4)
{
this.brain1 = _brain1;
this.brain2 = _brain2;
this.brain3 = _brain3;
this.brain4 = _brain4;
}
private void RedrawGraph(NeatGenome genome)
{
IBlackBox box = _experiment.GetBlackBox(genome);
Color trueColor = panPlot.TrueColor;
Color falseColor = panPlot.FalseColor;
var samples = Enumerable.Range(0, 1000).
Select(o =>
{
Vector3D pos = Math3D.GetRandomVector(new Vector3D(0, 0, 0), new Vector3D(1, 1, 1));
box.ResetState();
//box.InputSignalArray.CopyFrom()
box.InputSignalArray[0] = pos.X;
box.InputSignalArray[1] = pos.Y;
box.InputSignalArray[2] = pos.Z;
box.Activate();
double percent = box.OutputSignalArray[0];
Color color = UtilityWPF.AlphaBlend(trueColor, falseColor, percent);
return(Tuple.Create(pos.ToPoint(), color));
});
panPlot.ClearFrame();
panPlot.AddDots(samples, .01, false);
}
/// <summary>
/// Refresh/update the view with the provided genome.
/// </summary>
public override void RefreshView(object genome)
{
NeatGenome neatGenome = genome as NeatGenome;
if (null == neatGenome)
{
return;
}
// Decode genome.
IBlackBox box = _genomeDecoder.Decode(neatGenome);
// Probe the black box.
_blackBoxProbe.Probe(box, _yArrTarget);
// Update plot points.
double[] xArr = _paramSamplingInfo._xArr;
for (int i = 0; i < xArr.Length; i++)
{
if (!_generativeMode)
{
box.ResetState();
box.InputSignalArray[0] = xArr[i];
}
box.Activate();
_plotPointListResponse[i].Y = _yArrTarget[i];
}
// Trigger graph to redraw.
zed.AxisChange();
Refresh();
}
private EvaluationInfo Test(IBlackBox phenome, int iterations)
{
Controller.NoveltySearch = NoveltySearchInfo;
Environment.NoveltySearch = NoveltySearchInfo;
Controller.Phenome = phenome;
// Deactivate novelty search for testing
bool novelty = NoveltySearchInfo.ScoreNovelty;
NoveltySearchInfo.ScoreNovelty = false;
EvaluationInfo evaluation = new EvaluationInfo();
evaluation.Iterations = iterations;
evaluation.ObjectiveFitnessIt = new double[iterations];
EvaluateRecord(Controller, iterations, ref evaluation);
CalculateTestStats(ref evaluation);
TearDownTest();
NoveltySearchInfo.ScoreNovelty = novelty;
Controller.Phenome = null;
return(evaluation);
}
private static void SingleInput_WeightOne_Inner(
IBlackBox <double> net,
IActivationFunction <double> actFn)
{
// Activate and test.
net.InputVector[0] = 0.0;
for (int i = 0; i < 10; i++)
{
net.Activate();
Assert.Equal(0.5, net.OutputVector[0]);
}
// Activate and test.
net.InputVector[0] = 1.0;
for (int i = 0; i < 10; i++)
{
net.Activate();
Assert.Equal(actFn.Fn(1), net.OutputVector[0]);
}
// Activate and test.
net.InputVector[0] = 10.0;
for (int i = 0; i < 10; i++)
{
net.Activate();
Assert.Equal(actFn.Fn(10), net.OutputVector[0]);
}
}
/// <summary>
/// IBlackBox can't be directly deserialized, so it takes a village of properties to construct it
/// </summary>
private static (IBlackBox phenome, NeatGenome genome)? DeserializeBrain(BrainNEATDNA dna)
{
if (dna == null)
{
return(null);
}
else if (string.IsNullOrEmpty(dna.Genome) || dna.Activation == null)
{
return(null);
}
List <NeatGenome> genomeList = null;
try
{
// Deserialize the genome (sharpneat library has a custom serialize/deserialize)
if (dna.Hyper == null)
{
genomeList = ExperimentNEATBase.LoadPopulation(dna.Genome, dna.Activation, dna.NEATPositions_Input.Length, dna.NEATPositions_Output.Length);
}
else
{
genomeList = ExperimentNEATBase.LoadPopulation(dna.Genome, dna.Activation, dna.Hyper);
}
if (genomeList == null || genomeList.Count == 0)
{
return(null);
}
// Construct the phenome (the actual instance of a neural net)
IBlackBox phenome = ExperimentNEATBase.GetBlackBox(genomeList[0], dna.Activation, dna.Hyper);
return(phenome, genomeList[0]);
public NeuralAI(int inps, int outps, Random rand, IBlackBox net)
{
random = rand;
inputsNum = inps;
outputsNum = outps;
network = net;
}
/// <summary>
/// Refresh/update the view with the provided genome.
/// </summary>
public override void RefreshView(object genome)
{
NeatGenome neatGenome = genome as NeatGenome;
if (null == neatGenome)
{
return;
}
// Decode genome.
IBlackBox box = _genomeDecoder.Decode(neatGenome);
// Update plot points.
double x = _xMin;
for (int i = 0; i < _sampleCount; i++, x += _xIncr)
{
box.ResetState();
box.InputSignalArray[0] = x;
box.Activate();
_plotPointListResponse[i].Y = box.OutputSignalArray[0];
}
// Trigger graph to redraw.
zed.AxisChange();
Refresh();
}
public FitnessInfo Test(IBlackBox box)
{
double fitness, altFitness;
//these are our inputs and outputs
ISignalArray inputArr = box.InputSignalArray;
ISignalArray outputArr = box.OutputSignalArray;
List <Sample> sampleList = LEEAParams.DOMAIN.getTestSamples();
int sampleCount = sampleList.Count;
fitness = sampleList.Count;
foreach (Sample sample in sampleList)
{
inputArr.CopyFrom(sample.Inputs, 0);
box.ResetState();
box.Activate();
fitness -= Math.Abs(outputArr[0] - sample.Outputs[0]) * Math.Abs(outputArr[0] - sample.Outputs[0]);
}
fitness = Math.Max(fitness, LEEAParams.MINFITNESS) + LEEAParams.FITNESSBASE;
return(new FitnessInfo(fitness, 0));
}
/// <summary>
/// Invoke any required control logic in the Box2D world.
/// </summary>
protected override void InvokeController()
{
// Provide state info to the black box inputs.
InvertedDoublePendulumWorld simWorld = (InvertedDoublePendulumWorld)_simWorld;
// _box is updated by other threads so copy the reference so that we know we are workign with the same IBlackBox within this method.
IBlackBox box = _box;
box.InputSignalArray[0] = simWorld.CartPosX / __TrackLengthHalf; // CartPosX range is +-trackLengthHalf. Here we normalize it to [-1,1].
box.InputSignalArray[1] = simWorld.CartVelocityX; // Cart track velocity x is typically +-0.75.
box.InputSignalArray[2] = simWorld.CartJointAngle / __TwelveDegrees; // Rescale angle to match range of values during balancing.
box.InputSignalArray[3] = simWorld.CartJointAngularVelocity; // Pole angular velocity is typically +-1.0 radians. No scaling required.
box.InputSignalArray[4] = simWorld.ElbowJointAngle / __TwelveDegrees;
box.InputSignalArray[5] = simWorld.ElbowJointAngularVelocity;
// Activate the network.
box.Activate();
// Read the network's force signal output.
// FIXME: Force mag should be configurable somewhere.
float force = (float)(box.OutputSignalArray[0] - 0.5f) * __MaxForceNewtonsX2;
simWorld.SetCartForce(force);
}
private static void TwoInputs_WeightHalf_Inner(
IBlackBox <double> net,
IActivationFunction <double> actFn)
{
double x;
var inputs = net.Inputs.Span;
var outputs = net.Outputs.Span;
// Activate and test.
inputs[0] = 0.0;
inputs[1] = 0.0;
net.Activate();
Assert.Equal(0.5, outputs[0]);
// Activate and test.
inputs[0] = 1.0;
inputs[1] = 2.0;
net.Activate();
x = 1.5; actFn.Fn(ref x);
Assert.Equal(x, outputs[0]);
// Activate and test.
inputs[0] = 10.0;
inputs[1] = 20.0;
net.Activate();
x = 15.0; actFn.Fn(ref x);
Assert.Equal(x, outputs[0]);
}
/// <summary>
/// Run a single cart-pole simulation/trial on the given black box, and with the given initial model state.
/// </summary>
/// <param name="box">The black box (neural net) to evaluate.</param>
/// <param name="cartPos">Cart position on the track.</param>
/// <param name="poleAngle1">Pole 1 angle in radians.</param>
/// <param name="poleAngle2">Pole 2 angle in radians.</param>
/// <returns>Fitness score.</returns>
public float RunTrial(
IBlackBox <double> box,
float cartPos,
float poleAngle1,
float poleAngle2)
{
// Reset black box state.
box.ResetState();
// Reset model state.
_physics.ResetState(cartPos, poleAngle1, poleAngle2);
// Get a local variable ref to the internal model state array.
float[] state = _physics.State;
// Run the cart-pole simulation.
int timestep = 0;
for (; timestep < _maxTimesteps; timestep++)
{
// Provide model state to the black box inputs (normalised to +-1.0).
box.InputVector[0] = 1.0; // Bias input.
box.InputVector[1] = state[0] * __TrackLengthHalf_Reciprocal; // Cart X position range is +-__TrackLengthHalf; here we normalize to [-1,1].
box.InputVector[2] = state[1]; // Cart velocity. Typical range is approx. +-10.
box.InputVector[3] = state[2] * __MaxPoleAngle_Reciprocal; // Pole 1 angle. Range is +-__MaxPoleAngle radians; here we normalize to [-1,1].
box.InputVector[4] = state[3] * 0.2; // Pole 1 angular velocity. Typical range is approx. +-5.
box.InputVector[5] = state[4] * __MaxPoleAngle_Reciprocal; // Pole 2 angle.
box.InputVector[6] = state[5] * 0.2; // Pole 2 angular velocity.
// Activate the network.
box.Activate();
// Read the output to determine the force to be applied to the cart by the controller.
float force = (float)(box.OutputVector[0] - 0.5) * 2f;
ClipForce(ref force);
force *= __MaxForce;
// Update model state, i.e. move the model forward by one timestep.
_physics.Update(force);
// Check for failure state. I.e. has the cart run off the ends of the track, or has either of the pole
// angles exceeded the defined threshold.
if (MathF.Abs(state[0]) > __TrackLengthHalf || MathF.Abs(state[2]) > __MaxPoleAngle || MathF.Abs(state[4]) > __MaxPoleAngle)
{
break;
}
}
// Fitness is given by the combination of two fitness components:
// 1) Amount of simulation time that elapsed before the pole angle and/or cart position threshold was exceeded. Max score is 99 if the
// end of the trial is reached without exceeding any thresholds.
// 2) Cart position component. Max fitness of 1.0 for a cart position of zero (i.e. the cart is in the middle of the track range);
//
// Therefore the maximum possible fitness is 100.0.
float fitness =
(timestep * _maxTimesteps_Reciprocal * 99f)
+ (1f - (MathF.Min(MathF.Abs(state[0]), __TrackLengthHalf) * __TrackLengthHalf_Reciprocal));
return(fitness);
}
private double[] activate(IBlackBox box, IList <double> inputs, double[] outputs)
{
box.ResetState();
// Give inputs to the network
var nodeId = 0;
foreach (var input in inputs)
{
box.InputSignalArray[nodeId++] = input;
}
// Activate the network and get outputs back
box.Activate();
box.OutputSignalArray.CopyTo(outputs, 0);
// Normalize outputs when using NEAT
if (phenotype == Phenotype.Neat)
{
NormalizeOutputs(outputs);
}
else
{
Debug.Assert(outputs.All(x => x >= 0.0 && x <= 1.0));
}
return(outputs);
}
/// <summary>
/// Evaluation unit constructor.
/// </summary>
/// <param name="mazeStructure">The maze structure on which the agent is evaluated.</param>
/// <param name="agentPhenome">The agent ANN phenome.</param>
/// <param name="mazeId">The unique maze identifier.</param>
/// <param name="agentId">The unique agent identifier.</param>
public MazeNavigatorEvaluationUnit(MazeStructure mazeStructure, IBlackBox agentPhenome, int mazeId, int agentId)
{
MazePhenome = mazeStructure;
AgentPhenome = agentPhenome;
MazeId = mazeId;
AgentId = agentId;
}
/// <summary>
/// Determine the agent's position in the world relative to the prey and walls, and set its sensor inputs accordingly.
/// </summary>
/// <param name="agent"></param>
public void SetAgentInputsAndActivate(IBlackBox agent)
{
// Calc prey's position relative to the agent (in polar coordinate system).
PolarPoint relPos = CartesianToPolar(_preyPos - _agentPos);
// Determine agent sensor input values.
// Reset all inputs.
agent.InputSignalArray.Reset();
// Test if prey is in sensor range.
if (relPos.RadialSquared <= _sensorRangeSquared)
{
// Determine which sensor segment the prey is within - [0,7]. There are eight segments and they are tilted 22.5 degrees (half a segment)
// such that due North, East South and West are each in the centre of a sensor segment (rather than on a segment boundary).
double thetaAdjusted = relPos.Theta - PiDiv8;
if (thetaAdjusted < 0.0)
{
thetaAdjusted += 2.0 * Math.PI;
}
int segmentIdx = (int)Math.Floor(thetaAdjusted / PiDiv4);
// Set sensor segment's input.
agent.InputSignalArray[segmentIdx] = 1.0;
}
// Prey closeness detector.
agent.InputSignalArray[8] = relPos.RadialSquared <= 4.0 ? 1.0 : 0.0;
// Wall detectors - N,E,S,W.
// North.
int d = (_gridSize - 1) - _agentPos._y;
if (d <= 4)
{
agent.InputSignalArray[9] = (4 - d) / 4.0;
}
// East.
d = (_gridSize - 1) - _agentPos._x;
if (d <= 4)
{
agent.InputSignalArray[10] = (4 - d) / 4.0;
}
// South.
if (_agentPos._y <= 4)
{
agent.InputSignalArray[11] = (4 - _agentPos._y) / 4.0;
}
// West.
if (_agentPos._x <= 4)
{
agent.InputSignalArray[12] = (4 - _agentPos._x) / 4.0;
}
// Activate agent.
agent.Activate();
}
/// <summary>
/// Evaluate the provided IBlackBox.
/// </summary>
public FitnessInfo Evaluate(IBlackBox box)
{
int nbSamples = dataset.InputSamples.Count();
var outputs = new double[nbSamples][];
var selectedClusters = new int[nbSamples];
var centers = new double[nbClusters][];
for (var i = 0; i < nbClusters; i++)
{
centers[i] = new double[dataset.InputCount];
}
// Evaluate each samples of the dataset
for (var i = 0; i < nbSamples; i++)
{
var inputs = dataset.InputSamples[i];
outputs[i] = new double[nbClusters];
activate(box, inputs, outputs[i]);
selectedClusters[i] = outputs[i].MaxIndex();
for (var j = 0; j < dataset.InputCount; j++)
{
centers[selectedClusters[i]][j] += inputs[j];
}
}
// Compute center of each cluster
for (var i = 0; i < nbClusters; i++)
{
for (var j = 0; j < dataset.InputCount; j++)
{
centers[i][j] /= nbSamples;
}
}
// Compute inter and intra cluster distances
var distancesIntra = new double[nbClusters];
var distancesInter = 0.0;
for (var i = 0; i < nbSamples; i++)
{
var cluster = selectedClusters[i];
distancesIntra[cluster] += distance(dataset.InputSamples[i], centers[cluster]);
}
for (var i = 0; i < nbClusters - 1; i++)
{
for (var j = i; j < nbClusters; j++)
{
distancesInter += distance(centers[i], centers[j]);
}
}
double intra = distancesIntra.Mean();
double inter = distancesInter / nbClusters;
_evalCount++;
return(new FitnessInfo(inter / intra, intra));
}
public float GetFitness(IBlackBox box)
{
if (ControllerMap.ContainsKey(box))
{
return(ControllerMap[box].GetFitness());
}
return(0);
}
public NeuralAgent(int id, int speciesId, IBlackBox brain,
bool navigationEnabled, bool hidingEnabled) : base(id)
{
Brain = brain;
SpeciesId = speciesId;
NavigationEnabled = navigationEnabled;
HidingEnabled = hidingEnabled;
}
public float GetFitness(IBlackBox box)
{
if (ControllerMap.ContainsKey(box))
{
return ControllerMap[box].GetFitness();
}
return 0;
}
public override void SetEvolvedBrain(IBlackBox blackBox, NeatGenome genome)
{
this.genome = genome;
if (evolvedPlayer != null)
{
evolvedPlayer.SetBrain(blackBox);
}
}
public void Update(ICardAdapter pocket, ICardAdapter communityCards, IGetTurnContext context, IBlackBox box)
{
this.pocket = pocket;
this.normalization.Update(pocket, communityCards, context);
this.GetTurnContext = context;
this.BlackBox = box;
this.isUpdated = true;
}
public RecurrentNeuralAcceptability(IBlackBox brain, double acceptThreshold = 0.8, double rewardNormalizer = 100)
{
Brain = brain;
AcceptThreshold = acceptThreshold;
RewardNormalizer = rewardNormalizer;
Debug.Assert(brain.OutputCount == 1);
}
/// <summary>
/// Initializes a new instance of the <see cref="BlackBoxController"/> class.
/// </summary>
/// <param name="blackBox">The black box.</param>
/// <exception cref="System.ArgumentNullException">blackBox</exception>
public BlackBoxController(IBlackBox blackBox)
{
if (blackBox == null)
{
throw new ArgumentNullException("blackBox");
}
_blackBox = blackBox;
}
public void Evaluate(IBlackBox box)
{
GameObject obj = Instantiate(Unit, Unit.transform.position, Unit.transform.rotation) as GameObject;
UnitController controller = obj.GetComponent<UnitController>();
ControllerMap.Add(box, controller);
controller.Activate(box);
}
/// <summary>
/// Sets the black box.
/// </summary>
/// <param name="blackBox">The black box.</param>
/// <exception cref="System.ArgumentNullException">blackBox</exception>
public void SetBlackBox(IBlackBox blackBox)
{
if (blackBox == null)
{
throw new ArgumentNullException("blackBox");
}
_blackBox = blackBox;
}
/// <summary>
/// Evaluate the provided IBlackBox.
/// </summary>
public FitnessInfo Evaluate(IBlackBox box)
{
int nbSamples = dataset.InputSamples.Count();
var outputs = new double[nbSamples][];
var selectedClusters = new int[nbSamples];
var centers = new double[nbClusters][];
for (var i = 0; i < nbClusters; i++)
{
centers[i] = new double[dataset.InputCount];
}
// Evaluate each samples of the dataset
for (var i = 0; i < nbSamples; i++)
{
var inputs = dataset.InputSamples[i];
outputs[i] = new double[nbClusters];
activate(box, inputs, outputs[i]);
selectedClusters[i] = outputs[i].MaxIndex();
for (var j = 0; j < dataset.InputCount; j++)
{
centers[selectedClusters[i]][j] += inputs[j];
}
}
// Compute center of each cluster
for (var i = 0; i < nbClusters; i++)
{
for (var j = 0; j < dataset.InputCount; j++)
{
centers[i][j] /= nbSamples;
}
}
// Compute inter and intra cluster distances
var distancesIntra = new double[nbClusters];
var distancesInter = 0.0;
for (var i = 0; i < nbSamples; i++)
{
var cluster = selectedClusters[i];
distancesIntra[cluster] += distance(dataset.InputSamples[i], centers[cluster]);
}
for (var i = 0; i < nbClusters - 1; i++)
{
for (var j = i; j < nbClusters; j++)
{
distancesInter += distance(centers[i], centers[j]);
}
}
double intra = distancesIntra.Mean();
double inter = distancesInter / nbClusters;
_evalCount++;
return new FitnessInfo(inter / intra, intra);
}
public MctsNeatAgent(int id,
CheckGameOver check,
GetValidNextMoves valid,
IBlackBox brain,
ApplyMove applyMove,
GridGameParameters parameters) : base(id, check, valid, applyMove, parameters)
{
Brain = brain;
}
/// <summary>
/// Constructs a social learning teacher controlled by the brain and using the give acceptability function.
/// </summary>
public SocialAgent(int id, int speciesId, IBlackBox brain, bool agentsNavigate, bool agentsHide,
IAcceptabilityFunction accept) : base(id, speciesId, brain, agentsNavigate, agentsHide)
{
MemorySize = DEFAULT_MEMORY_SIZE;
Memory = new LinkedList<StateActionReward>();
LearningRate = DEFAULT_LEARNING_RATE;
Momentum = DEFAULT_MOMENTUM_RATE;
AcceptabilityFn = accept;
}
public BlondieAgent(int id,
MinimaxAgent.CheckGameOver check,
MinimaxAgent.GetValidNextMoves valid,
IBlackBox brain,
MinimaxAgent.ApplyMove apply,
GridGameParameters parameters)
: base(id)
{
Brain = brain;
_params = parameters;
_minimax = new MinimaxAgent(id, check, valid, Evaluate, apply, parameters);
}
/// <summary>
/// Evaluate the provided IBlackBox.
/// </summary>
public override FitnessInfo Evaluate(IBlackBox box)
{
_evalCount++;
// [0] - Cart Position (meters).
// [1] - Cart velocity (m/s).
// [2] - Pole 1 angle (radians)
// [3] - Pole 1 angular velocity (radians/sec).
// [4] - Pole 2 angle (radians)
// [5] - Pole 2 angular velocity (radians/sec).
double[] state = new double[6];
state[2] = FourDegrees;
// Run the pole-balancing simulation.
int timestep = 0;
for(; timestep < _maxTimesteps; timestep++)
{
// Provide state info to the network (normalised to +-1.0).
// Markovian (With velocity info)
box.InputSignalArray[0] = state[0] / _trackLengthHalf; // Cart Position is +-trackLengthHalfed
box.InputSignalArray[1] = state[2] / ThirtySixDegrees; // Pole Angle is +-thirtysix_degrees. Values outside of this range stop the simulation.
box.InputSignalArray[2] = state[4] / ThirtySixDegrees; // Pole Angle is +-thirtysix_degrees. Values outside of this range stop the simulation.
// Activate the black box.
box.Activate();
// Get black box response and calc next timestep state.
performAction(state, box.OutputSignalArray[0]);
// Check for failure state. Has the cart run off the ends of the track or has the pole
// angle gone beyond the threshold.
if( (state[0]< -_trackLengthHalf) || (state[0]> _trackLengthHalf)
|| (state[2] > _poleAngleThreshold) || (state[2] < -_poleAngleThreshold)
|| (state[4] > _poleAngleThreshold) || (state[4] < -_poleAngleThreshold))
{
break;
}
}
if(timestep == _maxTimesteps) {
_stopConditionSatisfied = true;
}
// The controller's fitness is defined as the number of timesteps that elapse before failure.
double fitness = timestep;
return new FitnessInfo(fitness, fitness);
}
public override void RefreshView(object genome)
{
currentBox = decoder.Decode(genome as NeatGenome);
plotChart.Series.Clear();
for (var cluster = 0; cluster < nbClusters; cluster++)
{
var serie = plotChart.Series.Add("Cluster #" + cluster);
serie.ChartType = System.Windows.Forms.DataVisualization.Charting.SeriesChartType.Point;
}
var outputs = new double[nbClusters];
double xmin = double.PositiveInfinity;
double xmax = 0;
double ymin = double.PositiveInfinity;
double ymax = 0;
for (var i = 0; i < dataset.InputSamples.Count(); i++)
{
for (var j = 0; j < dataset.InputCount; j++)
{
currentBox.InputSignalArray[j] = dataset.InputSamples[i][j];
}
currentBox.Activate();
currentBox.OutputSignalArray.CopyTo(outputs, 0);
var x = dataset.InputSamples[i][0];
var y = dataset.InputSamples[i][1];
var cluster = outputs.MaxIndex();
if (x < xmin) xmin = x;
if (x > xmax) xmax = x;
if (y < ymin) ymin = y;
if (y > ymax) ymax = y;
plotChart.Series[cluster].Points.AddXY(x, y);
}
plotChart.ChartAreas[0].AxisX.Minimum = xmin;
plotChart.ChartAreas[0].AxisX.Maximum = xmax;
plotChart.ChartAreas[0].AxisY.Minimum = ymin;
plotChart.ChartAreas[0].AxisY.Maximum = xmax;
}
/// <summary>
/// Creates a new Q-Learning teacher.
/// </summary>
/// <param name="id">The unique ID of this teacher.</param>
/// <param name="brain">The neural network value function for this teacher. It should have (2 + # of sensors) input nodes and 1 output node.</param>
/// <param name="numOrientationActions">The number of buckets to discretize the orientation action spacer into.</param>
/// <param name="numVelocityActions">The number of buckets to discretize the velocity action spacer into.</param>
/// <param name="world">The world this teacher will be evaluated in.</param>
public QLearningAgent(int id, int speciesId, IBlackBox brain, bool agentsNavigate, bool agentsHide,
int numOrientationActions, int numVelocityActions, World world)
: base(id, speciesId, brain, agentsNavigate, agentsHide)
{
Debug.Assert(brain.OutputCount == 1, "Incorrect number of outputs in neural network!");
_numVelocityActions = numVelocityActions;
_numOrientationActions = numOrientationActions;
_random = new Random();
_prevState = new double[brain.InputCount];
_observedValue = new double[1];
world.PlantEaten += new World.PlantEatenHandler(world_PlantEaten);
MaxReward = 200;
LearningRate = DEFAULT_LEARNING_RATE;
DiscountFactor = DEFAULT_DISCOUNT_FACTOR;
Epsilon = DEFAULT_EPSILON;
// The backprop learning rate is equivalent to the Q-Learning learning rate.
((FastCyclicNetwork)Brain).BackpropLearningRate = LearningRate;
}
private double[] activate(IBlackBox box, IList<double> inputs, double[] outputs)
{
box.ResetState();
// Give inputs to the network
var nodeId = 0;
foreach (var input in inputs)
{
box.InputSignalArray[nodeId++] = input;
}
// Activate the network and get outputs back
box.Activate();
box.OutputSignalArray.CopyTo(outputs, 0);
// Normalize outputs when using NEAT
if (phenotype == Phenotype.Neat)
{
NormalizeOutputs(outputs);
}
else
{
Debug.Assert(outputs.All(x => x >= 0.0 && x <= 1.0));
}
return outputs;
}
/// <summary>
/// Evaluate the provided IBlackBox.
/// </summary>
public FitnessInfo Evaluate(IBlackBox box)
{
int nbSamples = dataset.InputSamples.Count();
var results = new ResultType[nbSamples][];
var squaredErrors = new double[nbSamples][];
// Evaluate each samples of the dataset
for (var i = 0; i < nbSamples; i++)
{
var inputs = dataset.InputSamples[i];
var expected = dataset.OutputSamples[i];
var outputs = new double[dataset.OutputCount];
activate(box, dataset.InputSamples[i], outputs);
results[i] = outputs.Zip(expected, (o, e) => getResultType(o, e)).ToArray();
squaredErrors[i] = outputs.Zip(expected, (o, e) => Math.Pow(e - o, 2.0)).ToArray();
}
// Compute per-column sums
var TPs = new int[dataset.OutputCount];
var TNs = new int[dataset.OutputCount];
var FPs = new int[dataset.OutputCount];
var FNs = new int[dataset.OutputCount];
var sumSquaredErrors = new double[dataset.OutputCount];
for (var i = 0; i < dataset.OutputCount; i++)
{
for (var j = 0; j < nbSamples; j++)
{
TPs[i] += (results[j][i] == ResultType.TP) ? 1 : 0;
TNs[i] += (results[j][i] == ResultType.TN) ? 1 : 0;
FPs[i] += (results[j][i] == ResultType.FP) ? 1 : 0;
FNs[i] += (results[j][i] == ResultType.FN) ? 1 : 0;
sumSquaredErrors[i] += squaredErrors[j][i];
}
}
// Compute fitness measures
var TP = TPs.Mean();
var TN = TNs.Mean();
var FP = FPs.Mean();
var FN = FNs.Mean();
var RMSE = sumSquaredErrors.Select(x => Math.Pow(2.0, -Math.Sqrt(x))).Mean();
// Compute final fitness value
var fitness = new double[4];
var weights = _weights.ToList();
Debug.Assert(weights.Count == 4, "weights must correspond to { accuracy, sensitivity, specificity, rmse }");
// accuracy
fitness[0] = (TP + TN) / (TP + TN + FP + FN);
// sensitivity
fitness[1] = (TP > 0) ? TP / (TP + FN) : 0;
// specificity
fitness[2] = (TN > 0) ? TN / (TN + FP) : 0;
// rmse
fitness[3] = RMSE;
var score = fitness.Zip(weights, (f, w) => f * w).Sum() / weights.Sum();
_evalCount++;
return new FitnessInfo(score, fitness[0]);
}
private void SetResolution(int visualFieldResolution)
{
_visualFieldResolution = visualFieldResolution;
_visualPixelSize = BoxesVisualDiscriminationEvaluator.VisualFieldEdgeLength / _visualFieldResolution;
_visualOriginPixelXY = -1 + (_visualPixelSize/2.0);
_genomeDecoder = _experiment.CreateGenomeDecoder(visualFieldResolution, _experiment.LengthCppnInput);
_box = null;
// If we have a cached genome then re-decode it using the new decoder (with the updated resolution).
if(null != _neatGenome) {
_box = _genomeDecoder.Decode(_neatGenome);
}
}
private void PaintView(IBlackBox box)
{
if(_initializing) {
return;
}
Graphics g = Graphics.FromImage(_image);
g.FillRectangle(_brushBackground, 0, 0, _image.Width, _image.Height);
g.SmoothingMode = SmoothingMode.AntiAlias;
// Get control width and height.
int width = Width;
int height = Height;
// Determine smallest dimension. Use that as the edge length of the square grid.
width = height = Math.Min(width, height);
// Pixel size is calculated using integer division to produce cleaner lines when drawing.
// The inherent rounding down may produce a grid 1 pixel smaller then the determined edge length.
// Also make room for a combo box above the grid. This will be used allow the user to change the grid resolution.
int visualFieldPixelSize = (height-GridTop) / _visualFieldResolution;
width = height = visualFieldPixelSize * _visualFieldResolution;
// Paint pixel outline grid.
// Vertical lines.
int x = GridLeft;
for(int i=0; i<=_visualFieldResolution; i++, x+=visualFieldPixelSize) {
g.DrawLine(__penGrey, x, GridTop, x, GridTop+height);
}
// Horizontal lines.
int y = GridTop;
for(int i=0; i<=_visualFieldResolution; i++, y+=visualFieldPixelSize) {
g.DrawLine(__penGrey, GridLeft, y, GridLeft+width, y);
}
// Apply test case to black box's visual field sensor inputs.
BoxesVisualDiscriminationEvaluator.ApplyVisualFieldToBlackBox(_testCaseField, box, _visualFieldResolution, _visualOriginPixelXY, _visualPixelSize);
// Activate the black box.
box.ResetState();
box.Activate();
// Read output responses and paint them to the pixel grid.
// Also, we determine the range of values so that we can normalize the range of pixel shades used.
double low, high;
low = high = box.OutputSignalArray[0];
for(int i=1; i<box.OutputSignalArray.Length; i++)
{
double val = box.OutputSignalArray[i];
if(val > high) {
high = val;
} else if(val <low) {
low = val;
}
}
double colorScaleFactor;
if(0.0 == (high-low)) {
colorScaleFactor = 1.0;
} else {
colorScaleFactor = 1.0 / (high-low);
}
IntPoint maxActivationPoint = new IntPoint(0,0);
double maxActivation = -1.0;
int outputIdx = 0;
y = GridTop;
for(int i=0; i<_visualFieldResolution; i++, y+=visualFieldPixelSize)
{
x = GridLeft;
for(int j=0; j<_visualFieldResolution; j++, x+=visualFieldPixelSize, outputIdx++)
{
double output = box.OutputSignalArray[outputIdx];
if(output > maxActivation)
{
maxActivation = output;
maxActivationPoint._x = j;
maxActivationPoint._y = i;
}
Color color = GetResponseColor((output-low)*colorScaleFactor);
Brush brush = new SolidBrush(color);
g.FillRectangle(brush, x+1, y+1, visualFieldPixelSize-2, visualFieldPixelSize-2);
}
}
// Paint lines around the small and large test case boxes to highlight them.
// Small box.
int testFieldPixelSize = (_visualFieldResolution / TestCaseField.TestFieldResolution) * visualFieldPixelSize;
g.DrawRectangle(__penBoxOutline,
GridLeft + (_testCaseField.SmallBoxTopLeft._x * testFieldPixelSize),
GridTop + (_testCaseField.SmallBoxTopLeft._y * testFieldPixelSize),
testFieldPixelSize, testFieldPixelSize);
// Large box.
g.DrawRectangle(__penBoxOutline,
GridLeft + (_testCaseField.LargeBoxTopLeft._x * testFieldPixelSize),
GridTop + (_testCaseField.LargeBoxTopLeft._y * testFieldPixelSize),
testFieldPixelSize*3, testFieldPixelSize*3);
// Draw red line around pixel with highest activation.
g.DrawRectangle(__penSelectedPixel,
GridLeft + (maxActivationPoint._x * visualFieldPixelSize),
GridTop + (maxActivationPoint._y * visualFieldPixelSize),
visualFieldPixelSize, visualFieldPixelSize);
Refresh();
}
/// <summary>
/// Allow the agent to move one square based on its decision. Note that the agent can choose to not move.
/// </summary>
public void MoveAgent(IBlackBox agent)
{
// Selected output is highest signal at or above 0.5. Tied signals result in no result.
double maxSig = agent.OutputSignalArray[0];
int maxSigIdx = 0;
for(int i=1; i<4; i++)
{
if(agent.OutputSignalArray[i] > maxSig)
{
maxSig = agent.OutputSignalArray[i];
maxSigIdx = i;
}
else if(agent.OutputSignalArray[i] == maxSig)
{ // Tie. Two equally high outputs.
maxSigIdx = -1;
}
}
if(-1 == maxSigIdx || maxSig < 0.5)
{ // No action.
return;
}
switch(maxSigIdx)
{
case 0: // Move north.
_agentPos._y = Math.Min(_agentPos._y + 1, _gridSize - 1);
break;
case 1: // Move east.
_agentPos._x = Math.Min(_agentPos._x + 1, _gridSize - 1);
break;
case 2: // Move south.
_agentPos._y = Math.Max(_agentPos._y - 1, 0);
break;
case 3: // Move west (is the best?)
_agentPos._x = Math.Max(_agentPos._x - 1, 0);
break;
}
}
/// <summary>
/// Refresh/update the view with the provided genome.
/// </summary>
public override void RefreshView(object genome)
{
_box = null;
_neatGenome = genome as NeatGenome;
if(null == _neatGenome) {
return;
}
// Decode genome.
_box = _genomeDecoder.Decode(_neatGenome);
// Generate new test case.
_testCaseField.InitTestCase(_largeBoxTestCase++ % 3);
// Paint test case and network response.
PaintView(_box);
}
/// <summary>
/// Adding new bb
/// </summary>
/// <param name="param">BlackBox</param>
public void SaveData(IBlackBox param)
{
bbParams.Add(param);
}
public NeatAiPlayerController(IBlackBox brain, GameStateSerializer gameStateSerializer)
{
_brain = brain;
_gameStateSerializer = gameStateSerializer;
}
public NeuralAgent(int id, IBlackBox brain)
: base(id)
{
Brain = brain;
}
/// <summary>
/// Determine the agent's position in the world relative to the prey and walls, and set its sensor inputs accordingly.
/// </summary>
/// <param name="agent"></param>
public void SetAgentInputsAndActivate(IBlackBox agent)
{
// Calc prey's position relative to the agent (in polar coordinate system).
PolarPoint relPos = PolarPoint.FromCartesian(_preyPos - _agentPos);
// Determine agent sensor input values.
// Reset all inputs.
agent.InputSignalArray.Reset();
// Test if prey is in sensor range.
if(relPos.Radial <= _sensorRange)
{
// Determine which sensor segment the prey is within - [0,7]. There are eight segments and they are tilted 22.5 degrees (half a segment)
// such that due North, East South and West are each in the center of a sensor segment (rather than on a segment boundary).
int segmentIdx = (int)Math.Floor((relPos.Theta / PiDiv4) + PiDiv8);
if(8==segmentIdx) {
segmentIdx = 0;
}
// Set sensor segment's input.
agent.InputSignalArray[segmentIdx] = 1.0;
}
// Prey closeness detector.
agent.InputSignalArray[8] = relPos.Radial > 2.0 ? 0.0 : 1.0;
// Wall detectors - N,E,S,W.
// North.
int d = (_gridSize-1) - _agentPos._y;
if(d < 4) { agent.InputSignalArray[9] = (4-d) / 4.0; }
// East.
d = (_gridSize-1) - _agentPos._x;
if(d < 4) { agent.InputSignalArray[10] = (4-d) / 4.0; }
// South.
if(_agentPos._y < 4) { agent.InputSignalArray[11] = (4 - _agentPos._y) / 4.0; }
// West.
if(_agentPos._x < 4) { agent.InputSignalArray[12] = (4 - _agentPos._x) / 4.0; }
// Activate agent.
agent.Activate();
}
public override void RefreshView(object genome)
{
// Zero indicates that the simulation is not currently running.
if (0 == Interlocked.Exchange(ref _simRunningFlag, 1))
{
// We got the lock. Decode the genome and store resuly in an instance field.
var neatGenome = genome as NeatGenome;
_agent = _genomeDecoder.Decode(neatGenome);
// Signal simulation thread to start running a simulation.
_simStartEvent.Set();
}
}
public void Init(IBlackBox blackBox)
{
this.blackBox = blackBox;
}
public void LoadMazeNavigator(string navigatorGenomeFile)
{
_navigatorAnnController =
_mazeSimulationIoController.ReadNavigatorGenomeFile(navigatorGenomeFile);
}
public EventstoreBoardprovider(IBlackBox blackBox)
{
this.blackBox = blackBox;
}
/// <summary>
/// Runs one trial of the provided agent in the world. Returns true if the agent captures the prey within
/// the maximum number of timesteps allowed.
/// </summary>
public bool RunTrial(IBlackBox agent)
{
// Init world state.
InitPositions();
// Clear any prior agent state.
agent.ResetState();
// Prey gets a head start (simulate world as normal but agent is frozen).
int t = 0;
for(; t<_preyInitMoves; t++)
{
SetAgentInputsAndActivate(agent);
MovePrey();
}
// Let the chase begin!
for(; t<_maxTimesteps; t++)
{
SetAgentInputsAndActivate(agent);
MoveAgent(agent);
if(IsPreyCaptured()) {
return true;
}
MovePrey();
if(IsPreyCaptured())
{ // The prey walked directly into the agent.
return true;
}
}
// Agent failed to capture prey in the alloted time.
return false;
}
/// <summary>
/// Evaluate the provided IBlackBox.
/// </summary>
public override FitnessInfo Evaluate(IBlackBox box)
{
_evalCount++;
// [0] - Cart Position (meters).
// [1] - Cart velocity (m/s).
// [2] - Pole 1 angle (radians)
// [3] - Pole 1 angular velocity (radians/sec).
// [4] - Pole 2 angle (radians)
// [5] - Pole 2 angular velocity (radians/sec).
double[] state = new double[6];
state[2] = FourDegrees;
JiggleBuffer jiggleBuffer1 = new JiggleBuffer(100);
JiggleBuffer jiggleBuffer2 = new JiggleBuffer(100);
// Run the pole-balancing simulation.
int timestep = 0;
for(; timestep < _maxTimesteps; timestep++)
{
// Provide state info to the network (normalised to +-1.0).
// Markovian (With velocity info)
box.InputSignalArray[0] = state[0] / _trackLengthHalf; // Cart Position is +-trackLengthHalfed
box.InputSignalArray[1] = state[2] / ThirtySixDegrees; // Pole Angle is +-thirtysix_degrees. Values outside of this range stop the simulation.
box.InputSignalArray[2] = state[4] / ThirtySixDegrees; // Pole Angle is +-thirtysix_degrees. Values outside of this range stop the simulation.
// Activate the black box.
box.Activate();
// Get black box response and calc next timestep state.
performAction(state, box.OutputSignalArray[0]);
// Jiggle buffer updates..
if(100 == jiggleBuffer1.Length)
{ // Feed an old value from buffer 1 into buffer2.
jiggleBuffer2.Enqueue(jiggleBuffer1.Dequeue());
}
// Place the latest jiggle value into buffer1.
jiggleBuffer1.Enqueue( Math.Abs(state[0]) + Math.Abs(state[1]) +
Math.Abs(state[2]) + Math.Abs(state[3]));
// Check for failure state. Has the cart run off the ends of the track or has the pole
// angle gone beyond the threshold.
if( (state[0]< -_trackLengthHalf) || (state[0]> _trackLengthHalf)
|| (state[2] > _poleAngleThreshold) || (state[2] < -_poleAngleThreshold)
|| (state[4] > _poleAngleThreshold) || (state[4] < -_poleAngleThreshold))
{
break;
}
// Give the simulation at least 500 timesteps(5secs) to stabilise before penalising instability.
if(timestep > 499 && jiggleBuffer2.Total > 30.0)
{ // Too much wiggling. Stop simulation early (30 was an experimentally determined value).
break;
}
}
double fitness;
if(timestep > 499 && timestep < 600)
{ // For the 100(1 sec) steps after the 500(5 secs) mark we punish wiggling based
// on the values from the 1 sec just gone. This is on the basis that the values
// in jiggleBuffer2 (from 2 to 1 sec ago) will refelct the large amount of
// wiggling that occurs at the start of the simulation when the system is still stabilising.
fitness = timestep + (10.0 / Math.Max(1.0, jiggleBuffer1.Total));
}
else if(timestep > 599)
{ // After 600 steps we use jiggleBuffer2 to punsih wiggling, this contains data from between
// 2 and 1 secs ago. This is on the basis that when the system becomes unstable and causes
// the simulation to terminate prematurely, the immediately prior 1 secs data will reflect that
// instability, which may not be indicative of the overall stability of the system up to that time.
fitness = timestep + (10.0 / Math.Max(1.0, jiggleBuffer2.Total));
}
else
{ // Return just the currentTimestep without any extra fitness component.
fitness = timestep;
}
// Max fitness is # of timesteps + max antiwiggle factor of 10 (which is actually impossible - some wiggle is necessary to balance the poles).
if(timestep == _maxTimesteps + 10.0) {
_stopConditionSatisfied = true;
}
return new FitnessInfo(fitness, fitness);
}
public NeatAiPlayerControllerFactory(IBlackBox brain, int numberOfNeuromon)
{
_brain = brain;
_gameStateSerializer = new GameStateSerializer(numberOfNeuromon);
}