public override void BackPropagate(TrainingRun trainingRun)
{
var inputError = _weights.Transpose() * trainingRun.OutputError.Value;
trainingRun.InputError = inputError.SubVector(1, inputError.Count - 1);
trainingRun.WeightsDelta = CalcWeightsDelta(trainingRun.Input, trainingRun.OutputError);
}
public float Tick(TrainingRun run, float timer, float dt, out bool earlyOut)
{
timer += dt;
earlyOut = false;
var active = false;
foreach (var obj in run.objects)
{
var gann = obj.GetComponent <GANNBehaviour>();
gann.Tick();
if (gann.IsActive())
{
active = true;
}
}
if (!active)
{
earlyOut = true;
}
return(timer);
}
public override void BackPropagate(TrainingRun trainingRun)
{
var outputErrors = trainingRun.OutputError
.ToMatrices()
.Select(m => _isTranspose ? m.Rotate180() : m)
.ToArray();
trainingRun.WeightsDelta = Matrix <float> .Build.Dense(this._weights.RowCount, this._weights.ColumnCount);
var weightsDelta = CartesianConvolve(
outputErrors,
trainingRun.Input.ToMatrices()
);
trainingRun.InputError = MatrixwiseConvolveTrans(
_weights.SplitByColumn(_kernelCount),
outputErrors
);
for (var r = 0; r < this._weights.RowCount / weightsDelta.RowCount; r++)
{
for (var c = 0; c < this._weights.ColumnCount / weightsDelta.ColumnCount; c++)
{
trainingRun.WeightsDelta.SetSubMatrix(r, c * weightsDelta.ColumnCount, weightsDelta);
}
}
}
public void should_back_propagate_correctly()
{
// Arrange
var input = new float[, ] {
{ 1, 2, 3 },
}.ToMatrix();
_sut.SetWeights(new float[, ] {
{ 0.2f, 0.7f },
}.ToMatrix());
var trainingRun = new TrainingRun(1)
{
Input = input,
Output = _sut.FeedForwards(new Tensor(input)),
OutputError = new float[, ] {
{ 0.2f, -0.5f }
}
};
var expectedInputError = new float[, ] {
{ 0.04f, 0.04f, -0.35f },
}.ToMatrix();
var expectedWeightsDelta = new float[, ] {
{ -0.8f, -1.1f },
}.ToMatrix();
// Act
_sut.BackPropagate(trainingRun);
// Assert
Assert.That(trainingRun.WeightsDelta.AlmostEqual(expectedWeightsDelta, 0.001f), Is.True);
Assert.That(trainingRun.InputError.ToMatrix().AlmostEqual(expectedInputError, 0.001f), Is.True);
}
public void should_back_propagate_correctly()
{
// Arrange
var sigmoid = new Sigmoid(new Size(2, 2));
var trainingRun = new TrainingRun(1)
{
Input = new float[]
{
0.31f, 0.61f, 0.27f, 0.19f
},
OutputError = new float[]
{
0.25f * 0.61f + -0.15f * 0.02f,
0.25f * 0.96f + -0.15f * 0.23f,
0.25f * 0.82f + -0.15f * -0.50f,
0.25f * -1.00f + -0.15f * 0.17f
}
};
var expected = new float[, ]
{
{ 0.0364182f, 0.068628f },
{ 0.04675125f, -0.06818625f }
};
// Act
sigmoid.BackPropagate(trainingRun);
// Assert
var actual = (trainingRun.InputError.ToMatrix() * 100).PointwiseRound() / 100;
var expectedMatrix = (Matrix <float> .Build.DenseOfArray(expected) * 100).PointwiseRound() / 100;
Assert.That(actual, Is.EqualTo(expectedMatrix));
}
public override void BackPropagate(TrainingRun trainingRun)
{
for (int i = Layers.Count - 1; i >= 0; i--)
{
trainingRun.Counter = i;
Layers[i].BackPropagate(trainingRun);
}
}
public override void BackPropagate(TrainingRun trainingRun)
{
var derivative = trainingRun.Input.Value.Map(_gradientFn);
trainingRun.InputError = new Tensor(
InputSize.Clone(),
trainingRun.OutputError.Value.PointwiseMultiply(derivative)
);
}
public void should_back_propagate_correctly()
{
// Arrange
var input = new float[, ] {
{ 1 },
}.ToMatrix();
_sut.SetWeights(new float[, ] {
{ 0.2f, 0.5f },
{ 0.3f, 0.4f },
}.ToMatrix());
var trainingRun = new TrainingRun(1)
{
Input = input,
Output = _sut.FeedForwards(new Tensor(input)),
OutputError = new float[, ] {
{ 0.1f, 0 },
{ 0, 0 }
}
};
var expectedInputError = new float[, ] {
{ 0.04f },
}.ToMatrix();
var expectedWeightsDelta = new float[, ] {
{ 0.1f, 0 },
{ 0, 0 },
}.ToMatrix();
// Check application of delta fixes error
_sut.SetWeights(new float[, ] {
{ 0.3f, 0.5f },
{ 0.3f, 0.4f },
}.ToMatrix());
var expectedNewOutput = new float[, ] {
{ 0.3f, 0.5f },
{ 0.3f, 0.4f },
}.ToMatrix();
var newOutput = _sut.FeedForwards(new Tensor(input));
// Act
_sut.BackPropagate(trainingRun);
// Assert
Assert.That(trainingRun.WeightsDelta.AlmostEqual(expectedWeightsDelta, 0.001f), Is.True);
Assert.That(trainingRun.InputError.ToMatrix().AlmostEqual(expectedInputError, 0.001f), Is.True);
Assert.That(newOutput.ToMatrix().AlmostEqual(expectedNewOutput, 0.001f), Is.True);
}
public void ApplyTraining(TrainingRun trainingRun, float learningRate)
{
Parallel.For(0, Layers.Count, i =>
{
if (Layers[i] is IHaveWeights layer)
{
// TODO: Regularization var regularisation = layer.GetWeights() / trainingRun.BatchSize;
var delta = trainingRun.WeightsDeltas[i] * (learningRate / trainingRun.BatchSize); // + regularisation;
layer.UpdateWeights(weights => weights - delta);
}
});
}
public TrainingRun FeedForwardsTraining(Tensor input)
{
var trainingRun = new TrainingRun(Layers.Count)
{
Input = input,
BatchSize = 1
};
// TODO: Guard clauses for when Tensors are unexpected shape
for (int i = 0; i < Layers.Count; i++)
{
trainingRun.Counter = i;
trainingRun.Output = Layers[i].FeedForwards(trainingRun.Input);
}
return(trainingRun);
}
public void should_backpropagate_tensor_of_correct_size(int inputSize, int weightLength, int outputSize)
{
// Arrange
_sut = (ConvolutionalLayer) new Net(inputSize, inputSize)
.ConvolutionTranspose(new[] { weightLength, weightLength })
.Layers[0];
var trainingRun = new TrainingRun(1)
{
Input = new float[inputSize, inputSize].ToMatrix(),
Output = _sut.FeedForwards(new float[inputSize, inputSize].ToMatrix()),
OutputError = new float[outputSize, outputSize].ToMatrix()
};
// Act
_sut.BackPropagate(trainingRun);
// Assert
Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size));
}
public void should_back_propagate_correctly()
{
// Arrange
var layer = new DenseLayer(3, 2);
layer.SetWeights(new float[2, 4]
{
{ 0.61f, 0.82f, 0.96f, -1 },
{ 0.02f, -0.5f, 0.23f, 0.17f },
});
var input = new float[3]
{
0.57f, 0.65f, 0.55f
};
var trainingRun = new TrainingRun(1)
{
Input = input,
Output = layer.FeedForwards(input),
OutputError = new float[2] {
0.25f, -0.68f
}
};
var expected = new float[3]
{
0.82f * 0.25f + -0.5f * -0.68f,
0.96f * 0.25f + 0.23f * -0.68f,
-1 * 0.25f + 0.17f * -0.68f,
}.ToVector();
// Act
layer.BackPropagate(trainingRun);
// Assert
var actual = (trainingRun.InputError.Value * 100).PointwiseRound() / 100;
expected = (expected * 100).PointwiseRound() / 100;
Assert.That(actual, Is.EqualTo(expected));
}
public void should_backpropagate_tensor_of_correct_kernel_size(int kernelSize)
{
// Arrange
_sut = (ConvolutionalLayer) new Net(1, 1)
.ConvolutionTranspose(new[] { 5, 5 }, kernelSize)
.Layers[0];
var input = new float[, ] {
{ 0 }
}.ToMatrix();
var trainingRun = new TrainingRun(1)
{
Input = input,
Output = _sut.FeedForwards(input),
OutputError = new Tensor(new Size(5, 5, kernelSize), new float[5 * 5 * kernelSize])
};
// Act
_sut.BackPropagate(trainingRun);
// Assert
Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size));
}
public IEnumerator DoTraining()
{
Debug.LogFormat("GANNTrainer: training starting...");
var count = parallelCount;
runs = new List <TrainingRun>();
while (true)
{
var run = new TrainingRun();
run.holder = new GameObject(string.Format("Run{0}", runs.Count));
run.objects = new List <GameObject>();
run.scores = new List <float>();
for (int i = 0; i < parallelCount; ++i)
{
var obj = GameObject.Instantiate(prefab);
var gann = obj.GetComponent <GANNBehaviour>();
// want to build from the previous winning trainer
var best = (GANNBehaviour)null;
if (best != null)
{
gann.Duplicate(best);
gann.Mutate();
}
else
{
gann.Rebuild(3, 3, 3);
}
obj.transform.SetParent(run.holder.transform);
obj.name = string.Format("GANN{0}", i);
run.objects.Add(obj);
}
var remaining = trainingRunTime;
while (remaining > 0.0f)
{
var timer = 0.0f;
var dt = Time.fixedDeltaTime;
var skip = false;
if (trainingTimeScale)
{
Time.timeScale = trainingTimeScaleValue;
}
else
{
Time.timeScale = 1.0f;
}
if (trainingStep)
{
trainingPause = true;
trainingStep = false;
}
Physics.autoSimulation = false;
Physics.Simulate(dt);
timer = Tick(run, timer, dt, out skip);
if (skip)
{
break;
}
remaining -= dt * Time.timeScale;
yield return(null);
}
run.holder.SetActive(false);
runs.Add(run);
yield return(null);
}
}
public void Load()
{ // Load Coordinates from DB, Model (from flat file)
double[] eigenvalues;
lock (_modelLockObject)
{
var db = new DataClasses1DataContext(_connectionString);
_model = db.TrainingRuns.Where(x => x.Name.Equals(_modelName)).Single();
String tmpPath = _storageSystem.downloadModelFlatFile(_modelName);
//throw new ApplicationException("File is " + tmpPath);
_dimensions = CAPI.TrainModel(tmpPath, true, -1, _model.Rows, _model.Cols, (int)CAPI.ImageMode.Undefined, (int)_modelType);
if (_model.Dimensions != _dimensions) throw new ApplicationException("Number of dimensions in db dont match number of dimensions in model flat file.");
var deletedFaces = new HashSet<int>(db.NewFaces.Where(x=>x.Deleted).Select(x => x.FaceUKey));
var slurp = db.Coordinates.Where(x => x.TrainingRunID == _model.TrainingRunID)
//.ToList().Where(x => !deletedFaces.Contains(x.FaceUKey))
.ToLookup(x => x.FaceUKey);
lock (_faceCoordsLockObject)
{
_faceCoords = new Dictionary<int, List<double>>();
_faceCoordsDeletedFaces = new Dictionary<int, List<double>>();
foreach (var item in slurp)
{
int ukey = item.Key;
var list = new List<double>(_dimensions);
if (deletedFaces.Contains(ukey))
{
if (_faceCoordsDeletedFaces.ContainsKey(ukey)) throw new ApplicationException("Duplicate Key is " + ukey);
_faceCoordsDeletedFaces.Add(ukey, list);
}
else
_faceCoords.Add(ukey, list);
int i = 0;
foreach (Coordinate c in item.OrderBy(x => x.Dimension))
{
if (i++ != c.Dimension) throw new ApplicationException("Missing coordinate date for face = " + item.Key);
list.Add(c.Coordinate1);
}
}
_allFaceUKeys = _faceCoords.Keys.ToList();
updateMinsAndMaxes();
}
eigenvalues = new double[_dimensions];
CAPI.GetEigenValues(eigenvalues);
}
lock (_eigenvaluesLockObject)
_eigenvalues = eigenvalues.ToList();
}
public abstract void BackPropagate(TrainingRun trainingRun);
