public override void BackPropagate(TrainingRun trainingRun) { var inputError = _weights.Transpose() * trainingRun.OutputError.Value; trainingRun.InputError = inputError.SubVector(1, inputError.Count - 1); trainingRun.WeightsDelta = CalcWeightsDelta(trainingRun.Input, trainingRun.OutputError); }
public float Tick(TrainingRun run, float timer, float dt, out bool earlyOut) { timer += dt; earlyOut = false; var active = false; foreach (var obj in run.objects) { var gann = obj.GetComponent <GANNBehaviour>(); gann.Tick(); if (gann.IsActive()) { active = true; } } if (!active) { earlyOut = true; } return(timer); }
public override void BackPropagate(TrainingRun trainingRun) { var outputErrors = trainingRun.OutputError .ToMatrices() .Select(m => _isTranspose ? m.Rotate180() : m) .ToArray(); trainingRun.WeightsDelta = Matrix <float> .Build.Dense(this._weights.RowCount, this._weights.ColumnCount); var weightsDelta = CartesianConvolve( outputErrors, trainingRun.Input.ToMatrices() ); trainingRun.InputError = MatrixwiseConvolveTrans( _weights.SplitByColumn(_kernelCount), outputErrors ); for (var r = 0; r < this._weights.RowCount / weightsDelta.RowCount; r++) { for (var c = 0; c < this._weights.ColumnCount / weightsDelta.ColumnCount; c++) { trainingRun.WeightsDelta.SetSubMatrix(r, c * weightsDelta.ColumnCount, weightsDelta); } } }
public void should_back_propagate_correctly() { // Arrange var input = new float[, ] { { 1, 2, 3 }, }.ToMatrix(); _sut.SetWeights(new float[, ] { { 0.2f, 0.7f }, }.ToMatrix()); var trainingRun = new TrainingRun(1) { Input = input, Output = _sut.FeedForwards(new Tensor(input)), OutputError = new float[, ] { { 0.2f, -0.5f } } }; var expectedInputError = new float[, ] { { 0.04f, 0.04f, -0.35f }, }.ToMatrix(); var expectedWeightsDelta = new float[, ] { { -0.8f, -1.1f }, }.ToMatrix(); // Act _sut.BackPropagate(trainingRun); // Assert Assert.That(trainingRun.WeightsDelta.AlmostEqual(expectedWeightsDelta, 0.001f), Is.True); Assert.That(trainingRun.InputError.ToMatrix().AlmostEqual(expectedInputError, 0.001f), Is.True); }
public void should_back_propagate_correctly() { // Arrange var sigmoid = new Sigmoid(new Size(2, 2)); var trainingRun = new TrainingRun(1) { Input = new float[] { 0.31f, 0.61f, 0.27f, 0.19f }, OutputError = new float[] { 0.25f * 0.61f + -0.15f * 0.02f, 0.25f * 0.96f + -0.15f * 0.23f, 0.25f * 0.82f + -0.15f * -0.50f, 0.25f * -1.00f + -0.15f * 0.17f } }; var expected = new float[, ] { { 0.0364182f, 0.068628f }, { 0.04675125f, -0.06818625f } }; // Act sigmoid.BackPropagate(trainingRun); // Assert var actual = (trainingRun.InputError.ToMatrix() * 100).PointwiseRound() / 100; var expectedMatrix = (Matrix <float> .Build.DenseOfArray(expected) * 100).PointwiseRound() / 100; Assert.That(actual, Is.EqualTo(expectedMatrix)); }
public override void BackPropagate(TrainingRun trainingRun) { for (int i = Layers.Count - 1; i >= 0; i--) { trainingRun.Counter = i; Layers[i].BackPropagate(trainingRun); } }
public override void BackPropagate(TrainingRun trainingRun) { var derivative = trainingRun.Input.Value.Map(_gradientFn); trainingRun.InputError = new Tensor( InputSize.Clone(), trainingRun.OutputError.Value.PointwiseMultiply(derivative) ); }
public void should_back_propagate_correctly() { // Arrange var input = new float[, ] { { 1 }, }.ToMatrix(); _sut.SetWeights(new float[, ] { { 0.2f, 0.5f }, { 0.3f, 0.4f }, }.ToMatrix()); var trainingRun = new TrainingRun(1) { Input = input, Output = _sut.FeedForwards(new Tensor(input)), OutputError = new float[, ] { { 0.1f, 0 }, { 0, 0 } } }; var expectedInputError = new float[, ] { { 0.04f }, }.ToMatrix(); var expectedWeightsDelta = new float[, ] { { 0.1f, 0 }, { 0, 0 }, }.ToMatrix(); // Check application of delta fixes error _sut.SetWeights(new float[, ] { { 0.3f, 0.5f }, { 0.3f, 0.4f }, }.ToMatrix()); var expectedNewOutput = new float[, ] { { 0.3f, 0.5f }, { 0.3f, 0.4f }, }.ToMatrix(); var newOutput = _sut.FeedForwards(new Tensor(input)); // Act _sut.BackPropagate(trainingRun); // Assert Assert.That(trainingRun.WeightsDelta.AlmostEqual(expectedWeightsDelta, 0.001f), Is.True); Assert.That(trainingRun.InputError.ToMatrix().AlmostEqual(expectedInputError, 0.001f), Is.True); Assert.That(newOutput.ToMatrix().AlmostEqual(expectedNewOutput, 0.001f), Is.True); }
public void ApplyTraining(TrainingRun trainingRun, float learningRate) { Parallel.For(0, Layers.Count, i => { if (Layers[i] is IHaveWeights layer) { // TODO: Regularization var regularisation = layer.GetWeights() / trainingRun.BatchSize; var delta = trainingRun.WeightsDeltas[i] * (learningRate / trainingRun.BatchSize); // + regularisation; layer.UpdateWeights(weights => weights - delta); } }); }
public TrainingRun FeedForwardsTraining(Tensor input) { var trainingRun = new TrainingRun(Layers.Count) { Input = input, BatchSize = 1 }; // TODO: Guard clauses for when Tensors are unexpected shape for (int i = 0; i < Layers.Count; i++) { trainingRun.Counter = i; trainingRun.Output = Layers[i].FeedForwards(trainingRun.Input); } return(trainingRun); }
public void should_backpropagate_tensor_of_correct_size(int inputSize, int weightLength, int outputSize) { // Arrange _sut = (ConvolutionalLayer) new Net(inputSize, inputSize) .ConvolutionTranspose(new[] { weightLength, weightLength }) .Layers[0]; var trainingRun = new TrainingRun(1) { Input = new float[inputSize, inputSize].ToMatrix(), Output = _sut.FeedForwards(new float[inputSize, inputSize].ToMatrix()), OutputError = new float[outputSize, outputSize].ToMatrix() }; // Act _sut.BackPropagate(trainingRun); // Assert Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size)); }
public void should_back_propagate_correctly() { // Arrange var layer = new DenseLayer(3, 2); layer.SetWeights(new float[2, 4] { { 0.61f, 0.82f, 0.96f, -1 }, { 0.02f, -0.5f, 0.23f, 0.17f }, }); var input = new float[3] { 0.57f, 0.65f, 0.55f }; var trainingRun = new TrainingRun(1) { Input = input, Output = layer.FeedForwards(input), OutputError = new float[2] { 0.25f, -0.68f } }; var expected = new float[3] { 0.82f * 0.25f + -0.5f * -0.68f, 0.96f * 0.25f + 0.23f * -0.68f, -1 * 0.25f + 0.17f * -0.68f, }.ToVector(); // Act layer.BackPropagate(trainingRun); // Assert var actual = (trainingRun.InputError.Value * 100).PointwiseRound() / 100; expected = (expected * 100).PointwiseRound() / 100; Assert.That(actual, Is.EqualTo(expected)); }
public void should_backpropagate_tensor_of_correct_kernel_size(int kernelSize) { // Arrange _sut = (ConvolutionalLayer) new Net(1, 1) .ConvolutionTranspose(new[] { 5, 5 }, kernelSize) .Layers[0]; var input = new float[, ] { { 0 } }.ToMatrix(); var trainingRun = new TrainingRun(1) { Input = input, Output = _sut.FeedForwards(input), OutputError = new Tensor(new Size(5, 5, kernelSize), new float[5 * 5 * kernelSize]) }; // Act _sut.BackPropagate(trainingRun); // Assert Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size)); }
public IEnumerator DoTraining() { Debug.LogFormat("GANNTrainer: training starting..."); var count = parallelCount; runs = new List <TrainingRun>(); while (true) { var run = new TrainingRun(); run.holder = new GameObject(string.Format("Run{0}", runs.Count)); run.objects = new List <GameObject>(); run.scores = new List <float>(); for (int i = 0; i < parallelCount; ++i) { var obj = GameObject.Instantiate(prefab); var gann = obj.GetComponent <GANNBehaviour>(); // want to build from the previous winning trainer var best = (GANNBehaviour)null; if (best != null) { gann.Duplicate(best); gann.Mutate(); } else { gann.Rebuild(3, 3, 3); } obj.transform.SetParent(run.holder.transform); obj.name = string.Format("GANN{0}", i); run.objects.Add(obj); } var remaining = trainingRunTime; while (remaining > 0.0f) { var timer = 0.0f; var dt = Time.fixedDeltaTime; var skip = false; if (trainingTimeScale) { Time.timeScale = trainingTimeScaleValue; } else { Time.timeScale = 1.0f; } if (trainingStep) { trainingPause = true; trainingStep = false; } Physics.autoSimulation = false; Physics.Simulate(dt); timer = Tick(run, timer, dt, out skip); if (skip) { break; } remaining -= dt * Time.timeScale; yield return(null); } run.holder.SetActive(false); runs.Add(run); yield return(null); } }
public void Load() { // Load Coordinates from DB, Model (from flat file) double[] eigenvalues; lock (_modelLockObject) { var db = new DataClasses1DataContext(_connectionString); _model = db.TrainingRuns.Where(x => x.Name.Equals(_modelName)).Single(); String tmpPath = _storageSystem.downloadModelFlatFile(_modelName); //throw new ApplicationException("File is " + tmpPath); _dimensions = CAPI.TrainModel(tmpPath, true, -1, _model.Rows, _model.Cols, (int)CAPI.ImageMode.Undefined, (int)_modelType); if (_model.Dimensions != _dimensions) throw new ApplicationException("Number of dimensions in db dont match number of dimensions in model flat file."); var deletedFaces = new HashSet<int>(db.NewFaces.Where(x=>x.Deleted).Select(x => x.FaceUKey)); var slurp = db.Coordinates.Where(x => x.TrainingRunID == _model.TrainingRunID) //.ToList().Where(x => !deletedFaces.Contains(x.FaceUKey)) .ToLookup(x => x.FaceUKey); lock (_faceCoordsLockObject) { _faceCoords = new Dictionary<int, List<double>>(); _faceCoordsDeletedFaces = new Dictionary<int, List<double>>(); foreach (var item in slurp) { int ukey = item.Key; var list = new List<double>(_dimensions); if (deletedFaces.Contains(ukey)) { if (_faceCoordsDeletedFaces.ContainsKey(ukey)) throw new ApplicationException("Duplicate Key is " + ukey); _faceCoordsDeletedFaces.Add(ukey, list); } else _faceCoords.Add(ukey, list); int i = 0; foreach (Coordinate c in item.OrderBy(x => x.Dimension)) { if (i++ != c.Dimension) throw new ApplicationException("Missing coordinate date for face = " + item.Key); list.Add(c.Coordinate1); } } _allFaceUKeys = _faceCoords.Keys.ToList(); updateMinsAndMaxes(); } eigenvalues = new double[_dimensions]; CAPI.GetEigenValues(eigenvalues); } lock (_eigenvaluesLockObject) _eigenvalues = eigenvalues.ToList(); }
public abstract void BackPropagate(TrainingRun trainingRun);