Beispiel #1
0
        public override void BackPropagate(TrainingRun trainingRun)
        {
            var inputError = _weights.Transpose() * trainingRun.OutputError.Value;

            trainingRun.InputError   = inputError.SubVector(1, inputError.Count - 1);
            trainingRun.WeightsDelta = CalcWeightsDelta(trainingRun.Input, trainingRun.OutputError);
        }
Beispiel #2
0
    public float Tick(TrainingRun run, float timer, float dt, out bool earlyOut)
    {
        timer += dt;

        earlyOut = false;

        var active = false;

        foreach (var obj in run.objects)
        {
            var gann = obj.GetComponent <GANNBehaviour>();

            gann.Tick();

            if (gann.IsActive())
            {
                active = true;
            }
        }

        if (!active)
        {
            earlyOut = true;
        }

        return(timer);
    }
        public override void BackPropagate(TrainingRun trainingRun)
        {
            var outputErrors = trainingRun.OutputError
                               .ToMatrices()
                               .Select(m => _isTranspose ? m.Rotate180() : m)
                               .ToArray();

            trainingRun.WeightsDelta = Matrix <float> .Build.Dense(this._weights.RowCount, this._weights.ColumnCount);

            var weightsDelta = CartesianConvolve(
                outputErrors,
                trainingRun.Input.ToMatrices()
                );

            trainingRun.InputError = MatrixwiseConvolveTrans(
                _weights.SplitByColumn(_kernelCount),
                outputErrors
                );

            for (var r = 0; r < this._weights.RowCount / weightsDelta.RowCount; r++)
            {
                for (var c = 0; c < this._weights.ColumnCount / weightsDelta.ColumnCount; c++)
                {
                    trainingRun.WeightsDelta.SetSubMatrix(r, c * weightsDelta.ColumnCount, weightsDelta);
                }
            }
        }
        public void should_back_propagate_correctly()
        {
            // Arrange
            var input = new float[, ] {
                { 1, 2, 3 },
            }.ToMatrix();

            _sut.SetWeights(new float[, ] {
                { 0.2f, 0.7f },
            }.ToMatrix());

            var trainingRun = new TrainingRun(1)
            {
                Input       = input,
                Output      = _sut.FeedForwards(new Tensor(input)),
                OutputError = new float[, ] {
                    { 0.2f, -0.5f }
                }
            };

            var expectedInputError = new float[, ] {
                { 0.04f, 0.04f, -0.35f },
            }.ToMatrix();

            var expectedWeightsDelta = new float[, ] {
                { -0.8f, -1.1f },
            }.ToMatrix();

            // Act
            _sut.BackPropagate(trainingRun);

            // Assert
            Assert.That(trainingRun.WeightsDelta.AlmostEqual(expectedWeightsDelta, 0.001f), Is.True);
            Assert.That(trainingRun.InputError.ToMatrix().AlmostEqual(expectedInputError, 0.001f), Is.True);
        }
        public void should_back_propagate_correctly()
        {
            // Arrange
            var sigmoid = new Sigmoid(new Size(2, 2));

            var trainingRun = new TrainingRun(1)
            {
                Input = new float[]
                {
                    0.31f, 0.61f, 0.27f, 0.19f
                },
                OutputError = new float[]
                {
                    0.25f * 0.61f + -0.15f * 0.02f,
                    0.25f * 0.96f + -0.15f * 0.23f,
                    0.25f * 0.82f + -0.15f * -0.50f,
                    0.25f * -1.00f + -0.15f * 0.17f
                }
            };

            var expected = new float[, ]
            {
                { 0.0364182f, 0.068628f },
                { 0.04675125f, -0.06818625f }
            };

            // Act
            sigmoid.BackPropagate(trainingRun);

            // Assert
            var actual         = (trainingRun.InputError.ToMatrix() * 100).PointwiseRound() / 100;
            var expectedMatrix = (Matrix <float> .Build.DenseOfArray(expected) * 100).PointwiseRound() / 100;

            Assert.That(actual, Is.EqualTo(expectedMatrix));
        }
 public override void BackPropagate(TrainingRun trainingRun)
 {
     for (int i = Layers.Count - 1; i >= 0; i--)
     {
         trainingRun.Counter = i;
         Layers[i].BackPropagate(trainingRun);
     }
 }
        public override void BackPropagate(TrainingRun trainingRun)
        {
            var derivative = trainingRun.Input.Value.Map(_gradientFn);

            trainingRun.InputError = new Tensor(
                InputSize.Clone(),
                trainingRun.OutputError.Value.PointwiseMultiply(derivative)
                );
        }
Beispiel #8
0
        public void should_back_propagate_correctly()
        {
            // Arrange
            var input = new float[, ] {
                { 1 },
            }.ToMatrix();

            _sut.SetWeights(new float[, ] {
                { 0.2f, 0.5f },
                { 0.3f, 0.4f },
            }.ToMatrix());

            var trainingRun = new TrainingRun(1)
            {
                Input       = input,
                Output      = _sut.FeedForwards(new Tensor(input)),
                OutputError = new float[, ] {
                    { 0.1f, 0 },
                    { 0, 0 }
                }
            };

            var expectedInputError = new float[, ] {
                { 0.04f },
            }.ToMatrix();

            var expectedWeightsDelta = new float[, ] {
                { 0.1f, 0 },
                { 0, 0 },
            }.ToMatrix();

            // Check application of delta fixes error
            _sut.SetWeights(new float[, ] {
                { 0.3f, 0.5f },
                { 0.3f, 0.4f },
            }.ToMatrix());

            var expectedNewOutput = new float[, ] {
                { 0.3f, 0.5f },
                { 0.3f, 0.4f },
            }.ToMatrix();

            var newOutput = _sut.FeedForwards(new Tensor(input));

            // Act
            _sut.BackPropagate(trainingRun);

            // Assert
            Assert.That(trainingRun.WeightsDelta.AlmostEqual(expectedWeightsDelta, 0.001f), Is.True);
            Assert.That(trainingRun.InputError.ToMatrix().AlmostEqual(expectedInputError, 0.001f), Is.True);

            Assert.That(newOutput.ToMatrix().AlmostEqual(expectedNewOutput, 0.001f), Is.True);
        }
        public void ApplyTraining(TrainingRun trainingRun, float learningRate)
        {
            Parallel.For(0, Layers.Count, i =>
            {
                if (Layers[i] is IHaveWeights layer)
                {
                    // TODO: Regularization var regularisation = layer.GetWeights() / trainingRun.BatchSize;
                    var delta = trainingRun.WeightsDeltas[i] * (learningRate / trainingRun.BatchSize); // + regularisation;

                    layer.UpdateWeights(weights => weights - delta);
                }
            });
        }
        public TrainingRun FeedForwardsTraining(Tensor input)
        {
            var trainingRun = new TrainingRun(Layers.Count)
            {
                Input     = input,
                BatchSize = 1
            };

            // TODO: Guard clauses for when Tensors are unexpected shape
            for (int i = 0; i < Layers.Count; i++)
            {
                trainingRun.Counter = i;
                trainingRun.Output  = Layers[i].FeedForwards(trainingRun.Input);
            }

            return(trainingRun);
        }
        public void should_backpropagate_tensor_of_correct_size(int inputSize, int weightLength, int outputSize)
        {
            // Arrange
            _sut = (ConvolutionalLayer) new Net(inputSize, inputSize)
                   .ConvolutionTranspose(new[] { weightLength, weightLength })
                   .Layers[0];

            var trainingRun = new TrainingRun(1)
            {
                Input       = new float[inputSize, inputSize].ToMatrix(),
                Output      = _sut.FeedForwards(new float[inputSize, inputSize].ToMatrix()),
                OutputError = new float[outputSize, outputSize].ToMatrix()
            };

            // Act
            _sut.BackPropagate(trainingRun);

            // Assert
            Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size));
        }
Beispiel #12
0
        public void should_back_propagate_correctly()
        {
            // Arrange
            var layer = new DenseLayer(3, 2);

            layer.SetWeights(new float[2, 4]
            {
                { 0.61f, 0.82f, 0.96f, -1 },
                { 0.02f, -0.5f, 0.23f, 0.17f },
            });

            var input = new float[3]
            {
                0.57f, 0.65f, 0.55f
            };

            var trainingRun = new TrainingRun(1)
            {
                Input       = input,
                Output      = layer.FeedForwards(input),
                OutputError = new float[2] {
                    0.25f, -0.68f
                }
            };

            var expected = new float[3]
            {
                0.82f * 0.25f + -0.5f * -0.68f,
                0.96f * 0.25f + 0.23f * -0.68f,
                -1 * 0.25f + 0.17f * -0.68f,
            }.ToVector();

            // Act
            layer.BackPropagate(trainingRun);

            // Assert
            var actual = (trainingRun.InputError.Value * 100).PointwiseRound() / 100;

            expected = (expected * 100).PointwiseRound() / 100;
            Assert.That(actual, Is.EqualTo(expected));
        }
        public void should_backpropagate_tensor_of_correct_kernel_size(int kernelSize)
        {
            // Arrange
            _sut = (ConvolutionalLayer) new Net(1, 1)
                   .ConvolutionTranspose(new[] { 5, 5 }, kernelSize)
                   .Layers[0];
            var input = new float[, ] {
                { 0 }
            }.ToMatrix();

            var trainingRun = new TrainingRun(1)
            {
                Input       = input,
                Output      = _sut.FeedForwards(input),
                OutputError = new Tensor(new Size(5, 5, kernelSize), new float[5 * 5 * kernelSize])
            };

            // Act
            _sut.BackPropagate(trainingRun);

            // Assert
            Assert.That(trainingRun.InputError.Size, Is.EqualTo(trainingRun.Input.Size));
        }
Beispiel #14
0
    public IEnumerator DoTraining()
    {
        Debug.LogFormat("GANNTrainer: training starting...");

        var count = parallelCount;

        runs = new List <TrainingRun>();

        while (true)
        {
            var run = new TrainingRun();

            run.holder  = new GameObject(string.Format("Run{0}", runs.Count));
            run.objects = new List <GameObject>();
            run.scores  = new List <float>();

            for (int i = 0; i < parallelCount; ++i)
            {
                var obj  = GameObject.Instantiate(prefab);
                var gann = obj.GetComponent <GANNBehaviour>();

                // want to build from the previous winning trainer
                var best = (GANNBehaviour)null;

                if (best != null)
                {
                    gann.Duplicate(best);
                    gann.Mutate();
                }
                else
                {
                    gann.Rebuild(3, 3, 3);
                }

                obj.transform.SetParent(run.holder.transform);
                obj.name = string.Format("GANN{0}", i);

                run.objects.Add(obj);
            }

            var remaining = trainingRunTime;
            while (remaining > 0.0f)
            {
                var timer = 0.0f;
                var dt    = Time.fixedDeltaTime;
                var skip  = false;

                if (trainingTimeScale)
                {
                    Time.timeScale = trainingTimeScaleValue;
                }
                else
                {
                    Time.timeScale = 1.0f;
                }

                if (trainingStep)
                {
                    trainingPause = true;
                    trainingStep  = false;
                }

                Physics.autoSimulation = false;
                Physics.Simulate(dt);
                timer = Tick(run, timer, dt, out skip);
                if (skip)
                {
                    break;
                }

                remaining -= dt * Time.timeScale;
                yield return(null);
            }

            run.holder.SetActive(false);

            runs.Add(run);

            yield return(null);
        }
    }
Beispiel #15
0
        public void Load()
        {            // Load Coordinates from DB, Model (from flat file)
            double[] eigenvalues;
            lock (_modelLockObject)
            {
                var db = new DataClasses1DataContext(_connectionString);
                _model = db.TrainingRuns.Where(x => x.Name.Equals(_modelName)).Single();

                String tmpPath = _storageSystem.downloadModelFlatFile(_modelName);
                //throw new ApplicationException("File is " + tmpPath);
                _dimensions = CAPI.TrainModel(tmpPath, true, -1, _model.Rows, _model.Cols, (int)CAPI.ImageMode.Undefined, (int)_modelType);
                if (_model.Dimensions != _dimensions) throw new ApplicationException("Number of dimensions in db dont match number of dimensions in model flat file.");

                var deletedFaces = new HashSet<int>(db.NewFaces.Where(x=>x.Deleted).Select(x => x.FaceUKey));
                var slurp = db.Coordinates.Where(x => x.TrainingRunID == _model.TrainingRunID)
                                //.ToList().Where(x => !deletedFaces.Contains(x.FaceUKey))
                                .ToLookup(x => x.FaceUKey);

                lock (_faceCoordsLockObject)
                {
                    _faceCoords = new Dictionary<int, List<double>>();
                    _faceCoordsDeletedFaces = new Dictionary<int, List<double>>();
                    foreach (var item in slurp)
                    {
                        int ukey = item.Key;
                        var list = new List<double>(_dimensions);
                        if (deletedFaces.Contains(ukey))
                        {
                            if (_faceCoordsDeletedFaces.ContainsKey(ukey)) throw new ApplicationException("Duplicate Key is " + ukey);
                            _faceCoordsDeletedFaces.Add(ukey, list);
                        }
                        else
                            _faceCoords.Add(ukey, list);
                        int i = 0;
                        foreach (Coordinate c in item.OrderBy(x => x.Dimension))
                        {
                            if (i++ != c.Dimension) throw new ApplicationException("Missing coordinate date for face = " + item.Key);
                            list.Add(c.Coordinate1);
                        }
                    }
                    _allFaceUKeys = _faceCoords.Keys.ToList();
                    updateMinsAndMaxes();
                }

                eigenvalues = new double[_dimensions];
                CAPI.GetEigenValues(eigenvalues);
            }
            lock (_eigenvaluesLockObject)
                _eigenvalues = eigenvalues.ToList();
        }
 public abstract void BackPropagate(TrainingRun trainingRun);