Esempio n. 1
0
        public void TestRecurrent()
        {
            var data        = BinaryIntegers.Addition(100, false).Split(0);
            var graph       = new GraphFactory(_lap);
            var errorMetric = graph.ErrorMetric.BinaryClassification;

            graph.CurrentPropertySet
            .Use(graph.GradientDescent.Adam)
            .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));

            // create the engine
            var trainingData = graph.CreateDataSource(data.Training);
            var testData     = trainingData.CloneWith(data.Test);
            var engine       = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);

            // build the network
            const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5;
            var       memory  = new float[HIDDEN_LAYER_SIZE];
            var       network = graph.Connect(engine)
                                .AddSimpleRecurrent(graph.ReluActivation(), memory)
                                .AddFeedForward(engine.DataSource.OutputSize)
                                .Add(graph.ReluActivation())
                                .AddBackpropagationThroughTime(errorMetric)
            ;

            // train the network for twenty iterations, saving the model on each improvement
            BrightWire.Models.ExecutionGraph bestGraph = null;
            engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);

            // export the graph and verify it against some unseen integers on the best model
            var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
            var testData2       = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
            var results         = executionEngine.Execute(testData2);
        }
Esempio n. 2
0
        public void BidirectionalAddition()
        {
            var trainingSet = BinaryIntegers.Addition(10, false).Select(l => l.ToArray()).ToList();

            const int HIDDEN_SIZE = 16, NUM_EPOCHS = 100, BATCH_SIZE = 32;
            var       errorMetric = ErrorMetricType.BinaryClassification.Create();

            var layerTemplate = new LayerDescriptor(0.1f)
            {
                Activation           = ActivationType.LeakyRelu,
                WeightInitialisation = WeightInitialisationType.Gaussian,
                DecayRate            = 0.99f
            };

            var recurrentTemplate = layerTemplate.Clone();

            recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian;

            var trainingDataProvider = _lap.NN.CreateSequentialTrainingDataProvider(trainingSet);
            var layers = new INeuralNetworkBidirectionalLayer[] {
                _lap.NN.CreateBidirectionalLayer(
                    _lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate),
                    _lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate)
                    ),
                _lap.NN.CreateBidirectionalLayer(_lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE * 2, trainingDataProvider.OutputSize, layerTemplate))
            };
            BidirectionalNetwork networkData = null;

            using (var trainer = _lap.NN.CreateBidirectionalBatchTrainer(layers)) {
                var forwardMemory   = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
                var backwardMemory  = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
                var trainingContext = _lap.NN.CreateTrainingContext(errorMetric, 0.1f, BATCH_SIZE);
                trainingContext.RecurrentEpochComplete += (tc, rtc) => {
                    Debug.WriteLine(tc.LastTrainingError);
                };
                trainer.Train(trainingDataProvider, forwardMemory, backwardMemory, NUM_EPOCHS, _lap.NN.CreateRecurrentTrainingContext(trainingContext));
                networkData = trainer.NetworkInfo;
                networkData.ForwardMemory = new FloatArray {
                    Data = forwardMemory
                };
                networkData.BackwardMemory = new FloatArray {
                    Data = backwardMemory
                };
            }

            var network = _lap.NN.CreateBidirectional(networkData);

            foreach (var sequence in trainingSet)
            {
                var result = network.Execute(sequence.Select(d => d.Input).ToList());
            }
        }
Esempio n. 3
0
        public static void IntegerAddition()
        {
            // generate 1000 random integer additions (split into training and test sets)
            var data = BinaryIntegers.Addition(1000, false).Split(0);

            using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) {
                var graph = new GraphFactory(lap);

                // binary classification rounds each output to either 0 or 1
                var errorMetric = graph.ErrorMetric.BinaryClassification;

                // configure the network properties
                graph.CurrentPropertySet
                .Use(graph.GradientDescent.Adam)
                .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N))
                ;

                // create the engine
                var trainingData = graph.CreateDataSource(data.Training);
                var testData     = trainingData.CloneWith(data.Test);
                var engine       = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);

                // build the network
                const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30;
                var       memory  = new float[HIDDEN_LAYER_SIZE];
                var       network = graph.Connect(engine)
                                    .AddSimpleRecurrent(graph.ReluActivation(), memory)
                                    .AddFeedForward(engine.DataSource.OutputSize)
                                    .Add(graph.ReluActivation())
                                    .AddBackpropagationThroughTime(errorMetric)
                ;

                // train the network for twenty iterations, saving the model on each improvement
                Models.ExecutionGraph bestGraph = null;
                engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);

                // export the graph and verify it against some unseen integers on the best model
                var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
                var testData2       = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
                var results         = executionEngine.Execute(testData2);

                // group the output
                var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
Esempio n. 4
0
        public static void IntegerAddition()
        {
            // generate 1000 random integer additions
            var dataSet = BinaryIntegers.Addition(1000, false)
                          .Select(l => l.ToArray())
                          .ToList()
            ;

            // split the numbers into training and test sets
            int split        = Convert.ToInt32(dataSet.Count * 0.8);
            var trainingData = dataSet.Take(split).ToList();
            var testData     = dataSet.Skip(split).ToList();

            // neural network hyper parameters
            const int   HIDDEN_SIZE = 32, NUM_EPOCHS = 25, BATCH_SIZE = 16;
            const float TRAINING_RATE = 0.001f;
            var         errorMetric   = ErrorMetricType.BinaryClassification.Create();
            var         layerTemplate = new LayerDescriptor(0.3f)
            {
                Activation           = ActivationType.Relu,
                WeightInitialisation = WeightInitialisationType.Xavier,
                WeightUpdate         = WeightUpdateType.RMSprop
            };
            var recurrentTemplate = layerTemplate.Clone();

            recurrentTemplate.WeightInitialisation = WeightInitialisationType.Gaussian;

            using (var lap = Provider.CreateLinearAlgebra()) {
                // create training data providers
                var trainingDataProvider = lap.NN.CreateSequentialTrainingDataProvider(trainingData);
                var testDataProvider     = lap.NN.CreateSequentialTrainingDataProvider(testData);
                var layers = new INeuralNetworkRecurrentLayer[] {
                    lap.NN.CreateSimpleRecurrentLayer(trainingDataProvider.InputSize, HIDDEN_SIZE, recurrentTemplate),
                    lap.NN.CreateFeedForwardRecurrentLayer(HIDDEN_SIZE, trainingDataProvider.OutputSize, layerTemplate)
                };

                // train the network
                RecurrentNetwork networkData = null;
                using (var trainer = lap.NN.CreateRecurrentBatchTrainer(layers)) {
                    var memory          = Enumerable.Range(0, HIDDEN_SIZE).Select(i => 0f).ToArray();
                    var trainingContext = lap.NN.CreateTrainingContext(errorMetric, TRAINING_RATE, BATCH_SIZE);
                    trainingContext.RecurrentEpochComplete += (tc, rtc) => {
                        var testError = trainer.Execute(testDataProvider, memory, rtc).SelectMany(s => s.Select(d => errorMetric.Compute(d.Output, d.ExpectedOutput))).Average();
                        Console.WriteLine($"Epoch {tc.CurrentEpoch} - score: {testError:P}");
                    };
                    trainer.Train(trainingDataProvider, memory, NUM_EPOCHS, lap.NN.CreateRecurrentTrainingContext(trainingContext));
                    networkData        = trainer.NetworkInfo;
                    networkData.Memory = new FloatArray {
                        Data = memory
                    };
                }

                // evaluate the network on some freshly generated data
                var network = lap.NN.CreateRecurrent(networkData);
                foreach (var sequence in BinaryIntegers.Addition(8, true))
                {
                    var result = network.Execute(sequence.Select(d => d.Input).ToList());
                    Console.Write("First:     ");
                    foreach (var item in sequence)
                    {
                        _WriteBinary(item.Input[0]);
                    }
                    Console.WriteLine();

                    Console.Write("Second:    ");
                    foreach (var item in sequence)
                    {
                        _WriteBinary(item.Input[1]);
                    }
                    Console.WriteLine();
                    Console.WriteLine("           --------------------------------");

                    Console.Write("Expected:  ");
                    foreach (var item in sequence)
                    {
                        _WriteBinary(item.Output[0]);
                    }
                    Console.WriteLine();

                    Console.Write("Predicted: ");
                    foreach (var item in result)
                    {
                        _WriteBinary(item.Output[0]);
                    }
                    Console.WriteLine();
                    Console.WriteLine();
                }
            }
        }
Esempio n. 5
0
        public static void IntegerAddition()
        {
            // generate 1000 random integer additions (split into training and test sets)
            var data = BinaryIntegers.Addition(1000, false).Split(0);

            using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) {
                var graph       = new GraphFactory(lap);
                var errorMetric = graph.ErrorMetric.BinaryClassification;

                // modify the property set
                var propertySet = graph.CurrentPropertySet
                                  .Use(graph.GradientDescent.Adam)
                                  .Use(graph.WeightInitialisation.Xavier)
                ;

                // create the engine
                var trainingData = graph.CreateDataSource(data.Training);
                var testData     = trainingData.CloneWith(data.Test);
                var engine       = graph.CreateTrainingEngine(trainingData, 0.01f, 16);

                // build the network
                const int HIDDEN_LAYER_SIZE = 32;
                var       memory            = new float[HIDDEN_LAYER_SIZE];
                var       network           = graph.Connect(engine)
                                              .AddSimpleRecurrent(graph.ReluActivation(), memory)
                                              .AddFeedForward(engine.DataSource.OutputSize)
                                              .Add(graph.ReluActivation())
                                              .AddBackpropagationThroughTime(errorMetric)
                ;

                // train the network
                GraphModel bestNetwork = null;
                engine.Train(30, testData, errorMetric, bn => bestNetwork = bn);

                // export the graph and verify it against some unseen integers
                var executionEngine = graph.CreateEngine(bestNetwork.Graph);
                var testData2       = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
                var results         = executionEngine.Execute(testData2);

                // group the output
                var groupedResults = new Tuple <FloatVector[], FloatVector[], FloatVector[]> [8];
                for (var i = 0; i < 8; i++)
                {
                    var input  = new FloatVector[32];
                    var target = new FloatVector[32];
                    var output = new FloatVector[32];
                    for (var j = 0; j < 32; j++)
                    {
                        input[j]  = results[j].Input[0][i];
                        target[j] = results[j].Target[i];
                        output[j] = results[j].Output[i];
                    }
                    groupedResults[i] = Tuple.Create(input, target, output);
                }

                // write the results
                foreach (var result in groupedResults)
                {
                    Console.Write("First:     ");
                    foreach (var item in result.Item1)
                    {
                        _WriteBinary(item.Data[0]);
                    }
                    Console.WriteLine();

                    Console.Write("Second:    ");
                    foreach (var item in result.Item1)
                    {
                        _WriteBinary(item.Data[1]);
                    }
                    Console.WriteLine();
                    Console.WriteLine("           --------------------------------");

                    Console.Write("Expected:  ");
                    foreach (var item in result.Item2)
                    {
                        _WriteBinary(item.Data[0]);
                    }
                    Console.WriteLine();

                    Console.Write("Predicted: ");
                    foreach (var item in result.Item3)
                    {
                        _WriteBinary(item.Data[0]);
                    }
                    Console.WriteLine();
                    Console.WriteLine();
                }
            }
        }