public static void TrainWithSelu(string dataFilesPath) { using (var lap = BrightWireProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); // parse the iris CSV into a data table and normalise var dataTable = new StreamReader(new MemoryStream(File.ReadAllBytes(dataFilesPath))).ParseCSV(',').Normalise(NormalisationType.Standard); // split the data table into training and test tables var split = dataTable.Split(0); var trainingData = graph.CreateDataSource(split.Training); var testData = graph.CreateDataSource(split.Test); // one hot encoding uses the index of the output vector's maximum value as the classification label var errorMetric = graph.ErrorMetric.OneHotEncoding; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.RmsProp) .Use(graph.GaussianWeightInitialisation(true, 0.1f, GaussianVarianceCalibration.SquareRoot2N, GaussianVarianceCount.FanInFanOut)) ; // create the training engine and schedule a training rate change const float TRAINING_RATE = 0.1f; var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, batchSize: 128); const int LAYER_SIZE = 64; Func <INode> activation = () => new SeluActivation(); //Func<INode> activation = () => graph.ReluActivation(); // create the network with the custom activation function graph.Connect(engine) .AddFeedForward(LAYER_SIZE) .AddBatchNormalisation() .Add(activation()) .AddFeedForward(LAYER_SIZE) .AddBatchNormalisation() .Add(activation()) .AddFeedForward(LAYER_SIZE) .AddBatchNormalisation() .Add(activation()) .AddFeedForward(LAYER_SIZE) .AddBatchNormalisation() .Add(activation()) .AddFeedForward(LAYER_SIZE) .AddBatchNormalisation() .Add(activation()) .AddFeedForward(LAYER_SIZE) .AddBatchNormalisation() .Add(activation()) .AddFeedForward(trainingData.OutputSize) .Add(graph.SoftMaxActivation()) .AddBackpropagation(errorMetric) ; const int TRAINING_ITERATIONS = 500; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, null, 50); } }
public void TestRecurrent() { var data = BinaryIntegers.Addition(100, false).Split(0); var graph = new GraphFactory(_lap); var errorMetric = graph.ErrorMetric.BinaryClassification; graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement BrightWire.Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); }
public static IGraphTrainingEngine BuildNetwork(NetworkConfig config, GraphFactory graph, DataSet dataset, string outputModelPath = null) { graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(config.ZERO_BIAS, config.STANDARD_DEVIATION, config.GAUSSIAN_VARIANCE_CALIBRATION)); var engine = graph.CreateTrainingEngine(dataset.TrainData, config.LEARNING_RATE, config.BATCH_SIZE); if (!String.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath)) { engine = LoadTrainingNetwork(outputModelPath, graph, config, dataset); } else { graph = CreateStandardNetwork(engine, graph, config, dataset); } engine.LearningContext.ScheduleLearningRate(15, config.LEARNING_RATE / 2); return(engine); }
/// <summary> /// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> public static void MNIST(string dataFilesPath) { using (var lap = BrightWireProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte")); var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte")); Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); // one hot encoding uses the index of the output vector's maximum value as the classification label var errorMetric = graph.ErrorMetric.OneHotEncoding; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)) ; // create the training engine and schedule a training rate change const float TRAINING_RATE = 0.1f; var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128); engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3); // create the network graph.Connect(engine) .AddFeedForward(outputSize: 1024) .Add(graph.LeakyReluActivation()) .AddDropOut(dropOutPercentage: 0.5f) .AddFeedForward(outputSize: trainingData.OutputSize) .Add(graph.SoftMaxActivation()) .AddBackpropagation(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(20, testData, errorMetric, model => bestGraph = model.Graph); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}"); } }
public static void IntegerAddition() { // generate 1000 random integer additions (split into training and test sets) var data = BinaryIntegers.Addition(1000, false).Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); // binary classification rounds each output to either 0 or 1 var errorMetric = graph.ErrorMetric.BinaryClassification; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); // group the output var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
public static void TrainWithSelu(string dataFilesPath) { using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); // parse the iris CSV into a data table and normalise var dataTable = new StreamReader(new MemoryStream(File.ReadAllBytes(dataFilesPath))).ParseCSV(',').Normalise(NormalisationType.Standard); // split the data table into training and test tables var split = dataTable.Split(0); var trainingData = graph.CreateDataSource(split.Training); var testData = graph.CreateDataSource(split.Test); // use a one hot encoding error metric, rmsprop gradient descent and xavier weight initialisation var errorMetric = graph.ErrorMetric.OneHotEncoding; var propertySet = graph.CurrentPropertySet .Use(graph.GradientDescent.RmsProp) .Use(graph.GaussianWeightInitialisation(true, 0.1f, GaussianVarianceCalibration.SquareRoot2N, GaussianVarianceCount.FanInFanOut)) ; // create the training engine and schedule a training rate change const float TRAINING_RATE = 0.01f; var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128); // create the network graph.Connect(engine) .AddFeedForward(32) .Add(new SeluActivation()) .AddFeedForward(trainingData.OutputSize) .Add(graph.SigmoidActivation()) .AddBackpropagation(errorMetric) ; // train the network engine.Train(1000, testData, errorMetric, null, 50); } }
/// <summary> /// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> /// <param name="outputModelPath">Optional path to save the best model to</param> static void MNISTConvolutional(string dataFilesPath, string outputModelPath = null) { using var lap = BrightWireGpuProvider.CreateLinearAlgebra(); var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var mnistTraining = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"); var mnistTest = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"); var trainingData = _BuildTensors(graph, null, mnistTraining /*.Where(d => d.Label < 2).ToList()*/); var testData = _BuildTensors(graph, trainingData, mnistTest /*.Where(d => d.Label < 2).ToList()*/); Console.WriteLine( $"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); // one hot encoding uses the index of the output vector's maximum value as the classification label var errorMetric = graph.ErrorMetric.OneHotEncoding; // configure the network properties graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use( graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); // create the network const int HIDDEN_LAYER_SIZE = 1024, TRAINING_ITERATIONS = 20; const float LEARNING_RATE = 0.05f; var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE); if (!string.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath)) { Console.WriteLine("Loading existing model from: " + outputModelPath); using var file = new FileStream(outputModelPath, FileMode.Open, FileAccess.Read); var model = Serializer.Deserialize <GraphModel>(file); engine = graph.CreateTrainingEngine(trainingData, model.Graph, LEARNING_RATE); } else { graph.Connect(engine). AddConvolutional(filterCount: 16, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1, yStride: 1, shouldBackpropagate: false).Add(graph.LeakyReluActivation()). AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2). AddConvolutional(filterCount: 32, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1, yStride: 1).Add(graph.LeakyReluActivation()). AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).Transpose(). AddFeedForward(HIDDEN_LAYER_SIZE).Add(graph.LeakyReluActivation()). AddDropOut(dropOutPercentage: 0.5f).AddFeedForward(trainingData.OutputSize). Add(graph.SoftMaxActivation()).AddBackpropagation(errorMetric); } // lower the learning rate over time engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 2); // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model => { bestGraph = model.Graph; if (!string.IsNullOrWhiteSpace(outputModelPath)) { using var file = new FileStream(outputModelPath, FileMode.Create, FileAccess.Write); Serializer.Serialize(file, model); } }); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}"); // execute the model with a single image var tensor = mnistTest.First().AsFloatTensor.Tensor; var singleData = graph.CreateDataSource(new[] { tensor }); var result = executionEngine.Execute(singleData); var prediction = result.Single().Output.Single().MaximumIndex(); }
public static void CrazyTest() { using (var la = BrightWireProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(la); graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); var dtbuilder = BrightWireProvider.CreateDataTableBuilder(); dtbuilder.AddColumn(ColumnType.Tensor, "image", false); dtbuilder.AddColumn(ColumnType.Vector, "expectedoutput", true); // DATEN GENERIEREN for (int i = 0; i < 666; i++) { var fv = new BrightWire.Models.FloatVector { Data = new float[666], }; var fv2 = new BrightWire.Models.FloatVector { Data = new float[666], }; dtbuilder.Add(fv, fv2); } var datatable = dtbuilder.Build(); Console.WriteLine($"####### TRAINING START #######"); // create the engine var(Training, Test) = datatable.Split(); var trainingData = graph.CreateDataSource(Training); var testData = graph.CreateDataSource(Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network var errorMetric = graph.ErrorMetric.Quadratic; const int HIDDEN_LAYER_SIZE = 2000, TRAINING_ITERATIONS = 2000; var network = graph.Connect(engine) .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.ReluActivation()) //.AddBackpropagation(errorMetric) .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.ReluActivation()) //.AddBackpropagation(errorMetric) .AddDropOut(.2f) .AddFeedForward(engine.DataSource.OutputSize) //.Add(graph.ReluActivation()) .AddBackpropagation(errorMetric) //.AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement BrightWire.Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, ((modelGraph) => bestGraph = modelGraph.Graph)); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); } }