static void ManyToOne() { var grammar = new SequenceClassification(dictionarySize: 10, minSize: 5, maxSize: 5, noRepeat: true, isStochastic: false); var sequences = grammar.GenerateSequences().Take(1000).ToList(); var builder = BrightWireProvider.CreateDataTableBuilder(); builder.AddColumn(ColumnType.Matrix, "Sequence"); builder.AddColumn(ColumnType.Vector, "Summary"); foreach (var sequence in sequences) { var list = new List <FloatVector>(); var charSet = new HashSet <char>(); foreach (var ch in sequence) { charSet.Add(ch); var row = grammar.Encode(charSet.Select(ch2 => (ch2, 1f))); list.Add(row); } builder.Add(new FloatMatrix { Row = list.ToArray() }, list.Last()); } var data = builder.Build().Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.BinaryClassification; // create the property set var propertySet = graph.CurrentPropertySet .Use(graph.GradientDescent.RmsProp) .Use(graph.WeightInitialisation.Xavier) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, 0.03f, 8); // build the network const int HIDDEN_LAYER_SIZE = 128; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddLstm(memory) //.AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.SigmoidActivation()) .AddBackpropagationThroughTime(errorMetric) ; engine.Train(10, testData, errorMetric); var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); var output = executionEngine.Execute(testData); Console.WriteLine(output.Where(o => o.Target != null).Average(o => o.CalculateError(errorMetric))); } }
public void TestRecurrent() { var data = BinaryIntegers.Addition(100, false).Split(0); var graph = new GraphFactory(_lap); var errorMetric = graph.ErrorMetric.BinaryClassification; graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement BrightWire.Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); }
public static IGraphEngine LoadTestingNetwork(string path, GraphFactory graph) { IGraphEngine engine = null; using (var file = new FileStream(path, FileMode.Open, FileAccess.Read)) { var model = Serializer.Deserialize <GraphModel>(file); engine = graph.CreateEngine(model.Graph); } return(engine); }
static void OneToMany() { var grammar = new SequenceGenerator(dictionarySize: 10, minSize: 5, maxSize: 5, noRepeat: true, isStochastic: false); var sequences = grammar.GenerateSequences().Take(1000).ToList(); var builder = BrightWireProvider.CreateDataTableBuilder(); builder.AddColumn(ColumnType.Vector, "Summary"); builder.AddColumn(ColumnType.Matrix, "Sequence"); foreach (var sequence in sequences) { var sequenceData = sequence.GroupBy(ch => ch).Select(g => (g.Key, g.Count())). ToDictionary(d => d.Item1, d => (float)d.Item2); var summary = grammar.Encode(sequenceData.Select(kv => (kv.Key, kv.Value))); var list = new List <FloatVector>(); foreach (var item in sequenceData.OrderBy(kv => kv.Key)) { var row = grammar.Encode(item.Key, item.Value); list.Add(row); } builder.Add(summary, FloatMatrix.Create(list.ToArray())); } var data = builder.Build().Split(0); using var lap = BrightWireProvider.CreateLinearAlgebra(false); var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.BinaryClassification; // create the property set var propertySet = graph.CurrentPropertySet.Use(graph.GradientDescent.RmsProp). Use(graph.WeightInitialisation.Xavier); // create the engine const float TRAINING_RATE = 0.1f; var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 8); engine.LearningContext.ScheduleLearningRate(30, TRAINING_RATE / 3); // build the network const int HIDDEN_LAYER_SIZE = 128; graph.Connect(engine).AddLstm(HIDDEN_LAYER_SIZE).AddFeedForward(engine.DataSource.OutputSize). Add(graph.SigmoidActivation()).AddBackpropagation(errorMetric); engine.Train(40, testData, errorMetric); var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); var output = executionEngine.Execute(testData); Console.WriteLine(output.Average(o => o.CalculateError(errorMetric))); }
public static int TestItems(string modelPath, Bitmap testImage) { using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); DataSet testDataset = BuildTestSet(graph, testImage); var errorMetric = graph.ErrorMetric.OneHotEncoding; var config = new NetworkConfig(); config.ERROR_METRIC = errorMetric; var engine = LoadTestingNetwork(modelPath, graph); var executionEngine = graph.CreateEngine(engine.Graph); var output = executionEngine.Execute(testDataset.TestData); return(GetLargestPercent(output[0])); } }
public static float TrainCNN(string dataFolderPath, string outputModelPath) { using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); var dataset = CreateDataset(graph, dataFolderPath); var errorMetric = graph.ErrorMetric.OneHotEncoding; var config = new NetworkConfig(); config.ERROR_METRIC = errorMetric; var engine = BuildNetwork(config, graph, dataset, outputModelPath); var bestGraph = TrainModel(engine, config, dataset, outputModelPath); var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(dataset.TestData); return(output.Average(o => o.CalculateError(errorMetric))); } }
public void TiedAutoEncoder() { const int DATA_SIZE = 1000, REDUCED_SIZE = 200; // create some random data var rand = new Random(); var builder = BrightWireProvider.CreateDataTableBuilder(); builder.AddVectorColumn(DATA_SIZE, "Input"); builder.AddVectorColumn(DATA_SIZE, "Output", true); for (var i = 0; i < 100; i++) { var vector = new FloatVector { Data = Enumerable.Range(0, DATA_SIZE).Select(j => Convert.ToSingle(rand.NextDouble())).ToArray() }; builder.Add(vector, vector); } var dataTable = builder.Build(); // build the autoencoder with tied weights var graph = new GraphFactory(_lap); var dataSource = graph.CreateDataSource(dataTable); var engine = graph.CreateTrainingEngine(dataSource, 0.03f, 32); var errorMetric = graph.ErrorMetric.Quadratic; graph.CurrentPropertySet .Use(graph.RmsProp()) .Use(graph.WeightInitialisation.Xavier) ; graph.Connect(engine) .AddFeedForward(REDUCED_SIZE, "layer") .Add(graph.TanhActivation()) .AddTiedFeedForward(engine.Start.FindByName("layer") as IFeedForward) .Add(graph.TanhActivation()) .AddBackpropagation(errorMetric) ; using (var executionContext = graph.CreateExecutionContext()) { for (var i = 0; i < 2; i++) { var trainingError = engine.Train(executionContext); } } var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); var results = executionEngine.Execute(dataTable.GetRow(0).GetField <FloatVector>(0).Data); }
static void MNISTConvolutional(string dataFilesPath) { using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var trainingData = _BuildTensors(graph, null, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte")); var testData = _BuildTensors(graph, trainingData, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte")); Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); var errorMetric = graph.ErrorMetric.OneHotEncoding; var propertySet = graph.CurrentPropertySet .Use(graph.GradientDescent.RmsProp) .Use(graph.WeightInitialisation.Xavier) ; // create the network const int HIDDEN_LAYER_SIZE = 1024; const float LEARNING_RATE = 0.05f; var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE, 32); engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 3); graph.Connect(engine) .AddConvolutional(8, 2, 5, 5, 1, false) .Add(graph.ReluActivation()) .AddDropOut(0.5f) .AddMaxPooling(2, 2, 2) .AddConvolutional(16, 2, 5, 5, 1) .Add(graph.ReluActivation()) .AddMaxPooling(2, 2, 2) .Transpose() .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.ReluActivation()) .AddDropOut(0.5f) .AddFeedForward(trainingData.OutputSize) .Add(graph.SoftMaxActivation()) .AddBackpropagation(errorMetric) ; engine.Train(20, testData, errorMetric); // export the graph and verify that the error is the same var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); var output = executionEngine.Execute(testData); Console.WriteLine(output.Average(o => o.CalculateError(errorMetric))); } }
/// <summary> /// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> public static void MNIST(string dataFilesPath) { using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte")); var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte")); Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); // one hot encoding uses the index of the output vector's maximum value as the classification label var errorMetric = graph.ErrorMetric.OneHotEncoding; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.RmsProp) .Use(graph.WeightInitialisation.Xavier) ; // create the training engine and schedule a training rate change const float TRAINING_RATE = 0.1f; var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128); engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3); // create the network graph.Connect(engine) .AddFeedForward(outputSize: 1024) .Add(graph.LeakyReluActivation()) .AddDropOut(dropOutPercentage: 0.5f) .AddFeedForward(outputSize: trainingData.OutputSize) .Add(graph.SigmoidActivation()) .AddBackpropagation(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(20, testData, errorMetric, model => bestGraph = model.Graph); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}"); } }
public static void IntegerAddition() { // generate 1000 random integer additions (split into training and test sets) var data = BinaryIntegers.Addition(1000, false).Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); // binary classification rounds each output to either 0 or 1 var errorMetric = graph.ErrorMetric.BinaryClassification; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); // group the output var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
/// <summary> /// Trains a neural net on the MNIST database (digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> public static void MNIST(string dataFilesPath) { using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte")); var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte")); Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); // use a one hot encoding error metric, rmsprop gradient descent and xavier weight initialisation var errorMetric = graph.ErrorMetric.OneHotEncoding; var propertySet = graph.CurrentPropertySet //.Use(graph.Regularisation.L2) .Use(graph.GradientDescent.RmsProp) .Use(graph.WeightInitialisation.Xavier) ; // create the training engine and schedule a training rate change const float TRAINING_RATE = 0.1f; var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128); engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3); // create the network graph.Connect(engine) .AddFeedForward(1024) .Add(graph.LeakyReluActivation()) .AddDropOut(0.5f) .AddFeedForward(trainingData.OutputSize) .Add(graph.SigmoidActivation()) .AddBackpropagation(errorMetric) ; // train the network engine.Train(20, testData, errorMetric); // export the graph and verify that the error is the same var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); var output = executionEngine.Execute(testData); Console.WriteLine(output.Average(o => o.CalculateError(errorMetric))); } }
protected override void OnCreate(Bundle savedInstanceState) { base.OnCreate(savedInstanceState); var sb = new StringBuilder(); using (var lap = BrightWireProvider.CreateLinearAlgebra()) { // Create some training data that the network will learn. The XOR pattern looks like: // 0 0 => 0 // 1 0 => 1 // 0 1 => 1 // 1 1 => 0 var data = Xor.Get(); // create the graph var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.CrossEntropy; graph.CurrentPropertySet // use rmsprop gradient descent optimisation .Use(graph.GradientDescent.RmsProp) // and xavier weight initialisation .Use(graph.WeightInitialisation.Gaussian) ; // create the engine var testData = graph.CreateDataSource(data); var engine = graph.CreateTrainingEngine(testData, 0.1f, 4); // create the network const int HIDDEN_LAYER_SIZE = 6; graph.Connect(engine) // create a feed forward layer with sigmoid activation .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.SigmoidActivation()) // create a second feed forward layer with sigmoid activation .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.SigmoidActivation()) // backpropagate the error signal at the end of the graph .AddBackpropagation(errorMetric) ; // train the network var executionContext = graph.CreateExecutionContext(); for (var i = 0; i < 1000; i++) { var trainingError = engine.Train(executionContext); if (i % 100 == 0) { engine.Test(testData, errorMetric); } } engine.Test(testData, errorMetric); // create a new network to execute the learned network var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); var output = executionEngine.Execute(testData); // print the learnt values foreach (var item in output) { foreach (var index in item.MiniBatchSequence.MiniBatch.Rows) { var row = data.GetRow(index); var result = item.Output[index]; sb.AppendLine($"{row.GetField<int>(0)} XOR {row.GetField<int>(1)} = {result.Data[0]}"); } } } // Set our view from the "main" layout resource SetContentView(Resource.Layout.Main); var outputText = FindViewById <TextView>(Resource.Id.outputText); outputText.Text = sb.ToString(); }
public static void XOR() { using (var lap = BrightWireProvider.CreateLinearAlgebra()) { // Create some training data that the network will learn. The XOR pattern looks like: // 0 0 => 0 // 1 0 => 1 // 0 1 => 1 // 1 1 => 0 var data = Xor.Get(); // create the graph var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.CrossEntropy; graph.CurrentPropertySet // use rmsprop gradient descent optimisation .Use(graph.GradientDescent.RmsProp) // and xavier weight initialisation .Use(graph.WeightInitialisation.Gaussian) ; // create the engine var testData = graph.CreateDataSource(data); var engine = graph.CreateTrainingEngine(testData, 0.1f, 4); // create the network const int HIDDEN_LAYER_SIZE = 6; graph.Connect(engine) // create a feed forward layer with sigmoid activation .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.SigmoidActivation()) // create a second feed forward layer with sigmoid activation .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.SigmoidActivation()) // backpropagate the error signal at the end of the graph .AddBackpropagation(errorMetric) ; // train the network var executionContext = graph.CreateExecutionContext(); for (var i = 0; i < 1000; i++) { var trainingError = engine.Train(executionContext); if (i % 100 == 0) { engine.Test(testData, errorMetric); } } engine.Test(testData, errorMetric); // create a new network to execute the learned network var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); var output = executionEngine.Execute(testData); Console.WriteLine(output.Average(o => o.CalculateError(errorMetric))); // print the learnt values foreach (var item in output) { foreach (var index in item.MiniBatchSequence.MiniBatch.Rows) { var row = data.GetRow(index); var result = item.Output[index]; Console.WriteLine($"{row.GetField<int>(0)} XOR {row.GetField<int>(1)} = {result.Data[0]}"); } } } }
/// <summary> /// Classifies text into either positive or negative sentiment /// The data files can be downloaded from https://archive.ics.uci.edu/ml/datasets/Sentiment+Labelled+Sentences /// </summary> /// <param name="dataFilesPath">Path to extracted data files</param> public static void SentimentClassification(string dataFilesPath) { var files = new[] { "amazon_cells_labelled.txt", "imdb_labelled.txt", "yelp_labelled.txt" }; var LINE_SEPARATOR = "\n".ToCharArray(); var SEPARATOR = "\t".ToCharArray(); var stringTable = new StringTableBuilder(); var sentimentData = files.SelectMany(f => File.ReadAllText(dataFilesPath + f).Split(LINE_SEPARATOR). Where(l => !string.IsNullOrWhiteSpace(l)).Select(l => l.Split(SEPARATOR)). Select(s => Tuple.Create(Tokenise(s[0]), s[1][0] == '1' ? "positive" : "negative")). Where(d => d.Item1.Any())).Shuffle(0).ToList(); var splitSentimentData = sentimentData.Split(); // build training and test classification bag var trainingClassificationBag = BuildIndexedClassifications(splitSentimentData.Training, stringTable); var testClassificationBag = BuildIndexedClassifications(splitSentimentData.Test, stringTable); // train a bernoulli naive bayes classifier var bernoulli = trainingClassificationBag.TrainBernoulliNaiveBayes(); Console.WriteLine("Bernoulli accuracy: {0:P}", testClassificationBag.Classify(bernoulli.CreateClassifier()).Average(r => r.Score)); // train a multinomial naive bayes classifier var multinomial = trainingClassificationBag.TrainMultinomialNaiveBayes(); Console.WriteLine("Multinomial accuracy: {0:P}", testClassificationBag.Classify(multinomial.CreateClassifier()).Average(r => r.Score)); // convert the index lists to vectors and normalise along the way var sentimentDataTable = BuildIndexedClassifications(sentimentData, stringTable). ConvertToTable().Normalise(NormalisationType.Standard); var vectoriser = sentimentDataTable.GetVectoriser(); var sentimentDataSet = sentimentDataTable.Split(0); var dataTableAnalysis = sentimentDataTable.GetAnalysis(); using (var lap = BrightWireProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); var trainingData = graph.CreateDataSource(sentimentDataSet.Training, vectoriser); var testData = graph.CreateDataSource(sentimentDataSet.Test, vectoriser); var indexListEncoder = (IIndexListEncoder)trainingData; // use a one hot encoding error metric, rmsprop gradient descent and xavier weight initialisation var errorMetric = graph.ErrorMetric.OneHotEncoding; var propertySet = graph.CurrentPropertySet.Use(graph.GradientDescent.RmsProp). Use(graph.WeightInitialisation.Xavier); var engine = graph.CreateTrainingEngine(trainingData, 0.3f); engine.LearningContext.ScheduleLearningRate(5, 0.1f); engine.LearningContext.ScheduleLearningRate(11, 1f); engine.LearningContext.ScheduleLearningRate(15, 0.3f); // train a neural network classifier var neuralNetworkWire = graph.Connect(engine).AddFeedForward(512, "layer1") //.AddBatchNormalisation() .Add(graph.ReluActivation()).AddDropOut(0.5f). AddFeedForward(trainingData.OutputSize, "layer2").Add(graph.ReluActivation()). AddBackpropagation(errorMetric, "first-network"); // train the network Console.WriteLine("Training neural network classifier..."); const int TRAINING_ITERATIONS = 10; GraphModel bestNetwork = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, network => bestNetwork = network); if (bestNetwork != null) { engine.LoadParametersFrom(bestNetwork.Graph); } var firstClassifier = graph.CreateEngine(engine.Graph); // stop the backpropagation to the first neural network engine.LearningContext.EnableNodeUpdates(neuralNetworkWire.Find("layer1"), false); engine.LearningContext.EnableNodeUpdates(neuralNetworkWire.Find("layer2"), false); // create the bernoulli classifier wire var bernoulliClassifier = bernoulli.CreateClassifier(); var bernoulliWire = graph.Connect(engine).AddClassifier(bernoulliClassifier, sentimentDataSet.Training, dataTableAnalysis); // create the multinomial classifier wire var multinomialClassifier = multinomial.CreateClassifier(); var multinomialWire = graph.Connect(engine).AddClassifier(multinomialClassifier, sentimentDataSet.Training, dataTableAnalysis); // join the bernoulli, multinomial and neural network classification outputs var firstNetwork = neuralNetworkWire.Find("first-network"); var joined = graph.Join(multinomialWire, graph.Join(bernoulliWire, graph.Connect(trainingData.OutputSize, firstNetwork))); // train an additional classifier on the output of the previous three classifiers joined.AddFeedForward(outputSize: 64).Add(graph.ReluActivation()). AddDropOut(dropOutPercentage: 0.5f).AddFeedForward(trainingData.OutputSize). Add(graph.ReluActivation()).AddBackpropagation(errorMetric); // train the network again Console.WriteLine("Training stacked neural network classifier..."); GraphModel bestStackedNetwork = null; engine.Train(10, testData, errorMetric, network => bestStackedNetwork = network); if (bestStackedNetwork != null) { engine.LoadParametersFrom(bestStackedNetwork.Graph); } Console.WriteLine("Enter some text to test the classifiers..."); while (true) { Console.Write(">"); var line = Console.ReadLine(); if (string.IsNullOrWhiteSpace(line)) { break; } var tokens = Tokenise(line); var indexList = new List <uint>(); foreach (var token in tokens) { if (stringTable.TryGetIndex(token, out uint stringIndex)) { indexList.Add(stringIndex); } } if (indexList.Any()) { var queryTokens = indexList.GroupBy(d => d). Select(g => Tuple.Create(g.Key, (float)g.Count())).ToList(); var vector = new float[trainingData.InputSize]; foreach (var token in queryTokens) { vector[token.Item1] = token.Item2; } var indexList2 = IndexList.Create(indexList.ToArray()); var encodedInput = indexListEncoder.Encode(indexList2); Console.WriteLine("Bernoulli classification: " + bernoulliClassifier.Classify(indexList2).First().Label); Console.WriteLine("Multinomial classification: " + multinomialClassifier.Classify(indexList2).First().Label); var result = firstClassifier.Execute(encodedInput); var classification = vectoriser.GetOutputLabel(1, (result.Output[0].Data[0] > result.Output[0].Data[1]) ? 0 : 1); Console.WriteLine("Neural network classification: " + classification); var stackedResult = engine.Execute(encodedInput); var stackedClassification = vectoriser.GetOutputLabel(1, (stackedResult.Output[0].Data[0] > stackedResult.Output[0].Data[1]) ? 0 : 1); Console.WriteLine("Stack classification: " + stackedClassification); } else { Console.WriteLine("Sorry, none of those words have been seen before."); } Console.WriteLine(); } } Console.WriteLine(); }
static void ReberPrediction() { // generate 500 extended reber grammar training examples var grammar = new ReberGrammar(); var sequences = grammar.GetExtended(10, 16).Take(500).ToList(); // split the data into training and test sets var data = ReberGrammar.GetOneHot(sequences).Split(0); using var lap = BrightWireProvider.CreateLinearAlgebra(); var graph = new GraphFactory(lap); // binary classification rounds each output to either 0 or 1 var errorMetric = graph.ErrorMetric.BinaryClassification; // configure the network properties graph.CurrentPropertySet.Use(graph.GradientDescent.RmsProp). Use(graph.WeightInitialisation.Xavier); // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.1f, batchSize: 32); // build the network const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30; graph.Connect(engine).AddGru(HIDDEN_LAYER_SIZE).AddFeedForward(engine.DataSource.OutputSize). Add(graph.SigmoidActivation()).AddBackpropagationThroughTime(errorMetric); engine.Train(TRAINING_ITERATIONS, testData, errorMetric); // generate a sample sequence using the learned state transitions var networkGraph = engine.Graph; var executionEngine = graph.CreateEngine(networkGraph); Console.WriteLine("Generating new reber sequences from the observed state probabilities..."); for (var z = 0; z < 3; z++) { // prepare the first input var input = new float[ReberGrammar.Size]; input[ReberGrammar.GetIndex('B')] = 1f; Console.Write("B"); int index = 0, eCount = 0; using var executionContext = graph.CreateExecutionContext(); var result = executionEngine.ExecuteSequential(index++, input, executionContext, MiniBatchSequenceType.SequenceStart); for (var i = 0; i < 32; i++) { var next = result.Output[0].Data.Select((v, j) => ((double)v, j)). Where(d => d.Item1 >= 0.1f).ToList(); var distribution = new Categorical(next.Select(d => d.Item1).ToArray()); var nextIndex = next[distribution.Sample()].Item2; Console.Write(ReberGrammar.GetChar(nextIndex)); if (nextIndex == ReberGrammar.GetIndex('E') && ++eCount == 2) { break; } Array.Clear(input, 0, ReberGrammar.Size); input[nextIndex] = 1f; result = executionEngine.ExecuteSequential(index++, input, executionContext, MiniBatchSequenceType.Standard); } Console.WriteLine(); } }
/// <summary> /// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition) /// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/ /// </summary> /// <param name="dataFilesPath">The path to a directory with the four extracted data files</param> /// <param name="outputModelPath">Optional path to save the best model to</param> static void MNISTConvolutional(string dataFilesPath, string outputModelPath = null) { using var lap = BrightWireGpuProvider.CreateLinearAlgebra(); var graph = new GraphFactory(lap); Console.Write("Loading training data..."); var mnistTraining = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"); var mnistTest = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"); var trainingData = _BuildTensors(graph, null, mnistTraining /*.Where(d => d.Label < 2).ToList()*/); var testData = _BuildTensors(graph, trainingData, mnistTest /*.Where(d => d.Label < 2).ToList()*/); Console.WriteLine( $"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded"); // one hot encoding uses the index of the output vector's maximum value as the classification label var errorMetric = graph.ErrorMetric.OneHotEncoding; // configure the network properties graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use( graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); // create the network const int HIDDEN_LAYER_SIZE = 1024, TRAINING_ITERATIONS = 20; const float LEARNING_RATE = 0.05f; var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE); if (!string.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath)) { Console.WriteLine("Loading existing model from: " + outputModelPath); using var file = new FileStream(outputModelPath, FileMode.Open, FileAccess.Read); var model = Serializer.Deserialize <GraphModel>(file); engine = graph.CreateTrainingEngine(trainingData, model.Graph, LEARNING_RATE); } else { graph.Connect(engine). AddConvolutional(filterCount: 16, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1, yStride: 1, shouldBackpropagate: false).Add(graph.LeakyReluActivation()). AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2). AddConvolutional(filterCount: 32, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1, yStride: 1).Add(graph.LeakyReluActivation()). AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).Transpose(). AddFeedForward(HIDDEN_LAYER_SIZE).Add(graph.LeakyReluActivation()). AddDropOut(dropOutPercentage: 0.5f).AddFeedForward(trainingData.OutputSize). Add(graph.SoftMaxActivation()).AddBackpropagation(errorMetric); } // lower the learning rate over time engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 2); // train the network for twenty iterations, saving the model on each improvement Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model => { bestGraph = model.Graph; if (!string.IsNullOrWhiteSpace(outputModelPath)) { using var file = new FileStream(outputModelPath, FileMode.Create, FileAccess.Write); Serializer.Serialize(file, model); } }); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}"); // execute the model with a single image var tensor = mnistTest.First().AsFloatTensor.Tensor; var singleData = graph.CreateDataSource(new[] { tensor }); var result = executionEngine.Execute(singleData); var prediction = result.Single().Output.Single().MaximumIndex(); }
static void ReberPrediction() { var grammar = new ReberGrammar(false); var sequences = grammar.GetExtended(10).Take(500).ToList(); var data = ReberGrammar.GetOneHot(sequences).Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.BinaryClassification; var propertySet = graph.CurrentPropertySet .Use(graph.GradientDescent.RmsProp) .Use(graph.WeightInitialisation.Xavier) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, 0.1f, 32); // build the network const int HIDDEN_LAYER_SIZE = 64; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) //.AddSimpleRecurrent(graph.ReluActivation(), memory) .AddGru(memory) //.AddRan(memory) //.AddLstm(memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.TanhActivation()) .AddBackpropagationThroughTime(errorMetric) ; engine.Train(30, testData, errorMetric); // generate sample sequence var networkGraph = engine.Graph; using (var executionContext = graph.CreateExecutionContext()) { var executionEngine = graph.CreateEngine(networkGraph); Console.WriteLine("Generating a new reber sequence..."); var input = new float[ReberGrammar.Size]; input[ReberGrammar.GetIndex('B')] = 1f; Console.Write("B"); int index = 0, eCount = 0; var result = executionEngine.ExecuteSequential(index++, input, executionContext, MiniBatchSequenceType.SequenceStart); for (var i = 0; i < 32; i++) { var nextIndex = result.Output[0].Data .Select((v, j) => (v, j)) .Where(d => d.Item1 >= 0.5f) .Select(d => d.Item2) .Shuffle() .FirstOrDefault() ; if (index == 0) { break; } Console.Write(ReberGrammar.GetChar(nextIndex)); if (nextIndex == ReberGrammar.GetIndex('E') && ++eCount == 2) { break; } input = new float[ReberGrammar.Size]; input[nextIndex] = 1f; result = executionEngine.ExecuteSequential(index++, input, executionContext, MiniBatchSequenceType.Standard); } Console.WriteLine(); } } }
/// <summary> /// Uses a recurrent LSTM neural network to predict stock price movements /// Data can be downloaded from https://raw.githubusercontent.com/plotly/datasets/master/stockdata.csv /// </summary> static void StockData(string dataFilePath) { // load and normalise the data var dataSet = new StreamReader(dataFilePath).ParseCSV(',', true); var normalised = dataSet.Normalise(NormalisationType.FeatureScale); var rows = normalised.GetNumericRows(dataSet.Columns.Where(c => c.Name != "Date").Select(c => c.Index)); // build the data table with a window of input data and the prediction as the following value var builder = BrightWireProvider.CreateDataTableBuilder(); builder.AddColumn(ColumnType.Matrix, "Past"); builder.AddColumn(ColumnType.Vector, "Future"); const int LAST_X_DAYS = 14; for (var i = 0; i < rows.Count - LAST_X_DAYS - 1; i++) { var inputVector = new List <FloatVector>(); for (var j = 0; j < LAST_X_DAYS; j++) { inputVector.Add(FloatVector.Create(rows[i + j])); } var input = FloatMatrix.Create(inputVector.ToArray()); var target = FloatVector.Create(rows[i + LAST_X_DAYS + 1]); builder.Add(input, target); } var data = builder.Build().Split(trainingPercentage: 0.2); using (var lap = BrightWireProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.Quadratic; // create the property set graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.WeightInitialisation.Xavier); // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.03f, batchSize: 128); // build the network const int HIDDEN_LAYER_SIZE = 256; graph.Connect(engine) .AddLstm(HIDDEN_LAYER_SIZE) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.TanhActivation()) .AddBackpropagationThroughTime(errorMetric); // train the network and restore the best result GraphModel bestNetwork = null; engine.Train(50, testData, errorMetric, model => bestNetwork = model); if (bestNetwork != null) { // execute each row of the test data on an execution engine var executionEngine = graph.CreateEngine(bestNetwork.Graph); var results = executionEngine.Execute(testData).OrderSequentialOutput(); var expectedOutput = data.Test.GetColumn <FloatVector>(1); var score = results.Select((r, i) => errorMetric.Compute(r.Last(), expectedOutput[i])).Average(); Console.WriteLine(score); } } }
public static void IntegerAddition() { // generate 1000 random integer additions (split into training and test sets) var data = BinaryIntegers.Addition(1000, false).Split(0); using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); var errorMetric = graph.ErrorMetric.BinaryClassification; // modify the property set var propertySet = graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.WeightInitialisation.Xavier) ; // create the engine var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, 0.01f, 16); // build the network const int HIDDEN_LAYER_SIZE = 32; var memory = new float[HIDDEN_LAYER_SIZE]; var network = graph.Connect(engine) .AddSimpleRecurrent(graph.ReluActivation(), memory) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.ReluActivation()) .AddBackpropagationThroughTime(errorMetric) ; // train the network GraphModel bestNetwork = null; engine.Train(30, testData, errorMetric, bn => bestNetwork = bn); // export the graph and verify it against some unseen integers var executionEngine = graph.CreateEngine(bestNetwork.Graph); var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true)); var results = executionEngine.Execute(testData2); // group the output var groupedResults = new Tuple <FloatVector[], FloatVector[], FloatVector[]> [8]; for (var i = 0; i < 8; i++) { var input = new FloatVector[32]; var target = new FloatVector[32]; var output = new FloatVector[32]; for (var j = 0; j < 32; j++) { input[j] = results[j].Input[0][i]; target[j] = results[j].Target[i]; output[j] = results[j].Output[i]; } groupedResults[i] = Tuple.Create(input, target, output); } // write the results foreach (var result in groupedResults) { Console.Write("First: "); foreach (var item in result.Item1) { _WriteBinary(item.Data[0]); } Console.WriteLine(); Console.Write("Second: "); foreach (var item in result.Item1) { _WriteBinary(item.Data[1]); } Console.WriteLine(); Console.WriteLine(" --------------------------------"); Console.Write("Expected: "); foreach (var item in result.Item2) { _WriteBinary(item.Data[0]); } Console.WriteLine(); Console.Write("Predicted: "); foreach (var item in result.Item3) { _WriteBinary(item.Data[0]); } Console.WriteLine(); Console.WriteLine(); } } }
/// <summary> /// Trains a feed forward neural net on the emotion dataset /// http://lpis.csd.auth.gr/publications/tsoumakas-ismir08.pdf /// The data files can be downloaded from https://downloads.sourceforge.net/project/mulan/datasets/emotions.rar /// </summary> /// <param name="dataFilePath"></param> public static void MultiLabelSingleClassifier(string dataFilePath) { var emotionData = _LoadEmotionData(dataFilePath); var attributeColumns = Enumerable.Range(0, emotionData.ColumnCount - CLASSIFICATION_COUNT).ToList(); var classificationColumns = Enumerable.Range(emotionData.ColumnCount - CLASSIFICATION_COUNT, CLASSIFICATION_COUNT).ToList(); // create a new data table with a vector input column and a vector output column var dataTableBuilder = BrightWireProvider.CreateDataTableBuilder(); dataTableBuilder.AddColumn(ColumnType.Vector, "Attributes"); dataTableBuilder.AddColumn(ColumnType.Vector, "Target", isTarget: true); emotionData.ForEach(row => { var input = FloatVector.Create(row.GetFields <float>(attributeColumns).ToArray()); var target = FloatVector.Create(row.GetFields <float>(classificationColumns).ToArray()); dataTableBuilder.Add(input, target); return(true); }); var data = dataTableBuilder.Build().Split(0); // train a neural network using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) { var graph = new GraphFactory(lap); // binary classification rounds each output to 0 or 1 and compares each output against the binary classification targets var errorMetric = graph.ErrorMetric.BinaryClassification; // configure the network properties graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.WeightInitialisation.Xavier) ; // create a training engine const float TRAINING_RATE = 0.3f; var trainingData = graph.CreateDataSource(data.Training); var testData = trainingData.CloneWith(data.Test); var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128); // build the network const int HIDDEN_LAYER_SIZE = 64, TRAINING_ITERATIONS = 2000; var network = graph.Connect(engine) .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.SigmoidActivation()) .AddDropOut(dropOutPercentage: 0.5f) .AddFeedForward(engine.DataSource.OutputSize) .Add(graph.SigmoidActivation()) .AddBackpropagation(errorMetric) ; // train the network Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model => bestGraph = model.Graph, 50); // export the final model and execute it on the training set var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); var output = executionEngine.Execute(testData); // output the results var rowIndex = 0; foreach (var item in output) { var sb = new StringBuilder(); foreach (var classification in item.Output.Zip(item.Target, (o, t) => (Output: o, Target: t))) { var columnIndex = 0; sb.AppendLine($"{rowIndex++}) "); foreach (var column in classification.Output.Data.Zip(classification.Target.Data, (o, t) => (Output: o, Target: t))) { var prediction = column.Output >= 0.5f ? "true" : "false"; var actual = column.Target >= 0.5f ? "true" : "false"; sb.AppendLine($"\t{columnIndex++}) predicted {prediction} (expected {actual})"); } } Console.WriteLine(sb.ToString()); } } }
public static void CrazyTest() { using (var la = BrightWireProvider.CreateLinearAlgebra()) { var graph = new GraphFactory(la); graph.CurrentPropertySet .Use(graph.GradientDescent.Adam) .Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N)); var dtbuilder = BrightWireProvider.CreateDataTableBuilder(); dtbuilder.AddColumn(ColumnType.Tensor, "image", false); dtbuilder.AddColumn(ColumnType.Vector, "expectedoutput", true); // DATEN GENERIEREN for (int i = 0; i < 666; i++) { var fv = new BrightWire.Models.FloatVector { Data = new float[666], }; var fv2 = new BrightWire.Models.FloatVector { Data = new float[666], }; dtbuilder.Add(fv, fv2); } var datatable = dtbuilder.Build(); Console.WriteLine($"####### TRAINING START #######"); // create the engine var(Training, Test) = datatable.Split(); var trainingData = graph.CreateDataSource(Training); var testData = graph.CreateDataSource(Test); var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16); // build the network var errorMetric = graph.ErrorMetric.Quadratic; const int HIDDEN_LAYER_SIZE = 2000, TRAINING_ITERATIONS = 2000; var network = graph.Connect(engine) .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.ReluActivation()) //.AddBackpropagation(errorMetric) .AddFeedForward(HIDDEN_LAYER_SIZE) .Add(graph.ReluActivation()) //.AddBackpropagation(errorMetric) .AddDropOut(.2f) .AddFeedForward(engine.DataSource.OutputSize) //.Add(graph.ReluActivation()) .AddBackpropagation(errorMetric) //.AddBackpropagationThroughTime(errorMetric) ; // train the network for twenty iterations, saving the model on each improvement BrightWire.Models.ExecutionGraph bestGraph = null; engine.Train(TRAINING_ITERATIONS, testData, errorMetric, ((modelGraph) => bestGraph = modelGraph.Graph)); // export the graph and verify it against some unseen integers on the best model var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph); } }