public void ComputeTwiceGradientShouldYieldTheSameResult() { const int inputWidth = 20; const int inputHeight = 20; const int inputDepth = 2; var layer = new LeakyReluLayer(); layer.Init(inputWidth, inputHeight, inputDepth); // Forward pass var input = BuilderInstance.Volume.Random(new Shape(inputWidth, inputHeight, inputDepth)); var output = layer.DoForward(input, true); // Set output gradients to 1 var outputGradient = BuilderInstance.Volume.From(new double[output.Shape.TotalLength].Populate(1.0), output.Shape); // Backward pass to retrieve gradients layer.Backward(outputGradient); var step1 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray(); layer.Backward(outputGradient); var step2 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray(); Assert.IsTrue(step1.SequenceEqual(step2)); }
public static LeakyReluLayer<T> LeakyRelu<T>(this LayerBase<T> layer, T alpha) where T : struct, IEquatable<T>, IFormattable { var relu = new LeakyReluLayer<T>(alpha); relu.AcceptParent(layer); return relu; }
public static LeakyReluLayer<T> LeakyRelu<T>(this ConvLayer<T> layer, T alpha) where T : struct, IEquatable<T>, IFormattable { var relu = new LeakyReluLayer<T>(alpha); relu.AcceptParent(layer); layer.BiasPref = (T)Convert.ChangeType(0.1, typeof(T)); // can we do better? return relu; }
public void GradientWrtInputCheck() { const int inputWidth = 20; const int inputHeight = 20; const int inputDepth = 2; const int batchSize = 3; // Create layer var layer = new LeakyReluLayer(); GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth, batchSize, 1e-6); }
public NeuralNetwork(Random r, int[] layer_nodes) { layers = new Layer[layer_nodes.Length]; for (int i = 0; i < layers.Length; i++) { layers[i] = new LeakyReluLayer(layer_nodes[i]); if (i != layers.Length - 1) { layers[i].weights = new Matrix(layer_nodes[i + 1], layer_nodes[i]); layers[i].weights.randomize(r); } layers[i].bias = new Matrix(layer_nodes[i], 1); layers[i].bias.randomize(r); } }
public void LeakyReluLayerSerialization() { var layer = new LeakyReluLayer(); layer.Init(28, 24, 1); var data = layer.GetData(); Assert.AreEqual(28, data["InputWidth"]); Assert.AreEqual(24, data["InputHeight"]); Assert.AreEqual(1, data["InputDepth"]); var deserialized = LayerBase <double> .FromData(data) as LeakyReluLayer; Assert.IsNotNull(deserialized); Assert.AreEqual(28, deserialized.InputWidth); Assert.AreEqual(24, deserialized.InputHeight); Assert.AreEqual(1, deserialized.InputDepth); Assert.AreEqual(layer.OutputWidth, deserialized.OutputWidth); Assert.AreEqual(layer.OutputHeight, deserialized.OutputHeight); Assert.AreEqual(layer.OutputDepth, deserialized.OutputDepth); }
public void TerminalCommand(String command) { Console.WriteLine(command); try { command = command.ToLower(); if (String.IsNullOrEmpty(command) || String.IsNullOrWhiteSpace(command)) { Console.WriteLine("That's not a valid command... Type \'help\' if you don\'t know what command to use."); return; } String[] commands = command.Split(' '); String head = commands[0]; if (head == "neuralnetwork") { Model network = NeuralNetworkHandler.network; if (commands.Length == 1) { if (network == null) { Console.WriteLine("There is no Network currently loaded."); } else { String message = "Current loaded network:\n" + "\tName:\t" + network.Name + "\n" + "\tType:\t" + network.Type; Console.WriteLine(message); } } else if (commands[1] == "create") { if (network == null) { List <LayerBase> layers = new List <LayerBase>(); for (int i = 4; i < commands.Length; i += 2) { LayerBase l = null; if (commands[i] == "input") { l = new InputLayer(); } else if (commands[i] == "lrelu") { l = new LeakyReluLayer(); } else if (commands[i] == "sigmoid") { l = new SigmoidLayer(); } else { Console.Error.WriteLine(String.Format("Layer type {0} does not exist.", commands[i])); return; } l.nodes = int.Parse(commands[i + 1]); layers.Add(l); } NeuralNetworkHandler.CreateNetwork(layers.ToArray(), commands[2], commands[3]); Console.WriteLine("Neural Network successfully created."); return; } } else if (commands[1] == "load") { Console.WriteLine(NeuralNetworkHandler.LoadNetwork(commands[2])); } else if (commands[1] == "save") { if (commands.Length == 2) { Console.WriteLine(NeuralNetworkHandler.SaveNetwork()); } else { Console.WriteLine(NeuralNetworkHandler.SaveNetwork(commands[2])); } } else if (commands[1] == "randomize") { Console.WriteLine(NeuralNetworkHandler.RandomizeNetwork()); } else if (commands[1] == "train") { if (network == null) { Console.Error.WriteLine("No network loaded."); return; } if (NeuralNetworkHandler.keeper == null) { Console.Error.WriteLine("No dataset loaded."); return; } int epochs = 1; if (commands.Length != 2) { epochs = int.Parse(commands[2]); } NeuralNetworkHandler.train = new Thread(() => NeuralNetworkHandler.TrainNetwork(epochs)); NeuralNetworkHandler.train.IsBackground = true; NeuralNetworkHandler.train.Name = "Training Thread"; NeuralNetworkHandler.train.Start(); } else if (commands[1] == "learningrate") { if (commands.Length == 2) { Console.WriteLine("Learning rate: " + Model.LearningRate); } else { Model.LearningRate = float.Parse(commands[2]); } } } else if (head == "test") { Data[] data = new Data[3600]; float x = 0; for (int i = 0; i < data.Length; i++) { data[i] = new Data(); data[i].Inputs = new Matrix(1, 1) { data = new float[, ] { { (1f / 360f) * x } } }; data[i].Targets = new Matrix(1, 1) { data = new float[, ] { { (float)Math.Sin(x * (Math.PI / 180)) } } }; x += 0.1f; } NeuralNetworkHandler.keeper = new DataKeeper(); NeuralNetworkHandler.keeper.DataSet = data; NeuralNetworkHandler.keeper.Name = "sinusfunction"; } else if (head == "dataset") { if (commands.Length == 1) { if (NeuralNetworkHandler.keeper == null) { Console.WriteLine("There is no dataset loaded yet."); } else { Console.WriteLine(String.Format("The dataset with the name {0} is currently loaded.", NeuralNetworkHandler.keeper.Name)); } } else if (commands[1] == "load") { Console.WriteLine(DataKeeper.LoadDataSet(commands[2])); } else if (commands[1] == "save") { if (commands.Length == 2) { Console.WriteLine(DataKeeper.SaveDataSet()); } else { Console.WriteLine(DataKeeper.SaveDataSet(commands[2])); } } else if (commands[1] == "unload") { NeuralNetworkHandler.keeper = null; } else if (commands[1] == "view") { if (NeuralNetworkHandler.keeper == null) { Console.Error.WriteLine("No dataset loaded."); return; } for (int i = 0; i < NeuralNetworkHandler.keeper.DataSet.Length; i++) { Console.WriteLine("Data " + i + ":"); Console.WriteLine("Inputs:"); Console.WriteLine(NeuralNetworkHandler.keeper.DataSet[i].getInputString()); Console.WriteLine("Targets:"); Console.WriteLine(NeuralNetworkHandler.keeper.DataSet[i].getTargetsString()); Console.WriteLine(""); } } } else if (head == "googledrive") { if (commands[1] == "login") { GoogleDriveHandler.GoogleDriveLogin(HttpRuntime.AppDomainAppPath + "/credentials.json"); Console.WriteLine("Successfully logged in on Google Drive."); } else if (commands[1] == "logout") { GoogleDriveHandler.GoogleDriveLogout("credentials.json"); Console.WriteLine("Successfully logged out from Google Drive."); } else if (commands[1] == "list") { IList <Google.Apis.Drive.v3.Data.File> list = GoogleDriveHandler.GetFileList(); foreach (var file in list) { Console.WriteLine(String.Format("{0} ({1})", file.Name, file.Id)); } } } else { Console.Error.WriteLine("That's not a valid command... Type \'help\' if you don\'t know what command to use."); } } catch (Exception e) { Console.Error.WriteLine("Command resolved in an error."); Console.Error.WriteLine(e.Message); Console.Error.WriteLine(e.StackTrace); } }
public static NeuralNetwork loadNetwork(String file_name) { String text; String[] sections; Layer[] layers; using (StreamReader streamReader = new StreamReader("models\\" + file_name + ".network", Encoding.UTF8)) { text = streamReader.ReadToEnd(); sections = text.Split("#".ToCharArray()); } String[] values = sections[0].Split(";".ToCharArray()); layers = new Layer[values.Length]; for (int i = 0; i < sections.Length; i++) { if (i == 0) { for (int j = 0; j < layers.Length; j++) { String[] curText = values[j].Split(" ".ToCharArray()); if (curText[0] == "relu") { int layer_nodes = int.Parse(curText[1]); layers[j] = new ReluLayer(layer_nodes); } else if (curText[0] == "do") { int layer_nodes = int.Parse(curText[1]); layers[j] = new DropoutLayer(layer_nodes); } else if (curText[0] == "lrelu") { int layer_nodes = int.Parse(curText[1]); layers[j] = new LeakyReluLayer(layer_nodes); } if (j == layers.Length - 1) { layers[j].weights = new Matrix(1, 1); } } } else if (layers[i - 1].weights == null) { int index = i - 1; values = sections[i].Split(";".ToCharArray()); int rows = values.Length; int cols = values[0].Split(" ".ToCharArray()).Length; Matrix m = new Matrix(rows, cols); for (int j = 0; j < values.Length; j++) { String[] comp = values[j].Split(" ".ToCharArray()); for (int k = 0; k < comp.Length; k++) { m.data[j, k] = float.Parse(comp[k]); } } Matrix.table(m); layers[index].weights = m; } } NeuralNetwork nn = new NeuralNetwork(layers); return(nn); }