public void ComputeTwiceGradientShouldYieldTheSameResult()
{
const int inputWidth = 20;
const int inputHeight = 20;
const int inputDepth = 2;
var layer = new LeakyReluLayer();
layer.Init(inputWidth, inputHeight, inputDepth);
// Forward pass
var input = BuilderInstance.Volume.Random(new Shape(inputWidth, inputHeight, inputDepth));
var output = layer.DoForward(input, true);
// Set output gradients to 1
var outputGradient = BuilderInstance.Volume.From(new double[output.Shape.TotalLength].Populate(1.0), output.Shape);
// Backward pass to retrieve gradients
layer.Backward(outputGradient);
var step1 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray();
layer.Backward(outputGradient);
var step2 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray();
Assert.IsTrue(step1.SequenceEqual(step2));
}
public static LeakyReluLayer<T> LeakyRelu<T>(this LayerBase<T> layer, T alpha) where T : struct, IEquatable<T>, IFormattable
{
var relu = new LeakyReluLayer<T>(alpha);
relu.AcceptParent(layer);
return relu;
}
public static LeakyReluLayer<T> LeakyRelu<T>(this ConvLayer<T> layer, T alpha) where T : struct, IEquatable<T>, IFormattable
{
var relu = new LeakyReluLayer<T>(alpha);
relu.AcceptParent(layer);
layer.BiasPref = (T)Convert.ChangeType(0.1, typeof(T)); // can we do better?
return relu;
}
public void GradientWrtInputCheck()
{
const int inputWidth = 20;
const int inputHeight = 20;
const int inputDepth = 2;
const int batchSize = 3;
// Create layer
var layer = new LeakyReluLayer();
GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth, batchSize, 1e-6);
}
public NeuralNetwork(Random r, int[] layer_nodes)
{
layers = new Layer[layer_nodes.Length];
for (int i = 0; i < layers.Length; i++)
{
layers[i] = new LeakyReluLayer(layer_nodes[i]);
if (i != layers.Length - 1)
{
layers[i].weights = new Matrix(layer_nodes[i + 1], layer_nodes[i]);
layers[i].weights.randomize(r);
}
layers[i].bias = new Matrix(layer_nodes[i], 1);
layers[i].bias.randomize(r);
}
}
public void LeakyReluLayerSerialization()
{
var layer = new LeakyReluLayer();
layer.Init(28, 24, 1);
var data = layer.GetData();
Assert.AreEqual(28, data["InputWidth"]);
Assert.AreEqual(24, data["InputHeight"]);
Assert.AreEqual(1, data["InputDepth"]);
var deserialized = LayerBase <double> .FromData(data) as LeakyReluLayer;
Assert.IsNotNull(deserialized);
Assert.AreEqual(28, deserialized.InputWidth);
Assert.AreEqual(24, deserialized.InputHeight);
Assert.AreEqual(1, deserialized.InputDepth);
Assert.AreEqual(layer.OutputWidth, deserialized.OutputWidth);
Assert.AreEqual(layer.OutputHeight, deserialized.OutputHeight);
Assert.AreEqual(layer.OutputDepth, deserialized.OutputDepth);
}
public void TerminalCommand(String command)
{
Console.WriteLine(command);
try
{
command = command.ToLower();
if (String.IsNullOrEmpty(command) || String.IsNullOrWhiteSpace(command))
{
Console.WriteLine("That's not a valid command... Type \'help\' if you don\'t know what command to use.");
return;
}
String[] commands = command.Split(' ');
String head = commands[0];
if (head == "neuralnetwork")
{
Model network = NeuralNetworkHandler.network;
if (commands.Length == 1)
{
if (network == null)
{
Console.WriteLine("There is no Network currently loaded.");
}
else
{
String message =
"Current loaded network:\n" +
"\tName:\t" + network.Name + "\n" +
"\tType:\t" + network.Type;
Console.WriteLine(message);
}
}
else if (commands[1] == "create")
{
if (network == null)
{
List <LayerBase> layers = new List <LayerBase>();
for (int i = 4; i < commands.Length; i += 2)
{
LayerBase l = null;
if (commands[i] == "input")
{
l = new InputLayer();
}
else if (commands[i] == "lrelu")
{
l = new LeakyReluLayer();
}
else if (commands[i] == "sigmoid")
{
l = new SigmoidLayer();
}
else
{
Console.Error.WriteLine(String.Format("Layer type {0} does not exist.", commands[i]));
return;
}
l.nodes = int.Parse(commands[i + 1]);
layers.Add(l);
}
NeuralNetworkHandler.CreateNetwork(layers.ToArray(), commands[2], commands[3]);
Console.WriteLine("Neural Network successfully created.");
return;
}
}
else if (commands[1] == "load")
{
Console.WriteLine(NeuralNetworkHandler.LoadNetwork(commands[2]));
}
else if (commands[1] == "save")
{
if (commands.Length == 2)
{
Console.WriteLine(NeuralNetworkHandler.SaveNetwork());
}
else
{
Console.WriteLine(NeuralNetworkHandler.SaveNetwork(commands[2]));
}
}
else if (commands[1] == "randomize")
{
Console.WriteLine(NeuralNetworkHandler.RandomizeNetwork());
}
else if (commands[1] == "train")
{
if (network == null)
{
Console.Error.WriteLine("No network loaded.");
return;
}
if (NeuralNetworkHandler.keeper == null)
{
Console.Error.WriteLine("No dataset loaded.");
return;
}
int epochs = 1;
if (commands.Length != 2)
{
epochs = int.Parse(commands[2]);
}
NeuralNetworkHandler.train = new Thread(() => NeuralNetworkHandler.TrainNetwork(epochs));
NeuralNetworkHandler.train.IsBackground = true;
NeuralNetworkHandler.train.Name = "Training Thread";
NeuralNetworkHandler.train.Start();
}
else if (commands[1] == "learningrate")
{
if (commands.Length == 2)
{
Console.WriteLine("Learning rate: " + Model.LearningRate);
}
else
{
Model.LearningRate = float.Parse(commands[2]);
}
}
}
else if (head == "test")
{
Data[] data = new Data[3600];
float x = 0;
for (int i = 0; i < data.Length; i++)
{
data[i] = new Data();
data[i].Inputs = new Matrix(1, 1)
{
data = new float[, ] {
{ (1f / 360f) * x }
}
};
data[i].Targets = new Matrix(1, 1)
{
data = new float[, ] {
{ (float)Math.Sin(x * (Math.PI / 180)) }
}
};
x += 0.1f;
}
NeuralNetworkHandler.keeper = new DataKeeper();
NeuralNetworkHandler.keeper.DataSet = data;
NeuralNetworkHandler.keeper.Name = "sinusfunction";
}
else if (head == "dataset")
{
if (commands.Length == 1)
{
if (NeuralNetworkHandler.keeper == null)
{
Console.WriteLine("There is no dataset loaded yet.");
}
else
{
Console.WriteLine(String.Format("The dataset with the name {0} is currently loaded.", NeuralNetworkHandler.keeper.Name));
}
}
else if (commands[1] == "load")
{
Console.WriteLine(DataKeeper.LoadDataSet(commands[2]));
}
else if (commands[1] == "save")
{
if (commands.Length == 2)
{
Console.WriteLine(DataKeeper.SaveDataSet());
}
else
{
Console.WriteLine(DataKeeper.SaveDataSet(commands[2]));
}
}
else if (commands[1] == "unload")
{
NeuralNetworkHandler.keeper = null;
}
else if (commands[1] == "view")
{
if (NeuralNetworkHandler.keeper == null)
{
Console.Error.WriteLine("No dataset loaded.");
return;
}
for (int i = 0; i < NeuralNetworkHandler.keeper.DataSet.Length; i++)
{
Console.WriteLine("Data " + i + ":");
Console.WriteLine("Inputs:");
Console.WriteLine(NeuralNetworkHandler.keeper.DataSet[i].getInputString());
Console.WriteLine("Targets:");
Console.WriteLine(NeuralNetworkHandler.keeper.DataSet[i].getTargetsString());
Console.WriteLine("");
}
}
}
else if (head == "googledrive")
{
if (commands[1] == "login")
{
GoogleDriveHandler.GoogleDriveLogin(HttpRuntime.AppDomainAppPath + "/credentials.json");
Console.WriteLine("Successfully logged in on Google Drive.");
}
else if (commands[1] == "logout")
{
GoogleDriveHandler.GoogleDriveLogout("credentials.json");
Console.WriteLine("Successfully logged out from Google Drive.");
}
else if (commands[1] == "list")
{
IList <Google.Apis.Drive.v3.Data.File> list = GoogleDriveHandler.GetFileList();
foreach (var file in list)
{
Console.WriteLine(String.Format("{0} ({1})", file.Name, file.Id));
}
}
}
else
{
Console.Error.WriteLine("That's not a valid command... Type \'help\' if you don\'t know what command to use.");
}
} catch (Exception e)
{
Console.Error.WriteLine("Command resolved in an error.");
Console.Error.WriteLine(e.Message);
Console.Error.WriteLine(e.StackTrace);
}
}
public static NeuralNetwork loadNetwork(String file_name)
{
String text;
String[] sections;
Layer[] layers;
using (StreamReader streamReader = new StreamReader("models\\" + file_name + ".network", Encoding.UTF8))
{
text = streamReader.ReadToEnd();
sections = text.Split("#".ToCharArray());
}
String[] values = sections[0].Split(";".ToCharArray());
layers = new Layer[values.Length];
for (int i = 0; i < sections.Length; i++)
{
if (i == 0)
{
for (int j = 0; j < layers.Length; j++)
{
String[] curText = values[j].Split(" ".ToCharArray());
if (curText[0] == "relu")
{
int layer_nodes = int.Parse(curText[1]);
layers[j] = new ReluLayer(layer_nodes);
}
else if (curText[0] == "do")
{
int layer_nodes = int.Parse(curText[1]);
layers[j] = new DropoutLayer(layer_nodes);
}
else if (curText[0] == "lrelu")
{
int layer_nodes = int.Parse(curText[1]);
layers[j] = new LeakyReluLayer(layer_nodes);
}
if (j == layers.Length - 1)
{
layers[j].weights = new Matrix(1, 1);
}
}
}
else if (layers[i - 1].weights == null)
{
int index = i - 1;
values = sections[i].Split(";".ToCharArray());
int rows = values.Length;
int cols = values[0].Split(" ".ToCharArray()).Length;
Matrix m = new Matrix(rows, cols);
for (int j = 0; j < values.Length; j++)
{
String[] comp = values[j].Split(" ".ToCharArray());
for (int k = 0; k < comp.Length; k++)
{
m.data[j, k] = float.Parse(comp[k]);
}
}
Matrix.table(m);
layers[index].weights = m;
}
}
NeuralNetwork nn = new NeuralNetwork(layers);
return(nn);
}
