public void TestTrainingRegression_MeanSquared_L2()
{
float[] referenceOutput = new float[] { 2.466907E-12f, 0.2984998f, 3.949761E-08f, 0.000338129f, 4.667237E-06f };
Network network = Network.CreateNetworkFromJSON(Testing.Properties.Resources.ReferenceNetwork1JSON);
Utils.TestTraining(network, ComputeDeviceFactory.CreateFallbackComputeDevice(), referenceOutput, new MeanSquaredErrorFunction(), TrainingConfig.Regularization.L2, 10.0f, 0.01f);
}
public void TestOpenCLLayerCalc()
{
List <int> layerConfig = new List <int>();
layerConfig.Add(10);
layerConfig.Add(512);
layerConfig.Add(12);
layerConfig.Add(3);
layerConfig.Add(51);
layerConfig.Add(30);
Network networkReference = Network.CreateNetworkInitRandom(layerConfig.ToArray(), new SigmoidActivation());
var jsonData = networkReference.ExportToJSON();
Network networkCpuTrained = Network.CreateNetworkFromJSON(jsonData);
Network networkOpenCLTrained = Network.CreateNetworkFromJSON(jsonData);
ComputeDevice cpuCalculator = ComputeDeviceFactory.CreateFallbackComputeDevice();
ComputeDevice openCLCalculator = Utils.GetFirstOpenCLDevice();
float[] testInput = new float[layerConfig[0]];
var cpuTrainedOutput = networkCpuTrained.Compute(testInput, cpuCalculator);
var openCLTrainedOutput = networkOpenCLTrained.Compute(testInput, openCLCalculator);
Utils.CheckNetworkError(cpuTrainedOutput, openCLTrainedOutput);
}
public void TestTrainingRegression_CrossEntropy()
{
float[] referenceOutput = new float[] { 3.45829E-11f, 0.139514f, 3.661705E-05f, 0.3912812f, 1.076772E-06f };
Network network = Network.CreateNetworkFromJSON(Testing.Properties.Resources.ReferenceNetwork1JSON);
Utils.TestTraining(network, ComputeDeviceFactory.CreateFallbackComputeDevice(), referenceOutput, new CrossEntropyErrorFunction(), TrainingConfig.Regularization.None, 0.0f, 0.01f);
}
public void TestTrainingRegression_CrossEntropy_L1()
{
float[] referenceOutput = new float[] { 2.571606E-11f, 0.1361623f, 3.719596E-05f, 0.3950554f, 8.929147E-07f };
Network network = Network.CreateNetworkFromJSON(Testing.Properties.Resources.ReferenceNetwork1JSON);
Utils.TestTraining(network, ComputeDeviceFactory.CreateFallbackComputeDevice(), referenceOutput, new CrossEntropyErrorFunction(), TrainingConfig.Regularization.L1, 10.0f, 0.01f);
}
public void TestTrainingRegression_CrossEntropy_L2()
{
float[] referenceOutput = new float[] { 5.959933E-11f, 0.1458118f, 4.22751E-05f, 0.3619123f, 1.531221E-06f };
Network network = Network.CreateNetworkFromJSON(Testing.Properties.Resources.ReferenceNetwork1JSON);
Utils.TestTraining(network, ComputeDeviceFactory.CreateFallbackComputeDevice(), referenceOutput, new CrossEntropyErrorFunction(), TrainingConfig.Regularization.L2, 10.0f, 0.01f);
}
public void TestTrainingRegression_MeanSquared()
{
float[] referenceOutput = new float[] { 1.485432E-12f, 0.3179092f, 3.043587E-08f, 0.0003209518f, 4.003032E-06f };
Network network = Network.CreateNetworkFromJSON(Testing.Properties.Resources.ReferenceNetwork1JSON);
Utils.TestTraining(network, ComputeDeviceFactory.CreateFallbackComputeDevice(), referenceOutput, new MeanSquaredErrorFunction(), TrainingConfig.Regularization.None, 0.0f, 0.01f);
}
public void TestTrainingRegression_MeanSquared_L1()
{
float[] referenceOutput = new float[] { 1.059097E-12f, 0.3169f, 3.026366E-08f, 0.0003242506f, 3.843542E-06f };
Network network = Network.CreateNetworkFromJSON(Testing.Properties.Resources.ReferenceNetwork1JSON);
Utils.TestTraining(network, ComputeDeviceFactory.CreateFallbackComputeDevice(), referenceOutput, new MeanSquaredErrorFunction(), TrainingConfig.Regularization.L1, 10.0f, 0.01f);
}
public void TestRandomNetworkWithNoLayers()
{
int[] referenceLayerConf = new int[] { 3, 4 };
var network = Network.CreateNetworkInitRandom(referenceLayerConf, new SigmoidActivation());
float[] result = network.Compute(new float[] { 0.2f, 0.4f, 0.5f }, ComputeDeviceFactory.CreateFallbackComputeDevice());
Assert.AreEqual(referenceLayerConf[referenceLayerConf.Length - 1], result.Length);
}
private static void Train(ComputeDevice selectedDevice, int epochs)
{
List <TrainingSuite.TrainingData> trainingData = new List <TrainingSuite.TrainingData>();
for (int i = 0; i < 10000; i++)
{
float[] input = new float[1];
float[] desiredOutput = new float[1];
float rnd = (float)random.NextDouble();
input[0] = rnd;
desiredOutput[0] = (float)Math.Sin((rnd - 0.5f) * 4.0f) * 0.5f + 0.5f;
trainingData.Add(new TrainingSuite.TrainingData(input, desiredOutput));
}
TrainingSuite suite = new TrainingSuite(trainingData);
suite.config.epochs = epochs;
suite.config.shuffleTrainingData = true;
suite.config.miniBatchSize = 100;
suite.config.costFunction = new CrossEntropyErrorFunction();
suite.config.regularization = TrainingConfig.Regularization.L2;
suite.config.regularizationLambda = 0.01f;
suite.config.learningRate = 0.01f;
Console.WriteLine("Running training for {0} epochs!", epochs);
Stopwatch sw = Stopwatch.StartNew();
int progress = 0;
var promise = target_network.Train(suite, ComputeDeviceFactory.CreateFallbackComputeDevice());
Console.WriteLine("____________________");
while (!promise.IsReady())
{
int progress_rounded = (int)(promise.GetTotalProgress() * 20);
if (progress_rounded > progress)
{
++progress;
Console.Write("#");
}
Thread.Sleep(50);
}
sw.Stop();
Console.WriteLine("#");
Console.WriteLine("Training finished! Elapsed={0}ms", sw.Elapsed.TotalMilliseconds);
}
public void TestRandomNetwork()
{
int[] referenceLayerConf = new int[] { 3, 7, 5, 4 };
var network = Network.CreateNetworkInitRandom(referenceLayerConf, new SigmoidActivation());
Assert.AreEqual(3, network.GetLayerConfig()[0]);
Assert.AreEqual(7, network.GetLayerConfig()[1]);
Assert.AreEqual(5, network.GetLayerConfig()[2]);
Assert.AreEqual(4, network.GetLayerConfig()[3]);
float[] result = network.Compute(new float[] { 0.2f, 0.4f, 0.5f }, ComputeDeviceFactory.CreateFallbackComputeDevice());
Assert.AreEqual(referenceLayerConf[referenceLayerConf.Length - 1], result.Length);
}
private void Form1_Load(object sender, EventArgs e)
{
List <int> layerConfig = new List <int>(new int[] { 5, 32, 32, 5 });
solver = Network.CreateNetworkInitRandom(layerConfig.ToArray(), new SigmoidActivation(), new DefaultWeightInitializer());
calculator = ComputeDeviceFactory.CreateFallbackComputeDevice();
foreach (var device in ComputeDeviceFactory.GetComputeDevices())
{
string item = device.GetDeviceAccessType() + " - " + device.GetDeviceName();
comboBox1.Items.Add(item);
}
comboBox1.SelectedIndex = 0;
}
public static void TestTraining(Network network, float[] referenceOutput, IErrorFunction errorFunc, TrainingSuite.TrainingConfig.Regularization regularization, float regularizationLambda, float learningRate)
{
List <int> layerConfig = new List <int>();
layerConfig.Add(5);
layerConfig.Add(33);
layerConfig.Add(12);
layerConfig.Add(51);
layerConfig.Add(5);
#region Training
List <TrainingSuite.TrainingData> trainingData = new List <TrainingSuite.TrainingData>();
for (int i = 0; i < 1000; i++)
{
float[] input = new float[layerConfig[0]];
float[] desiredOutput = new float[layerConfig[layerConfig.Count - 1]];
input[(i * 13426) % 5] = 1.0f;
desiredOutput[(i * 13426) % 5] = 1.0f;
trainingData.Add(new TrainingSuite.TrainingData(input, desiredOutput));
}
TrainingSuite suite = new TrainingSuite(trainingData);
suite.config.epochs = 2;
suite.config.shuffleTrainingData = false;
suite.config.miniBatchSize = 13;
suite.config.costFunction = errorFunc;
suite.config.regularization = regularization;
suite.config.regularizationLambda = regularizationLambda;
suite.config.learningRate = learningRate;
var promise = network.Train(suite, ComputeDeviceFactory.CreateFallbackComputeDevice());
promise.Await();
#endregion
float[] testInput = new float[] { 0.3f, 0.4f, 0.6f, 0.1f, 0.5f };
var result = network.Compute(testInput, ComputeDeviceFactory.CreateFallbackComputeDevice());
Utils.CheckNetworkError(referenceOutput, result);
}
public void TestJSONExportImport()
{
List <int> layerConfig = new List <int>();
layerConfig.Add(5);
layerConfig.Add(33);
layerConfig.Add(12);
layerConfig.Add(51);
layerConfig.Add(5);
float[] testInput = new float[] { 0.1f, 0.5f, 0.2f, 0.14f, 0.54f };
Network networkReference = Network.CreateNetworkInitRandom(layerConfig.ToArray(), new SigmoidActivation());
Network networkFromJSON = Network.CreateNetworkFromJSON(networkReference.ExportToJSON());
float[] outputRef = networkReference.Compute(testInput, ComputeDeviceFactory.CreateFallbackComputeDevice());
float[] outputJS = networkFromJSON.Compute(testInput, ComputeDeviceFactory.CreateFallbackComputeDevice());
Utils.CheckNetworkError(outputRef, outputJS, 0.00000001);
}
private void Form1_Load(object sender, EventArgs e)
{
comboRegularization.SelectedIndex = 2;
comboCostFunction.SelectedIndex = 1;
calculator = ComputeDeviceFactory.CreateFallbackComputeDevice();
bitmap = new Bitmap(targetWidth, targetHeight, System.Drawing.Imaging.PixelFormat.Format16bppRgb565);
bitmapDownscaled = new Bitmap(downScaleWidth, downScaleHeight, System.Drawing.Imaging.PixelFormat.Format16bppRgb565);
ClearBitmap();
pictureBox1.Image = bitmap;
foreach (var device in ComputeDeviceFactory.GetComputeDevices())
{
string item = device.GetDeviceAccessType() + " - " + device.GetDeviceName();
comboBox1.Items.Add(item);
}
comboBox1.SelectedIndex = 0;
trainingtimer.Interval = 300;
trainingtimer.Tick += Trainingtimer_Tick;
InitRandomNetwork();
}
public static void TestOpenCLTrainingWithConfig(IErrorFunction errorFunc, TrainingSuite.TrainingConfig.Regularization regularization, float regularizationLambda, float learningRate)
{
List <int> layerConfig = new List <int>();
layerConfig.Add(10);
layerConfig.Add(512);
layerConfig.Add(12);
layerConfig.Add(3);
layerConfig.Add(51);
layerConfig.Add(30);
Network networkReference = Network.CreateNetworkInitRandom(layerConfig.ToArray(), new SigmoidActivation());
var jsonData = networkReference.ExportToJSON();
Network networkCpuTrained = Network.CreateNetworkFromJSON(jsonData);
Network networkOpenCLTrained = Network.CreateNetworkFromJSON(jsonData);
var cpuCalculator = ComputeDeviceFactory.CreateFallbackComputeDevice();
var openCLCalculator = GetFirstOpenCLDevice();
var rnd = new Random();
List <TrainingSuite.TrainingData> trainingData = new List <TrainingSuite.TrainingData>();
for (int i = 0; i < 1000; i++)
{
float[] input = new float[layerConfig[0]];
float[] output = new float[layerConfig[layerConfig.Count - 1]];
var idx = rnd.Next(0, input.Length);
input[rnd.Next(0, input.Length)] = 1.0f;
for (int j = 0; j < 10; j++)
{
output[j * 3 + 0] = idx * 0.1f;
output[j * 3 + 1] = 1.0f - (idx * 0.1f);
output[j * 3 + 2] = idx * 0.05f;
}
trainingData.Add(new TrainingSuite.TrainingData(input, output));
}
TrainingSuite suite = new TrainingSuite(trainingData);
suite.config.epochs = 1;
suite.config.shuffleTrainingData = false;
suite.config.miniBatchSize = 13;
suite.config.costFunction = errorFunc;
suite.config.regularization = regularization;
suite.config.regularizationLambda = regularizationLambda;
suite.config.learningRate = learningRate;
var promise1 = networkCpuTrained.Train(suite, cpuCalculator);
var promise2 = networkOpenCLTrained.Train(suite, openCLCalculator);
promise1.Await();
promise2.Await();
Assert.IsTrue(promise1.IsReady() && promise2.IsReady());
float[] testInput = new float[layerConfig[0]];
var cpuTrainedOutput = networkCpuTrained.Compute(testInput, cpuCalculator);
var openCLTrainedOutput = networkOpenCLTrained.Compute(testInput, cpuCalculator);
CheckNetworkError(cpuTrainedOutput, openCLTrainedOutput);
}
public static void TestOpenCLTrainingWithConfig(IErrorFunction errorFunc, TrainingConfig.Regularization regularization, float regularizationLambda, float learningRate, bool mix_activations = false)
{
IActivationFunction alternateActivation = new SigmoidActivation();
if (mix_activations)
{
alternateActivation = new ReLUActivation();
}
int input_neurons = 10;
var layer_config = new List <Tuple <IActivationFunction, int> >();
layer_config.Add(new Tuple <IActivationFunction, int>(new SigmoidActivation(), 512));
layer_config.Add(new Tuple <IActivationFunction, int>(alternateActivation, 12));
layer_config.Add(new Tuple <IActivationFunction, int>(new SigmoidActivation(), 3));
layer_config.Add(new Tuple <IActivationFunction, int>(alternateActivation, 51));
layer_config.Add(new Tuple <IActivationFunction, int>(new SigmoidActivation(), 30));
Network networkReference = Network.CreateNetworkInitRandom(input_neurons, layer_config);
var jsonData = networkReference.ExportToJSON();
Network networkCpuTrained = Network.CreateNetworkFromJSON(jsonData);
Network networkOpenCLTrained = Network.CreateNetworkFromJSON(jsonData);
var cpuCalculator = ComputeDeviceFactory.CreateFallbackComputeDevice();
var openCLCalculator = GetFirstOpenCLDevice();
var rnd = new Random();
List <TrainingSuite.TrainingData> trainingData = new List <TrainingSuite.TrainingData>();
for (int i = 0; i < 1000; i++)
{
float[] input = new float[input_neurons];
float[] output = new float[layer_config.Last().Item2];
var idx = rnd.Next(0, input.Length);
input[rnd.Next(0, input.Length)] = 1.0f;
for (int j = 0; j < 10; j++)
{
output[j * 3 + 0] = idx * 0.1f;
output[j * 3 + 1] = 1.0f - (idx * 0.1f);
output[j * 3 + 2] = idx * 0.05f;
}
trainingData.Add(new TrainingSuite.TrainingData(input, output));
}
TrainingSuite suite = new TrainingSuite(trainingData);
suite.config.epochs = 1;
suite.config.shuffleTrainingData = false;
suite.config.miniBatchSize = 13;
suite.config.costFunction = errorFunc;
suite.config.regularization = regularization;
suite.config.regularizationLambda = regularizationLambda;
suite.config.learningRate = learningRate;
var promise1 = networkCpuTrained.Train(suite, cpuCalculator);
var promise2 = networkOpenCLTrained.Train(suite, openCLCalculator);
promise1.Await();
promise2.Await();
Assert.IsTrue(promise1.IsReady() && promise2.IsReady());
float[] testInput = new float[input_neurons];
var cpuTrainedOutput = networkCpuTrained.Compute(testInput, cpuCalculator);
var openCLTrainedOutput = networkOpenCLTrained.Compute(testInput, cpuCalculator);
ValidateFloatArray(cpuTrainedOutput, openCLTrainedOutput);
}
static void Main(string[] args)
{
Console.WriteLine(" ### macademy test console ");
ComputeDevice selectedDevice = ComputeDeviceFactory.CreateFallbackComputeDevice();
Generate();
while (true)
{
Console.WriteLine("");
Console.Write("> ");
string rawCommand = Console.ReadLine().Trim();
var commands = rawCommand.Split(' ');
if (commands.Length == 0)
{
continue;
}
var nextCommand = commands[0];
try
{
if (nextCommand == "exit")
{
break;
}
else if (nextCommand == "help")
{
Console.WriteLine("General");
Console.WriteLine(" help - Displays this help message");
Console.WriteLine(" exit - Exits the app");
Console.WriteLine("");
Console.WriteLine("Device selection");
Console.WriteLine(" devices - Displays available devices");
Console.WriteLine(" select (n) - Selectes the devices with the given id");
Console.WriteLine(" info - Displays information about the selected device");
Console.WriteLine(" quicktest - Performs a quick test on the selected device");
Console.WriteLine(" benchmark [n] - Performs a benchmark on the selected device, on the given difficulty (1-10, default: 2)");
Console.WriteLine("");
Console.WriteLine("Model training and testing");
Console.WriteLine(" generate - Generates a new network");
Console.WriteLine(" train - Trains the network");
Console.WriteLine(" eval (i) - Evaluates the output of the network to the given input");
Console.WriteLine(" export [f] - Exports the current network to the filename provided (default: 'output.json')");
Console.WriteLine(" import [f] - Imports the network from the filename provided (default: 'output.json')");
}
else if (nextCommand == "devices")
{
var devices = ComputeDeviceFactory.GetComputeDevices();
System.Console.WriteLine(String.Format("Found a total of {0} devices!", devices.Count));
int i = 0;
foreach (var dev in devices)
{
Console.WriteLine(String.Format((i++).ToString() + ": [{0}] {1}", dev.GetDeviceAccessType(), dev.GetDeviceName()));
}
}
else if (nextCommand.StartsWith("select"))
{
if (commands.Length >= 2)
{
var devices = ComputeDeviceFactory.GetComputeDevices();
int selectedDeviceId = 0;
if (int.TryParse(commands[1], out selectedDeviceId))
{
if (selectedDeviceId < 0 || selectedDeviceId >= devices.Count)
{
Console.WriteLine("No such device: " + selectedDeviceId);
continue;
}
selectedDevice = ComputeDeviceFactory.CreateComputeDevice(devices[selectedDeviceId]);
Console.WriteLine("Selected device: " + selectedDeviceId + ": " + selectedDevice.GetName());
}
else
{
Console.WriteLine("Invalid device id given!");
}
}
else
{
Console.WriteLine("No device id given!");
}
}
else if (nextCommand == "quicktest")
{
Console.WriteLine("Testing on device: " + selectedDevice.GetName());
TestDevice(selectedDevice);
}
else if (nextCommand == "train")
{
int epochs = 1;
if (commands.Length >= 2)
{
int.TryParse(commands[1], out epochs);
}
Train(selectedDevice, epochs);
}
else if (nextCommand == "eval")
{
if (commands.Length < 2)
{
Console.WriteLine("No input given!");
continue;
}
List <float> input_values = new List <float>();
for (int i = 1; i < commands.Length; ++i)
{
float input = 0;
if (!float.TryParse(commands[i], out input))
{
Console.WriteLine("Invalid input given at index {0}", (i - 1));
}
input_values.Add(input);
}
float[] result = Eval(input_values.ToArray(), selectedDevice);
Console.WriteLine("Network output: [" + string.Join(", ", result) + "]");
}
else if (nextCommand == "generate")
{
Generate();
Console.WriteLine("A new network has been generated!");
}
else if (nextCommand == "benchmark")
{
int level = 2;
if (commands.Length >= 2)
{
try
{
level = Math.Max(Math.Min(10, Int32.Parse(commands[1])), 1);
}
catch (System.Exception)
{
}
}
BenchmarkDevice(selectedDevice, level);
}
else if (nextCommand == "export")
{
string filename = "output.json";
if (commands.Length >= 2)
{
filename = commands[1];
}
try
{
System.IO.File.WriteAllText(filename, target_network.ExportToJSON());
Console.WriteLine("Model exported to: '{0}'", filename);
}
catch (Exception exc)
{
Console.WriteLine("Error: " + exc.Message);
}
}
else if (nextCommand == "import")
{
string filename = "output.json";
if (commands.Length >= 2)
{
filename = commands[1];
}
try
{
string json_network = System.IO.File.ReadAllText(filename);
target_network = Network.CreateNetworkFromJSON(json_network);
Console.WriteLine("Model imported from: '{0}'", filename);
}
catch (Exception exc)
{
Console.WriteLine("Error: " + exc.Message);
}
}
else if (nextCommand == "info")
{
if (selectedDevice != null)
{
Console.WriteLine("Device Name: " + selectedDevice.GetName());
Console.WriteLine("Device Access: " + selectedDevice.GetDeviceAccessMode());
Console.WriteLine("Core count: " + selectedDevice.GetDeviceCoreCount());
Console.WriteLine("Memory: " + selectedDevice.GetDeviceMemorySize());
}
else
{
Console.WriteLine("CPU Fallback device is selected!");
}
}
}
catch (System.Exception exc)
{
Console.WriteLine("An error occured when running the command! " + exc.ToString());
}
}
}
static void Main(string[] args)
{
Console.WriteLine(" ### macademy test console ");
ComputeDevice selectedDevice = ComputeDeviceFactory.CreateFallbackComputeDevice();
while (true)
{
Console.WriteLine("");
string rawCommand = Console.ReadLine().Trim();
var commands = rawCommand.Split(' ');
if (commands.Length == 0)
{
continue;
}
var nextCommand = commands[0];
try
{
if (nextCommand == "exit")
{
break;
}
else if (nextCommand == "help")
{
Console.WriteLine("help - Displays this help message");
Console.WriteLine("devices - Displays available devices");
Console.WriteLine("select (n) - Selectes the devices with the given id");
Console.WriteLine("info - Displays information about the selected device");
Console.WriteLine("test - Performs a quick test on the selected device");
Console.WriteLine("exit - Exits the app");
}
else if (nextCommand == "devices")
{
var devices = ComputeDeviceFactory.GetComputeDevices();
System.Console.WriteLine(String.Format("Found a total of {0} devices!", devices.Count));
int i = 0;
foreach (var dev in devices)
{
Console.WriteLine(String.Format((i++).ToString() + ": [{0}] {1}", dev.GetDeviceAccessType(), dev.GetDeviceName()));
}
}
else if (nextCommand.StartsWith("select"))
{
if (commands.Length >= 2)
{
var devices = ComputeDeviceFactory.GetComputeDevices();
int selectedDeviceId = 0;
if (int.TryParse(commands[1], out selectedDeviceId))
{
if (selectedDeviceId < 0 || selectedDeviceId >= devices.Count)
{
Console.WriteLine("No such device: " + selectedDeviceId);
continue;
}
selectedDevice = ComputeDeviceFactory.CreateComputeDevice(devices[selectedDeviceId]);
Console.WriteLine("Selected device: " + selectedDeviceId + ": " + selectedDevice.GetName());
}
else
{
Console.WriteLine("Invalid device id given!");
}
}
else
{
Console.WriteLine("No device id given!");
}
}
else if (nextCommand == "test")
{
Console.WriteLine("Testing on device: " + selectedDevice.GetName());
TestDevice(selectedDevice);
}
else if (nextCommand == "info")
{
if (selectedDevice != null)
{
Console.WriteLine("Device Name: " + selectedDevice.GetName());
Console.WriteLine("Device Access: " + selectedDevice.GetDeviceAccessMode());
Console.WriteLine("Core count: " + selectedDevice.GetDeviceCoreCount());
Console.WriteLine("Memory: " + selectedDevice.GetDeviceMemorySize());
}
else
{
Console.WriteLine("CPU Fallback device is selected!");
}
}
}
catch (System.Exception exc)
{
Console.WriteLine("An error occured when running the command! " + exc.ToString());
}
}
}