public static void TrainWithSelu(string dataFilesPath)
{
using (var lap = BrightWireProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
// parse the iris CSV into a data table and normalise
var dataTable = new StreamReader(new MemoryStream(File.ReadAllBytes(dataFilesPath))).ParseCSV(',').Normalise(NormalisationType.Standard);
// split the data table into training and test tables
var split = dataTable.Split(0);
var trainingData = graph.CreateDataSource(split.Training);
var testData = graph.CreateDataSource(split.Test);
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.GaussianWeightInitialisation(true, 0.1f, GaussianVarianceCalibration.SquareRoot2N, GaussianVarianceCount.FanInFanOut))
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.1f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, batchSize: 128);
const int LAYER_SIZE = 64;
Func <INode> activation = () => new SeluActivation();
//Func<INode> activation = () => graph.ReluActivation();
// create the network with the custom activation function
graph.Connect(engine)
.AddFeedForward(LAYER_SIZE)
.AddBatchNormalisation()
.Add(activation())
.AddFeedForward(LAYER_SIZE)
.AddBatchNormalisation()
.Add(activation())
.AddFeedForward(LAYER_SIZE)
.AddBatchNormalisation()
.Add(activation())
.AddFeedForward(LAYER_SIZE)
.AddBatchNormalisation()
.Add(activation())
.AddFeedForward(LAYER_SIZE)
.AddBatchNormalisation()
.Add(activation())
.AddFeedForward(LAYER_SIZE)
.AddBatchNormalisation()
.Add(activation())
.AddFeedForward(trainingData.OutputSize)
.Add(graph.SoftMaxActivation())
.AddBackpropagation(errorMetric)
;
const int TRAINING_ITERATIONS = 500;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, null, 50);
}
}
public void TestRecurrent()
{
var data = BinaryIntegers.Addition(100, false).Split(0);
var graph = new GraphFactory(_lap);
var errorMetric = graph.ErrorMetric.BinaryClassification;
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));
// create the engine
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);
// build the network
const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 5;
var memory = new float[HIDDEN_LAYER_SIZE];
var network = graph.Connect(engine)
.AddSimpleRecurrent(graph.ReluActivation(), memory)
.AddFeedForward(engine.DataSource.OutputSize)
.Add(graph.ReluActivation())
.AddBackpropagationThroughTime(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
BrightWire.Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);
// export the graph and verify it against some unseen integers on the best model
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
var results = executionEngine.Execute(testData2);
}
public static IGraphTrainingEngine BuildNetwork(NetworkConfig config, GraphFactory graph, DataSet dataset, string outputModelPath = null)
{
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(config.ZERO_BIAS, config.STANDARD_DEVIATION, config.GAUSSIAN_VARIANCE_CALIBRATION));
var engine = graph.CreateTrainingEngine(dataset.TrainData, config.LEARNING_RATE, config.BATCH_SIZE);
if (!String.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath))
{
engine = LoadTrainingNetwork(outputModelPath, graph, config, dataset);
}
else
{
graph = CreateStandardNetwork(engine, graph, config, dataset);
}
engine.LearningContext.ScheduleLearningRate(15, config.LEARNING_RATE / 2);
return(engine);
}
/// <summary>
/// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
public static void MNIST(string dataFilesPath)
{
using (var lap = BrightWireProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var trainingData = _BuildVectors(null, graph, Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte", dataFilesPath + "train-images.idx3-ubyte"));
var testData = _BuildVectors(trainingData, graph, Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte", dataFilesPath + "t10k-images.idx3-ubyte"));
Console.WriteLine($"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N))
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.1f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128);
engine.LearningContext.ScheduleLearningRate(15, TRAINING_RATE / 3);
// create the network
graph.Connect(engine)
.AddFeedForward(outputSize: 1024)
.Add(graph.LeakyReluActivation())
.AddDropOut(dropOutPercentage: 0.5f)
.AddFeedForward(outputSize: trainingData.OutputSize)
.Add(graph.SoftMaxActivation())
.AddBackpropagation(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(20, testData, errorMetric, model => bestGraph = model.Graph);
// export the final model and execute it on the training set
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var output = executionEngine.Execute(testData);
Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}");
}
}
public static void IntegerAddition()
{
// generate 1000 random integer additions (split into training and test sets)
var data = BinaryIntegers.Addition(1000, false).Split(0);
using (var lap = BrightWireProvider.CreateLinearAlgebra(false)) {
var graph = new GraphFactory(lap);
// binary classification rounds each output to either 0 or 1
var errorMetric = graph.ErrorMetric.BinaryClassification;
// configure the network properties
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N))
;
// create the engine
var trainingData = graph.CreateDataSource(data.Training);
var testData = trainingData.CloneWith(data.Test);
var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);
// build the network
const int HIDDEN_LAYER_SIZE = 32, TRAINING_ITERATIONS = 30;
var memory = new float[HIDDEN_LAYER_SIZE];
var network = graph.Connect(engine)
.AddSimpleRecurrent(graph.ReluActivation(), memory)
.AddFeedForward(engine.DataSource.OutputSize)
.Add(graph.ReluActivation())
.AddBackpropagationThroughTime(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, bn => bestGraph = bn.Graph);
// export the graph and verify it against some unseen integers on the best model
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var testData2 = graph.CreateDataSource(BinaryIntegers.Addition(8, true));
var results = executionEngine.Execute(testData2);
// group the output
var groupedResults = new (FloatVector[] Input, FloatVector[] Target, FloatVector[] Output)[8];
public static void TrainWithSelu(string dataFilesPath)
{
using (var lap = BrightWireGpuProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
// parse the iris CSV into a data table and normalise
var dataTable = new StreamReader(new MemoryStream(File.ReadAllBytes(dataFilesPath))).ParseCSV(',').Normalise(NormalisationType.Standard);
// split the data table into training and test tables
var split = dataTable.Split(0);
var trainingData = graph.CreateDataSource(split.Training);
var testData = graph.CreateDataSource(split.Test);
// use a one hot encoding error metric, rmsprop gradient descent and xavier weight initialisation
var errorMetric = graph.ErrorMetric.OneHotEncoding;
var propertySet = graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.GaussianWeightInitialisation(true, 0.1f, GaussianVarianceCalibration.SquareRoot2N, GaussianVarianceCount.FanInFanOut))
;
// create the training engine and schedule a training rate change
const float TRAINING_RATE = 0.01f;
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, 128);
// create the network
graph.Connect(engine)
.AddFeedForward(32)
.Add(new SeluActivation())
.AddFeedForward(trainingData.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagation(errorMetric)
;
// train the network
engine.Train(1000, testData, errorMetric, null, 50);
}
}
/// <summary>
/// Trains a feed forward neural net on the MNIST data set (handwritten digit recognition)
/// The data files can be downloaded from http://yann.lecun.com/exdb/mnist/
/// </summary>
/// <param name="dataFilesPath">The path to a directory with the four extracted data files</param>
/// <param name="outputModelPath">Optional path to save the best model to</param>
static void MNISTConvolutional(string dataFilesPath, string outputModelPath = null)
{
using var lap = BrightWireGpuProvider.CreateLinearAlgebra();
var graph = new GraphFactory(lap);
Console.Write("Loading training data...");
var mnistTraining = Mnist.Load(dataFilesPath + "train-labels.idx1-ubyte",
dataFilesPath + "train-images.idx3-ubyte");
var mnistTest = Mnist.Load(dataFilesPath + "t10k-labels.idx1-ubyte",
dataFilesPath + "t10k-images.idx3-ubyte");
var trainingData =
_BuildTensors(graph, null, mnistTraining /*.Where(d => d.Label < 2).ToList()*/);
var testData = _BuildTensors(graph, trainingData,
mnistTest /*.Where(d => d.Label < 2).ToList()*/);
Console.WriteLine(
$"done - {trainingData.RowCount} training images and {testData.RowCount} test images loaded");
// one hot encoding uses the index of the output vector's maximum value as the classification label
var errorMetric = graph.ErrorMetric.OneHotEncoding;
// configure the network properties
graph.CurrentPropertySet.Use(graph.GradientDescent.Adam).Use(
graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));
// create the network
const int HIDDEN_LAYER_SIZE = 1024, TRAINING_ITERATIONS = 20;
const float LEARNING_RATE = 0.05f;
var engine = graph.CreateTrainingEngine(trainingData, LEARNING_RATE);
if (!string.IsNullOrWhiteSpace(outputModelPath) && File.Exists(outputModelPath))
{
Console.WriteLine("Loading existing model from: " + outputModelPath);
using var file = new FileStream(outputModelPath, FileMode.Open, FileAccess.Read);
var model = Serializer.Deserialize <GraphModel>(file);
engine = graph.CreateTrainingEngine(trainingData, model.Graph, LEARNING_RATE);
}
else
{
graph.Connect(engine).
AddConvolutional(filterCount: 16, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1,
yStride: 1, shouldBackpropagate: false).Add(graph.LeakyReluActivation()).
AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).
AddConvolutional(filterCount: 32, padding: 2, filterWidth: 5, filterHeight: 5, xStride: 1,
yStride: 1).Add(graph.LeakyReluActivation()).
AddMaxPooling(filterWidth: 2, filterHeight: 2, xStride: 2, yStride: 2).Transpose().
AddFeedForward(HIDDEN_LAYER_SIZE).Add(graph.LeakyReluActivation()).
AddDropOut(dropOutPercentage: 0.5f).AddFeedForward(trainingData.OutputSize).
Add(graph.SoftMaxActivation()).AddBackpropagation(errorMetric);
}
// lower the learning rate over time
engine.LearningContext.ScheduleLearningRate(15, LEARNING_RATE / 2);
// train the network for twenty iterations, saving the model on each improvement
Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, model =>
{
bestGraph = model.Graph;
if (!string.IsNullOrWhiteSpace(outputModelPath))
{
using var file = new FileStream(outputModelPath, FileMode.Create, FileAccess.Write);
Serializer.Serialize(file, model);
}
});
// export the final model and execute it on the training set
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
var output = executionEngine.Execute(testData);
Console.WriteLine($"Final accuracy: {output.Average(o => o.CalculateError(errorMetric)):P2}");
// execute the model with a single image
var tensor = mnistTest.First().AsFloatTensor.Tensor;
var singleData = graph.CreateDataSource(new[] { tensor });
var result = executionEngine.Execute(singleData);
var prediction = result.Single().Output.Single().MaximumIndex();
}
public static void CrazyTest()
{
using (var la = BrightWireProvider.CreateLinearAlgebra())
{
var graph = new GraphFactory(la);
graph.CurrentPropertySet
.Use(graph.GradientDescent.Adam)
.Use(graph.GaussianWeightInitialisation(false, 0.1f, GaussianVarianceCalibration.SquareRoot2N));
var dtbuilder = BrightWireProvider.CreateDataTableBuilder();
dtbuilder.AddColumn(ColumnType.Tensor, "image", false);
dtbuilder.AddColumn(ColumnType.Vector, "expectedoutput", true);
// DATEN GENERIEREN
for (int i = 0; i < 666; i++)
{
var fv = new BrightWire.Models.FloatVector
{
Data = new float[666],
};
var fv2 = new BrightWire.Models.FloatVector
{
Data = new float[666],
};
dtbuilder.Add(fv, fv2);
}
var datatable = dtbuilder.Build();
Console.WriteLine($"####### TRAINING START #######");
// create the engine
var(Training, Test) = datatable.Split();
var trainingData = graph.CreateDataSource(Training);
var testData = graph.CreateDataSource(Test);
var engine = graph.CreateTrainingEngine(trainingData, learningRate: 0.01f, batchSize: 16);
// build the network
var errorMetric = graph.ErrorMetric.Quadratic;
const int HIDDEN_LAYER_SIZE = 2000, TRAINING_ITERATIONS = 2000;
var network = graph.Connect(engine)
.AddFeedForward(HIDDEN_LAYER_SIZE)
.Add(graph.ReluActivation())
//.AddBackpropagation(errorMetric)
.AddFeedForward(HIDDEN_LAYER_SIZE)
.Add(graph.ReluActivation())
//.AddBackpropagation(errorMetric)
.AddDropOut(.2f)
.AddFeedForward(engine.DataSource.OutputSize)
//.Add(graph.ReluActivation())
.AddBackpropagation(errorMetric)
//.AddBackpropagationThroughTime(errorMetric)
;
// train the network for twenty iterations, saving the model on each improvement
BrightWire.Models.ExecutionGraph bestGraph = null;
engine.Train(TRAINING_ITERATIONS, testData, errorMetric, ((modelGraph) => bestGraph = modelGraph.Graph));
// export the graph and verify it against some unseen integers on the best model
var executionEngine = graph.CreateEngine(bestGraph ?? engine.Graph);
}
}