/// <summary>
/// Create a MNIST trainer (writing recognition) that will be added to an environemnt.
/// </summary>
/// <param name="sigma">The sigma environemnt this trainer will be assigned to.</param>
/// <returns>The newly created trainer.</returns>
private static ITrainer CreateMnistTrainer(SigmaEnvironment sigma)
{
IDataset dataset = Defaults.Datasets.Mnist();
ITrainer trainer = sigma.CreateTrainer("mnist-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(28, 28)
+ DropoutLayer.Construct(0.2)
+ FullyConnectedLayer.Construct(1000, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(800, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(10, activation: "sigmoid")
+ OutputLayer.Construct(10)
+ SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(100, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(Defaults.Datasets.MnistValidation()));
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.02);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new ValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration), reportEpochIteration: true)
.On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(1, TimeScale.Epoch), 4));
return(trainer);
}
/// <summary>
/// Create a MNIST trainer (writing recognition) will be added to an environemnt.
/// </summary>
/// <param name="sigma">The sigma environemnt this trainer will be assigned to.</param>
/// <returns>The newly created trainer.</returns>
private static ITrainer CreateMnistTrainer(SigmaEnvironment sigma)
{
ByteRecordReader mnistImageReader = new ByteRecordReader(headerLengthBytes: 16, recordSizeBytes: 28 * 28, source: new CompressedSource(new MultiSource(new FileSource("train-images-idx3-ubyte.gz"), new UrlSource("http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz"))));
IRecordExtractor mnistImageExtractor = mnistImageReader.Extractor("inputs", new[] { 0L, 0L }, new[] { 28L, 28L }).Preprocess(new NormalisingPreprocessor(0, 255));
ByteRecordReader mnistTargetReader = new ByteRecordReader(headerLengthBytes: 8, recordSizeBytes: 1, source: new CompressedSource(new MultiSource(new FileSource("train-labels-idx1-ubyte.gz"), new UrlSource("http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz"))));
IRecordExtractor mnistTargetExtractor = mnistTargetReader.Extractor("targets", new[] { 0L }, new[] { 1L }).Preprocess(new OneHotPreprocessor(minValue: 0, maxValue: 9));
IDataset dataset = new Dataset("mnist-training", Dataset.BlockSizeAuto, mnistImageExtractor, mnistTargetExtractor);
ITrainer trainer = sigma.CreateTrainer("test");
trainer.Network = new Network
{
Architecture = InputLayer.Construct(28, 28)
+ 2 * FullyConnectedLayer.Construct(28 * 28)
+ FullyConnectedLayer.Construct(10)
+ OutputLayer.Construct(10)
+ SoftMaxCrossEntropyCostLayer.Construct()
};
trainer.TrainingDataIterator = new MinibatchIterator(8, dataset);
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.02);
trainer.Operator = new CpuSinglethreadedOperator();
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.05f));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.01f, mean: 0.03f));
trainer.AddGlobalHook(new CurrentEpochIterationReporter(TimeStep.Every(1, TimeScale.Iteration)));
return(trainer);
}
public static ITrainer CreateTicTacToeTrainer(SigmaEnvironment sigma)
{
IDataset dataset = Defaults.Datasets.TicTacToe();
ITrainer trainer = sigma.CreateTrainer("tictactoe-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(9)
+ FullyConnectedLayer.Construct(72, "tanh")
+ FullyConnectedLayer.Construct(99, "tanh")
+ FullyConnectedLayer.Construct(3, "tanh")
+ OutputLayer.Construct(3)
+ SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(21, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new MomentumGradientOptimiser(learningRate: 0.01, momentum: 0.9);
trainer.Operator = new CpuSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)));
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", TimeStep.Every(1, TimeScale.Epoch), tops: new[] { 1, 2 }));
trainer.AddGlobalHook(new DiskSaviorHook <INetwork>(TimeStep.Every(1, TimeScale.Epoch), "network.self", Namers.Static("tictactoe.sgnet"), verbose: true)
.On(new ExtremaCriteria("shared.classification_accuracy_top1", ExtremaTarget.Max)));
return(trainer);
}
private static ITrainer CreateIrisTrainer(SigmaEnvironment sigma)
{
IDataset dataset = Defaults.Datasets.Iris();
ITrainer trainer = sigma.CreateTrainer("iris-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(4)
+ FullyConnectedLayer.Construct(12)
+ FullyConnectedLayer.Construct(3)
+ OutputLayer.Construct(3)
+ SquaredDifferenceCostLayer.Construct();
//trainer.Network = Serialisation.ReadBinaryFileIfExists("iris.sgnet", trainer.Network);
trainer.TrainingDataIterator = new MinibatchIterator(50, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.06);
trainer.Operator = new CpuSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
//trainer.AddGlobalHook(new StopTrainingHook(atEpoch: 100));
//trainer.AddLocalHook(new EarlyStopperHook("optimiser.cost_total", 20, target: ExtremaTarget.Min));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch), reportEpochIteration: true));
//.On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
//trainer.AddLocalHook(new DiskSaviorHook<INetwork>("network.self", Namers.Dynamic("iris_epoch{0}.sgnet", "epoch"), verbose: true)
// .On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", TimeStep.Every(1, TimeScale.Epoch), tops: 1));
return(trainer);
}
private static ITrainer CreateParkinsonsTrainer(SigmaEnvironment sigma)
{
IDataset dataset = Defaults.Datasets.Parkinsons();
ITrainer trainer = sigma.CreateTrainer("parkinsons-trainer");
trainer.Network = new Network
{
Architecture = InputLayer.Construct(22)
+ FullyConnectedLayer.Construct(140)
+ FullyConnectedLayer.Construct(20)
+ FullyConnectedLayer.Construct(1)
+ OutputLayer.Construct(1)
+ SquaredDifferenceCostLayer.Construct()
};
trainer.TrainingDataIterator = new MinibatchIterator(10, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.01);
trainer.Operator = new CpuSinglethreadedOperator(new DebugHandler(new CpuFloat32Handler()));
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)));
trainer.AddHook(new UniClassificationAccuracyReporter("validation", 0.5, TimeStep.Every(1, TimeScale.Epoch)));
return(trainer);
}
private static void SampleXor()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("logical");
sigma.SetRandomSeed(0);
sigma.Prepare();
RawDataset dataset = new RawDataset("xor");
dataset.AddRecords("inputs", new[] { 0, 0 }, new[] { 0, 1 }, new[] { 1, 0 }, new[] { 1, 1 });
dataset.AddRecords("targets", new[] { 0 }, new[] { 0 }, new[] { 0 }, new[] { 1 });
ITrainer trainer = sigma.CreateTrainer("xor-trainer");
trainer.Network.Architecture = InputLayer.Construct(2) + FullyConnectedLayer.Construct(2) + FullyConnectedLayer.Construct(1) + OutputLayer.Construct(1) + SquaredDifferenceCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(1, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.1);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new StopTrainingHook(atEpoch: 10000));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch), averageValues: true));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Stop), averageValues: true));
trainer.AddLocalHook(new ValueReporter("network.layers.*<external_output>._outputs.default.activations", TimeStep.Every(1, TimeScale.Stop)));
trainer.AddLocalHook(new ValueReporter("network.layers.*-fullyconnected.weights", TimeStep.Every(1, TimeScale.Stop)));
trainer.AddLocalHook(new ValueReporter("network.layers.*-fullyconnected.biases", TimeStep.Every(1, TimeScale.Stop)));
sigma.Run();
}
private static void SampleHutter()
{
const long timeWindowSize = 10L;
SigmaEnvironment sigma = SigmaEnvironment.Create("recurrent");
IDataSource source = new MultiSource(new FileSource("enwik8"), new CompressedSource(new MultiSource(new FileSource("enwik8.zip"), new UrlSource("http://mattmahoney.net/dc/enwik8.zip"))));
IRecordExtractor extractor = new CharacterRecordReader(source, (int)(timeWindowSize + 1), Encoding.ASCII)
.Extractor(new ArrayRecordExtractor <short>(ArrayRecordExtractor <short>
.ParseExtractorParameters("inputs", new[] { 0L }, new[] { timeWindowSize }, "targets", new[] { 0L }, new[] { timeWindowSize }))
.Offset("targets", 1L))
.Preprocess(new PermutePreprocessor(0, 2, 1))
.Preprocess(new OneHotPreprocessor(0, 255));
IDataset dataset = new ExtractedDataset("hutter", ExtractedDataset.BlockSizeAuto, false, extractor);
ITrainer trainer = sigma.CreateTrainer("hutter");
trainer.Network.Architecture = InputLayer.Construct(256) + RecurrentLayer.Construct(256) + OutputLayer.Construct(256) + SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(32, dataset);
trainer.AddNamedDataIterator("validation", new MinibatchIterator(100, dataset));
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.07);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration), averageValues: true));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(10, TimeScale.Iteration)));
sigma.PrepareAndRun();
}
private static void SampleNetworkArchitecture()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
IComputationHandler handler = new CpuFloat32Handler();
ITrainer trainer = sigma.CreateTrainer("test_trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(2, 2) +
ElementwiseLayer.Construct(2 * 2) +
FullyConnectedLayer.Construct(2) +
2 * (FullyConnectedLayer.Construct(4) + FullyConnectedLayer.Construct(2)) +
OutputLayer.Construct(2);
trainer.Network = (INetwork)trainer.Network.DeepCopy();
trainer.Operator = new CpuMultithreadedOperator(10);
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.1f));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.01f, mean: 0.03f));
trainer.Initialise(handler);
trainer.Network = (INetwork)trainer.Network.DeepCopy();
Console.WriteLine(trainer.Network.Registry);
IRegistryResolver resolver = new RegistryResolver(trainer.Network.Registry);
Console.WriteLine("===============");
object[] weights = resolver.ResolveGet <object>("layers.*.weights");
Console.WriteLine(string.Join("\n", weights));
Console.WriteLine("===============");
//foreach (ILayerBuffer buffer in trainer.Network.YieldLayerBuffersOrdered())
//{
// Console.WriteLine(buffer.Layer.Name + ": ");
// Console.WriteLine("inputs:");
// foreach (string input in buffer.Inputs.Keys)
// {
// Console.WriteLine($"\t{input}: {buffer.Inputs[input].GetHashCode()}");
// }
// Console.WriteLine("outputs:");
// foreach (string output in buffer.Outputs.Keys)
// {
// Console.WriteLine($"\t{output}: {buffer.Outputs[output].GetHashCode()}");
// }
//}
}
private static void SampleMnist()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("mnist");
sigma.SetRandomSeed(0);
IDataset dataset = Defaults.Datasets.Mnist();
ITrainer trainer = sigma.CreateTrainer("mnist-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(28, 28)
+ DropoutLayer.Construct(0.2)
+ FullyConnectedLayer.Construct(1000, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(800, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(10, activation: "sigmoid")
+ OutputLayer.Construct(10)
+ SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(100, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(Defaults.Datasets.MnistValidation()));
//trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.01);
//trainer.Optimiser = new MomentumGradientOptimiser(learningRate: 0.01, momentum: 0.9);
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.02);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new ValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration), reportEpochIteration: true)
.On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
var validationTimeStep = TimeStep.Every(1, TimeScale.Epoch);
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", validationTimeStep, tops: new[] { 1, 2, 3 }));
for (int i = 0; i < 10; i++)
{
trainer.AddGlobalHook(new TargetMaximisationReporter(trainer.Operator.Handler.NDArray(ArrayUtils.OneHot(i, 10), 10), TimeStep.Every(1, TimeScale.Epoch)));
}
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(10, TimeScale.Iteration), 32));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(1, TimeScale.Epoch), 4));
trainer.AddHook(new StopTrainingHook(atEpoch: 10));
sigma.PrepareAndRun();
}
/// <summary>
/// Create an IRIS trainer that observers the current epoch and iteration
/// </summary>
/// <param name="sigma">The sigma environemnt.</param>
/// <returns>The newly created trainer that can be added to the environemnt.</returns>
private static ITrainer CreateIrisTrainer(SigmaEnvironment sigma)
{
CsvRecordReader irisReader = new CsvRecordReader(new MultiSource(new FileSource("iris.data"), new UrlSource("http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data")));
IRecordExtractor irisExtractor = irisReader.Extractor("inputs", new[] { 0, 3 }, "targets", 4).AddValueMapping(4, "Iris-setosa", "Iris-versicolor", "Iris-virginica");
irisExtractor = irisExtractor.Preprocess(new OneHotPreprocessor(sectionName: "targets", minValue: 0, maxValue: 2));
irisExtractor = irisExtractor.Preprocess(new PerIndexNormalisingPreprocessor(0, 1, "inputs", 0, 4.3, 7.9, 1, 2.0, 4.4, 2, 1.0, 6.9, 3, 0.1, 2.5));
Dataset dataset = new Dataset("iris", Dataset.BlockSizeAuto, irisExtractor);
IDataset trainingDataset = dataset;
IDataset validationDataset = dataset;
ITrainer trainer = sigma.CreateTrainer("test");
trainer.Network = new Network
{
Architecture = InputLayer.Construct(4)
+ FullyConnectedLayer.Construct(10)
+ FullyConnectedLayer.Construct(20)
+ FullyConnectedLayer.Construct(10)
+ FullyConnectedLayer.Construct(3)
+ OutputLayer.Construct(3)
+ SquaredDifferenceCostLayer.Construct()
};
trainer.TrainingDataIterator = new MinibatchIterator(4, trainingDataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(validationDataset));
trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.002);
trainer.Operator = new CpuSinglethreadedOperator();
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.4));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.01, mean: 0.05));
trainer.AddHook(new ValueReporterHook("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)));
trainer.AddHook(new ValidationAccuracyReporter("validation", TimeStep.Every(1, TimeScale.Epoch), tops: 1));
trainer.AddLocalHook(new CurrentEpochIterationReporter(TimeStep.Every(1, TimeScale.Epoch)));
return(trainer);
}
private static ITrainer CreateXorTrainer(SigmaEnvironment sigma)
{
RawDataset dataset = new RawDataset("xor");
dataset.AddRecords("inputs", new[] { 0, 0 }, new[] { 0, 1 }, new[] { 1, 0 }, new[] { 1, 1 });
dataset.AddRecords("targets", new[] { 0 }, new[] { 1 }, new[] { 1 }, new[] { 0 });
ITrainer trainer = sigma.CreateTrainer("xor-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(2) + FullyConnectedLayer.Construct(1) + OutputLayer.Construct(1) + SquaredDifferenceCostLayer.Construct();
trainer.TrainingDataIterator = new UndividedIterator(dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Operator = new CpuSinglethreadedOperator();
trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.01);
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch), reportEpochIteration: true));
trainer.AddLocalHook(new ValueReporter("network.layers.1-fullyconnected._outputs.default.activations", TimeStep.Every(1, TimeScale.Epoch)));
return(trainer);
}
private static void SampleNetworkMerging()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("merge_test");
ITrainer[] trainers = new ITrainer[3];
int[] constantValues = { 2, 10, 70 };
//INetworkMerger merger = new WeightedNetworkMerger(10d, 10d, 1d);
INetworkMerger merger = new AverageNetworkMerger();
IComputationHandler handler = new CpuFloat32Handler();
for (int i = 0; i < trainers.Length; i++)
{
trainers[i] = sigma.CreateTrainer($"MergeTrainer{i}");
trainers[i].Network = new Network($"{i}");
trainers[i].Network.Architecture = InputLayer.Construct(2, 2) + ElementwiseLayer.Construct(2 * 2) + OutputLayer.Construct(2);
trainers[i].AddInitialiser("*.weights", new ConstantValueInitialiser(constantValues[i]));
trainers[i].Operator = new CpuMultithreadedOperator(5);
trainers[i].Initialise(handler);
}
foreach (ITrainer trainer in trainers)
{
Console.WriteLine(trainer.Network.Registry);
}
merger.AddMergeEntry("layers.*.weights");
merger.Merge(trainers[1].Network, trainers[2].Network, handler);
Console.WriteLine("*******************");
foreach (ITrainer trainer in trainers)
{
Console.WriteLine(trainer.Network.Registry);
}
}
public static INetwork GenerateNetwork(double number)
{
if (_environment == null)
{
_environment = SigmaEnvironment.Create("TestAverageNetworkMergerEnvironment");
}
ITrainer trainer = _environment.CreateTrainer($"trainer{_count++}");
Network net = new Network();
net.Architecture = InputLayer.Construct(2, 2) + FullyConnectedLayer.Construct(2 * 2) + OutputLayer.Construct(2);
trainer.Network = net;
trainer.AddInitialiser("*.weights", new ConstantValueInitialiser(number));
trainer.Operator = new CpuSinglethreadedOperator();
trainer.Initialise(new CpuFloat32Handler());
SigmaEnvironment.Clear();
return(net);
}