private static void SampleParkinsons()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("parkinsons");
IDataset dataset = Defaults.Datasets.Parkinsons();
ITrainer trainer = sigma.CreateGhostTrainer("parkinsons-trainer");
trainer.Network.Architecture = InputLayer.Construct(22)
+ FullyConnectedLayer.Construct(140)
+ FullyConnectedLayer.Construct(20)
+ FullyConnectedLayer.Construct(1)
+ OutputLayer.Construct(1)
+ SquaredDifferenceCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(10, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.01);
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)));
trainer.AddHook(new UniClassificationAccuracyReporter("validation", 0.5, TimeStep.Every(1, TimeScale.Epoch)));
sigma.AddTrainer(trainer);
sigma.PrepareAndRun();
}
private static void SampleWdbc()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("wdbc");
IDataset dataset = Defaults.Datasets.Wdbc();
ITrainer trainer = sigma.CreateGhostTrainer("wdbc-trainer");
trainer.Network.Architecture = InputLayer.Construct(30)
+ FullyConnectedLayer.Construct(42)
+ FullyConnectedLayer.Construct(24)
+ FullyConnectedLayer.Construct(1)
+ OutputLayer.Construct(1)
+ SquaredDifferenceCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(72, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.005);
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)));
trainer.AddHook(new UniClassificationAccuracyReporter("validation", 0.5, TimeStep.Every(1, TimeScale.Epoch)));
sigma.AddTrainer(trainer);
sigma.AddMonitor(new HttpMonitor("http://+:8080/sigma/"));
sigma.PrepareAndRun();
}
private static void SampleHutter()
{
const long timeWindowSize = 10L;
SigmaEnvironment sigma = SigmaEnvironment.Create("recurrent");
IDataSource source = new MultiSource(new FileSource("enwik8"), new CompressedSource(new MultiSource(new FileSource("enwik8.zip"), new UrlSource("http://mattmahoney.net/dc/enwik8.zip"))));
IRecordExtractor extractor = new CharacterRecordReader(source, (int)(timeWindowSize + 1), Encoding.ASCII)
.Extractor(new ArrayRecordExtractor <short>(ArrayRecordExtractor <short>
.ParseExtractorParameters("inputs", new[] { 0L }, new[] { timeWindowSize }, "targets", new[] { 0L }, new[] { timeWindowSize }))
.Offset("targets", 1L))
.Preprocess(new PermutePreprocessor(0, 2, 1))
.Preprocess(new OneHotPreprocessor(0, 255));
IDataset dataset = new ExtractedDataset("hutter", ExtractedDataset.BlockSizeAuto, false, extractor);
ITrainer trainer = sigma.CreateTrainer("hutter");
trainer.Network.Architecture = InputLayer.Construct(256) + RecurrentLayer.Construct(256) + OutputLayer.Construct(256) + SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(32, dataset);
trainer.AddNamedDataIterator("validation", new MinibatchIterator(100, dataset));
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.07);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration), averageValues: true));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(10, TimeScale.Iteration)));
sigma.PrepareAndRun();
}
public void TestUnifiedIteratorYield()
{
string filename = ".unittestfile" + nameof(TestUnifiedIteratorYield);
CreateCsvTempFile(filename);
SigmaEnvironment.Clear();
FileSource source = new FileSource(filename, Path.GetTempPath());
CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> {
["inputs"] = new[] { new[] { 0 } }
}));
ExtractedDataset dataset = new ExtractedDataset("test", 2, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUnifiedIteratorYield)), true, extractor);
UnifiedIterator iterator = new UnifiedIterator(dataset);
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
IComputationHandler handler = new CpuFloat32Handler();
foreach (var block in iterator.Yield(handler, sigma))
{
Assert.AreEqual(new[] { 5.1f, 4.9f, 4.7f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 3));
}
dataset.Dispose();
DeleteTempFile(filename);
}
private static void SampleXor()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("logical");
sigma.SetRandomSeed(0);
sigma.Prepare();
RawDataset dataset = new RawDataset("xor");
dataset.AddRecords("inputs", new[] { 0, 0 }, new[] { 0, 1 }, new[] { 1, 0 }, new[] { 1, 1 });
dataset.AddRecords("targets", new[] { 0 }, new[] { 0 }, new[] { 0 }, new[] { 1 });
ITrainer trainer = sigma.CreateTrainer("xor-trainer");
trainer.Network.Architecture = InputLayer.Construct(2) + FullyConnectedLayer.Construct(2) + FullyConnectedLayer.Construct(1) + OutputLayer.Construct(1) + SquaredDifferenceCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(1, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.1);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new StopTrainingHook(atEpoch: 10000));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch), averageValues: true));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Stop), averageValues: true));
trainer.AddLocalHook(new ValueReporter("network.layers.*<external_output>._outputs.default.activations", TimeStep.Every(1, TimeScale.Stop)));
trainer.AddLocalHook(new ValueReporter("network.layers.*-fullyconnected.weights", TimeStep.Every(1, TimeScale.Stop)));
trainer.AddLocalHook(new ValueReporter("network.layers.*-fullyconnected.biases", TimeStep.Every(1, TimeScale.Stop)));
sigma.Run();
}
private static void SampleTicTacToe()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("tictactoe");
IDataset dataset = Defaults.Datasets.TicTacToe();
ITrainer trainer = sigma.CreateTrainer("tictactoe-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(9)
+ FullyConnectedLayer.Construct(63, "tanh")
+ FullyConnectedLayer.Construct(90, "tanh")
+ FullyConnectedLayer.Construct(3, "tanh")
+ OutputLayer.Construct(3)
+ SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(21, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new MomentumGradientOptimiser(learningRate: 0.01, momentum: 0.9);
trainer.Operator = new CpuSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)));
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", TimeStep.Every(1, TimeScale.Epoch), tops: new[] { 1, 2 }));
trainer.AddGlobalHook(new DiskSaviorHook <INetwork>(TimeStep.Every(1, TimeScale.Epoch), "network.self", Namers.Static("tictactoe.sgnet"), verbose: true)
.On(new ExtremaCriteria("shared.classification_accuracy_top1", ExtremaTarget.Max)));
sigma.PrepareAndRun();
}
public void TestSigmaEnvironmentCreate()
{
SigmaEnvironment.Clear();
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
Assert.AreEqual("test", sigma.Name);
}
public void TestSigmaEnvironmentAlreadyCreated()
{
SigmaEnvironment.Clear();
SigmaEnvironment.Create("test");
Assert.Throws <ArgumentException>(() => SigmaEnvironment.Create("test"));
}
private static void Main()
{
SigmaEnvironment.EnableLogging();
SigmaEnvironment sigma = SigmaEnvironment.Create("Sigma-IRIS");
ITrainer trainer = CreateIrisTrainer(sigma);
}
private static void SampleLoadExtractIterate()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
sigma.Prepare();
//var irisReader = new CsvRecordReader(new MultiSource(new FileSource("iris.data"), new UrlSource("http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data")));
//IRecordExtractor irisExtractor = irisReader.Extractor("inputs2", new[] { 0, 3 }, "targets2", 4).AddValueMapping(4, "Iris-setosa", "Iris-versicolor", "Iris-virginica");
//irisExtractor = irisExtractor.Preprocess(new OneHotPreprocessor(sectionName: "targets2", minValue: 0, maxValue: 2), new NormalisingPreprocessor(sectionNames: "inputs2", minInputValue: 0, maxInputValue: 6));
ByteRecordReader mnistImageReader = new ByteRecordReader(headerLengthBytes: 16, recordSizeBytes: 28 * 28, source: new CompressedSource(new MultiSource(new FileSource("train-images-idx3-ubyte.gz"), new UrlSource("http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz"))));
IRecordExtractor mnistImageExtractor = mnistImageReader.Extractor("inputs", new[] { 0L, 0L }, new[] { 28L, 28L }).Preprocess(new NormalisingPreprocessor(0, 255));
ByteRecordReader mnistTargetReader = new ByteRecordReader(headerLengthBytes: 8, recordSizeBytes: 1, source: new CompressedSource(new MultiSource(new FileSource("train-labels-idx1-ubyte.gz"), new UrlSource("http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz"))));
IRecordExtractor mnistTargetExtractor = mnistTargetReader.Extractor("targets", new[] { 0L }, new[] { 1L }).Preprocess(new OneHotPreprocessor(minValue: 0, maxValue: 9));
IComputationHandler handler = new CpuFloat32Handler();
ExtractedDataset dataset = new ExtractedDataset("mnist-training", ExtractedDataset.BlockSizeAuto, mnistImageExtractor, mnistTargetExtractor);
IDataset[] slices = dataset.SplitRecordwise(0.8, 0.2);
IDataset trainingData = slices[0];
IDataset validationData = slices[1];
MinibatchIterator trainingIterator = new MinibatchIterator(1, trainingData);
MinibatchIterator validationIterator = new MinibatchIterator(1, validationData);
while (true)
{
foreach (var block in trainingIterator.Yield(handler, sigma))
{
Thread.Sleep(100);
PrintFormattedBlock(block, PrintUtils.AsciiGreyscalePalette);
Thread.Sleep(1000);
}
}
//Random random = new Random();
//INDArray array = new ADNDArray<float>(3, 1, 2, 2);
//new GaussianInitialiser(0.05, 0.05).Initialise(array, Handler, random);
//Console.WriteLine(array);
//new ConstantValueInitialiser(1).Initialise(array, Handler, random);
//Console.WriteLine(array);
//dataset.InvalidateAndClearCaches();
}
private static void SampleNetworkArchitecture()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
IComputationHandler handler = new CpuFloat32Handler();
ITrainer trainer = sigma.CreateTrainer("test_trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(2, 2) +
ElementwiseLayer.Construct(2 * 2) +
FullyConnectedLayer.Construct(2) +
2 * (FullyConnectedLayer.Construct(4) + FullyConnectedLayer.Construct(2)) +
OutputLayer.Construct(2);
trainer.Network = (INetwork)trainer.Network.DeepCopy();
trainer.Operator = new CpuMultithreadedOperator(10);
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.1f));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.01f, mean: 0.03f));
trainer.Initialise(handler);
trainer.Network = (INetwork)trainer.Network.DeepCopy();
Console.WriteLine(trainer.Network.Registry);
IRegistryResolver resolver = new RegistryResolver(trainer.Network.Registry);
Console.WriteLine("===============");
object[] weights = resolver.ResolveGet <object>("layers.*.weights");
Console.WriteLine(string.Join("\n", weights));
Console.WriteLine("===============");
//foreach (ILayerBuffer buffer in trainer.Network.YieldLayerBuffersOrdered())
//{
// Console.WriteLine(buffer.Layer.Name + ": ");
// Console.WriteLine("inputs:");
// foreach (string input in buffer.Inputs.Keys)
// {
// Console.WriteLine($"\t{input}: {buffer.Inputs[input].GetHashCode()}");
// }
// Console.WriteLine("outputs:");
// foreach (string output in buffer.Outputs.Keys)
// {
// Console.WriteLine($"\t{output}: {buffer.Outputs[output].GetHashCode()}");
// }
//}
}
private static void SampleIris()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("iris");
sigma.SetRandomSeed(0);
sigma.Prepare();
IDataset dataset = Defaults.Datasets.Iris();
ITrainer trainer = sigma.CreateGhostTrainer("iris-trainer");
trainer.Network.Architecture = InputLayer.Construct(4)
+ FullyConnectedLayer.Construct(12)
+ FullyConnectedLayer.Construct(3)
+ OutputLayer.Construct(3)
+ SquaredDifferenceCostLayer.Construct();
//trainer.Network = Serialisation.ReadBinaryFileIfExists("iris.sgnet", trainer.Network);
trainer.TrainingDataIterator = new MinibatchIterator(50, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.06);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
//trainer.AddGlobalHook(new StopTrainingHook(atEpoch: 100));
//trainer.AddLocalHook(new EarlyStopperHook("optimiser.cost_total", 20, target: ExtremaTarget.Min));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch), reportEpochIteration: true));
//.On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
//trainer.AddLocalHook(new DiskSaviorHook<INetwork>("network.self", Namers.Dynamic("iris_epoch{0}.sgnet", "epoch"), verbose: true)
// .On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", TimeStep.Every(1, TimeScale.Epoch), tops: 1));
trainer.AddHook(new StopTrainingHook(new ThresholdCriteria("shared.classification_accuracy_top1", ComparisonTarget.GreaterThanEquals, 0.98)));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(599, TimeScale.Iteration), 128));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(1, TimeScale.Epoch), 4));
//Serialisation.WriteBinaryFile(trainer, "trainer.sgtrainer");
//trainer = Serialisation.ReadBinaryFile<ITrainer>("trainer.sgtrainer");
sigma.AddTrainer(trainer);
sigma.AddMonitor(new HttpMonitor("http://+:8080/sigma/"));
sigma.PrepareAndRun();
}
private static void SampleMnist()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("mnist");
sigma.SetRandomSeed(0);
IDataset dataset = Defaults.Datasets.Mnist();
ITrainer trainer = sigma.CreateTrainer("mnist-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(28, 28)
+ DropoutLayer.Construct(0.2)
+ FullyConnectedLayer.Construct(1000, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(800, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(10, activation: "sigmoid")
+ OutputLayer.Construct(10)
+ SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(100, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(Defaults.Datasets.MnistValidation()));
//trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.01);
//trainer.Optimiser = new MomentumGradientOptimiser(learningRate: 0.01, momentum: 0.9);
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.02);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new ValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration), reportEpochIteration: true)
.On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
var validationTimeStep = TimeStep.Every(1, TimeScale.Epoch);
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", validationTimeStep, tops: new[] { 1, 2, 3 }));
for (int i = 0; i < 10; i++)
{
trainer.AddGlobalHook(new TargetMaximisationReporter(trainer.Operator.Handler.NDArray(ArrayUtils.OneHot(i, 10), 10), TimeStep.Every(1, TimeScale.Epoch)));
}
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(10, TimeScale.Iteration), 32));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(1, TimeScale.Epoch), 4));
trainer.AddHook(new StopTrainingHook(atEpoch: 10));
sigma.PrepareAndRun();
}
private static void Main()
{
SigmaEnvironment.EnableLogging();
SigmaEnvironment sigma = SigmaEnvironment.Create("Sigma-MNIST");
// create a new mnist trainer
ITrainer trainer = CreateMnistTrainer(sigma);
// for the UI we have to activate more features
if (UI)
{
// create and attach a new UI framework
WPFMonitor gui = sigma.AddMonitor(new WPFMonitor("MNIST"));
// create a tab
gui.AddTabs("Overview");
// access the window inside the ui thread
gui.WindowDispatcher(window =>
{
// enable initialisation
window.IsInitializing = true;
// add a panel that controls the learning process
window.TabControl["Overview"].AddCumulativePanel(new ControlPanel("Control", trainer));
// create an accuracy cost that updates every iteration
var cost = new TrainerChartPanel <CartesianChart, LineSeries, TickChartValues <double>, double>("Cost", trainer, "optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration));
// improve the chart performance
cost.Fast();
// add the newly created panel
window.TabControl["Overview"].AddCumulativePanel(cost);
// finish initialisation
window.IsInitializing = false;
});
// the operators should not run instantly but when the user clicks play
sigma.StartOperatorsOnRun = false;
}
sigma.Prepare();
sigma.Run();
}
public void TestUndividedIteratorYield()
{
string filename = ".unittestfile" + nameof(TestUndividedIteratorCreate);
CreateCsvTempFile(filename);
SigmaEnvironment.Clear();
FileSource source = new FileSource(filename, Path.GetTempPath());
CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> {
["inputs"] = new[] { new[] { 0 } }
}));
ExtractedDataset dataset = new ExtractedDataset("test", 2, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestUndividedIteratorCreate)), true, extractor);
UndividedIterator iterator = new UndividedIterator(dataset);
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
IComputationHandler handler = new CpuFloat32Handler();
int index = 0;
foreach (var block in iterator.Yield(handler, sigma))
{
if (index == 0)
{
Assert.AreEqual(new float[] { 5.1f, 4.9f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 2));
}
else if (index == 1)
{
Assert.AreEqual(new float[] { 4.7f }, block["inputs"].GetDataAs <float>().GetValuesArrayAs <float>(0, 1));
}
else
{
Assert.Fail("There can be a maximum of two iterations, but this is yield iteration 3 (index 2).");
}
index++;
}
dataset.Dispose();
DeleteTempFile(filename);
}
private static void SampleCachedFastIteration()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
IDataSource dataSource = new CompressedSource(new MultiSource(new FileSource("train-images-idx3-ubyte.gz"), new UrlSource("http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz")));
ByteRecordReader mnistImageReader = new ByteRecordReader(headerLengthBytes: 16, recordSizeBytes: 28 * 28, source: dataSource);
IRecordExtractor mnistImageExtractor = mnistImageReader.Extractor("inputs", new[] { 0L, 0L }, new[] { 28L, 28L }).Preprocess(new NormalisingPreprocessor(0, 255));
IDataset dataset = new ExtractedDataset("mnist-training", ExtractedDataset.BlockSizeAuto, mnistImageExtractor);
IDataset[] slices = dataset.SplitRecordwise(0.8, 0.2);
IDataset trainingData = slices[0];
Stopwatch stopwatch = Stopwatch.StartNew();
IDataIterator iterator = new MinibatchIterator(10, trainingData);
foreach (var block in iterator.Yield(new CpuFloat32Handler(), sigma))
{
//PrintFormattedBlock(block, PrintUtils.AsciiGreyscalePalette);
}
Console.Write("\nFirst iteration took " + stopwatch.Elapsed + "\n+=+ Iterating over dataset again +=+ Dramatic pause...");
ArrayUtils.Range(1, 10).ToList().ForEach(i =>
{
Thread.Sleep(500);
Console.Write(".");
});
stopwatch.Restart();
foreach (var block in iterator.Yield(new CpuFloat32Handler(), sigma))
{
//PrintFormattedBlock(block, PrintUtils.AsciiGreyscalePalette);
}
Console.WriteLine("Second iteration took " + stopwatch.Elapsed);
}
public void TestMinibatchIteratorYield(int minibatchSize)
{
string filename = ".unittestfile" + nameof(TestMinibatchIteratorYield);
CreateCsvTempFile(filename);
SigmaEnvironment.Clear();
FileSource source = new FileSource(filename, Path.GetTempPath());
CsvRecordExtractor extractor = (CsvRecordExtractor) new CsvRecordReader(source).Extractor(new CsvRecordExtractor(new Dictionary <string, int[][]> {
["inputs"] = new[] { new[] { 0 } }
}));
ExtractedDataset dataset = new ExtractedDataset("test", 1, new DiskCacheProvider(Path.GetTempPath() + "/" + nameof(TestMinibatchIteratorYield)), true, extractor);
MinibatchIterator iterator = new MinibatchIterator(minibatchSize, dataset);
IComputationHandler handler = new CpuFloat32Handler();
SigmaEnvironment sigma = SigmaEnvironment.Create("test");
Assert.Throws <ArgumentNullException>(() => iterator.Yield(null, null).GetEnumerator().MoveNext());
Assert.Throws <ArgumentNullException>(() => iterator.Yield(handler, null).GetEnumerator().MoveNext());
Assert.Throws <ArgumentNullException>(() => iterator.Yield(null, sigma).GetEnumerator().MoveNext());
int index = 0;
foreach (var block in iterator.Yield(handler, sigma))
{
//pass through each more than 5 times to ensure consistency
if (index++ > 20)
{
break;
}
Assert.Contains(block["inputs"].GetValue <float>(0, 0, 0), new float[] { 5.1f, 4.9f, 4.7f });
}
dataset.Dispose();
DeleteTempFile(filename);
}
private static void SampleNetworkMerging()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("merge_test");
ITrainer[] trainers = new ITrainer[3];
int[] constantValues = { 2, 10, 70 };
//INetworkMerger merger = new WeightedNetworkMerger(10d, 10d, 1d);
INetworkMerger merger = new AverageNetworkMerger();
IComputationHandler handler = new CpuFloat32Handler();
for (int i = 0; i < trainers.Length; i++)
{
trainers[i] = sigma.CreateTrainer($"MergeTrainer{i}");
trainers[i].Network = new Network($"{i}");
trainers[i].Network.Architecture = InputLayer.Construct(2, 2) + ElementwiseLayer.Construct(2 * 2) + OutputLayer.Construct(2);
trainers[i].AddInitialiser("*.weights", new ConstantValueInitialiser(constantValues[i]));
trainers[i].Operator = new CpuMultithreadedOperator(5);
trainers[i].Initialise(handler);
}
foreach (ITrainer trainer in trainers)
{
Console.WriteLine(trainer.Network.Registry);
}
merger.AddMergeEntry("layers.*.weights");
merger.Merge(trainers[1].Network, trainers[2].Network, handler);
Console.WriteLine("*******************");
foreach (ITrainer trainer in trainers)
{
Console.WriteLine(trainer.Network.Registry);
}
}
public static INetwork GenerateNetwork(double number)
{
if (_environment == null)
{
_environment = SigmaEnvironment.Create("TestAverageNetworkMergerEnvironment");
}
ITrainer trainer = _environment.CreateTrainer($"trainer{_count++}");
Network net = new Network();
net.Architecture = InputLayer.Construct(2, 2) + FullyConnectedLayer.Construct(2 * 2) + OutputLayer.Construct(2);
trainer.Network = net;
trainer.AddInitialiser("*.weights", new ConstantValueInitialiser(number));
trainer.Operator = new CpuSinglethreadedOperator();
trainer.Initialise(new CpuFloat32Handler());
SigmaEnvironment.Clear();
return(net);
}
private static SigmaEnvironment ClearAndCreate(string identifier)
{
SigmaEnvironment.Clear();
return(SigmaEnvironment.Create(identifier));
}
private static void Main()
{
SigmaEnvironment.EnableLogging();
SigmaEnvironment sigma = SigmaEnvironment.Create("sigma_demo");
// create a new mnist trainer
string name = DemoMode.Name;
ITrainer trainer = DemoMode.CreateTrainer(sigma);
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.weights", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_weights_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.weights", (a, h) => h.StandardDeviation(a), "shared.network_weights_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.biases", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_biases_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.biases", (a, h) => h.StandardDeviation(a), "shared.network_biases_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("optimiser.updates", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.optimiser_updates_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("optimiser.updates", (a, h) => h.StandardDeviation(a), "shared.optimiser_updates_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*<external_output>._outputs.default.activations", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_activations_mean"));
// create and attach a new UI framework
WPFMonitor gui = sigma.AddMonitor(new WPFMonitor(name, DemoMode.Language));
gui.ColourManager.Dark = DemoMode.Dark;
gui.ColourManager.PrimaryColor = DemoMode.PrimarySwatch;
StatusBarLegendInfo iris = new StatusBarLegendInfo(name, MaterialColour.Blue);
StatusBarLegendInfo general = new StatusBarLegendInfo("General", MaterialColour.Grey);
gui.AddLegend(iris);
gui.AddLegend(general);
// create a tab
gui.AddTabs("Overview", "Metrics", "Validation", "Maximisation", "Reproduction", "Update");
// access the window inside the ui thread
gui.WindowDispatcher(window =>
{
// enable initialisation
window.IsInitializing = true;
window.TabControl["Metrics"].GridSize = new GridSize(2, 4);
window.TabControl["Validation"].GridSize = new GridSize(2, 5);
window.TabControl["Maximisation"].GridSize = new GridSize(2, 5);
window.TabControl["Reproduction"].GridSize = new GridSize(2, 5);
window.TabControl["Update"].GridSize = new GridSize(1, 1);
window.TabControl["Overview"].GridSize.Rows -= 1;
window.TabControl["Overview"].GridSize.Columns -= 1;
// add a panel that controls the learning process
window.TabControl["Overview"].AddCumulativePanel(new ControlPanel("Control", trainer), legend: iris);
ITimeStep reportTimeStep = DemoMode.Slow ? TimeStep.Every(1, TimeScale.Iteration) : TimeStep.Every(10, TimeScale.Epoch);
var cost1 = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Cost / Epoch", trainer, "optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)).Linearify();
var cost2 = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Cost / Epoch", trainer, "optimiser.cost_total", reportTimeStep);
var weightAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Weights / Epoch", trainer, "shared.network_weights_average", reportTimeStep, averageMode: true).Linearify();
var weightStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Weights / Epoch", trainer, "shared.network_weights_stddev", reportTimeStep, averageMode: true).Linearify();
var biasesAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Biases / Epoch", trainer, "shared.network_biases_average", reportTimeStep, averageMode: true).Linearify();
var biasesStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Biases / Epoch", trainer, "shared.network_biases_stddev", reportTimeStep, averageMode: true).Linearify();
var updateAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Parameter Updates / Epoch", trainer, "shared.optimiser_updates_average", reportTimeStep, averageMode: true).Linearify();
var updateStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Parameter Updates / Epoch", trainer, "shared.optimiser_updates_stddev", reportTimeStep, averageMode: true).Linearify();
var outputActivationsMean = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Output Activations", trainer, "shared.network_activations_mean", reportTimeStep, averageMode: true).Linearify();
AccuracyPanel accuracy1 = null, accuracy2 = null;
if (DemoMode != DemoType.Wdbc && DemoMode != DemoType.Parkinsons)
{
accuracy1 = new AccuracyPanel("Validation Accuracy", trainer, DemoMode.Slow ? TimeStep.Every(1, TimeScale.Epoch) : reportTimeStep, null, 1, 2);
accuracy1.Fast().Linearify();
accuracy2 = new AccuracyPanel("Validation Accuracy", trainer, DemoMode.Slow ? TimeStep.Every(1, TimeScale.Epoch) : reportTimeStep, null, 1, 2);
accuracy2.Fast().Linearify();
}
IRegistry regTest = new Registry();
regTest.Add("test", DateTime.Now);
var parameter = new ParameterPanel("Parameters", sigma, window);
parameter.Add("Time", typeof(DateTime), regTest, "test");
ValueSourceReporter valueHook = new ValueSourceReporter(TimeStep.Every(1, TimeScale.Epoch), "optimiser.cost_total");
trainer.AddGlobalHook(valueHook);
sigma.SynchronisationHandler.AddSynchronisationSource(valueHook);
var costBlock = (UserControlParameterVisualiser)parameter.Content.Add("Cost", typeof(double), trainer.Operator.Registry, "optimiser.cost_total");
costBlock.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Epoch));
var learningBlock = (UserControlParameterVisualiser)parameter.Content.Add("Learning rate", typeof(double), trainer.Operator.Registry, "optimiser.learning_rate");
learningBlock.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Epoch));
var paramCount = (UserControlParameterVisualiser)parameter.Content.Add("Parameter count", typeof(long), trainer.Operator.Registry, "network.parameter_count");
paramCount.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Start));
window.TabControl["Overview"].AddCumulativePanel(cost1, 1, 2, legend: iris);
window.TabControl["Overview"].AddCumulativePanel(parameter);
//window.TabControl["Overview"].AddCumulativePanel(accuracy1, 1, 2, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(cost2, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(weightAverage, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(biasesAverage, legend: iris);
window.TabControl["Update"].AddCumulativePanel(updateAverage, legend: iris);
if (accuracy2 != null)
{
window.TabControl["Metrics"].AddCumulativePanel(accuracy2, legend: iris);
}
window.TabControl["Metrics"].AddCumulativePanel(weightStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(biasesStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(updateStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(outputActivationsMean, legend: iris);
if (DemoMode == DemoType.Mnist)
{
NumberPanel outputpanel = new NumberPanel("Numbers", trainer);
DrawPanel drawPanel = new DrawPanel("Draw", trainer, 560, 560, 20, outputpanel);
window.TabControl["Validation"].AddCumulativePanel(drawPanel, 2, 3);
window.TabControl["Validation"].AddCumulativePanel(outputpanel, 2);
window.TabControl["Validation"].AddCumulativePanel(weightAverage);
window.TabControl["Validation"].AddCumulativePanel(biasesAverage);
for (int i = 0; i < 10; i++)
{
window.TabControl["Maximisation"].AddCumulativePanel(new MnistBitmapHookPanel($"Target Maximisation {i}", i, trainer, TimeStep.Every(1, TimeScale.Epoch)));
}
}
if (DemoMode == DemoType.TicTacToe)
{
window.TabControl["Overview"].AddCumulativePanel(new TicTacToePanel("Play TicTacToe!", trainer));
}
//for (int i = 0; i < 10; i++)
//{
// window.TabControl["Reproduction"].AddCumulativePanel(new MnistBitmapHookPanel($"Target Maximisation 7-{i}", 8, 28, 28, trainer, TimeStep.Every(1, TimeScale.Start)));
//}
});
if (DemoMode == DemoType.Mnist)
{
sigma.AddMonitor(new HttpMonitor("http://+:8080/sigma/"));
}
// the operators should not run instantly but when the user clicks play
sigma.StartOperatorsOnRun = false;
sigma.Prepare();
sigma.RunAsync();
gui.WindowDispatcher(window => window.IsInitializing = false);
}