/// <summary>
/// Create a MNIST trainer (writing recognition) will be added to an environemnt.
/// </summary>
/// <param name="sigma">The sigma environemnt this trainer will be assigned to.</param>
/// <returns>The newly created trainer.</returns>
private static ITrainer CreateMnistTrainer(SigmaEnvironment sigma)
{
ByteRecordReader mnistImageReader = new ByteRecordReader(headerLengthBytes: 16, recordSizeBytes: 28 * 28, source: new CompressedSource(new MultiSource(new FileSource("train-images-idx3-ubyte.gz"), new UrlSource("http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz"))));
IRecordExtractor mnistImageExtractor = mnistImageReader.Extractor("inputs", new[] { 0L, 0L }, new[] { 28L, 28L }).Preprocess(new NormalisingPreprocessor(0, 255));
ByteRecordReader mnistTargetReader = new ByteRecordReader(headerLengthBytes: 8, recordSizeBytes: 1, source: new CompressedSource(new MultiSource(new FileSource("train-labels-idx1-ubyte.gz"), new UrlSource("http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz"))));
IRecordExtractor mnistTargetExtractor = mnistTargetReader.Extractor("targets", new[] { 0L }, new[] { 1L }).Preprocess(new OneHotPreprocessor(minValue: 0, maxValue: 9));
IDataset dataset = new Dataset("mnist-training", Dataset.BlockSizeAuto, mnistImageExtractor, mnistTargetExtractor);
ITrainer trainer = sigma.CreateTrainer("test");
trainer.Network = new Network
{
Architecture = InputLayer.Construct(28, 28)
+ 2 * FullyConnectedLayer.Construct(28 * 28)
+ FullyConnectedLayer.Construct(10)
+ OutputLayer.Construct(10)
+ SoftMaxCrossEntropyCostLayer.Construct()
};
trainer.TrainingDataIterator = new MinibatchIterator(8, dataset);
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.02);
trainer.Operator = new CpuSinglethreadedOperator();
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.05f));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.01f, mean: 0.03f));
trainer.AddGlobalHook(new CurrentEpochIterationReporter(TimeStep.Every(1, TimeScale.Iteration)));
return(trainer);
}
public static ITrainer CreateTicTacToeTrainer(SigmaEnvironment sigma)
{
IDataset dataset = Defaults.Datasets.TicTacToe();
ITrainer trainer = sigma.CreateTrainer("tictactoe-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(9)
+ FullyConnectedLayer.Construct(72, "tanh")
+ FullyConnectedLayer.Construct(99, "tanh")
+ FullyConnectedLayer.Construct(3, "tanh")
+ OutputLayer.Construct(3)
+ SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(21, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(dataset));
trainer.Optimiser = new MomentumGradientOptimiser(learningRate: 0.01, momentum: 0.9);
trainer.Operator = new CpuSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)));
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", TimeStep.Every(1, TimeScale.Epoch), tops: new[] { 1, 2 }));
trainer.AddGlobalHook(new DiskSaviorHook <INetwork>(TimeStep.Every(1, TimeScale.Epoch), "network.self", Namers.Static("tictactoe.sgnet"), verbose: true)
.On(new ExtremaCriteria("shared.classification_accuracy_top1", ExtremaTarget.Max)));
return(trainer);
}
/// <summary>
/// This method will be called after the panel has been added (window, monitor set...)
/// </summary>
protected override void OnInitialise(SigmaWindow window)
{
if (!Trainers.Contains(Trainer))
{
ValueSourceReporter valueHook = new ValueSourceReporter(TimeStep.Every(1, TimeScale.Epoch), "runtime_millis");
_trainer.AddGlobalHook(valueHook);
Monitor.Sigma.SynchronisationHandler.AddSynchronisationSource(valueHook);
Trainers.Add(Trainer);
valueHook = new ValueSourceReporter(TimeStep.Every(1, TimeScale.Iteration), "iteration");
_trainer.AddLocalHook(valueHook);
Monitor.Sigma.SynchronisationHandler.AddSynchronisationSource(valueHook);
}
//TODO: style?
_playbackControl = new SigmaPlaybackControl {
Trainer = Trainer, Margin = new Thickness(0, 0, 0, 20), HorizontalAlignment = HorizontalAlignment.Center
};
Content.Children.Add(_playbackControl);
_parameterView = new ParameterView(Monitor.Sigma, window);
SigmaTextBlock timeBox = (SigmaTextBlock)_parameterView.Add(Properties.Resources.RunningTime, typeof(object), _trainer.Operator.Registry, "runtime_millis");
timeBox.AutoPollValues(_trainer, TimeStep.Every(1, TimeScale.Epoch));
timeBox.Postfix = " ms";
UserControlParameterVisualiser epochBox = (UserControlParameterVisualiser)_parameterView.Add(Properties.Resources.CurrentEpoch, typeof(object), _trainer.Operator.Registry, "epoch");
epochBox.AutoPollValues(_trainer, TimeStep.Every(1, TimeScale.Epoch));
UserControlParameterVisualiser iterationBox = (UserControlParameterVisualiser)_parameterView.Add(Properties.Resources.CurrentIteration, typeof(object), _trainer.Operator.Registry, "iteration");
iterationBox.AutoPollValues(_trainer, TimeStep.Every(1, TimeScale.Iteration));
IRegistry registry = new Registry
{
{ "operator", Trainer.Operator.GetType().Name },
{ "optimiser", Trainer.Optimiser.GetType().Name }
};
_parameterView.Add(Properties.Resources.CurrentOperator, typeof(object), registry, "operator");
_parameterView.Add(Properties.Resources.CurrentOptimiser, typeof(object), registry, "optimiser");
//TODO: completely hardcoded activation function
UserControlParameterVisualiser activationBox = (UserControlParameterVisualiser)_parameterView.Add(Properties.Resources.CurrentActivationFunction, typeof(object), _trainer.Operator.Registry, "network.layers.2-fullyconnected.activation");
activationBox.AutoPollValues(_trainer, TimeStep.Every(1, TimeScale.Start));
Trainer.AddGlobalHook(new LambdaHook(TimeStep.Every(1, TimeScale.Reset), (registry1, resolver) =>
{
epochBox.Read();
iterationBox.Read();
}));
Content.Children.Add(_parameterView);
}
/// <summary>
/// Set the trainer and hook. Attach the hook.
/// </summary>
/// <param name="trainer">The trainer that will be set.</param>
/// <param name="hook">The hook that will be applied.</param>
protected void Init(ITrainer trainer, VisualAccumulatedValueReporterHook hook)
{
Trainer = trainer;
AttachedHook = hook;
Trainer.AddHook(hook);
Trainer.AddGlobalHook(new LambdaHook(TimeStep.Every(1, TimeScale.Stop), (registry, resolver) => Clear()));
// TODO: is a formatter the best solution?
AxisX.LabelFormatter = number => (number * hook.TimeStep.Interval).ToString(CultureInfo.InvariantCulture);
AxisX.Unit = hook.TimeStep.Interval;
}
/// <summary>
/// Enables the automatic polling of values (call Read on every TimeStep).
/// <c>null</c> if no automatic polling should be enabled.
/// </summary>
/// <param name="trainer">The trainer on which the poll will be performed.</param>
/// <param name="step">The TimeStep on when the parameter should update.</param>
public virtual void AutoPollValues(ITrainer trainer, ITimeStep step)
{
if (ActiveHook != null)
{
trainer.Operator.DetachGlobalHook(ActiveHook);
}
ActiveHook = new PollParameterHook(step, this);
trainer.AddGlobalHook(ActiveHook);
}
/// <summary>
/// Create a BitmapPanel that can easily be updated by a hook. This hook will be automatically attached to a given trainer.
/// This panel is optimised for MNIST with one hot of 10 and a given number.
/// </summary>
/// <param name="title">The given tile.</param>
/// <param name="timestep">The timestep this panel updates.</param>
/// <param name="trainer">The trainer the hook will be applied to.</param>
/// <param name="headerContent">The content for the header. If <c>null</c> is passed, the title will be used.</param>
/// <param name="number">The number this panel tries to visualise.</param>
public MnistBitmapHookPanel(string title, int number, ITrainer trainer, ITimeStep timestep, object headerContent = null) : base(title, 28, 28, headerContent)
{
if (trainer == null)
{
throw new ArgumentNullException(nameof(trainer));
}
UseLoadingIndicator = true;
VisualTargetMaximisationReporter hook = new VisualTargetMaximisationReporter(this, trainer.Operator.Handler.NDArray(ArrayUtils.OneHot(number, 10), 10L), timestep);
trainer.AddGlobalHook(hook);
}
private static void SampleMnist()
{
SigmaEnvironment sigma = SigmaEnvironment.Create("mnist");
sigma.SetRandomSeed(0);
IDataset dataset = Defaults.Datasets.Mnist();
ITrainer trainer = sigma.CreateTrainer("mnist-trainer");
trainer.Network = new Network();
trainer.Network.Architecture = InputLayer.Construct(28, 28)
+ DropoutLayer.Construct(0.2)
+ FullyConnectedLayer.Construct(1000, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(800, activation: "rel")
+ DropoutLayer.Construct(0.4)
+ FullyConnectedLayer.Construct(10, activation: "sigmoid")
+ OutputLayer.Construct(10)
+ SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(100, dataset);
trainer.AddNamedDataIterator("validation", new UndividedIterator(Defaults.Datasets.MnistValidation()));
//trainer.Optimiser = new GradientDescentOptimiser(learningRate: 0.01);
//trainer.Optimiser = new MomentumGradientOptimiser(learningRate: 0.01, momentum: 0.9);
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.02);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.weights", new GaussianInitialiser(standardDeviation: 0.1));
trainer.AddInitialiser("*.bias*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new ValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration), reportEpochIteration: true)
.On(new ExtremaCriteria("optimiser.cost_total", ExtremaTarget.Min)));
var validationTimeStep = TimeStep.Every(1, TimeScale.Epoch);
trainer.AddHook(new MultiClassificationAccuracyReporter("validation", validationTimeStep, tops: new[] { 1, 2, 3 }));
for (int i = 0; i < 10; i++)
{
trainer.AddGlobalHook(new TargetMaximisationReporter(trainer.Operator.Handler.NDArray(ArrayUtils.OneHot(i, 10), 10), TimeStep.Every(1, TimeScale.Epoch)));
}
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(10, TimeScale.Iteration), 32));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(1, TimeScale.Epoch), 4));
trainer.AddHook(new StopTrainingHook(atEpoch: 10));
sigma.PrepareAndRun();
}
/// <summary>
/// Create an AccuracyPanel with a given title. It displays given accuracies per epoch.
/// If a title is not sufficient modify <see cref="SigmaPanel.Header" />.
/// </summary>
/// <param name="title">The given tile.</param>
/// <param name="trainer"></param>
/// <param name="headerContent">The content for the header. If <c>null</c> is passed,
/// the title will be used.</param>
/// <param name="tops"></param>
public AccuracyPanel(string title, ITrainer trainer, ITimeStep timeStep, object headerContent = null, params int[] tops) : base(title, headerContent)
{
if (timeStep == null)
{
throw new ArgumentNullException(nameof(timeStep));
}
// skip the first since its automatically generated
for (int i = 1; i < tops.Length; i++)
{
AddSeries(new LineSeries());
}
trainer.AddHook(new ChartValidationAccuracyReport(this, "validation", timeStep, tops));
trainer.AddGlobalHook(new LambdaHook(TimeStep.Every(1, TimeScale.Stop), (registry, resolver) => Clear()));
AxisY.MinValue = 0;
AxisY.MaxValue = 100;
}
/// <summary>
/// This method will be called once the window is initialising (after it has been added).
/// Do not store a reference of the window unless you properly dispose it (remove reference once not required).
/// </summary>
/// <param name="window">The wpf window this panel will be added to.</param>
protected override void OnInitialise(SigmaWindow window)
{
Content = new TicTacToeField(Handler, Monitor);
_moveOrder = new List <int>(9);
Block = new Dictionary <string, INDArray>();
PassNetworkHook = new PassNetworkHook(this, Block, TimeStep.Every(1, TimeScale.Epoch));
PassNetworkHook.On(new ExternalCriteria(registerHoldFunc: action => InvokePass = action));
UpdateBlock();
Content.AiMove += AIMoveRequest;
Trainer.AddGlobalHook(PassNetworkHook);
//Content.FieldChange += FieldChange;
IsReady = true;
}
private static void Main()
{
SigmaEnvironment.EnableLogging();
SigmaEnvironment sigma = SigmaEnvironment.Create("sigma_demo");
// create a new mnist trainer
string name = DemoMode.Name;
ITrainer trainer = DemoMode.CreateTrainer(sigma);
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.weights", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_weights_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.weights", (a, h) => h.StandardDeviation(a), "shared.network_weights_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.biases", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_biases_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.biases", (a, h) => h.StandardDeviation(a), "shared.network_biases_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("optimiser.updates", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.optimiser_updates_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("optimiser.updates", (a, h) => h.StandardDeviation(a), "shared.optimiser_updates_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*<external_output>._outputs.default.activations", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_activations_mean"));
// create and attach a new UI framework
WPFMonitor gui = sigma.AddMonitor(new WPFMonitor(name, DemoMode.Language));
gui.ColourManager.Dark = DemoMode.Dark;
gui.ColourManager.PrimaryColor = DemoMode.PrimarySwatch;
StatusBarLegendInfo iris = new StatusBarLegendInfo(name, MaterialColour.Blue);
StatusBarLegendInfo general = new StatusBarLegendInfo("General", MaterialColour.Grey);
gui.AddLegend(iris);
gui.AddLegend(general);
// create a tab
gui.AddTabs("Overview", "Metrics", "Validation", "Maximisation", "Reproduction", "Update");
// access the window inside the ui thread
gui.WindowDispatcher(window =>
{
// enable initialisation
window.IsInitializing = true;
window.TabControl["Metrics"].GridSize = new GridSize(2, 4);
window.TabControl["Validation"].GridSize = new GridSize(2, 5);
window.TabControl["Maximisation"].GridSize = new GridSize(2, 5);
window.TabControl["Reproduction"].GridSize = new GridSize(2, 5);
window.TabControl["Update"].GridSize = new GridSize(1, 1);
window.TabControl["Overview"].GridSize.Rows -= 1;
window.TabControl["Overview"].GridSize.Columns -= 1;
// add a panel that controls the learning process
window.TabControl["Overview"].AddCumulativePanel(new ControlPanel("Control", trainer), legend: iris);
ITimeStep reportTimeStep = DemoMode.Slow ? TimeStep.Every(1, TimeScale.Iteration) : TimeStep.Every(10, TimeScale.Epoch);
var cost1 = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Cost / Epoch", trainer, "optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)).Linearify();
var cost2 = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Cost / Epoch", trainer, "optimiser.cost_total", reportTimeStep);
var weightAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Weights / Epoch", trainer, "shared.network_weights_average", reportTimeStep, averageMode: true).Linearify();
var weightStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Weights / Epoch", trainer, "shared.network_weights_stddev", reportTimeStep, averageMode: true).Linearify();
var biasesAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Biases / Epoch", trainer, "shared.network_biases_average", reportTimeStep, averageMode: true).Linearify();
var biasesStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Biases / Epoch", trainer, "shared.network_biases_stddev", reportTimeStep, averageMode: true).Linearify();
var updateAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Parameter Updates / Epoch", trainer, "shared.optimiser_updates_average", reportTimeStep, averageMode: true).Linearify();
var updateStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Parameter Updates / Epoch", trainer, "shared.optimiser_updates_stddev", reportTimeStep, averageMode: true).Linearify();
var outputActivationsMean = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Output Activations", trainer, "shared.network_activations_mean", reportTimeStep, averageMode: true).Linearify();
AccuracyPanel accuracy1 = null, accuracy2 = null;
if (DemoMode != DemoType.Wdbc && DemoMode != DemoType.Parkinsons)
{
accuracy1 = new AccuracyPanel("Validation Accuracy", trainer, DemoMode.Slow ? TimeStep.Every(1, TimeScale.Epoch) : reportTimeStep, null, 1, 2);
accuracy1.Fast().Linearify();
accuracy2 = new AccuracyPanel("Validation Accuracy", trainer, DemoMode.Slow ? TimeStep.Every(1, TimeScale.Epoch) : reportTimeStep, null, 1, 2);
accuracy2.Fast().Linearify();
}
IRegistry regTest = new Registry();
regTest.Add("test", DateTime.Now);
var parameter = new ParameterPanel("Parameters", sigma, window);
parameter.Add("Time", typeof(DateTime), regTest, "test");
ValueSourceReporter valueHook = new ValueSourceReporter(TimeStep.Every(1, TimeScale.Epoch), "optimiser.cost_total");
trainer.AddGlobalHook(valueHook);
sigma.SynchronisationHandler.AddSynchronisationSource(valueHook);
var costBlock = (UserControlParameterVisualiser)parameter.Content.Add("Cost", typeof(double), trainer.Operator.Registry, "optimiser.cost_total");
costBlock.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Epoch));
var learningBlock = (UserControlParameterVisualiser)parameter.Content.Add("Learning rate", typeof(double), trainer.Operator.Registry, "optimiser.learning_rate");
learningBlock.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Epoch));
var paramCount = (UserControlParameterVisualiser)parameter.Content.Add("Parameter count", typeof(long), trainer.Operator.Registry, "network.parameter_count");
paramCount.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Start));
window.TabControl["Overview"].AddCumulativePanel(cost1, 1, 2, legend: iris);
window.TabControl["Overview"].AddCumulativePanel(parameter);
//window.TabControl["Overview"].AddCumulativePanel(accuracy1, 1, 2, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(cost2, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(weightAverage, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(biasesAverage, legend: iris);
window.TabControl["Update"].AddCumulativePanel(updateAverage, legend: iris);
if (accuracy2 != null)
{
window.TabControl["Metrics"].AddCumulativePanel(accuracy2, legend: iris);
}
window.TabControl["Metrics"].AddCumulativePanel(weightStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(biasesStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(updateStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(outputActivationsMean, legend: iris);
if (DemoMode == DemoType.Mnist)
{
NumberPanel outputpanel = new NumberPanel("Numbers", trainer);
DrawPanel drawPanel = new DrawPanel("Draw", trainer, 560, 560, 20, outputpanel);
window.TabControl["Validation"].AddCumulativePanel(drawPanel, 2, 3);
window.TabControl["Validation"].AddCumulativePanel(outputpanel, 2);
window.TabControl["Validation"].AddCumulativePanel(weightAverage);
window.TabControl["Validation"].AddCumulativePanel(biasesAverage);
for (int i = 0; i < 10; i++)
{
window.TabControl["Maximisation"].AddCumulativePanel(new MnistBitmapHookPanel($"Target Maximisation {i}", i, trainer, TimeStep.Every(1, TimeScale.Epoch)));
}
}
if (DemoMode == DemoType.TicTacToe)
{
window.TabControl["Overview"].AddCumulativePanel(new TicTacToePanel("Play TicTacToe!", trainer));
}
//for (int i = 0; i < 10; i++)
//{
// window.TabControl["Reproduction"].AddCumulativePanel(new MnistBitmapHookPanel($"Target Maximisation 7-{i}", 8, 28, 28, trainer, TimeStep.Every(1, TimeScale.Start)));
//}
});
if (DemoMode == DemoType.Mnist)
{
sigma.AddMonitor(new HttpMonitor("http://+:8080/sigma/"));
}
// the operators should not run instantly but when the user clicks play
sigma.StartOperatorsOnRun = false;
sigma.Prepare();
sigma.RunAsync();
gui.WindowDispatcher(window => window.IsInitializing = false);
}