RecurrentLayerComponent _ReadComponent(RecurrentLayer network, ActivationType activation)
{
var c1 = _ReadFeedForward(network.Layer[0]);
var c2 = _ReadFeedForward(network.Layer[1]);
return(new RecurrentLayerComponent(c1, c2, _activation[activation]));
}
IRecurrentLayerExecution _GetRecurrentExecution(RecurrentLayer network)
{
var type = network.Type;
if (type == RecurrentLayerType.SimpleRecurrent)
{
return(new SimpleRecurrent(_ReadComponent(network, network.Activation)));
}
else if (type == RecurrentLayerType.Lstm)
{
var c = _ReadComponent(network, network.Activation);
var i = _ReadComponent(network, ActivationType.Sigmoid);
var f = _ReadComponent(network, ActivationType.Sigmoid);
var o = _ReadComponent(network, ActivationType.Sigmoid);
return(new Lstm(c, i, f, o, _activation[network.Activation]));
}
else if (type == RecurrentLayerType.FeedForward)
{
return(new RecurrentFeedForward(_ReadFeedForward(network.Layer[0])));
}
else
{
throw new Exception("Unknown recurrent type: " + type.ToString());
}
}
private static void SampleHutter()
{
const long timeWindowSize = 10L;
SigmaEnvironment sigma = SigmaEnvironment.Create("recurrent");
IDataSource source = new MultiSource(new FileSource("enwik8"), new CompressedSource(new MultiSource(new FileSource("enwik8.zip"), new UrlSource("http://mattmahoney.net/dc/enwik8.zip"))));
IRecordExtractor extractor = new CharacterRecordReader(source, (int)(timeWindowSize + 1), Encoding.ASCII)
.Extractor(new ArrayRecordExtractor <short>(ArrayRecordExtractor <short>
.ParseExtractorParameters("inputs", new[] { 0L }, new[] { timeWindowSize }, "targets", new[] { 0L }, new[] { timeWindowSize }))
.Offset("targets", 1L))
.Preprocess(new PermutePreprocessor(0, 2, 1))
.Preprocess(new OneHotPreprocessor(0, 255));
IDataset dataset = new ExtractedDataset("hutter", ExtractedDataset.BlockSizeAuto, false, extractor);
ITrainer trainer = sigma.CreateTrainer("hutter");
trainer.Network.Architecture = InputLayer.Construct(256) + RecurrentLayer.Construct(256) + OutputLayer.Construct(256) + SoftMaxCrossEntropyCostLayer.Construct();
trainer.TrainingDataIterator = new MinibatchIterator(32, dataset);
trainer.AddNamedDataIterator("validation", new MinibatchIterator(100, dataset));
trainer.Optimiser = new AdagradOptimiser(baseLearningRate: 0.07);
trainer.Operator = new CudaSinglethreadedOperator();
trainer.AddInitialiser("*.*", new GaussianInitialiser(standardDeviation: 0.05));
trainer.AddLocalHook(new AccumulatedValueReporter("optimiser.cost_total", TimeStep.Every(1, TimeScale.Iteration), averageValues: true));
trainer.AddLocalHook(new RunningTimeReporter(TimeStep.Every(10, TimeScale.Iteration)));
sigma.PrepareAndRun();
}
