static void SequenceToSequence()
{
const int SEQUENCE_LENGTH = 5;
var grammar = new SequenceClassification(8, SEQUENCE_LENGTH, SEQUENCE_LENGTH, true, false);
var sequences = grammar.GenerateSequences().Take(2000).ToList();
var builder = BrightWireProvider.CreateDataTableBuilder();
builder.AddColumn(ColumnType.Matrix, "Input");
builder.AddColumn(ColumnType.Matrix, "Output");
foreach (var sequence in sequences)
{
var encodedSequence = grammar.Encode(sequence);
var reversedSequence = new FloatMatrix {
Row = encodedSequence.Row.Reverse().Take(SEQUENCE_LENGTH - 1).ToArray()
};
builder.Add(encodedSequence, reversedSequence);
}
var data = builder.Build().Split(0);
using (var lap = BrightWireProvider.CreateLinearAlgebra()) {
var graph = new GraphFactory(lap);
var errorMetric = graph.ErrorMetric.BinaryClassification;
// create the property set
var propertySet = graph.CurrentPropertySet
.Use(graph.GradientDescent.RmsProp)
.Use(graph.WeightInitialisation.Xavier)
;
const int BATCH_SIZE = 16;
int HIDDEN_LAYER_SIZE = 64;
const float TRAINING_RATE = 0.1f;
// create the encoder
var encoderLearningContext = graph.CreateLearningContext(TRAINING_RATE, BATCH_SIZE, TrainingErrorCalculation.Fast, true);
var encoderMemory = new float[HIDDEN_LAYER_SIZE];
var trainingData = graph.CreateDataSource(data.Training, encoderLearningContext, wb => wb
.AddLstm(encoderMemory, "encoder")
.WriteNodeMemoryToSlot("shared-memory", wb.Find("encoder") as IHaveMemoryNode)
.AddFeedForward(grammar.DictionarySize)
.Add(graph.SigmoidActivation())
.AddBackpropagationThroughTime(errorMetric)
);
var testData = trainingData.CloneWith(data.Test);
// create the engine
var engine = graph.CreateTrainingEngine(trainingData, TRAINING_RATE, BATCH_SIZE);
engine.LearningContext.ScheduleLearningRate(30, TRAINING_RATE / 3);
engine.LearningContext.ScheduleLearningRate(40, TRAINING_RATE / 9);
// create the decoder
var decoderMemory = new float[HIDDEN_LAYER_SIZE];
var wb2 = graph.Connect(engine);
wb2
.JoinInputWithMemory("shared-memory")
.IncrementSizeBy(HIDDEN_LAYER_SIZE)
.AddLstm(decoderMemory, "decoder")
.AddFeedForward(trainingData.OutputSize)
.Add(graph.SigmoidActivation())
.AddBackpropagationThroughTime(errorMetric)
;
engine.Train(50, testData, errorMetric);
//var dataSourceModel = (trainingData as IAdaptiveDataSource).GetModel();
//var testData2 = graph.CreateDataSource(data.Test, dataSourceModel);
//var networkGraph = engine.Graph;
//var executionEngine = graph.CreateEngine(networkGraph);
//var output = executionEngine.Execute(testData2);
//Console.WriteLine(output.Average(o => o.CalculateError(errorMetric)));
}
}
