void _Create(GraphFactory graph, int inputSize, float[] memory, INode activation, string memoryId)
{
_inputSize = inputSize;
_activation = activation;
int hiddenLayerSize = memory.Length;
_memory = new MemoryFeeder(memory, null, memoryId);
_input = new FlowThrough();
var inputChannel = graph.Connect(inputSize, _input)
.AddFeedForward(hiddenLayerSize, "Wh");
var memoryChannel = graph.Connect(hiddenLayerSize, _memory)
.AddFeedForward(hiddenLayerSize, "Uh");
_output = graph.Add(inputChannel, memoryChannel)
.AddBackwardAction(new ConstrainSignal())
.Add(activation)
.AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode;
;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
{
int hiddenLayerSize = memory.Length;
_inputSize = inputSize;
_memory = new MemoryFeeder(memory, null, memoryId);
_input = new FlowThrough();
var Wx = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wx").LastNode;
var Wi = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wi");
var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui");
var Wf = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wf");
var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf");
var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation());
var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation());
_output = graph
.Add(graph.Multiply(hiddenLayerSize, Wx, It.LastNode), graph.Multiply(hiddenLayerSize, _memory, Ft.LastNode))
.AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode
;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
{
_inputSize = inputSize;
int hiddenLayerSize = memory.Length;
_memory = new MemoryFeeder(memory, null, memoryId);
_state = new MemoryFeeder(new float[memory.Length]);
_input = new FlowThrough();
var Wf = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wf");
var Wi = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wi");
var Wo = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wo");
var Wc = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wc");
var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf");
var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui");
var Uo = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uo");
var Uc = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uc");
var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation("Ft"));
var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation("It"));
var Ot = graph.Add(Wo, Uo).Add(graph.SigmoidActivation("Ot"));
var ftCt1 = graph.Multiply(hiddenLayerSize, Ft.LastNode, _state);
var Ct = graph.Add(ftCt1, graph.Multiply(It, graph.Add(Wc, Uc).Add(graph.TanhActivation()))).
AddForwardAction(_state.SetMemoryAction);
_output = graph.Multiply(Ot, Ct.Add(graph.TanhActivation())).
AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
{
_inputSize = inputSize;
int hiddenLayerSize = memory.Length;
_memory = new MemoryFeeder(memory, null, memoryId);
_input = new FlowThrough();
var Wz = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wz");
var Uz = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uz");
var Wr = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wr");
var Ur = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ur");
// add sigmoids to the gates
var Rt = graph.Add(Wr, Ur).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Rt")).LastNode;
var Zt = graph.Add(Wz, Uz).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Zt")).LastNode;
// h1 = tanh(Wh(x) + Uh(Ht1xRt))
var Wh = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wh");
var Uh = graph.Multiply(hiddenLayerSize, Rt, _memory).AddFeedForward(hiddenLayerSize, "Uh");
var h1 = graph.Add(Wh, Uh).AddBackwardAction(new ConstrainSignal()).Add(graph.TanhActivation());
// h2 = h1x(1-Zt)
var h2 = graph.Multiply(h1, graph.Connect(hiddenLayerSize, Zt).Add(graph.CreateOneMinusInput()));
// h = h1xh2
var previous = graph.Multiply(hiddenLayerSize, Zt, _memory);
_output = graph
.Add(h2, previous)
.AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode
;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
