void _Create(GraphFactory graph, int inputSize, float[] memory, INode activation, string memoryId) { _inputSize = inputSize; _activation = activation; int hiddenLayerSize = memory.Length; _memory = new MemoryFeeder(memory, null, memoryId); _input = new FlowThrough(); var inputChannel = graph.Connect(inputSize, _input) .AddFeedForward(hiddenLayerSize, "Wh"); var memoryChannel = graph.Connect(hiddenLayerSize, _memory) .AddFeedForward(hiddenLayerSize, "Uh"); _output = graph.Add(inputChannel, memoryChannel) .AddBackwardAction(new ConstrainSignal()) .Add(activation) .AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode; ; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId) { int hiddenLayerSize = memory.Length; _inputSize = inputSize; _memory = new MemoryFeeder(memory, null, memoryId); _input = new FlowThrough(); var Wx = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wx").LastNode; var Wi = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wi"); var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui"); var Wf = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wf"); var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf"); var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation()); var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation()); _output = graph .Add(graph.Multiply(hiddenLayerSize, Wx, It.LastNode), graph.Multiply(hiddenLayerSize, _memory, Ft.LastNode)) .AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode ; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId) { _inputSize = inputSize; int hiddenLayerSize = memory.Length; _memory = new MemoryFeeder(memory, null, memoryId); _state = new MemoryFeeder(new float[memory.Length]); _input = new FlowThrough(); var Wf = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wf"); var Wi = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wi"); var Wo = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wo"); var Wc = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wc"); var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf"); var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui"); var Uo = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uo"); var Uc = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uc"); var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation("Ft")); var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation("It")); var Ot = graph.Add(Wo, Uo).Add(graph.SigmoidActivation("Ot")); var ftCt1 = graph.Multiply(hiddenLayerSize, Ft.LastNode, _state); var Ct = graph.Add(ftCt1, graph.Multiply(It, graph.Add(Wc, Uc).Add(graph.TanhActivation()))). AddForwardAction(_state.SetMemoryAction); _output = graph.Multiply(Ot, Ct.Add(graph.TanhActivation())). AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId) { _inputSize = inputSize; int hiddenLayerSize = memory.Length; _memory = new MemoryFeeder(memory, null, memoryId); _input = new FlowThrough(); var Wz = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wz"); var Uz = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uz"); var Wr = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wr"); var Ur = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ur"); // add sigmoids to the gates var Rt = graph.Add(Wr, Ur).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Rt")).LastNode; var Zt = graph.Add(Wz, Uz).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Zt")).LastNode; // h1 = tanh(Wh(x) + Uh(Ht1xRt)) var Wh = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wh"); var Uh = graph.Multiply(hiddenLayerSize, Rt, _memory).AddFeedForward(hiddenLayerSize, "Uh"); var h1 = graph.Add(Wh, Uh).AddBackwardAction(new ConstrainSignal()).Add(graph.TanhActivation()); // h2 = h1x(1-Zt) var h2 = graph.Multiply(h1, graph.Connect(hiddenLayerSize, Zt).Add(graph.CreateOneMinusInput())); // h = h1xh2 var previous = graph.Multiply(hiddenLayerSize, Zt, _memory); _output = graph .Add(h2, previous) .AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode ; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }