void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId) { int hiddenLayerSize = memory.Length; _inputSize = inputSize; _memory = new MemoryFeeder(memory, null, memoryId); _input = new FlowThrough(); var Wx = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wx").LastNode; var Wi = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wi"); var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui"); var Wf = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wf"); var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf"); var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation()); var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation()); _output = graph .Add(graph.Multiply(hiddenLayerSize, Wx, It.LastNode), graph.Multiply(hiddenLayerSize, _memory, Ft.LastNode)) .AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode ; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId) { _inputSize = inputSize; int hiddenLayerSize = memory.Length; _memory = new MemoryFeeder(memory, null, memoryId); _state = new MemoryFeeder(new float[memory.Length]); _input = new FlowThrough(); var Wf = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wf"); var Wi = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wi"); var Wo = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wo"); var Wc = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wc"); var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf"); var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui"); var Uo = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uo"); var Uc = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uc"); var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation("Ft")); var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation("It")); var Ot = graph.Add(Wo, Uo).Add(graph.SigmoidActivation("Ot")); var ftCt1 = graph.Multiply(hiddenLayerSize, Ft.LastNode, _state); var Ct = graph.Add(ftCt1, graph.Multiply(It, graph.Add(Wc, Uc).Add(graph.TanhActivation()))). AddForwardAction(_state.SetMemoryAction); _output = graph.Multiply(Ot, Ct.Add(graph.TanhActivation())). AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }
void _Create(GraphFactory graph, int inputSize, float[] memory, INode activation, string memoryId) { _inputSize = inputSize; _activation = activation; int hiddenLayerSize = memory.Length; _memory = new MemoryFeeder(memory, null, memoryId); _input = new FlowThrough(); var inputChannel = graph.Connect(inputSize, _input) .AddFeedForward(hiddenLayerSize, "Wh"); var memoryChannel = graph.Connect(hiddenLayerSize, _memory) .AddFeedForward(hiddenLayerSize, "Uh"); _output = graph.Add(inputChannel, memoryChannel) .AddBackwardAction(new ConstrainSignal()) .Add(activation) .AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode; ; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId) { _inputSize = inputSize; int hiddenLayerSize = memory.Length; _memory = new MemoryFeeder(memory, null, memoryId); _input = new FlowThrough(); var Wz = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wz"); var Uz = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uz"); var Wr = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wr"); var Ur = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ur"); // add sigmoids to the gates var Rt = graph.Add(Wr, Ur).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Rt")).LastNode; var Zt = graph.Add(Wz, Uz).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Zt")).LastNode; // h1 = tanh(Wh(x) + Uh(Ht1xRt)) var Wh = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wh"); var Uh = graph.Multiply(hiddenLayerSize, Rt, _memory).AddFeedForward(hiddenLayerSize, "Uh"); var h1 = graph.Add(Wh, Uh).AddBackwardAction(new ConstrainSignal()).Add(graph.TanhActivation()); // h2 = h1x(1-Zt) var h2 = graph.Multiply(h1, graph.Connect(hiddenLayerSize, Zt).Add(graph.CreateOneMinusInput())); // h = h1xh2 var previous = graph.Multiply(hiddenLayerSize, Zt, _memory); _output = graph .Add(h2, previous) .AddForwardAction(_memory.SetMemoryAction) //.Add(new HookErrorSignal(context => { // if (_lastBackpropagation != null) { // foreach (var item in _lastBackpropagation) // context.AppendErrorSignal(item.Value, item.Key); // _lastBackpropagation = null; // } //})) .LastNode ; _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp); }
void _Create(GraphFactory graph, int inputSize, float[] gamma, float[] beta, float[] derivedMean, float[] derivedM2, int count) { _inputSize = inputSize; _statistics = new VectorBasedStatistics(graph.LinearAlgebraProvider, inputSize, derivedMean, derivedM2, count); if (gamma == null) { gamma = new float[inputSize]; for (var i = 0; i < inputSize; i++) { gamma[i] = 1f; } } if (beta == null) { beta = new float[inputSize]; } _input = new FlowThrough(); _gamma = new VectorInput(gamma, "gamma"); _beta = new VectorInput(beta, "beta"); var mean = graph.Connect(inputSize, _input).Add(new BatchMean()); var subtractMean = graph.Subtract(inputSize, _input, mean.LastNode); var subtractMeanOutput = subtractMean.LastNode; var stdDev = subtractMean .Add(new InputSquared()) .Add(new BatchMean()) .Add(new SquareRootOfInput()) .Add(new OneDividedByInput()); var normalised = graph.Multiply(_inputSize, subtractMeanOutput, stdDev.LastNode) /*.Add(new InspectSignals(data => { * var matrix = data.GetMatrix().AsIndexable(); * var statistics = matrix.Columns.Select(v => (v, v.Average())).Select(v2 => (v2.Item2, v2.Item1.StdDev(v2.Item2))).ToList(); * }))*/; var adjusted = graph.Multiply(_inputSize, _gamma, normalised.LastNode); _output = graph.Add(_inputSize, _beta, adjusted.LastNode).LastNode; _start = new OneToMany(SubNodes, bp => { }); }
void _Create(GraphFactory graph, float gamma, float beta) { _input = new FlowThrough(); _input2 = new CalculateVariance(); var main = graph.Subtract(_inputSize, _input, _input2); var mainNode = main.LastNode; main.Add(new InputSquared()); main.Add(new CalculateVariance()); main.Add(new SquareRootOfInput()); main.Add(new OneDividedByInput()); main = graph.Multiply(_inputSize, mainNode, main.LastNode); _gamma = new MultiplyWithParameter(gamma); _beta = new MultiplyWithParameter(beta); main = graph.Multiply(_inputSize, _gamma, main.LastNode); _output = graph.Add(_inputSize, _beta, main.LastNode).LastNode; _start = new OneToMany(SubNodes, null); }