Ejemplo n.º 1
0
        void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
        {
            int hiddenLayerSize = memory.Length;

            _inputSize = inputSize;

            _memory = new MemoryFeeder(memory, null, memoryId);
            _input  = new FlowThrough();

            var Wx = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wx").LastNode;

            var Wi = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wi");
            var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui");

            var Wf = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wf");
            var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf");

            var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation());
            var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation());

            _output = graph
                      .Add(graph.Multiply(hiddenLayerSize, Wx, It.LastNode), graph.Multiply(hiddenLayerSize, _memory, Ft.LastNode))
                      .AddForwardAction(_memory.SetMemoryAction)
                      //.Add(new HookErrorSignal(context => {
                      //    if (_lastBackpropagation != null) {
                      //        foreach (var item in _lastBackpropagation)
                      //            context.AppendErrorSignal(item.Value, item.Key);
                      //        _lastBackpropagation = null;
                      //    }
                      //}))
                      .LastNode
            ;
            _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
        }
Ejemplo n.º 2
0
        void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
        {
            _inputSize = inputSize;
            int hiddenLayerSize = memory.Length;

            _memory = new MemoryFeeder(memory, null, memoryId);
            _state  = new MemoryFeeder(new float[memory.Length]);
            _input  = new FlowThrough();
            var Wf    = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wf");
            var Wi    = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wi");
            var Wo    = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wo");
            var Wc    = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wc");
            var Uf    = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf");
            var Ui    = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui");
            var Uo    = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uo");
            var Uc    = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uc");
            var Ft    = graph.Add(Wf, Uf).Add(graph.SigmoidActivation("Ft"));
            var It    = graph.Add(Wi, Ui).Add(graph.SigmoidActivation("It"));
            var Ot    = graph.Add(Wo, Uo).Add(graph.SigmoidActivation("Ot"));
            var ftCt1 = graph.Multiply(hiddenLayerSize, Ft.LastNode, _state);
            var Ct    = graph.Add(ftCt1, graph.Multiply(It, graph.Add(Wc, Uc).Add(graph.TanhActivation()))).
                        AddForwardAction(_state.SetMemoryAction);

            _output = graph.Multiply(Ot, Ct.Add(graph.TanhActivation())).
                      AddForwardAction(_memory.SetMemoryAction)
                      //.Add(new HookErrorSignal(context => {
                      //    if (_lastBackpropagation != null) {
                      //        foreach (var item in _lastBackpropagation)
                      //            context.AppendErrorSignal(item.Value, item.Key);
                      //        _lastBackpropagation = null;
                      //    }
                      //}))
                      .LastNode;
            _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
        }
Ejemplo n.º 3
0
        void _Create(GraphFactory graph, int inputSize, float[] memory, INode activation, string memoryId)
        {
            _inputSize  = inputSize;
            _activation = activation;
            int hiddenLayerSize = memory.Length;

            _memory = new MemoryFeeder(memory, null, memoryId);
            _input  = new FlowThrough();

            var inputChannel = graph.Connect(inputSize, _input)
                               .AddFeedForward(hiddenLayerSize, "Wh");
            var memoryChannel = graph.Connect(hiddenLayerSize, _memory)
                                .AddFeedForward(hiddenLayerSize, "Uh");

            _output = graph.Add(inputChannel, memoryChannel)
                      .AddBackwardAction(new ConstrainSignal())
                      .Add(activation)
                      .AddForwardAction(_memory.SetMemoryAction)
                      //.Add(new HookErrorSignal(context => {
                      //    if (_lastBackpropagation != null) {
                      //        foreach (var item in _lastBackpropagation)
                      //            context.AppendErrorSignal(item.Value, item.Key);
                      //        _lastBackpropagation = null;
                      //    }
                      //}))
                      .LastNode;
            ;
            _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
        }
Ejemplo n.º 4
0
        void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
        {
            _inputSize = inputSize;
            int hiddenLayerSize = memory.Length;

            _memory = new MemoryFeeder(memory, null, memoryId);
            _input  = new FlowThrough();

            var Wz = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wz");
            var Uz = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uz");

            var Wr = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wr");
            var Ur = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ur");

            // add sigmoids to the gates
            var Rt = graph.Add(Wr, Ur).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Rt")).LastNode;
            var Zt = graph.Add(Wz, Uz).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Zt")).LastNode;

            // h1 = tanh(Wh(x) + Uh(Ht1xRt))
            var Wh = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wh");
            var Uh = graph.Multiply(hiddenLayerSize, Rt, _memory).AddFeedForward(hiddenLayerSize, "Uh");
            var h1 = graph.Add(Wh, Uh).AddBackwardAction(new ConstrainSignal()).Add(graph.TanhActivation());

            // h2 = h1x(1-Zt)
            var h2 = graph.Multiply(h1, graph.Connect(hiddenLayerSize, Zt).Add(graph.CreateOneMinusInput()));

            // h = h1xh2
            var previous = graph.Multiply(hiddenLayerSize, Zt, _memory);

            _output = graph
                      .Add(h2, previous)
                      .AddForwardAction(_memory.SetMemoryAction)
                      //.Add(new HookErrorSignal(context => {
                      //    if (_lastBackpropagation != null) {
                      //        foreach (var item in _lastBackpropagation)
                      //            context.AppendErrorSignal(item.Value, item.Key);
                      //        _lastBackpropagation = null;
                      //    }
                      //}))
                      .LastNode
            ;
            _start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
        }
Ejemplo n.º 5
0
        void _Create(GraphFactory graph, int inputSize, float[] gamma, float[] beta, float[] derivedMean, float[] derivedM2, int count)
        {
            _inputSize  = inputSize;
            _statistics = new VectorBasedStatistics(graph.LinearAlgebraProvider, inputSize, derivedMean, derivedM2, count);

            if (gamma == null)
            {
                gamma = new float[inputSize];
                for (var i = 0; i < inputSize; i++)
                {
                    gamma[i] = 1f;
                }
            }
            if (beta == null)
            {
                beta = new float[inputSize];
            }

            _input = new FlowThrough();
            _gamma = new VectorInput(gamma, "gamma");
            _beta  = new VectorInput(beta, "beta");

            var mean               = graph.Connect(inputSize, _input).Add(new BatchMean());
            var subtractMean       = graph.Subtract(inputSize, _input, mean.LastNode);
            var subtractMeanOutput = subtractMean.LastNode;

            var stdDev = subtractMean
                         .Add(new InputSquared())
                         .Add(new BatchMean())
                         .Add(new SquareRootOfInput())
                         .Add(new OneDividedByInput());

            var normalised = graph.Multiply(_inputSize, subtractMeanOutput, stdDev.LastNode) /*.Add(new InspectSignals(data => {
                                                                                              * var matrix = data.GetMatrix().AsIndexable();
                                                                                              * var statistics = matrix.Columns.Select(v => (v, v.Average())).Select(v2 => (v2.Item2, v2.Item1.StdDev(v2.Item2))).ToList();
                                                                                              * }))*/;
            var adjusted   = graph.Multiply(_inputSize, _gamma, normalised.LastNode);

            _output = graph.Add(_inputSize, _beta, adjusted.LastNode).LastNode;

            _start = new OneToMany(SubNodes, bp => { });
        }
Ejemplo n.º 6
0
        void _Create(GraphFactory graph, float gamma, float beta)
        {
            _input  = new FlowThrough();
            _input2 = new CalculateVariance();
            var main     = graph.Subtract(_inputSize, _input, _input2);
            var mainNode = main.LastNode;

            main.Add(new InputSquared());
            main.Add(new CalculateVariance());
            main.Add(new SquareRootOfInput());
            main.Add(new OneDividedByInput());
            main = graph.Multiply(_inputSize, mainNode, main.LastNode);

            _gamma = new MultiplyWithParameter(gamma);
            _beta  = new MultiplyWithParameter(beta);

            main    = graph.Multiply(_inputSize, _gamma, main.LastNode);
            _output = graph.Add(_inputSize, _beta, main.LastNode).LastNode;
            _start  = new OneToMany(SubNodes, null);
        }