Exemplo n.º 1
0
        //IMPLEMENTATION OF SHIFT - page 9
        internal ShiftedAddressing(Unit shift, GatedAddressing gatedAddressing)
        {
            _shift = shift;
            GatedAddressing = gatedAddressing;
            _gatedVector = GatedAddressing.GatedVector;
            _cellCount = _gatedVector.Length;

            ShiftedVector = UnitFactory.GetVector(_cellCount);
            double cellCountDbl = _cellCount;

            //Max shift is from range -1 to 1
            _shiftWeight = Sigmoid.GetValue(_shift.Value);
            double maxShift = ((2 * _shiftWeight) - 1);
            double convolutionDbl = (maxShift + cellCountDbl) % cellCountDbl;

            _simj = 1 - (convolutionDbl - Math.Floor(convolutionDbl));
            _oneMinusSimj = (1 - _simj);
            _convolution = (int)convolutionDbl;

            for (int i = 0; i < _cellCount; i++)
            {
                int imj = (i + _convolution) % _cellCount;

                Unit vectorItem = ShiftedVector[i];

                vectorItem.Value = (_gatedVector[imj].Value * _simj) +
                                   (_gatedVector[(imj + 1) % _cellCount].Value * _oneMinusSimj);
                if (vectorItem.Value < 0 || double.IsNaN(vectorItem.Value))
                {
                    throw new Exception("Error - weight should not be smaller than zero or nan");
                }
            }
        }
Exemplo n.º 2
0
 internal SimilarityMeasure(ISimilarityFunction similarityFunction, Unit[] u, Unit[] v)
 {
     _similarityFunction = similarityFunction;
     _u = u;
     _v = v;
     Similarity = similarityFunction.Calculate(u, v);
 }
Exemplo n.º 3
0
        internal HeadSetting(Unit gamma, ShiftedAddressing shiftedVector)
        {
            _gamma = gamma;
            ShiftedVector = shiftedVector;
            _shiftedVector = ShiftedVector.ShiftedVector;
            _cellCount = _shiftedVector.Length;
            AddressingVector = UnitFactory.GetVector(_cellCount);

            //NO CLUE IN PAPER HOW TO IMPLEMENT - ONLY RESTRICTION IS THAT IT HAS TO BE LARGER THAN 1
            //(Page 9, Part 3.3.2. Focusing by location)
            _gammaIndex = Math.Log(Math.Exp(gamma.Value) + 1) + 1;

            double sum = 0;
            for (int i = 0; i < _cellCount; i++)
            {
                Unit unit = AddressingVector[i];
                unit.Value = Math.Pow(_shiftedVector[i].Value, _gammaIndex);
                sum += unit.Value;
            }

            foreach (Unit unit in AddressingVector)
            {
                unit.Value = unit.Value / sum;
                if (double.IsNaN(unit.Value))
                {
                    throw new Exception("Should not be NaN - Error");
                }
            }
        }
Exemplo n.º 4
0
 internal BetaSimilarity(Unit beta, SimilarityMeasure similarity)
 {
     _beta = beta;
     Similarity = similarity;
     //Ensuring that beta will be positive
     _b = Math.Exp(_beta.Value);
     BetaSimilarityMeasure = new Unit(_b * Similarity.Similarity.Value);
 }
Exemplo n.º 5
0
 public override void UpdateWeight(Unit unit)
 {
     _n[_i] = (GradientMomentum * _n[_i]) + ((1 - GradientMomentum) * unit.Gradient * unit.Gradient);
     _g[_i] = (GradientMomentum * _g[_i]) + ((1 - GradientMomentum) * unit.Gradient);
     _delta[_i] = (DeltaMomentum * _delta[_i]) - (ChangeMultiplier * (unit.Gradient / Math.Sqrt(_n[_i] - (_g[_i] * _g[_i]) + ChangeAddConstant)));
     unit.Value += _delta[_i];
     _i++;
 }
Exemplo n.º 6
0
Arquivo: Head.cs Projeto: weizh/NTM-1
 public Head(int memoryRowSize)
 {
     _memoryRowSize = memoryRowSize;
     _eraseVector = UnitFactory.GetVector(memoryRowSize);
     _addVector = UnitFactory.GetVector(memoryRowSize);
     _keyVector = UnitFactory.GetVector(memoryRowSize);
     _beta = new Unit();
     _gate = new Unit();
     _shift = new Unit();
     _gama = new Unit();
 }
Exemplo n.º 7
0
 private OutputLayer(Unit[][] hiddenToOutputLayerWeights, Unit[][][] hiddenToHeadsWeights, Unit[] outputLayerNeurons, Head[] headsNeurons, int headCount, int outputSize, int controllerSize, int memoryUnitSizeM, int headUnitSize)
 {
     _hiddenToOutputLayerWeights = hiddenToOutputLayerWeights;
     _hiddenToHeadsWeights = hiddenToHeadsWeights;
     HeadsNeurons = headsNeurons;
     _controllerSize = controllerSize;
     _outputSize = outputSize;
     OutputLayerNeurons = outputLayerNeurons;
     _headCount = headCount;
     _memoryUnitSizeM = memoryUnitSizeM;
     _headUnitSize = headUnitSize;
 }
Exemplo n.º 8
0
 private HiddenLayer(Unit[][][] readDataToHiddenLayerWeights, Unit[][] inputToHiddenLayerWeights, Unit[] hiddenLayerThresholds, Unit[] hiddenLayer, int controllerSize, int inputSize, int headCount, int memoryUnitSizeM, IDifferentiableFunction activationFunction)
 {
     _readDataToHiddenLayerWeights = readDataToHiddenLayerWeights;
     _inputToHiddenLayerWeights = inputToHiddenLayerWeights;
     _hiddenLayerThresholds = hiddenLayerThresholds;
     HiddenLayerNeurons = hiddenLayer;
     _controllerSize = controllerSize;
     _inputSize = inputSize;
     _headCount = headCount;
     _memoryUnitSizeM = memoryUnitSizeM;
     _activationFunction = activationFunction;
 }
Exemplo n.º 9
0
        public void Differentiate(Unit similarity, Unit[] uVector, Unit[] vVector)
        {
            double uvuu = _uv / (_normalizedU * _normalizedU);
            double uvvv = _uv / (_normalizedV * _normalizedV);
            double uvg = similarity.Gradient / (_normalizedU * _normalizedV);
            for (int i = 0; i < uVector.Length; i++)
            {
                double u = uVector[i].Value;
                double v = vVector[i].Value;

                uVector[i].Gradient += (v - (u * uvuu)) * uvg;
                vVector[i].Gradient += (u - (v * uvvv)) * uvg;
            }
        }
Exemplo n.º 10
0
        internal GatedAddressing(Unit gate, ContentAddressing contentAddressing, HeadSetting oldHeadSettings)
        {
            _gate = gate;
            ContentVector = contentAddressing;
            _oldHeadSettings = oldHeadSettings;
            Unit[] contentVector = ContentVector.ContentVector;
            _memoryCellCount = contentVector.Length;
            GatedVector = UnitFactory.GetVector(_memoryCellCount);

            //Implementation of focusing by location - page 8 part 3.3.2. Focusing by Location
            _gt = Sigmoid.GetValue(_gate.Value);
            _oneminusgt = (1 - _gt);

            for (int i = 0; i < _memoryCellCount; i++)
            {
                GatedVector[i].Value = (_gt * contentVector[i].Value) + (_oneminusgt * _oldHeadSettings.AddressingVector[i].Value);
            }
        }
Exemplo n.º 11
0
        public Unit Calculate(Unit[] u, Unit[] v)
        {
            for (int i = 0; i < u.Length; i++)
            {
                _uv += u[i].Value * v[i].Value;
                _normalizedU += u[i].Value * u[i].Value;
                _normalizedV += v[i].Value * v[i].Value;
            }

            _normalizedU = Math.Sqrt(_normalizedU);
            _normalizedV = Math.Sqrt(_normalizedV);

            Unit data = new Unit(_uv / (_normalizedU * _normalizedV));
            if (double.IsNaN(data.Value))
            {
                throw new Exception("Cosine similarity is nan -> error");
            }

            return data;
        }
Exemplo n.º 12
0
        internal ReadData(HeadSetting headSetting, NTMMemory controllerMemory)
        {
            HeadSetting = headSetting;
            _controllerMemory = controllerMemory;
            _cellSize = _controllerMemory.CellSizeM;
            _cellCount = _controllerMemory.CellCountN;

            ReadVector = new Unit[_cellSize];

            for (int i = 0; i < _cellSize; i++)
            {
                double temp = 0;
                for (int j = 0; j < _cellCount; j++)
                {
                    temp += headSetting.AddressingVector[j].Value * controllerMemory.Data[j][i].Value;
                    //if (double.IsNaN(temp))
                    //{
                    //    throw new Exception("Memory error");
                    //}
                }
                ReadVector[i] = new Unit(temp);
            }
        }
Exemplo n.º 13
0
 internal BetaSimilarity()
 {
     _beta = new Unit();
     BetaSimilarityMeasure = new Unit();
 }
Exemplo n.º 14
0
 private void UpdateInputGradient(double hiddenLayerGradient, Unit[] inputToHiddenNeuronWeights, double[] input)
 {
     for (int inputIndex = 0; inputIndex < _inputSize; inputIndex++)
     {
         inputToHiddenNeuronWeights[inputIndex].Gradient += hiddenLayerGradient * input[inputIndex];
     }
 }
Exemplo n.º 15
0
 public override void UpdateWeight(Unit data)
 {
     data.Gradient = 0;
 }
Exemplo n.º 16
0
 public override void UpdateWeight(Unit data)
 {
     data.Value = _rand.NextDouble() - 0.5;
 }