Beispiel #1
0
        internal MemoryState Process(Head[] heads)
        {
            int headCount      = heads.Length;
            int memoryColumnsN = _memory.CellCountN;

            ReadData[]    newReadDatas    = new ReadData[headCount];
            HeadSetting[] newHeadSettings = new HeadSetting[headCount];
            for (int i = 0; i < headCount; i++)
            {
                Head             head         = heads[i];
                BetaSimilarity[] similarities = new BetaSimilarity[_memory.CellCountN];

                for (int j = 0; j < memoryColumnsN; j++)
                {
                    Unit[]            memoryColumn = _memory.Data[j];
                    SimilarityMeasure similarity   = new SimilarityMeasure(new CosineSimilarityFunction(), head.KeyVector, memoryColumn);
                    similarities[j] = new BetaSimilarity(head.Beta, similarity);
                }

                ContentAddressing ca = new ContentAddressing(similarities);
                GatedAddressing   ga = new GatedAddressing(head.Gate, ca, _headSettings[i]);
                ShiftedAddressing sa = new ShiftedAddressing(head.Shift, ga);

                newHeadSettings[i] = new HeadSetting(head.Gamma, sa);
                newReadDatas[i]    = new ReadData(newHeadSettings[i], _memory);
            }

            NTMMemory newMemory = new NTMMemory(newHeadSettings, heads, _memory);

            return(new MemoryState(newMemory, newHeadSettings, newReadDatas));
        }
Beispiel #2
0
 private void HeadSettingGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting)
 {
     //Gradient of head settings
     for (int j = 0; j < CellCountN; j++)
     {
         Unit[] row      = Data[j];
         Unit[] oldRow   = _oldMemory.Data[j];
         double gradient = 0;
         for (int k = 0; k < CellSizeM; k++)
         {
             Unit   data         = row[k];
             double oldDataValue = oldRow[k].Value;
             for (int q = 0; q < HeadCount; q++)
             {
                 if (q == headIndex)
                 {
                     continue;
                 }
                 HeadSetting setting = HeadSettings[q];
                 oldDataValue *= (1 - (setting.AddressingVector[j].Value * _erase[q][k]));
             }
             gradient += ((oldDataValue * (-erase[k])) + add[k]) * data.Gradient;
         }
         headSetting.AddressingVector[j].Gradient += gradient;
     }
 }
Beispiel #3
0
 public static HeadSetting[] GetVector(int x, Func <int, Tuple <int, ContentAddressing> > paramGetter)
 {
     HeadSetting[] vector = new HeadSetting[x];
     for (int i = 0; i < x; i++)
     {
         Tuple <int, ContentAddressing> parameters = paramGetter(i);
         vector[i] = new HeadSetting(parameters.Item1, parameters.Item2);
     }
     return(vector);
 }
Beispiel #4
0
 public static HeadSetting[] GetVector(int x, Func<int, Tuple<int, ContentAddressing>> paramGetter)
 {
     HeadSetting[] vector = new HeadSetting[x];
     for (int i = 0; i < x; i++)
     {
         Tuple<int, ContentAddressing> parameters = paramGetter(i);
         vector[i] = new HeadSetting(parameters.Item1, parameters.Item2);
     }
     return vector;
 }
Beispiel #5
0
        internal NTMMemory(HeadSetting[] headSettings, Head[] heads, NTMMemory memory)
        {
            CellCountN = memory.CellCountN;
            CellSizeM = memory.CellSizeM;
            HeadCount = memory.HeadCount;
            HeadSettings = headSettings;
            _heads = heads;
            _oldMemory = memory;
            Data = UnitFactory.GetTensor2(memory.CellCountN, memory.CellSizeM);

            _erase = GetTensor2(HeadCount, memory.CellSizeM);
            _add = GetTensor2(HeadCount, memory.CellSizeM);
            var erasures = GetTensor2(memory.CellCountN, memory.CellSizeM);

            for (int i = 0; i < HeadCount; i++)
            {
                Unit[] eraseVector = _heads[i].EraseVector;
                Unit[] addVector = _heads[i].AddVector;
                double[] erases = _erase[i];
                double[] adds = _add[i];

                for (int j = 0; j < CellSizeM; j++)
                {
                    erases[j] = Sigmoid.GetValue(eraseVector[j].Value);
                    adds[j] = Sigmoid.GetValue(addVector[j].Value);
                }
            }

            for (int i = 0; i < CellCountN; i++)
            {
                Unit[] oldRow = _oldMemory.Data[i];
                double[] erasure = erasures[i];
                Unit[] row = Data[i];

                for (int j = 0; j < CellSizeM; j++)
                {
                    Unit oldCell = oldRow[j];
                    double erase = 1;
                    double add = 0;
                    for (int k = 0; k < HeadCount; k++)
                    {
                        HeadSetting headSetting = HeadSettings[k];
                        double addressingValue = headSetting.AddressingVector[i].Value;
                        erase *= (1 - (addressingValue * _erase[k][j]));
                        add += addressingValue * _add[k][j];
                    }
                    erasure[j] = erase;
                    row[j].Value += (erase * oldCell.Value) + add;
                }
            }
        }
Beispiel #6
0
        internal NTMMemory(HeadSetting[] headSettings, Head[] heads, NTMMemory memory)
        {
            CellCountN   = memory.CellCountN;
            CellSizeM    = memory.CellSizeM;
            HeadCount    = memory.HeadCount;
            HeadSettings = headSettings;
            _heads       = heads;
            _oldMemory   = memory;
            Data         = UnitFactory.GetTensor2(memory.CellCountN, memory.CellSizeM);

            _erase = GetTensor2(HeadCount, memory.CellSizeM);
            _add   = GetTensor2(HeadCount, memory.CellSizeM);
            var erasures = GetTensor2(memory.CellCountN, memory.CellSizeM);

            for (int i = 0; i < HeadCount; i++)
            {
                Unit[]   eraseVector = _heads[i].EraseVector;
                Unit[]   addVector   = _heads[i].AddVector;
                double[] erases      = _erase[i];
                double[] adds        = _add[i];

                for (int j = 0; j < CellSizeM; j++)
                {
                    erases[j] = Sigmoid.GetValue(eraseVector[j].Value);
                    adds[j]   = Sigmoid.GetValue(addVector[j].Value);
                }
            }

            for (int i = 0; i < CellCountN; i++)
            {
                Unit[]   oldRow  = _oldMemory.Data[i];
                double[] erasure = erasures[i];
                Unit[]   row     = Data[i];

                for (int j = 0; j < CellSizeM; j++)
                {
                    Unit   oldCell = oldRow[j];
                    double erase   = 1;
                    double add     = 0;
                    for (int k = 0; k < HeadCount; k++)
                    {
                        HeadSetting headSetting     = HeadSettings[k];
                        double      addressingValue = headSetting.AddressingVector[i].Value;
                        erase *= (1 - (addressingValue * _erase[k][j]));
                        add   += addressingValue * _add[k][j];
                    }
                    erasure[j]    = erase;
                    row[j].Value += (erase * oldCell.Value) + add;
                }
            }
        }
Beispiel #7
0
        public void BackwardErrorPropagation()
        {
            for (int i = 0; i < HeadCount; i++)
            {
                HeadSetting headSetting = HeadSettings[i];
                double[]    erase       = _erase[i];
                double[]    add         = _add[i];
                Head        head        = _heads[i];

                HeadSettingGradientUpdate(i, erase, add, headSetting);
                EraseAndAddGradientUpdate(i, erase, add, headSetting, head);
            }

            MemoryGradientUpdate();
        }
Beispiel #8
0
        internal ReadData(HeadSetting headSetting, NTMMemory controllerMemory)
        {
            HeadSetting = headSetting;
            _controllerMemory = controllerMemory;
            _cellSize = _controllerMemory.CellSizeM;
            _cellCount = _controllerMemory.CellCountN;

            ReadVector = new Unit[_cellSize];

            for (int i = 0; i < _cellSize; i++)
            {
                double temp = 0;
                for (int j = 0; j < _cellCount; j++)
                {
                    temp += headSetting.AddressingVector[j].Value * controllerMemory.Data[j][i].Value;
                    //if (double.IsNaN(temp))
                    //{
                    //    throw new Exception("Memory error");
                    //}
                }
                ReadVector[i] = new Unit(temp);
            }
        }
Beispiel #9
0
        internal ReadData(HeadSetting headSetting, NTMMemory controllerMemory)
        {
            HeadSetting       = headSetting;
            _controllerMemory = controllerMemory;
            _cellSize         = _controllerMemory.CellSizeM;
            _cellCount        = _controllerMemory.CellCountN;

            ReadVector = new Unit[_cellSize];

            for (int i = 0; i < _cellSize; i++)
            {
                double temp = 0;
                for (int j = 0; j < _cellCount; j++)
                {
                    temp += headSetting.AddressingVector[j].Value * controllerMemory.Data[j][i].Value;
                    //if (double.IsNaN(temp))
                    //{
                    //    throw new Exception("Memory error");
                    //}
                }
                ReadVector[i] = new Unit(temp);
            }
        }
Beispiel #10
0
 private void HeadSettingGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting)
 {
     //Gradient of head settings
     for (int j = 0; j < CellCountN; j++)
     {
         Unit[] row = Data[j];
         Unit[] oldRow = _oldMemory.Data[j];
         double gradient = 0;
         for (int k = 0; k < CellSizeM; k++)
         {
             Unit data = row[k];
             double oldDataValue = oldRow[k].Value;
             for (int q = 0; q < HeadCount; q++)
             {
                 if (q == headIndex)
                 {
                     continue;
                 }
                 HeadSetting setting = HeadSettings[q];
                 oldDataValue *= (1 - (setting.AddressingVector[j].Value * _erase[q][k]));
             }
             gradient += ((oldDataValue * (-erase[k])) + add[k]) * data.Gradient;
         }
         headSetting.AddressingVector[j].Gradient += gradient;
     }
 }
Beispiel #11
0
        private void EraseAndAddGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting, Head head)
        {
            Unit[] addVector = head.AddVector;
            for (int j = 0; j < CellSizeM; j++)
            {
                double gradientErase = 0;
                double gradientAdd = 0;
                for (int k = 0; k < CellCountN; k++)
                {
                    Unit[] row = Data[k];
                    double itemGradient = row[j].Gradient;
                    double addressingVectorItemValue = headSetting.AddressingVector[k].Value;

                    //Gradient of Erase vector
                    double gradientErase2 = _oldMemory.Data[k][j].Value;
                    for (int q = 0; q < HeadCount; q++)
                    {
                        if (q == headIndex)
                        {
                            continue;
                        }
                        gradientErase2 *= 1 - (HeadSettings[q].AddressingVector[k].Value * _erase[q][j]);
                    }
                    gradientErase += itemGradient * gradientErase2 * (-addressingVectorItemValue);
                    //Gradient of Add vector
                    gradientAdd += itemGradient * addressingVectorItemValue;
                }

                double e = erase[j];
                head.EraseVector[j].Gradient += gradientErase * e * (1 - e);

                double a = add[j];
                addVector[j].Gradient += gradientAdd * a * (1 - a);
            }
        }
Beispiel #12
0
 internal void DoInitialReading()
 {
     _contentAddressings = _memory.GetContentAddressing();
     _headSettings       = HeadSetting.GetVector(_memory.HeadCount, i => new Tuple <int, ContentAddressing>(_memory.CellCountN, _contentAddressings[i]));
     ReadData            = Memory.ReadData.GetVector(_memory.HeadCount, i => new Tuple <HeadSetting, NTMMemory>(_headSettings[i], _memory));
 }
Beispiel #13
0
        private void EraseAndAddGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting, Head head)
        {
            Unit[] addVector = head.AddVector;
            for (int j = 0; j < CellSizeM; j++)
            {
                double gradientErase = 0;
                double gradientAdd   = 0;
                for (int k = 0; k < CellCountN; k++)
                {
                    Unit[] row          = Data[k];
                    double itemGradient = row[j].Gradient;
                    double addressingVectorItemValue = headSetting.AddressingVector[k].Value;

                    //Gradient of Erase vector
                    double gradientErase2 = _oldMemory.Data[k][j].Value;
                    for (int q = 0; q < HeadCount; q++)
                    {
                        if (q == headIndex)
                        {
                            continue;
                        }
                        gradientErase2 *= 1 - (HeadSettings[q].AddressingVector[k].Value * _erase[q][j]);
                    }
                    gradientErase += itemGradient * gradientErase2 * (-addressingVectorItemValue);
                    //Gradient of Add vector
                    gradientAdd += itemGradient * addressingVectorItemValue;
                }

                double e = erase[j];
                head.EraseVector[j].Gradient += gradientErase * e * (1 - e);

                double a = add[j];
                addVector[j].Gradient += gradientAdd * a * (1 - a);
            }
        }
Beispiel #14
0
        internal MemoryState Process(Head[] heads)
        {
            int headCount = heads.Length;
            int memoryColumnsN = _memory.CellCountN;

            ReadData[] newReadDatas = new ReadData[headCount];
            HeadSetting[] newHeadSettings = new HeadSetting[headCount];
            for (int i = 0; i < headCount; i++)
            {
                Head head = heads[i];
                BetaSimilarity[] similarities = new BetaSimilarity[_memory.CellCountN];

                for (int j = 0; j < memoryColumnsN; j++)
                {
                    Unit[] memoryColumn = _memory.Data[j];
                    SimilarityMeasure similarity = new SimilarityMeasure(new CosineSimilarityFunction(), head.KeyVector, memoryColumn);
                    similarities[j] = new BetaSimilarity(head.Beta, similarity);
                }

                ContentAddressing ca = new ContentAddressing(similarities);
                GatedAddressing ga = new GatedAddressing(head.Gate, ca, _headSettings[i]);
                ShiftedAddressing sa = new ShiftedAddressing(head.Shift, ga);

                newHeadSettings[i] = new HeadSetting(head.Gamma, sa);
                newReadDatas[i] = new ReadData(newHeadSettings[i], _memory);
            }

            NTMMemory newMemory = new NTMMemory(newHeadSettings, heads, _memory);

            return new MemoryState(newMemory, newHeadSettings, newReadDatas);
        }
Beispiel #15
0
 internal MemoryState(NTMMemory memory, HeadSetting[] headSettings, ReadData[] readDatas)
 {
     _memory = memory;
     _headSettings = headSettings;
     ReadData = readDatas;
 }