internal MemoryState Process(Head[] heads)
{
int headCount = heads.Length;
int memoryColumnsN = _memory.CellCountN;
ReadData[] newReadDatas = new ReadData[headCount];
HeadSetting[] newHeadSettings = new HeadSetting[headCount];
for (int i = 0; i < headCount; i++)
{
Head head = heads[i];
BetaSimilarity[] similarities = new BetaSimilarity[_memory.CellCountN];
for (int j = 0; j < memoryColumnsN; j++)
{
Unit[] memoryColumn = _memory.Data[j];
SimilarityMeasure similarity = new SimilarityMeasure(new CosineSimilarityFunction(), head.KeyVector, memoryColumn);
similarities[j] = new BetaSimilarity(head.Beta, similarity);
}
ContentAddressing ca = new ContentAddressing(similarities);
GatedAddressing ga = new GatedAddressing(head.Gate, ca, _headSettings[i]);
ShiftedAddressing sa = new ShiftedAddressing(head.Shift, ga);
newHeadSettings[i] = new HeadSetting(head.Gamma, sa);
newReadDatas[i] = new ReadData(newHeadSettings[i], _memory);
}
NTMMemory newMemory = new NTMMemory(newHeadSettings, heads, _memory);
return(new MemoryState(newMemory, newHeadSettings, newReadDatas));
}
private void HeadSettingGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting)
{
//Gradient of head settings
for (int j = 0; j < CellCountN; j++)
{
Unit[] row = Data[j];
Unit[] oldRow = _oldMemory.Data[j];
double gradient = 0;
for (int k = 0; k < CellSizeM; k++)
{
Unit data = row[k];
double oldDataValue = oldRow[k].Value;
for (int q = 0; q < HeadCount; q++)
{
if (q == headIndex)
{
continue;
}
HeadSetting setting = HeadSettings[q];
oldDataValue *= (1 - (setting.AddressingVector[j].Value * _erase[q][k]));
}
gradient += ((oldDataValue * (-erase[k])) + add[k]) * data.Gradient;
}
headSetting.AddressingVector[j].Gradient += gradient;
}
}
public static HeadSetting[] GetVector(int x, Func <int, Tuple <int, ContentAddressing> > paramGetter)
{
HeadSetting[] vector = new HeadSetting[x];
for (int i = 0; i < x; i++)
{
Tuple <int, ContentAddressing> parameters = paramGetter(i);
vector[i] = new HeadSetting(parameters.Item1, parameters.Item2);
}
return(vector);
}
public static HeadSetting[] GetVector(int x, Func<int, Tuple<int, ContentAddressing>> paramGetter)
{
HeadSetting[] vector = new HeadSetting[x];
for (int i = 0; i < x; i++)
{
Tuple<int, ContentAddressing> parameters = paramGetter(i);
vector[i] = new HeadSetting(parameters.Item1, parameters.Item2);
}
return vector;
}
internal NTMMemory(HeadSetting[] headSettings, Head[] heads, NTMMemory memory)
{
CellCountN = memory.CellCountN;
CellSizeM = memory.CellSizeM;
HeadCount = memory.HeadCount;
HeadSettings = headSettings;
_heads = heads;
_oldMemory = memory;
Data = UnitFactory.GetTensor2(memory.CellCountN, memory.CellSizeM);
_erase = GetTensor2(HeadCount, memory.CellSizeM);
_add = GetTensor2(HeadCount, memory.CellSizeM);
var erasures = GetTensor2(memory.CellCountN, memory.CellSizeM);
for (int i = 0; i < HeadCount; i++)
{
Unit[] eraseVector = _heads[i].EraseVector;
Unit[] addVector = _heads[i].AddVector;
double[] erases = _erase[i];
double[] adds = _add[i];
for (int j = 0; j < CellSizeM; j++)
{
erases[j] = Sigmoid.GetValue(eraseVector[j].Value);
adds[j] = Sigmoid.GetValue(addVector[j].Value);
}
}
for (int i = 0; i < CellCountN; i++)
{
Unit[] oldRow = _oldMemory.Data[i];
double[] erasure = erasures[i];
Unit[] row = Data[i];
for (int j = 0; j < CellSizeM; j++)
{
Unit oldCell = oldRow[j];
double erase = 1;
double add = 0;
for (int k = 0; k < HeadCount; k++)
{
HeadSetting headSetting = HeadSettings[k];
double addressingValue = headSetting.AddressingVector[i].Value;
erase *= (1 - (addressingValue * _erase[k][j]));
add += addressingValue * _add[k][j];
}
erasure[j] = erase;
row[j].Value += (erase * oldCell.Value) + add;
}
}
}
internal NTMMemory(HeadSetting[] headSettings, Head[] heads, NTMMemory memory)
{
CellCountN = memory.CellCountN;
CellSizeM = memory.CellSizeM;
HeadCount = memory.HeadCount;
HeadSettings = headSettings;
_heads = heads;
_oldMemory = memory;
Data = UnitFactory.GetTensor2(memory.CellCountN, memory.CellSizeM);
_erase = GetTensor2(HeadCount, memory.CellSizeM);
_add = GetTensor2(HeadCount, memory.CellSizeM);
var erasures = GetTensor2(memory.CellCountN, memory.CellSizeM);
for (int i = 0; i < HeadCount; i++)
{
Unit[] eraseVector = _heads[i].EraseVector;
Unit[] addVector = _heads[i].AddVector;
double[] erases = _erase[i];
double[] adds = _add[i];
for (int j = 0; j < CellSizeM; j++)
{
erases[j] = Sigmoid.GetValue(eraseVector[j].Value);
adds[j] = Sigmoid.GetValue(addVector[j].Value);
}
}
for (int i = 0; i < CellCountN; i++)
{
Unit[] oldRow = _oldMemory.Data[i];
double[] erasure = erasures[i];
Unit[] row = Data[i];
for (int j = 0; j < CellSizeM; j++)
{
Unit oldCell = oldRow[j];
double erase = 1;
double add = 0;
for (int k = 0; k < HeadCount; k++)
{
HeadSetting headSetting = HeadSettings[k];
double addressingValue = headSetting.AddressingVector[i].Value;
erase *= (1 - (addressingValue * _erase[k][j]));
add += addressingValue * _add[k][j];
}
erasure[j] = erase;
row[j].Value += (erase * oldCell.Value) + add;
}
}
}
public void BackwardErrorPropagation()
{
for (int i = 0; i < HeadCount; i++)
{
HeadSetting headSetting = HeadSettings[i];
double[] erase = _erase[i];
double[] add = _add[i];
Head head = _heads[i];
HeadSettingGradientUpdate(i, erase, add, headSetting);
EraseAndAddGradientUpdate(i, erase, add, headSetting, head);
}
MemoryGradientUpdate();
}
internal ReadData(HeadSetting headSetting, NTMMemory controllerMemory)
{
HeadSetting = headSetting;
_controllerMemory = controllerMemory;
_cellSize = _controllerMemory.CellSizeM;
_cellCount = _controllerMemory.CellCountN;
ReadVector = new Unit[_cellSize];
for (int i = 0; i < _cellSize; i++)
{
double temp = 0;
for (int j = 0; j < _cellCount; j++)
{
temp += headSetting.AddressingVector[j].Value * controllerMemory.Data[j][i].Value;
//if (double.IsNaN(temp))
//{
// throw new Exception("Memory error");
//}
}
ReadVector[i] = new Unit(temp);
}
}
internal ReadData(HeadSetting headSetting, NTMMemory controllerMemory)
{
HeadSetting = headSetting;
_controllerMemory = controllerMemory;
_cellSize = _controllerMemory.CellSizeM;
_cellCount = _controllerMemory.CellCountN;
ReadVector = new Unit[_cellSize];
for (int i = 0; i < _cellSize; i++)
{
double temp = 0;
for (int j = 0; j < _cellCount; j++)
{
temp += headSetting.AddressingVector[j].Value * controllerMemory.Data[j][i].Value;
//if (double.IsNaN(temp))
//{
// throw new Exception("Memory error");
//}
}
ReadVector[i] = new Unit(temp);
}
}
private void HeadSettingGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting)
{
//Gradient of head settings
for (int j = 0; j < CellCountN; j++)
{
Unit[] row = Data[j];
Unit[] oldRow = _oldMemory.Data[j];
double gradient = 0;
for (int k = 0; k < CellSizeM; k++)
{
Unit data = row[k];
double oldDataValue = oldRow[k].Value;
for (int q = 0; q < HeadCount; q++)
{
if (q == headIndex)
{
continue;
}
HeadSetting setting = HeadSettings[q];
oldDataValue *= (1 - (setting.AddressingVector[j].Value * _erase[q][k]));
}
gradient += ((oldDataValue * (-erase[k])) + add[k]) * data.Gradient;
}
headSetting.AddressingVector[j].Gradient += gradient;
}
}
private void EraseAndAddGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting, Head head)
{
Unit[] addVector = head.AddVector;
for (int j = 0; j < CellSizeM; j++)
{
double gradientErase = 0;
double gradientAdd = 0;
for (int k = 0; k < CellCountN; k++)
{
Unit[] row = Data[k];
double itemGradient = row[j].Gradient;
double addressingVectorItemValue = headSetting.AddressingVector[k].Value;
//Gradient of Erase vector
double gradientErase2 = _oldMemory.Data[k][j].Value;
for (int q = 0; q < HeadCount; q++)
{
if (q == headIndex)
{
continue;
}
gradientErase2 *= 1 - (HeadSettings[q].AddressingVector[k].Value * _erase[q][j]);
}
gradientErase += itemGradient * gradientErase2 * (-addressingVectorItemValue);
//Gradient of Add vector
gradientAdd += itemGradient * addressingVectorItemValue;
}
double e = erase[j];
head.EraseVector[j].Gradient += gradientErase * e * (1 - e);
double a = add[j];
addVector[j].Gradient += gradientAdd * a * (1 - a);
}
}
internal void DoInitialReading()
{
_contentAddressings = _memory.GetContentAddressing();
_headSettings = HeadSetting.GetVector(_memory.HeadCount, i => new Tuple <int, ContentAddressing>(_memory.CellCountN, _contentAddressings[i]));
ReadData = Memory.ReadData.GetVector(_memory.HeadCount, i => new Tuple <HeadSetting, NTMMemory>(_headSettings[i], _memory));
}
private void EraseAndAddGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting, Head head)
{
Unit[] addVector = head.AddVector;
for (int j = 0; j < CellSizeM; j++)
{
double gradientErase = 0;
double gradientAdd = 0;
for (int k = 0; k < CellCountN; k++)
{
Unit[] row = Data[k];
double itemGradient = row[j].Gradient;
double addressingVectorItemValue = headSetting.AddressingVector[k].Value;
//Gradient of Erase vector
double gradientErase2 = _oldMemory.Data[k][j].Value;
for (int q = 0; q < HeadCount; q++)
{
if (q == headIndex)
{
continue;
}
gradientErase2 *= 1 - (HeadSettings[q].AddressingVector[k].Value * _erase[q][j]);
}
gradientErase += itemGradient * gradientErase2 * (-addressingVectorItemValue);
//Gradient of Add vector
gradientAdd += itemGradient * addressingVectorItemValue;
}
double e = erase[j];
head.EraseVector[j].Gradient += gradientErase * e * (1 - e);
double a = add[j];
addVector[j].Gradient += gradientAdd * a * (1 - a);
}
}
internal MemoryState Process(Head[] heads)
{
int headCount = heads.Length;
int memoryColumnsN = _memory.CellCountN;
ReadData[] newReadDatas = new ReadData[headCount];
HeadSetting[] newHeadSettings = new HeadSetting[headCount];
for (int i = 0; i < headCount; i++)
{
Head head = heads[i];
BetaSimilarity[] similarities = new BetaSimilarity[_memory.CellCountN];
for (int j = 0; j < memoryColumnsN; j++)
{
Unit[] memoryColumn = _memory.Data[j];
SimilarityMeasure similarity = new SimilarityMeasure(new CosineSimilarityFunction(), head.KeyVector, memoryColumn);
similarities[j] = new BetaSimilarity(head.Beta, similarity);
}
ContentAddressing ca = new ContentAddressing(similarities);
GatedAddressing ga = new GatedAddressing(head.Gate, ca, _headSettings[i]);
ShiftedAddressing sa = new ShiftedAddressing(head.Shift, ga);
newHeadSettings[i] = new HeadSetting(head.Gamma, sa);
newReadDatas[i] = new ReadData(newHeadSettings[i], _memory);
}
NTMMemory newMemory = new NTMMemory(newHeadSettings, heads, _memory);
return new MemoryState(newMemory, newHeadSettings, newReadDatas);
}
internal MemoryState(NTMMemory memory, HeadSetting[] headSettings, ReadData[] readDatas)
{
_memory = memory;
_headSettings = headSettings;
ReadData = readDatas;
}