private OutputLayer(Unit[][] hiddenToOutputLayerWeights, Unit[][][] hiddenToHeadsWeights, Unit[] outputLayerNeurons, Head[] headsNeurons, int headCount, int outputSize, int controllerSize, int memoryUnitSizeM, int headUnitSize)
{
_hiddenToOutputLayerWeights = hiddenToOutputLayerWeights;
_hiddenToHeadsWeights = hiddenToHeadsWeights;
HeadsNeurons = headsNeurons;
_controllerSize = controllerSize;
_outputSize = outputSize;
OutputLayerNeurons = outputLayerNeurons;
_headCount = headCount;
_memoryUnitSizeM = memoryUnitSizeM;
_headUnitSize = headUnitSize;
}
internal NTMMemory(HeadSetting[] headSettings, Head[] heads, NTMMemory memory)
{
CellCountN = memory.CellCountN;
CellSizeM = memory.CellSizeM;
HeadCount = memory.HeadCount;
HeadSettings = headSettings;
_heads = heads;
_oldMemory = memory;
Data = UnitFactory.GetTensor2(memory.CellCountN, memory.CellSizeM);
_erase = GetTensor2(HeadCount, memory.CellSizeM);
_add = GetTensor2(HeadCount, memory.CellSizeM);
var erasures = GetTensor2(memory.CellCountN, memory.CellSizeM);
for (int i = 0; i < HeadCount; i++)
{
Unit[] eraseVector = _heads[i].EraseVector;
Unit[] addVector = _heads[i].AddVector;
double[] erases = _erase[i];
double[] adds = _add[i];
for (int j = 0; j < CellSizeM; j++)
{
erases[j] = Sigmoid.GetValue(eraseVector[j].Value);
adds[j] = Sigmoid.GetValue(addVector[j].Value);
}
}
for (int i = 0; i < CellCountN; i++)
{
Unit[] oldRow = _oldMemory.Data[i];
double[] erasure = erasures[i];
Unit[] row = Data[i];
for (int j = 0; j < CellSizeM; j++)
{
Unit oldCell = oldRow[j];
double erase = 1;
double add = 0;
for (int k = 0; k < HeadCount; k++)
{
HeadSetting headSetting = HeadSettings[k];
double addressingValue = headSetting.AddressingVector[i].Value;
erase *= (1 - (addressingValue * _erase[k][j]));
add += addressingValue * _add[k][j];
}
erasure[j] = erase;
row[j].Value += (erase * oldCell.Value) + add;
}
}
}
public static Head[] GetVector(int length, Func<int, int> constructorParamGetter)
{
Head[] vector = new Head[length];
for (int i = 0; i < length; i++)
{
vector[i] = new Head(constructorParamGetter(i));
}
return vector;
}
private void EraseAndAddGradientUpdate(int headIndex, double[] erase, double[] add, HeadSetting headSetting, Head head)
{
Unit[] addVector = head.AddVector;
for (int j = 0; j < CellSizeM; j++)
{
double gradientErase = 0;
double gradientAdd = 0;
for (int k = 0; k < CellCountN; k++)
{
Unit[] row = Data[k];
double itemGradient = row[j].Gradient;
double addressingVectorItemValue = headSetting.AddressingVector[k].Value;
//Gradient of Erase vector
double gradientErase2 = _oldMemory.Data[k][j].Value;
for (int q = 0; q < HeadCount; q++)
{
if (q == headIndex)
{
continue;
}
gradientErase2 *= 1 - (HeadSettings[q].AddressingVector[k].Value * _erase[q][j]);
}
gradientErase += itemGradient * gradientErase2 * (-addressingVectorItemValue);
//Gradient of Add vector
gradientAdd += itemGradient * addressingVectorItemValue;
}
double e = erase[j];
head.EraseVector[j].Gradient += gradientErase * e * (1 - e);
double a = add[j];
addVector[j].Gradient += gradientAdd * a * (1 - a);
}
}
internal MemoryState Process(Head[] heads)
{
int headCount = heads.Length;
int memoryColumnsN = _memory.CellCountN;
ReadData[] newReadDatas = new ReadData[headCount];
HeadSetting[] newHeadSettings = new HeadSetting[headCount];
for (int i = 0; i < headCount; i++)
{
Head head = heads[i];
BetaSimilarity[] similarities = new BetaSimilarity[_memory.CellCountN];
for (int j = 0; j < memoryColumnsN; j++)
{
Unit[] memoryColumn = _memory.Data[j];
SimilarityMeasure similarity = new SimilarityMeasure(new CosineSimilarityFunction(), head.KeyVector, memoryColumn);
similarities[j] = new BetaSimilarity(head.Beta, similarity);
}
ContentAddressing ca = new ContentAddressing(similarities);
GatedAddressing ga = new GatedAddressing(head.Gate, ca, _headSettings[i]);
ShiftedAddressing sa = new ShiftedAddressing(head.Shift, ga);
newHeadSettings[i] = new HeadSetting(head.Gamma, sa);
newReadDatas[i] = new ReadData(newHeadSettings[i], _memory);
}
NTMMemory newMemory = new NTMMemory(newHeadSettings, heads, _memory);
return new MemoryState(newMemory, newHeadSettings, newReadDatas);
}