//IMPLEMENTATION OF SHIFT - page 9
internal ShiftedAddressing(Unit shift, GatedAddressing gatedAddressing)
{
_shift = shift;
GatedAddressing = gatedAddressing;
_gatedVector = GatedAddressing.GatedVector;
_cellCount = _gatedVector.Length;
ShiftedVector = UnitFactory.GetVector(_cellCount);
double cellCountDbl = _cellCount;
//Max shift is from range -1 to 1
_shiftWeight = Sigmoid.GetValue(_shift.Value);
double maxShift = ((2 * _shiftWeight) - 1);
double convolutionDbl = (maxShift + cellCountDbl) % cellCountDbl;
_simj = 1 - (convolutionDbl - Math.Floor(convolutionDbl));
_oneMinusSimj = (1 - _simj);
_convolution = (int)convolutionDbl;
for (int i = 0; i < _cellCount; i++)
{
int imj = (i + _convolution) % _cellCount;
Unit vectorItem = ShiftedVector[i];
vectorItem.Value = (_gatedVector[imj].Value * _simj) +
(_gatedVector[(imj + 1) % _cellCount].Value * _oneMinusSimj);
if (vectorItem.Value < 0 || double.IsNaN(vectorItem.Value))
{
throw new Exception("Error - weight should not be smaller than zero or nan");
}
}
}
public void NN_activation_function_sigmoid()
{
Sigmoid aFunc = new Sigmoid();
double value = aFunc.GetValue(3.0);
Assert.IsTrue(SupportFunctions.DoubleCompare(value, 0.9526));
}
internal NTMMemory(HeadSetting[] headSettings, Head[] heads, NTMMemory memory)
{
CellCountN = memory.CellCountN;
CellSizeM = memory.CellSizeM;
HeadCount = memory.HeadCount;
HeadSettings = headSettings;
_heads = heads;
_oldMemory = memory;
Data = UnitFactory.GetTensor2(memory.CellCountN, memory.CellSizeM);
_erase = GetTensor2(HeadCount, memory.CellSizeM);
_add = GetTensor2(HeadCount, memory.CellSizeM);
var erasures = GetTensor2(memory.CellCountN, memory.CellSizeM);
for (int i = 0; i < HeadCount; i++)
{
Unit[] eraseVector = _heads[i].EraseVector;
Unit[] addVector = _heads[i].AddVector;
double[] erases = _erase[i];
double[] adds = _add[i];
for (int j = 0; j < CellSizeM; j++)
{
erases[j] = Sigmoid.GetValue(eraseVector[j].Value);
adds[j] = Sigmoid.GetValue(addVector[j].Value);
}
}
for (int i = 0; i < CellCountN; i++)
{
Unit[] oldRow = _oldMemory.Data[i];
double[] erasure = erasures[i];
Unit[] row = Data[i];
for (int j = 0; j < CellSizeM; j++)
{
Unit oldCell = oldRow[j];
double erase = 1;
double add = 0;
for (int k = 0; k < HeadCount; k++)
{
HeadSetting headSetting = HeadSettings[k];
double addressingValue = headSetting.AddressingVector[i].Value;
erase *= (1 - (addressingValue * _erase[k][j]));
add += addressingValue * _add[k][j];
}
erasure[j] = erase;
row[j].Value += (erase * oldCell.Value) + add;
}
}
}
internal GatedAddressing(Unit gate, ContentAddressing contentAddressing, HeadSetting oldHeadSettings)
{
_gate = gate;
ContentVector = contentAddressing;
_oldHeadSettings = oldHeadSettings;
Unit[] contentVector = ContentVector.ContentVector;
_memoryCellCount = contentVector.Length;
GatedVector = UnitFactory.GetVector(_memoryCellCount);
//Implementation of focusing by location - page 8 part 3.3.2. Focusing by Location
_gt = Sigmoid.GetValue(_gate.Value);
_oneminusgt = (1 - _gt);
for (int i = 0; i < _memoryCellCount; i++)
{
GatedVector[i].Value = (_gt * contentVector[i].Value) + (_oneminusgt * _oldHeadSettings.AddressingVector[i].Value);
}
}