//IMPLEMENTATION OF SHIFT - page 9
internal ShiftedAddressing(Unit shift, GatedAddressing gatedAddressing)
{
_shift = shift;
GatedAddressing = gatedAddressing;
_gatedVector = GatedAddressing.GatedVector;
_cellCount = _gatedVector.Length;
ShiftedVector = UnitFactory.GetVector(_cellCount);
double cellCountDbl = _cellCount;
//Max shift is from range -1 to 1
_shiftWeight = Sigmoid.GetValue(_shift.Value);
double maxShift = ((2 * _shiftWeight) - 1);
double convolutionDbl = (maxShift + cellCountDbl) % cellCountDbl;
_simj = 1 - (convolutionDbl - Math.Floor(convolutionDbl));
_oneMinusSimj = (1 - _simj);
_convolution = (int)convolutionDbl;
for (int i = 0; i < _cellCount; i++)
{
int imj = (i + _convolution) % _cellCount;
Unit vectorItem = ShiftedVector[i];
vectorItem.Value = (_gatedVector[imj].Value * _simj) +
(_gatedVector[(imj + 1) % _cellCount].Value * _oneMinusSimj);
if (vectorItem.Value < 0 || double.IsNaN(vectorItem.Value))
{
throw new Exception("Error - weight should not be smaller than zero or nan");
}
}
}
internal SimilarityMeasure(ISimilarityFunction similarityFunction, Unit[] u, Unit[] v)
{
_similarityFunction = similarityFunction;
_u = u;
_v = v;
Similarity = similarityFunction.Calculate(u, v);
}
internal HeadSetting(Unit gamma, ShiftedAddressing shiftedVector)
{
_gamma = gamma;
ShiftedVector = shiftedVector;
_shiftedVector = ShiftedVector.ShiftedVector;
_cellCount = _shiftedVector.Length;
AddressingVector = UnitFactory.GetVector(_cellCount);
//NO CLUE IN PAPER HOW TO IMPLEMENT - ONLY RESTRICTION IS THAT IT HAS TO BE LARGER THAN 1
//(Page 9, Part 3.3.2. Focusing by location)
_gammaIndex = Math.Log(Math.Exp(gamma.Value) + 1) + 1;
double sum = 0;
for (int i = 0; i < _cellCount; i++)
{
Unit unit = AddressingVector[i];
unit.Value = Math.Pow(_shiftedVector[i].Value, _gammaIndex);
sum += unit.Value;
}
foreach (Unit unit in AddressingVector)
{
unit.Value = unit.Value / sum;
if (double.IsNaN(unit.Value))
{
throw new Exception("Should not be NaN - Error");
}
}
}
internal BetaSimilarity(Unit beta, SimilarityMeasure similarity)
{
_beta = beta;
Similarity = similarity;
//Ensuring that beta will be positive
_b = Math.Exp(_beta.Value);
BetaSimilarityMeasure = new Unit(_b * Similarity.Similarity.Value);
}
public override void UpdateWeight(Unit unit)
{
_n[_i] = (GradientMomentum * _n[_i]) + ((1 - GradientMomentum) * unit.Gradient * unit.Gradient);
_g[_i] = (GradientMomentum * _g[_i]) + ((1 - GradientMomentum) * unit.Gradient);
_delta[_i] = (DeltaMomentum * _delta[_i]) - (ChangeMultiplier * (unit.Gradient / Math.Sqrt(_n[_i] - (_g[_i] * _g[_i]) + ChangeAddConstant)));
unit.Value += _delta[_i];
_i++;
}
public Head(int memoryRowSize)
{
_memoryRowSize = memoryRowSize;
_eraseVector = UnitFactory.GetVector(memoryRowSize);
_addVector = UnitFactory.GetVector(memoryRowSize);
_keyVector = UnitFactory.GetVector(memoryRowSize);
_beta = new Unit();
_gate = new Unit();
_shift = new Unit();
_gama = new Unit();
}
private OutputLayer(Unit[][] hiddenToOutputLayerWeights, Unit[][][] hiddenToHeadsWeights, Unit[] outputLayerNeurons, Head[] headsNeurons, int headCount, int outputSize, int controllerSize, int memoryUnitSizeM, int headUnitSize)
{
_hiddenToOutputLayerWeights = hiddenToOutputLayerWeights;
_hiddenToHeadsWeights = hiddenToHeadsWeights;
HeadsNeurons = headsNeurons;
_controllerSize = controllerSize;
_outputSize = outputSize;
OutputLayerNeurons = outputLayerNeurons;
_headCount = headCount;
_memoryUnitSizeM = memoryUnitSizeM;
_headUnitSize = headUnitSize;
}
private HiddenLayer(Unit[][][] readDataToHiddenLayerWeights, Unit[][] inputToHiddenLayerWeights, Unit[] hiddenLayerThresholds, Unit[] hiddenLayer, int controllerSize, int inputSize, int headCount, int memoryUnitSizeM, IDifferentiableFunction activationFunction)
{
_readDataToHiddenLayerWeights = readDataToHiddenLayerWeights;
_inputToHiddenLayerWeights = inputToHiddenLayerWeights;
_hiddenLayerThresholds = hiddenLayerThresholds;
HiddenLayerNeurons = hiddenLayer;
_controllerSize = controllerSize;
_inputSize = inputSize;
_headCount = headCount;
_memoryUnitSizeM = memoryUnitSizeM;
_activationFunction = activationFunction;
}
public void Differentiate(Unit similarity, Unit[] uVector, Unit[] vVector)
{
double uvuu = _uv / (_normalizedU * _normalizedU);
double uvvv = _uv / (_normalizedV * _normalizedV);
double uvg = similarity.Gradient / (_normalizedU * _normalizedV);
for (int i = 0; i < uVector.Length; i++)
{
double u = uVector[i].Value;
double v = vVector[i].Value;
uVector[i].Gradient += (v - (u * uvuu)) * uvg;
vVector[i].Gradient += (u - (v * uvvv)) * uvg;
}
}
internal GatedAddressing(Unit gate, ContentAddressing contentAddressing, HeadSetting oldHeadSettings)
{
_gate = gate;
ContentVector = contentAddressing;
_oldHeadSettings = oldHeadSettings;
Unit[] contentVector = ContentVector.ContentVector;
_memoryCellCount = contentVector.Length;
GatedVector = UnitFactory.GetVector(_memoryCellCount);
//Implementation of focusing by location - page 8 part 3.3.2. Focusing by Location
_gt = Sigmoid.GetValue(_gate.Value);
_oneminusgt = (1 - _gt);
for (int i = 0; i < _memoryCellCount; i++)
{
GatedVector[i].Value = (_gt * contentVector[i].Value) + (_oneminusgt * _oldHeadSettings.AddressingVector[i].Value);
}
}
public Unit Calculate(Unit[] u, Unit[] v)
{
for (int i = 0; i < u.Length; i++)
{
_uv += u[i].Value * v[i].Value;
_normalizedU += u[i].Value * u[i].Value;
_normalizedV += v[i].Value * v[i].Value;
}
_normalizedU = Math.Sqrt(_normalizedU);
_normalizedV = Math.Sqrt(_normalizedV);
Unit data = new Unit(_uv / (_normalizedU * _normalizedV));
if (double.IsNaN(data.Value))
{
throw new Exception("Cosine similarity is nan -> error");
}
return data;
}
internal ReadData(HeadSetting headSetting, NTMMemory controllerMemory)
{
HeadSetting = headSetting;
_controllerMemory = controllerMemory;
_cellSize = _controllerMemory.CellSizeM;
_cellCount = _controllerMemory.CellCountN;
ReadVector = new Unit[_cellSize];
for (int i = 0; i < _cellSize; i++)
{
double temp = 0;
for (int j = 0; j < _cellCount; j++)
{
temp += headSetting.AddressingVector[j].Value * controllerMemory.Data[j][i].Value;
//if (double.IsNaN(temp))
//{
// throw new Exception("Memory error");
//}
}
ReadVector[i] = new Unit(temp);
}
}
internal BetaSimilarity()
{
_beta = new Unit();
BetaSimilarityMeasure = new Unit();
}
private void UpdateInputGradient(double hiddenLayerGradient, Unit[] inputToHiddenNeuronWeights, double[] input)
{
for (int inputIndex = 0; inputIndex < _inputSize; inputIndex++)
{
inputToHiddenNeuronWeights[inputIndex].Gradient += hiddenLayerGradient * input[inputIndex];
}
}
public override void UpdateWeight(Unit data)
{
data.Gradient = 0;
}
public override void UpdateWeight(Unit data)
{
data.Value = _rand.NextDouble() - 0.5;
}
