public MyMatrix[] GetActivisions(float[] input)
{
MyMatrix[] activisions = new MyMatrix[m_layerCount];
MyMatrix activision = new MyMatrix(input);
activisions[0] = activision;
for (int layerIndex = 0; layerIndex < m_layerCount - 1; layerIndex++)
{
// compensate weights scale for dropout
MyMatrix weightsCompensated = null;
if (m_dropoutKeepRate < 1)
{
weightsCompensated = new MyMatrix(m_weights[layerIndex], true);
weightsCompensated.MultiplyByFactor(m_dropoutKeepRate);
}
else
{
weightsCompensated = m_weights[layerIndex];
}
activision = GetActivisionFunction(MyMatrix.AddMatrix(MyMatrix.Dot(weightsCompensated, activision), m_biases[layerIndex]), layerIndex);
activisions[layerIndex + 1] = activision;
}
return(activisions);
}
private void FillActivisions(float[] input)
{
m_activisionValues[0] = new MyMatrix(input);
for (int layerIndex = 0; layerIndex < m_layerCount - 1; layerIndex++)
{
m_rawValues[layerIndex] = MyMatrix.AddMatrix(MyMatrix.Dot(m_weights[layerIndex], m_activisionValues[layerIndex]), m_biases[layerIndex]);
m_activisionValues[layerIndex + 1] = GetActivisionFunction(m_rawValues[layerIndex], layerIndex);
if (m_dropoutKeepRate < 1 && layerIndex != m_layerCount - 2)
{
if (m_dropoutKeepRate <= 0 || m_dropoutKeepRate > 1)
{
Debug.Log("Warning: m_regularizationKeepRate was corrupt! (" + m_dropoutKeepRate + ")");
}
MyMatrix regularizationMask = new MyMatrix(m_activisionValues[layerIndex + 1].m_rowCountY, 1);// new float[m_layerLengths[layerIndex] + 1];
for (int i = 0; i < regularizationMask.m_rowCountY; i++)
{
float random = 0;
if (m_initDropoutSeed >= 0)
{
random = Utility.GetRandomWithSeed(0f, 1f, m_currentDropoutSeed++);
}
else
{
Random.Range(0f, 1f);
}
if (random < m_dropoutKeepRate)
{
regularizationMask.m_data[i][0] = 1;
}
}
m_rawValues[layerIndex] = MyMatrix.MultiplyElementWise(m_rawValues[layerIndex], regularizationMask);
m_activisionValues[layerIndex + 1] = MyMatrix.MultiplyElementWise(m_activisionValues[layerIndex + 1], regularizationMask);
}
}
}
public void PerformBackPropagation(float learnRate)
{
ClearMatrixArray(m_batchedGradientBiases);
ClearMatrixArray(m_batchedGradientWeights);
for (int batchIndex = 0; batchIndex < m_batchSize; batchIndex++)
{
// clear delta matrices
ClearMatrixArray(m_deltaBiases);
ClearMatrixArray(m_deltaWeights);
ClearMatrixArray(m_deltaActivision);
ClearMatrixArray(m_rawValues);
ClearMatrixArray(m_activisionValues);
// feed forward, get all raw values and activision values
FillActivisions(m_batchInputs[batchIndex]);
// back pass, start at the last layer (manually) and loop over the prelayers afterwards
MyMatrix delta = GetCostDerivative(m_rawValues[m_rawValues.Length - 1], m_batchDesiredOutputs[batchIndex], m_activisionValues[m_activisionValues.Length - 1]);// MyMatrix.MultiplyElementWise(GetCostDrivative(m_batchDesiredOutputs[batchIndex], m_activisionValues[m_activisionValues.Length - 1]), GetSigmoidPrime(m_rawValues[m_rawValues.Length - 1]));
m_deltaBiases[m_deltaBiases.Length - 1] = delta;
m_deltaWeights[m_deltaWeights.Length - 1] = MyMatrix.Dot(delta, MyMatrix.Transposed(m_activisionValues[m_activisionValues.Length - 2]));
for (int layerIndex = 2; layerIndex < m_layerCount; layerIndex++)
{
MyMatrix weightsTransposed = MyMatrix.Transposed(m_weights[m_weights.Length - layerIndex + 1]);
delta = MyMatrix.Dot(weightsTransposed, delta);
MyMatrix activisionsPrime = GetActivisionFunctionPrime(m_rawValues[m_rawValues.Length - layerIndex], m_layerCount - layerIndex - 1);
delta = MyMatrix.MultiplyElementWise(delta, activisionsPrime);
m_deltaBiases[m_deltaBiases.Length - layerIndex] = delta;
m_deltaWeights[m_deltaWeights.Length - layerIndex] = MyMatrix.Dot(delta, MyMatrix.Transposed(m_activisionValues[m_activisionValues.Length - layerIndex - 1]));
}
// finally add the gradient to the current sum of changes
for (int layerIndex = 0; layerIndex < m_batchedGradientBiases.Length; layerIndex++)
{
m_batchedGradientBiases[layerIndex].AddMatrix(m_deltaBiases[layerIndex]);
}
for (int layerIndex = 0; layerIndex < m_batchedGradientWeights.Length; layerIndex++)
{
m_batchedGradientWeights[layerIndex].AddMatrix(m_deltaWeights[layerIndex]);
}
}
// set the final values
learnRate = (learnRate <= 0 ? m_learnRate : learnRate);
float weightDecayFactor = 1f - learnRate * m_weightDecayRate / m_batchSize;
if (weightDecayFactor < 0 || weightDecayFactor > 1)
{
Debug.Log(string.Format("Warning: weightDecayFactor is corrupt: learnRate {0} + weightDecayRate {1} / batchSize {2} = {3} ", learnRate, m_weightDecayRate, m_batchSize, weightDecayFactor));
weightDecayFactor = Mathf.Clamp(weightDecayFactor, 0.00001f, 1f);
}
for (int layerIndex = 0; layerIndex < m_biases.Length; layerIndex++)
{
m_batchedGradientBiases[layerIndex].MultiplyByFactor(learnRate / m_batchSize);
m_biases[layerIndex].AddMatrix(m_batchedGradientBiases[layerIndex]);
}
for (int layerIndex = 0; layerIndex < m_weights.Length; layerIndex++)
{
m_batchedGradientWeights[layerIndex].MultiplyByFactor(learnRate / m_batchSize);
// apply weight decay
if (m_weightDecayRate > 0)
{
m_weights[layerIndex].MultiplyByFactor(weightDecayFactor);
}
m_weights[layerIndex].AddMatrix(m_batchedGradientWeights[layerIndex]);
}
}
