public MyMatrix[] GetActivisions(float[] input) { MyMatrix[] activisions = new MyMatrix[m_layerCount]; MyMatrix activision = new MyMatrix(input); activisions[0] = activision; for (int layerIndex = 0; layerIndex < m_layerCount - 1; layerIndex++) { // compensate weights scale for dropout MyMatrix weightsCompensated = null; if (m_dropoutKeepRate < 1) { weightsCompensated = new MyMatrix(m_weights[layerIndex], true); weightsCompensated.MultiplyByFactor(m_dropoutKeepRate); } else { weightsCompensated = m_weights[layerIndex]; } activision = GetActivisionFunction(MyMatrix.AddMatrix(MyMatrix.Dot(weightsCompensated, activision), m_biases[layerIndex]), layerIndex); activisions[layerIndex + 1] = activision; } return(activisions); }
private void FillActivisions(float[] input) { m_activisionValues[0] = new MyMatrix(input); for (int layerIndex = 0; layerIndex < m_layerCount - 1; layerIndex++) { m_rawValues[layerIndex] = MyMatrix.AddMatrix(MyMatrix.Dot(m_weights[layerIndex], m_activisionValues[layerIndex]), m_biases[layerIndex]); m_activisionValues[layerIndex + 1] = GetActivisionFunction(m_rawValues[layerIndex], layerIndex); if (m_dropoutKeepRate < 1 && layerIndex != m_layerCount - 2) { if (m_dropoutKeepRate <= 0 || m_dropoutKeepRate > 1) { Debug.Log("Warning: m_regularizationKeepRate was corrupt! (" + m_dropoutKeepRate + ")"); } MyMatrix regularizationMask = new MyMatrix(m_activisionValues[layerIndex + 1].m_rowCountY, 1);// new float[m_layerLengths[layerIndex] + 1]; for (int i = 0; i < regularizationMask.m_rowCountY; i++) { float random = 0; if (m_initDropoutSeed >= 0) { random = Utility.GetRandomWithSeed(0f, 1f, m_currentDropoutSeed++); } else { Random.Range(0f, 1f); } if (random < m_dropoutKeepRate) { regularizationMask.m_data[i][0] = 1; } } m_rawValues[layerIndex] = MyMatrix.MultiplyElementWise(m_rawValues[layerIndex], regularizationMask); m_activisionValues[layerIndex + 1] = MyMatrix.MultiplyElementWise(m_activisionValues[layerIndex + 1], regularizationMask); } } }
public void PerformBackPropagation(float learnRate) { ClearMatrixArray(m_batchedGradientBiases); ClearMatrixArray(m_batchedGradientWeights); for (int batchIndex = 0; batchIndex < m_batchSize; batchIndex++) { // clear delta matrices ClearMatrixArray(m_deltaBiases); ClearMatrixArray(m_deltaWeights); ClearMatrixArray(m_deltaActivision); ClearMatrixArray(m_rawValues); ClearMatrixArray(m_activisionValues); // feed forward, get all raw values and activision values FillActivisions(m_batchInputs[batchIndex]); // back pass, start at the last layer (manually) and loop over the prelayers afterwards MyMatrix delta = GetCostDerivative(m_rawValues[m_rawValues.Length - 1], m_batchDesiredOutputs[batchIndex], m_activisionValues[m_activisionValues.Length - 1]);// MyMatrix.MultiplyElementWise(GetCostDrivative(m_batchDesiredOutputs[batchIndex], m_activisionValues[m_activisionValues.Length - 1]), GetSigmoidPrime(m_rawValues[m_rawValues.Length - 1])); m_deltaBiases[m_deltaBiases.Length - 1] = delta; m_deltaWeights[m_deltaWeights.Length - 1] = MyMatrix.Dot(delta, MyMatrix.Transposed(m_activisionValues[m_activisionValues.Length - 2])); for (int layerIndex = 2; layerIndex < m_layerCount; layerIndex++) { MyMatrix weightsTransposed = MyMatrix.Transposed(m_weights[m_weights.Length - layerIndex + 1]); delta = MyMatrix.Dot(weightsTransposed, delta); MyMatrix activisionsPrime = GetActivisionFunctionPrime(m_rawValues[m_rawValues.Length - layerIndex], m_layerCount - layerIndex - 1); delta = MyMatrix.MultiplyElementWise(delta, activisionsPrime); m_deltaBiases[m_deltaBiases.Length - layerIndex] = delta; m_deltaWeights[m_deltaWeights.Length - layerIndex] = MyMatrix.Dot(delta, MyMatrix.Transposed(m_activisionValues[m_activisionValues.Length - layerIndex - 1])); } // finally add the gradient to the current sum of changes for (int layerIndex = 0; layerIndex < m_batchedGradientBiases.Length; layerIndex++) { m_batchedGradientBiases[layerIndex].AddMatrix(m_deltaBiases[layerIndex]); } for (int layerIndex = 0; layerIndex < m_batchedGradientWeights.Length; layerIndex++) { m_batchedGradientWeights[layerIndex].AddMatrix(m_deltaWeights[layerIndex]); } } // set the final values learnRate = (learnRate <= 0 ? m_learnRate : learnRate); float weightDecayFactor = 1f - learnRate * m_weightDecayRate / m_batchSize; if (weightDecayFactor < 0 || weightDecayFactor > 1) { Debug.Log(string.Format("Warning: weightDecayFactor is corrupt: learnRate {0} + weightDecayRate {1} / batchSize {2} = {3} ", learnRate, m_weightDecayRate, m_batchSize, weightDecayFactor)); weightDecayFactor = Mathf.Clamp(weightDecayFactor, 0.00001f, 1f); } for (int layerIndex = 0; layerIndex < m_biases.Length; layerIndex++) { m_batchedGradientBiases[layerIndex].MultiplyByFactor(learnRate / m_batchSize); m_biases[layerIndex].AddMatrix(m_batchedGradientBiases[layerIndex]); } for (int layerIndex = 0; layerIndex < m_weights.Length; layerIndex++) { m_batchedGradientWeights[layerIndex].MultiplyByFactor(learnRate / m_batchSize); // apply weight decay if (m_weightDecayRate > 0) { m_weights[layerIndex].MultiplyByFactor(weightDecayFactor); } m_weights[layerIndex].AddMatrix(m_batchedGradientWeights[layerIndex]); } }