public void InterConnectionWeightRenewal(double learningRate, double decayRate) { InterConnectionMatrix += LayerActivationMatrix.Transpose() * LayerErrorMatrix * learningRate; Parallel.For(0, InterConnectionMatrix.RowCount, index => { InterConnectionMatrix[index, index] = 0; }); if (decayRate > 0) { Parallel.For(0, InterConnectionMatrix.RowCount, rowIndex => { for (int columnIndex = 0; columnIndex < InterConnectionMatrix.ColumnCount; columnIndex++) { if (InterConnectionMatrix[rowIndex, columnIndex] > 0) { InterConnectionMatrix[rowIndex, columnIndex] -= decayRate; } else if (InterConnectionMatrix[rowIndex, columnIndex] < 0) { InterConnectionMatrix[rowIndex, columnIndex] += decayRate; } } }); InterConnectionMatrix.CoerceZero(decayRate); } }
public void CleanUpBackwordProcess(double momentum, double learningRate, double decayRate) { CleanupLayerErrorMatrix = (LayerErrorMatrix * CleanupToLayerWeightMatrix.Transpose()) .PointwiseMultiply(CleanupLayerActivationMatrix.PointwiseMultiply(1 - CleanupLayerActivationMatrix)); CleanupBiasMatrix += CleanupLayerErrorMatrix.ColumnSums().ToRowMatrix() * learningRate; CleanupToLayerWeightMatrix += CleanupLayerActivationMatrix.Transpose() * LayerErrorMatrix * learningRate; LayerToCleanupWeightMatrix += LayerActivationMatrix.Transpose() * CleanupLayerErrorMatrix * learningRate; if (decayRate > 0) { Parallel.For(0, CleanupBiasMatrix.ColumnCount, columnIndex => { if (CleanupBiasMatrix[0, columnIndex] > 0) { CleanupBiasMatrix[0, columnIndex] -= decayRate; } else if (CleanupBiasMatrix[0, columnIndex] < 0) { CleanupBiasMatrix[0, columnIndex] += decayRate; } }); CleanupBiasMatrix.CoerceZero(decayRate); Parallel.For(0, CleanupToLayerWeightMatrix.RowCount, rowIndex => { for (int columnIndex = 0; columnIndex < CleanupToLayerWeightMatrix.ColumnCount; columnIndex++) { if (CleanupToLayerWeightMatrix[rowIndex, columnIndex] > 0) { CleanupToLayerWeightMatrix[rowIndex, columnIndex] -= decayRate; } else if (CleanupToLayerWeightMatrix[rowIndex, columnIndex] < 0) { CleanupToLayerWeightMatrix[rowIndex, columnIndex] += decayRate; } } }); CleanupToLayerWeightMatrix.CoerceZero(decayRate); Parallel.For(0, LayerToCleanupWeightMatrix.RowCount, rowIndex => { for (int columnIndex = 0; columnIndex < LayerToCleanupWeightMatrix.ColumnCount; columnIndex++) { if (LayerToCleanupWeightMatrix[rowIndex, columnIndex] > 0) { LayerToCleanupWeightMatrix[rowIndex, columnIndex] -= decayRate; } else if (LayerToCleanupWeightMatrix[rowIndex, columnIndex] < 0) { LayerToCleanupWeightMatrix[rowIndex, columnIndex] += decayRate; } } }); LayerToCleanupWeightMatrix.CoerceZero(decayRate); } }