public virtual void computeLayer(SparseVector sparseFeature, double[] denseFeature, bool isTrain = true) { if (SparseFeatureSize > 0) { //Apply sparse features SparseFeature = sparseFeature; Parallel.For(0, LayerSize, parallelOption, b => { double score = 0; double[] vector_b = SparseWeights[b]; for (int i = 0; i < SparseFeature.Count; i++) { var entry = SparseFeature.GetEntry(i); score += entry.Value * vector_b[entry.Key]; } cellOutput[b] += score; }); } if (DenseFeatureSize > 0) { DenseFeature = denseFeature; RNNHelper.matrixXvectorADD(cellOutput, denseFeature, DenseWeights, LayerSize, DenseFeatureSize); } }
public virtual void LearnFeatureWeights(int numStates, int curState) { if (DenseFeatureSize > 0) { //Update hidden-output weights Parallel.For(0, LayerSize, parallelOption, c => { double er2 = er[c]; double[] vector_c = DenseWeights[c]; for (int a = 0; a < DenseFeatureSize; a++) { double delta = RNNHelper.NormalizeGradient(er2 * DenseFeature[a]); double newLearningRate = RNNHelper.UpdateLearningRate(DenseWeightsLearningRate, c, a, delta); vector_c[a] += newLearningRate * delta; } }); } if (SparseFeatureSize > 0) { //Update hidden-output weights Parallel.For(0, LayerSize, parallelOption, c => { double er2 = er[c]; double[] vector_c = SparseWeights[c]; for (int a = 0; a < SparseFeature.Count; a++) { int pos = SparseFeature.GetEntry(a).Key; double val = SparseFeature.GetEntry(a).Value; double delta = RNNHelper.NormalizeGradient(er2 * val); double newLearningRate = RNNHelper.UpdateLearningRate(SparseWeightsLearningRate, c, pos, delta); vector_c[pos] += newLearningRate * delta; } }); } }
// forward process. output layer consists of tag value public override void computeLayer(SparseVector sparseFeature, double[] denseFeature, bool isTrain = true) { //keep last hidden layer and erase activations cellOutput.CopyTo(previousCellOutput, 0); //Apply previous feature to current time //hidden(t-1) -> hidden(t) RNNHelper.matrixXvectorADD(cellOutput, previousCellOutput, BpttWeights, LayerSize, LayerSize); //Apply features on hidden layer SparseFeature = sparseFeature; DenseFeature = denseFeature; if (SparseFeatureSize > 0) { //Apply sparse features Parallel.For(0, LayerSize, parallelOption, b => { double score = 0; if (SparseFeatureSize > 0) { double[] vector_b = SparseWeights[b]; for (int i = 0; i < SparseFeature.Count; i++) { var entry = SparseFeature.GetEntry(i); score += entry.Value * vector_b[entry.Key]; } } cellOutput[b] += score; }); } if (DenseFeatureSize > 0) { //Apply dense features RNNHelper.matrixXvectorADD(cellOutput, DenseFeature, DenseWeights, LayerSize, DenseFeatureSize, false); } //activate layer activityLayer(); }
// forward process. output layer consists of tag value public override void computeLayer(SparseVector sparseFeature, double[] denseFeature, bool isTrain = true) { //keep last hidden layer and erase activations cellOutput.CopyTo(previousCellOutput, 0); //Apply previous feature to current time //hidden(t-1) -> hidden(t) RNNHelper.matrixXvectorADD(cellOutput, previousCellOutput, BpttWeights, LayerSize, LayerSize); //Apply features on hidden layer SparseFeature = sparseFeature; DenseFeature = denseFeature; if (SparseFeatureSize > 0) { //Apply sparse features Parallel.For(0, LayerSize, parallelOption, b => { double score = 0; if (SparseFeatureSize > 0) { double[] vector_b = SparseWeights[b]; for (int i = 0; i < SparseFeature.Count; i++) { var entry = SparseFeature.GetEntry(i); score += entry.Value * vector_b[entry.Key]; } } cellOutput[b] += score; }); } if (DenseFeatureSize > 0) { //Apply dense features RNNHelper.matrixXvectorADD(cellOutput, DenseFeature, DenseWeights, LayerSize, DenseFeatureSize, false); } //activate layer activityLayer(isTrain); }
// forward process. output layer consists of tag value public override void computeLayer(SparseVector sparseFeature, double[] denseFeature, bool isTrain = true) { //inputs(t) -> hidden(t) //Get sparse feature and apply it into hidden layer SparseFeature = sparseFeature; DenseFeature = denseFeature; Parallel.For(0, LayerSize, parallelOption, j => { LSTMCell cell_j = cell[j]; //hidden(t-1) -> hidden(t) cell_j.previousCellState = cell_j.cellState; previousCellOutput[j] = cellOutput[j]; Vector4 vecCell_j = Vector4.Zero; if (SparseFeatureSize > 0) { //Apply sparse weights Vector4[] weights = input2hidden[j]; for (int i = 0; i < SparseFeature.Count; i++) { var entry = SparseFeature.GetEntry(i); vecCell_j += weights[entry.Key] * entry.Value; } } //Apply dense weights if (DenseFeatureSize > 0) { Vector4[] weights = feature2hidden[j]; for (int i = 0; i < DenseFeatureSize; i++) { vecCell_j += weights[i] * (float)DenseFeature[i]; } } //rest the value of the net input to zero cell_j.netIn = vecCell_j.X; cell_j.netForget = vecCell_j.Y; //reset each netCell state to zero cell_j.netCellState = vecCell_j.Z; //reset each netOut to zero cell_j.netOut = vecCell_j.W; double cell_j_previousCellOutput = previousCellOutput[j]; //include internal connection multiplied by the previous cell state cell_j.netIn += cell_j.previousCellState * cell_j.wPeepholeIn + cell_j_previousCellOutput * cell_j.wCellIn; //squash input cell_j.yIn = Sigmoid(cell_j.netIn); //include internal connection multiplied by the previous cell state cell_j.netForget += cell_j.previousCellState * cell_j.wPeepholeForget + cell_j_previousCellOutput * cell_j.wCellForget; cell_j.yForget = Sigmoid(cell_j.netForget); cell_j.netCellState += cell_j_previousCellOutput * cell_j.wCellState; cell_j.yCellState = TanH(cell_j.netCellState); //cell state is equal to the previous cell state multipled by the forget gate and the cell inputs multiplied by the input gate cell_j.cellState = cell_j.yForget * cell_j.previousCellState + cell_j.yIn * cell_j.yCellState; ////include the internal connection multiplied by the CURRENT cell state cell_j.netOut += cell_j.cellState * cell_j.wPeepholeOut + cell_j_previousCellOutput * cell_j.wCellOut; //squash output gate cell_j.yOut = Sigmoid(cell_j.netOut); cellOutput[j] = TanH(cell_j.cellState) * cell_j.yOut; cell[j] = cell_j; }); }
// forward process. output layer consists of tag value public override void computeLayer(SparseVector sparseFeature, double[] denseFeature, bool isTrain = true) { //inputs(t) -> hidden(t) //Get sparse feature and apply it into hidden layer SparseFeature = sparseFeature; DenseFeature = denseFeature; Parallel.For(0, LayerSize, parallelOption, j => { LSTMCell cell_j = cell[j]; //hidden(t-1) -> hidden(t) cell_j.previousCellState = cell_j.cellState; previousCellOutput[j] = cellOutput[j]; Vector4 vecCell_j = Vector4.Zero; if (SparseFeatureSize > 0) { //Apply sparse weights Vector4[] weights = input2hidden[j]; for (int i = 0; i < SparseFeature.Count; i++) { var entry = SparseFeature.GetEntry(i); vecCell_j += weights[entry.Key] * entry.Value; } } //Apply dense weights if (DenseFeatureSize > 0) { Vector4[] weights = feature2hidden[j]; for (int i = 0; i < DenseFeatureSize; i++) { vecCell_j += weights[i] * (float)DenseFeature[i]; } } //rest the value of the net input to zero cell_j.netIn = vecCell_j.X; cell_j.netForget = vecCell_j.Y; //reset each netCell state to zero cell_j.netCellState = vecCell_j.Z; //reset each netOut to zero cell_j.netOut = vecCell_j.W; double cell_j_previousCellOutput = previousCellOutput[j]; //include internal connection multiplied by the previous cell state cell_j.netIn += cell_j.previousCellState * cell_j.wPeepholeIn + cell_j_previousCellOutput * cell_j.wCellIn; //squash input cell_j.yIn = Sigmoid(cell_j.netIn); //include internal connection multiplied by the previous cell state cell_j.netForget += cell_j.previousCellState * cell_j.wPeepholeForget + cell_j_previousCellOutput * cell_j.wCellForget; cell_j.yForget = Sigmoid(cell_j.netForget); cell_j.netCellState += cell_j_previousCellOutput * cell_j.wCellState; cell_j.yCellState = TanH(cell_j.netCellState); if (mask[j] == true) { cell_j.cellState = 0; } else { //cell state is equal to the previous cell state multipled by the forget gate and the cell inputs multiplied by the input gate cell_j.cellState = cell_j.yForget * cell_j.previousCellState + cell_j.yIn * cell_j.yCellState; } if (isTrain == false) { cell_j.cellState = cell_j.cellState * (1.0 - Dropout); } ////include the internal connection multiplied by the CURRENT cell state cell_j.netOut += cell_j.cellState * cell_j.wPeepholeOut + cell_j_previousCellOutput * cell_j.wCellOut; //squash output gate cell_j.yOut = Sigmoid(cell_j.netOut); cellOutput[j] = TanH(cell_j.cellState) * cell_j.yOut; cell[j] = cell_j; }); }