// forward process. output layer consists of tag value public override void computeNet(State state, double[] doutput) { //inputs(t) -> hidden(t) //Get sparse feature and apply it into hidden layer var sparse = state.GetSparseData(); int sparseFeatureSize = sparse.GetNumberOfEntries(); //loop through all input gates in hidden layer //for each hidden neuron Parallel.For(0, L1, parallelOption, j => { //rest the value of the net input to zero neuHidden[j].netIn = 0; //hidden(t-1) -> hidden(t) neuHidden[j].previousCellState = neuHidden[j].cellState; //for each input neuron for (int i = 0; i < sparseFeatureSize; i++) { var entry = sparse.GetEntry(i); neuHidden[j].netIn += entry.Value * mat_input2hidden[j][entry.Key].wInputInputGate; } }); //fea(t) -> hidden(t) if (fea_size > 0) { matrixXvectorADD(neuHidden, neuFeatures, mat_feature2hidden, 0, L1, 0, fea_size); } Parallel.For(0, L1, parallelOption, j => { LSTMCell cell_j = neuHidden[j]; //include internal connection multiplied by the previous cell state cell_j.netIn += cell_j.previousCellState * cell_j.wCellIn; //squash input cell_j.yIn = activationFunctionF(cell_j.netIn); cell_j.netForget = 0; //reset each netCell state to zero cell_j.netCellState = 0; //reset each netOut to zero cell_j.netOut = 0; for (int i = 0; i < sparseFeatureSize; i++) { var entry = sparse.GetEntry(i); LSTMWeight w = mat_input2hidden[j][entry.Key]; //loop through all forget gates in hiddden layer cell_j.netForget += entry.Value * w.wInputForgetGate; cell_j.netCellState += entry.Value * w.wInputCell; cell_j.netOut += entry.Value * w.wInputOutputGate; } if (fea_size > 0) { for (int i = 0; i < fea_size; i++) { LSTMWeight w = mat_feature2hidden[j][i]; cell_j.netForget += neuFeatures[i].ac * w.wInputForgetGate; cell_j.netCellState += neuFeatures[i].ac * w.wInputCell; cell_j.netOut += neuFeatures[i].ac * w.wInputOutputGate; } } //include internal connection multiplied by the previous cell state cell_j.netForget += cell_j.previousCellState * cell_j.wCellForget; cell_j.yForget = activationFunctionF(cell_j.netForget); //cell state is equal to the previous cell state multipled by the forget gate and the cell inputs multiplied by the input gate cell_j.cellState = cell_j.yForget * cell_j.previousCellState + cell_j.yIn * activationFunctionG(cell_j.netCellState); //include the internal connection multiplied by the CURRENT cell state cell_j.netOut += cell_j.cellState * cell_j.wCellOut; //squash output gate cell_j.yOut = activationFunctionF(cell_j.netOut); cell_j.cellOutput = activationFunctionH(cell_j.cellState) * cell_j.yOut; neuHidden[j] = cell_j; }); //initialize output nodes for (int c = 0; c < L2; c++) { neuOutput[c].ac = 0; } matrixXvectorADD(neuOutput, neuHidden, mat_hidden2output, 0, L2, 0, L1); if (doutput != null) { for (int i = 0; i < L2; i++) { doutput[i] = neuOutput[i].ac; } } //activation 2 --softmax on words double sum = 0; //sum is used for normalization: it's better to have larger precision as many numbers are summed together here for (int c = 0; c < L2; c++) { if (neuOutput[c].ac > 50) neuOutput[c].ac = 50; //for numerical stability if (neuOutput[c].ac < -50) neuOutput[c].ac = -50; //for numerical stability double val = Math.Exp(neuOutput[c].ac); sum += val; neuOutput[c].ac = val; } for (int c = 0; c < L2; c++) { neuOutput[c].ac /= sum; } }
public override void learnNet(State state, int timeat) { //create delta list double beta2 = beta * alpha; if (m_bCRFTraining == true) { //For RNN-CRF, use joint probability of output layer nodes and transition between contigous nodes for (int c = 0; c < L2; c++) { neuOutput[c].er = -m_Diff[timeat][c]; } neuOutput[state.GetLabel()].er = 1 - m_Diff[timeat][state.GetLabel()]; } else { //For standard RNN for (int c = 0; c < L2; c++) { neuOutput[c].er = -neuOutput[c].ac; } neuOutput[state.GetLabel()].er = 1 - neuOutput[state.GetLabel()].ac; } //Get sparse feature and apply it into hidden layer var sparse = state.GetSparseData(); int sparseFeatureSize = sparse.GetNumberOfEntries(); //put variables for derivaties in weight class and cell class Parallel.For(0, L1, parallelOption, i => { LSTMWeight[] w_i = mat_input2hidden[i]; LSTMCell c = neuHidden[i]; for (int k = 0; k < sparseFeatureSize; k++) { var entry = sparse.GetEntry(k); LSTMWeight w = w_i[entry.Key]; w_i[entry.Key].dSInputCell = w.dSInputCell * c.yForget + gPrime(c.netCellState) * c.yIn * entry.Value; w_i[entry.Key].dSInputInputGate = w.dSInputInputGate * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * entry.Value; w_i[entry.Key].dSInputForgetGate = w.dSInputForgetGate * c.yForget + c.previousCellState * fPrime(c.netForget) * entry.Value; } if (fea_size > 0) { w_i = mat_feature2hidden[i]; for (int j = 0; j < fea_size; j++) { LSTMWeight w = w_i[j]; w_i[j].dSInputCell = w.dSInputCell * c.yForget + gPrime(c.netCellState) * c.yIn * neuFeatures[j].ac; w_i[j].dSInputInputGate = w.dSInputInputGate * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * neuFeatures[j].ac; w_i[j].dSInputForgetGate = w.dSInputForgetGate * c.yForget + c.previousCellState * fPrime(c.netForget) * neuFeatures[j].ac; } } //partial derivatives for internal connections c.dSWCellIn = c.dSWCellIn * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * c.cellState; //partial derivatives for internal connections, initially zero as dS is zero and previous cell state is zero c.dSWCellForget = c.dSWCellForget * c.yForget + c.previousCellState * fPrime(c.netForget) * c.previousCellState; neuHidden[i] = c; }); //for all output neurons for (int k = 0; k < L2; k++) { //for each connection to the hidden layer double er = neuOutput[k].er; for (int j = 0; j <= L1; j++) { deltaHiddenOutput[j][k] = alpha * neuHidden[j].cellOutput * er; } } //for each hidden neuron Parallel.For(0, L1, parallelOption, i => { LSTMCell c = neuHidden[i]; //find the error by find the product of the output errors and their weight connection. double weightedSum = 0; for (int k = 0; k < L2; k++) { weightedSum += neuOutput[k].er * mat_hidden2output[i][k]; } //using the error find the gradient of the output gate c.gradientOutputGate = fPrime(c.netOut) * activationFunctionH(c.cellState) * weightedSum; //internal cell state error c.cellStateError = c.yOut * weightedSum * hPrime(c.cellState); //weight updates //already done the deltas for the hidden-output connections //output gates. for each connection to the hidden layer //to the input layer LSTMWeight[] w_i = mat_input2hidden[i]; for (int k = 0; k < sparseFeatureSize; k++) { var entry = sparse.GetEntry(k); //updates weights for input to hidden layer if ((counter % 10) == 0) //regularization is done every 10. step { w_i[entry.Key].wInputCell += alpha * c.cellStateError * w_i[entry.Key].dSInputCell - w_i[entry.Key].wInputCell * beta2; w_i[entry.Key].wInputInputGate += alpha * c.cellStateError * w_i[entry.Key].dSInputInputGate - w_i[entry.Key].wInputInputGate * beta2; w_i[entry.Key].wInputForgetGate += alpha * c.cellStateError * w_i[entry.Key].dSInputForgetGate - w_i[entry.Key].wInputForgetGate * beta2; w_i[entry.Key].wInputOutputGate += alpha * c.gradientOutputGate * entry.Value - w_i[entry.Key].wInputOutputGate * beta2; } else { w_i[entry.Key].wInputCell += alpha * c.cellStateError * w_i[entry.Key].dSInputCell; w_i[entry.Key].wInputInputGate += alpha * c.cellStateError * w_i[entry.Key].dSInputInputGate; w_i[entry.Key].wInputForgetGate += alpha * c.cellStateError * w_i[entry.Key].dSInputForgetGate; w_i[entry.Key].wInputOutputGate += alpha * c.gradientOutputGate * entry.Value; } } if (fea_size > 0) { w_i = mat_feature2hidden[i]; for (int j = 0; j < fea_size; j++) { //make the delta equal to the learning rate multiplied by the gradient multipled by the input for the connection //update connection weights if ((counter % 10) == 0) //regularization is done every 10. step { w_i[j].wInputCell += alpha * c.cellStateError * w_i[j].dSInputCell - w_i[j].wInputCell * beta2; w_i[j].wInputInputGate += alpha * c.cellStateError * w_i[j].dSInputInputGate - w_i[j].wInputInputGate * beta2; w_i[j].wInputForgetGate += alpha * c.cellStateError * w_i[j].dSInputForgetGate - w_i[j].wInputForgetGate * beta2; w_i[j].wInputOutputGate += alpha * c.gradientOutputGate * neuFeatures[j].ac - w_i[j].wInputOutputGate * beta2; } else { w_i[j].wInputCell += alpha * c.cellStateError * w_i[j].dSInputCell; w_i[j].wInputInputGate += alpha * c.cellStateError * w_i[j].dSInputInputGate; w_i[j].wInputForgetGate += alpha * c.cellStateError * w_i[j].dSInputForgetGate; w_i[j].wInputOutputGate += alpha * c.gradientOutputGate * neuFeatures[j].ac; } } } //for the internal connection double deltaOutputGateCell = alpha * c.gradientOutputGate * c.cellState; //using internal partial derivative double deltaInputGateCell = alpha * c.cellStateError * c.dSWCellIn; double deltaForgetGateCell = alpha * c.cellStateError * c.dSWCellForget; //update internal weights if ((counter % 10) == 0) //regularization is done every 10. step { c.wCellIn += deltaInputGateCell - c.wCellIn * beta2; c.wCellForget += deltaForgetGateCell - c.wCellForget * beta2; c.wCellOut += deltaOutputGateCell - c.wCellOut * beta2; } else { c.wCellIn += deltaInputGateCell; c.wCellForget += deltaForgetGateCell; c.wCellOut += deltaOutputGateCell; } neuHidden[i] = c; //update weights for hidden to output layer for (int k = 0; k < L2; k++) { if ((counter % 10) == 0) //regularization is done every 10. step { mat_hidden2output[i][k] += deltaHiddenOutput[i][k] - mat_hidden2output[i][k] * beta2; } else { mat_hidden2output[i][k] += deltaHiddenOutput[i][k]; } } }); }
public override void LearnBackTime(State state, int numStates, int curState) { if (bptt > 0) { //shift memory needed for bptt to next time step for (int a = bptt + bptt_block - 1; a > 0; a--) bptt_inputs[a] = bptt_inputs[a - 1]; bptt_inputs[0] = state.GetSparseData(); for (int a = bptt + bptt_block - 1; a > 0; a--) { for (int b = 0; b < L1; b++) { bptt_hidden[a * L1 + b] = bptt_hidden[(a - 1) * L1 + b]; } } for (int a = bptt + bptt_block - 1; a > 0; a--) { for (int b = 0; b < fea_size; b++) { bptt_fea[a * fea_size + b].ac = bptt_fea[(a - 1) * fea_size + b].ac; } } } //Save hidden and feature layer nodes values for bptt for (int b = 0; b < L1; b++) { bptt_hidden[b] = neuHidden[b]; } for (int b = 0; b < fea_size; b++) { bptt_fea[b].ac = neuFeatures[b].ac; } // time to learn bptt if (((counter % bptt_block) == 0) || (curState == numStates - 1)) { learnBptt(state); } }
// forward process. output layer consists of tag value public override void computeNet(State state, double[] doutput) { //erase activations for (int a = 0; a < L1; a++) neuHidden[a].ac = 0; //hidden(t-1) -> hidden(t) matrixXvectorADD(neuHidden, neuInput, mat_hiddenBpttWeight, 0, L1, L0 - L1, L0, 0); //inputs(t) -> hidden(t) //Get sparse feature and apply it into hidden layer var sparse = state.GetSparseData(); int n = sparse.GetNumberOfEntries(); for (int i = 0; i < n; i++) { var entry = sparse.GetEntry(i); for (int b = 0; b < L1; b++) { neuHidden[b].ac += entry.Value * mat_input2hidden[b][entry.Key]; } } //fea(t) -> hidden(t) if (fea_size > 0) { matrixXvectorADD(neuHidden, neuFeatures, mat_feature2hidden, 0, L1, 0, fea_size, 0); } //activate 1 --sigmoid computeHiddenActivity(); //initialize output nodes for (int c = 0; c < L2; c++) { neuOutput[c].ac = 0; } matrixXvectorADD(neuOutput, neuHidden, mat_hidden2output, 0, L2, 0, L1, 0); if (doutput != null) { for (int i = 0; i < L2; i++) { doutput[i] = neuOutput[i].ac; } } //activation 2 --softmax on words double sum = 0; //sum is used for normalization: it's better to have larger precision as many numbers are summed together here for (int c = 0; c < L2; c++) { if (neuOutput[c].ac > 50) neuOutput[c].ac = 50; //for numerical stability if (neuOutput[c].ac < -50) neuOutput[c].ac = -50; //for numerical stability double val = Math.Exp(neuOutput[c].ac); sum += val; neuOutput[c].ac = val; } for (int c = 0; c < L2; c++) { neuOutput[c].ac /= sum; } }
void ExtractSparseFeature(int currentState, int numStates, List<string[]> features, State pState) { Dictionary<int, double> sparseFeature = new Dictionary<int, double>(); int start = 0; var fc = m_FeatureConfiguration; //Extract TFeatures in given context window if (m_TFeaturizer != null) { if (fc.ContainsKey(TFEATURE_CONTEXT) == true) { List<int> v = fc[TFEATURE_CONTEXT]; for (int j = 0; j < v.Count; j++) { int offset = TruncPosition(currentState + v[j], 0, numStates); List<int> tfeatureList = m_TFeaturizer.GetFeatureIds(features, offset); foreach (int featureId in tfeatureList) { if (m_TFeatureWeightType == TFEATURE_WEIGHT_TYPE_ENUM.BINARY) { sparseFeature[start + featureId] = 1; } else { if (sparseFeature.ContainsKey(start + featureId) == false) { sparseFeature.Add(start + featureId, 1); } else { sparseFeature[start + featureId]++; } } } start += m_TFeaturizer.GetFeatureSize(); } } } // Create place hold for run time feature // The real feature value is calculated at run time if (fc.ContainsKey(RT_FEATURE_CONTEXT) == true) { List<int> v = fc[RT_FEATURE_CONTEXT]; pState.SetNumRuntimeFeature(v.Count); for (int j = 0; j < v.Count; j++) { if (v[j] < 0) { pState.AddRuntimeFeaturePlacehold(j, v[j], sparseFeature.Count, start); sparseFeature[start] = 0; //Placehold a position start += m_TagSet.GetSize(); } else { throw new Exception("The offset of run time feature should be negative."); } } } SparseVector spSparseFeature = pState.GetSparseData(); spSparseFeature.SetDimension(m_SparseDimension); spSparseFeature.SetData(sparseFeature); }