// forward process. output layer consists of tag value
public override void computeNet(State state, double[] doutput)
{
//inputs(t) -> hidden(t)
//Get sparse feature and apply it into hidden layer
var sparse = state.GetSparseData();
int sparseFeatureSize = sparse.GetNumberOfEntries();
//loop through all input gates in hidden layer
//for each hidden neuron
Parallel.For(0, L1, parallelOption, j =>
{
//rest the value of the net input to zero
neuHidden[j].netIn = 0;
//hidden(t-1) -> hidden(t)
neuHidden[j].previousCellState = neuHidden[j].cellState;
//for each input neuron
for (int i = 0; i < sparseFeatureSize; i++)
{
var entry = sparse.GetEntry(i);
neuHidden[j].netIn += entry.Value * mat_input2hidden[j][entry.Key].wInputInputGate;
}
});
//fea(t) -> hidden(t)
if (fea_size > 0)
{
matrixXvectorADD(neuHidden, neuFeatures, mat_feature2hidden, 0, L1, 0, fea_size);
}
Parallel.For(0, L1, parallelOption, j =>
{
LSTMCell cell_j = neuHidden[j];
//include internal connection multiplied by the previous cell state
cell_j.netIn += cell_j.previousCellState * cell_j.wCellIn;
//squash input
cell_j.yIn = activationFunctionF(cell_j.netIn);
cell_j.netForget = 0;
//reset each netCell state to zero
cell_j.netCellState = 0;
//reset each netOut to zero
cell_j.netOut = 0;
for (int i = 0; i < sparseFeatureSize; i++)
{
var entry = sparse.GetEntry(i);
LSTMWeight w = mat_input2hidden[j][entry.Key];
//loop through all forget gates in hiddden layer
cell_j.netForget += entry.Value * w.wInputForgetGate;
cell_j.netCellState += entry.Value * w.wInputCell;
cell_j.netOut += entry.Value * w.wInputOutputGate;
}
if (fea_size > 0)
{
for (int i = 0; i < fea_size; i++)
{
LSTMWeight w = mat_feature2hidden[j][i];
cell_j.netForget += neuFeatures[i].ac * w.wInputForgetGate;
cell_j.netCellState += neuFeatures[i].ac * w.wInputCell;
cell_j.netOut += neuFeatures[i].ac * w.wInputOutputGate;
}
}
//include internal connection multiplied by the previous cell state
cell_j.netForget += cell_j.previousCellState * cell_j.wCellForget;
cell_j.yForget = activationFunctionF(cell_j.netForget);
//cell state is equal to the previous cell state multipled by the forget gate and the cell inputs multiplied by the input gate
cell_j.cellState = cell_j.yForget * cell_j.previousCellState + cell_j.yIn * activationFunctionG(cell_j.netCellState);
//include the internal connection multiplied by the CURRENT cell state
cell_j.netOut += cell_j.cellState * cell_j.wCellOut;
//squash output gate
cell_j.yOut = activationFunctionF(cell_j.netOut);
cell_j.cellOutput = activationFunctionH(cell_j.cellState) * cell_j.yOut;
neuHidden[j] = cell_j;
});
//initialize output nodes
for (int c = 0; c < L2; c++)
{
neuOutput[c].ac = 0;
}
matrixXvectorADD(neuOutput, neuHidden, mat_hidden2output, 0, L2, 0, L1);
if (doutput != null)
{
for (int i = 0; i < L2; i++)
{
doutput[i] = neuOutput[i].ac;
}
}
//activation 2 --softmax on words
double sum = 0; //sum is used for normalization: it's better to have larger precision as many numbers are summed together here
for (int c = 0; c < L2; c++)
{
if (neuOutput[c].ac > 50) neuOutput[c].ac = 50; //for numerical stability
if (neuOutput[c].ac < -50) neuOutput[c].ac = -50; //for numerical stability
double val = Math.Exp(neuOutput[c].ac);
sum += val;
neuOutput[c].ac = val;
}
for (int c = 0; c < L2; c++)
{
neuOutput[c].ac /= sum;
}
}
public override void learnNet(State state, int timeat)
{
//create delta list
double beta2 = beta * alpha;
if (m_bCRFTraining == true)
{
//For RNN-CRF, use joint probability of output layer nodes and transition between contigous nodes
for (int c = 0; c < L2; c++)
{
neuOutput[c].er = -m_Diff[timeat][c];
}
neuOutput[state.GetLabel()].er = 1 - m_Diff[timeat][state.GetLabel()];
}
else
{
//For standard RNN
for (int c = 0; c < L2; c++)
{
neuOutput[c].er = -neuOutput[c].ac;
}
neuOutput[state.GetLabel()].er = 1 - neuOutput[state.GetLabel()].ac;
}
//Get sparse feature and apply it into hidden layer
var sparse = state.GetSparseData();
int sparseFeatureSize = sparse.GetNumberOfEntries();
//put variables for derivaties in weight class and cell class
Parallel.For(0, L1, parallelOption, i =>
{
LSTMWeight[] w_i = mat_input2hidden[i];
LSTMCell c = neuHidden[i];
for (int k = 0; k < sparseFeatureSize; k++)
{
var entry = sparse.GetEntry(k);
LSTMWeight w = w_i[entry.Key];
w_i[entry.Key].dSInputCell = w.dSInputCell * c.yForget + gPrime(c.netCellState) * c.yIn * entry.Value;
w_i[entry.Key].dSInputInputGate = w.dSInputInputGate * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * entry.Value;
w_i[entry.Key].dSInputForgetGate = w.dSInputForgetGate * c.yForget + c.previousCellState * fPrime(c.netForget) * entry.Value;
}
if (fea_size > 0)
{
w_i = mat_feature2hidden[i];
for (int j = 0; j < fea_size; j++)
{
LSTMWeight w = w_i[j];
w_i[j].dSInputCell = w.dSInputCell * c.yForget + gPrime(c.netCellState) * c.yIn * neuFeatures[j].ac;
w_i[j].dSInputInputGate = w.dSInputInputGate * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * neuFeatures[j].ac;
w_i[j].dSInputForgetGate = w.dSInputForgetGate * c.yForget + c.previousCellState * fPrime(c.netForget) * neuFeatures[j].ac;
}
}
//partial derivatives for internal connections
c.dSWCellIn = c.dSWCellIn * c.yForget + activationFunctionG(c.netCellState) * fPrime(c.netIn) * c.cellState;
//partial derivatives for internal connections, initially zero as dS is zero and previous cell state is zero
c.dSWCellForget = c.dSWCellForget * c.yForget + c.previousCellState * fPrime(c.netForget) * c.previousCellState;
neuHidden[i] = c;
});
//for all output neurons
for (int k = 0; k < L2; k++)
{
//for each connection to the hidden layer
double er = neuOutput[k].er;
for (int j = 0; j <= L1; j++)
{
deltaHiddenOutput[j][k] = alpha * neuHidden[j].cellOutput * er;
}
}
//for each hidden neuron
Parallel.For(0, L1, parallelOption, i =>
{
LSTMCell c = neuHidden[i];
//find the error by find the product of the output errors and their weight connection.
double weightedSum = 0;
for (int k = 0; k < L2; k++)
{
weightedSum += neuOutput[k].er * mat_hidden2output[i][k];
}
//using the error find the gradient of the output gate
c.gradientOutputGate = fPrime(c.netOut) * activationFunctionH(c.cellState) * weightedSum;
//internal cell state error
c.cellStateError = c.yOut * weightedSum * hPrime(c.cellState);
//weight updates
//already done the deltas for the hidden-output connections
//output gates. for each connection to the hidden layer
//to the input layer
LSTMWeight[] w_i = mat_input2hidden[i];
for (int k = 0; k < sparseFeatureSize; k++)
{
var entry = sparse.GetEntry(k);
//updates weights for input to hidden layer
if ((counter % 10) == 0) //regularization is done every 10. step
{
w_i[entry.Key].wInputCell += alpha * c.cellStateError * w_i[entry.Key].dSInputCell - w_i[entry.Key].wInputCell * beta2;
w_i[entry.Key].wInputInputGate += alpha * c.cellStateError * w_i[entry.Key].dSInputInputGate - w_i[entry.Key].wInputInputGate * beta2;
w_i[entry.Key].wInputForgetGate += alpha * c.cellStateError * w_i[entry.Key].dSInputForgetGate - w_i[entry.Key].wInputForgetGate * beta2;
w_i[entry.Key].wInputOutputGate += alpha * c.gradientOutputGate * entry.Value - w_i[entry.Key].wInputOutputGate * beta2;
}
else
{
w_i[entry.Key].wInputCell += alpha * c.cellStateError * w_i[entry.Key].dSInputCell;
w_i[entry.Key].wInputInputGate += alpha * c.cellStateError * w_i[entry.Key].dSInputInputGate;
w_i[entry.Key].wInputForgetGate += alpha * c.cellStateError * w_i[entry.Key].dSInputForgetGate;
w_i[entry.Key].wInputOutputGate += alpha * c.gradientOutputGate * entry.Value;
}
}
if (fea_size > 0)
{
w_i = mat_feature2hidden[i];
for (int j = 0; j < fea_size; j++)
{
//make the delta equal to the learning rate multiplied by the gradient multipled by the input for the connection
//update connection weights
if ((counter % 10) == 0) //regularization is done every 10. step
{
w_i[j].wInputCell += alpha * c.cellStateError * w_i[j].dSInputCell - w_i[j].wInputCell * beta2;
w_i[j].wInputInputGate += alpha * c.cellStateError * w_i[j].dSInputInputGate - w_i[j].wInputInputGate * beta2;
w_i[j].wInputForgetGate += alpha * c.cellStateError * w_i[j].dSInputForgetGate - w_i[j].wInputForgetGate * beta2;
w_i[j].wInputOutputGate += alpha * c.gradientOutputGate * neuFeatures[j].ac - w_i[j].wInputOutputGate * beta2;
}
else
{
w_i[j].wInputCell += alpha * c.cellStateError * w_i[j].dSInputCell;
w_i[j].wInputInputGate += alpha * c.cellStateError * w_i[j].dSInputInputGate;
w_i[j].wInputForgetGate += alpha * c.cellStateError * w_i[j].dSInputForgetGate;
w_i[j].wInputOutputGate += alpha * c.gradientOutputGate * neuFeatures[j].ac;
}
}
}
//for the internal connection
double deltaOutputGateCell = alpha * c.gradientOutputGate * c.cellState;
//using internal partial derivative
double deltaInputGateCell = alpha * c.cellStateError * c.dSWCellIn;
double deltaForgetGateCell = alpha * c.cellStateError * c.dSWCellForget;
//update internal weights
if ((counter % 10) == 0) //regularization is done every 10. step
{
c.wCellIn += deltaInputGateCell - c.wCellIn * beta2;
c.wCellForget += deltaForgetGateCell - c.wCellForget * beta2;
c.wCellOut += deltaOutputGateCell - c.wCellOut * beta2;
}
else
{
c.wCellIn += deltaInputGateCell;
c.wCellForget += deltaForgetGateCell;
c.wCellOut += deltaOutputGateCell;
}
neuHidden[i] = c;
//update weights for hidden to output layer
for (int k = 0; k < L2; k++)
{
if ((counter % 10) == 0) //regularization is done every 10. step
{
mat_hidden2output[i][k] += deltaHiddenOutput[i][k] - mat_hidden2output[i][k] * beta2;
}
else
{
mat_hidden2output[i][k] += deltaHiddenOutput[i][k];
}
}
});
}
public override void LearnBackTime(State state, int numStates, int curState)
{
if (bptt > 0)
{
//shift memory needed for bptt to next time step
for (int a = bptt + bptt_block - 1; a > 0; a--)
bptt_inputs[a] = bptt_inputs[a - 1];
bptt_inputs[0] = state.GetSparseData();
for (int a = bptt + bptt_block - 1; a > 0; a--)
{
for (int b = 0; b < L1; b++)
{
bptt_hidden[a * L1 + b] = bptt_hidden[(a - 1) * L1 + b];
}
}
for (int a = bptt + bptt_block - 1; a > 0; a--)
{
for (int b = 0; b < fea_size; b++)
{
bptt_fea[a * fea_size + b].ac = bptt_fea[(a - 1) * fea_size + b].ac;
}
}
}
//Save hidden and feature layer nodes values for bptt
for (int b = 0; b < L1; b++)
{
bptt_hidden[b] = neuHidden[b];
}
for (int b = 0; b < fea_size; b++)
{
bptt_fea[b].ac = neuFeatures[b].ac;
}
// time to learn bptt
if (((counter % bptt_block) == 0) || (curState == numStates - 1))
{
learnBptt(state);
}
}
// forward process. output layer consists of tag value
public override void computeNet(State state, double[] doutput)
{
//erase activations
for (int a = 0; a < L1; a++)
neuHidden[a].ac = 0;
//hidden(t-1) -> hidden(t)
matrixXvectorADD(neuHidden, neuInput, mat_hiddenBpttWeight, 0, L1, L0 - L1, L0, 0);
//inputs(t) -> hidden(t)
//Get sparse feature and apply it into hidden layer
var sparse = state.GetSparseData();
int n = sparse.GetNumberOfEntries();
for (int i = 0; i < n; i++)
{
var entry = sparse.GetEntry(i);
for (int b = 0; b < L1; b++)
{
neuHidden[b].ac += entry.Value * mat_input2hidden[b][entry.Key];
}
}
//fea(t) -> hidden(t)
if (fea_size > 0)
{
matrixXvectorADD(neuHidden, neuFeatures, mat_feature2hidden, 0, L1, 0, fea_size, 0);
}
//activate 1 --sigmoid
computeHiddenActivity();
//initialize output nodes
for (int c = 0; c < L2; c++)
{
neuOutput[c].ac = 0;
}
matrixXvectorADD(neuOutput, neuHidden, mat_hidden2output, 0, L2, 0, L1, 0);
if (doutput != null)
{
for (int i = 0; i < L2; i++)
{
doutput[i] = neuOutput[i].ac;
}
}
//activation 2 --softmax on words
double sum = 0; //sum is used for normalization: it's better to have larger precision as many numbers are summed together here
for (int c = 0; c < L2; c++)
{
if (neuOutput[c].ac > 50) neuOutput[c].ac = 50; //for numerical stability
if (neuOutput[c].ac < -50) neuOutput[c].ac = -50; //for numerical stability
double val = Math.Exp(neuOutput[c].ac);
sum += val;
neuOutput[c].ac = val;
}
for (int c = 0; c < L2; c++)
{
neuOutput[c].ac /= sum;
}
}
void ExtractSparseFeature(int currentState, int numStates, List<string[]> features, State pState)
{
Dictionary<int, double> sparseFeature = new Dictionary<int, double>();
int start = 0;
var fc = m_FeatureConfiguration;
//Extract TFeatures in given context window
if (m_TFeaturizer != null)
{
if (fc.ContainsKey(TFEATURE_CONTEXT) == true)
{
List<int> v = fc[TFEATURE_CONTEXT];
for (int j = 0; j < v.Count; j++)
{
int offset = TruncPosition(currentState + v[j], 0, numStates);
List<int> tfeatureList = m_TFeaturizer.GetFeatureIds(features, offset);
foreach (int featureId in tfeatureList)
{
if (m_TFeatureWeightType == TFEATURE_WEIGHT_TYPE_ENUM.BINARY)
{
sparseFeature[start + featureId] = 1;
}
else
{
if (sparseFeature.ContainsKey(start + featureId) == false)
{
sparseFeature.Add(start + featureId, 1);
}
else
{
sparseFeature[start + featureId]++;
}
}
}
start += m_TFeaturizer.GetFeatureSize();
}
}
}
// Create place hold for run time feature
// The real feature value is calculated at run time
if (fc.ContainsKey(RT_FEATURE_CONTEXT) == true)
{
List<int> v = fc[RT_FEATURE_CONTEXT];
pState.SetNumRuntimeFeature(v.Count);
for (int j = 0; j < v.Count; j++)
{
if (v[j] < 0)
{
pState.AddRuntimeFeaturePlacehold(j, v[j], sparseFeature.Count, start);
sparseFeature[start] = 0; //Placehold a position
start += m_TagSet.GetSize();
}
else
{
throw new Exception("The offset of run time feature should be negative.");
}
}
}
SparseVector spSparseFeature = pState.GetSparseData();
spSparseFeature.SetDimension(m_SparseDimension);
spSparseFeature.SetData(sparseFeature);
}
