public static double CalculateMedianPerplexity(ILayer layer, List<DataSequence> sequences)
{
double temperature = 1.0;
List<Double> ppls = new List<Double>();
foreach (DataSequence seq in sequences) {
double n = 0;
double neglog2Ppl = 0;
Graph g = new Graph(false);
layer.ResetState();
foreach (DataStep step in seq.Steps) {
Matrix logprobs = layer.Activate(step.Input, g);
Matrix probs = GetSoftmaxProbs(logprobs, temperature);
int targetIndex = GetTargetIndex(step.TargetOutput);
double probOfCorrect = probs.W[targetIndex];
double log2Prob = Math.Log(probOfCorrect)/Math.Log(2); //change-of-base
neglog2Ppl += -log2Prob;
n += 1;
}
n -= 1; //don't count first symbol of sentence
double ppl = Math.Pow(2, (neglog2Ppl/(n-1)));
ppls.Add(ppl);
}
return Util.Util.Median(ppls);
}
public Matrix Activate(Matrix input, Graph g)
{
Matrix prev = input;
foreach (ILayer layer in _layers)
{
prev = layer.Activate(prev, g);
}
return prev;
}
public Matrix Activate(Matrix input, Graph g)
{
Matrix concat = g.ConcatVectors(input, _context);
Matrix sum = g.Mul(_w, concat); sum = g.Add(sum, _b);
Matrix output = g.Nonlin(_f, sum);
//rollover activations for next iteration
_context = output;
return output;
}
public static void Run()
{
Random rng = new Random();
DataSet data = new XorDataSetGenerator();
int inputDimension = 2;
int hiddenDimension = 3;
int outputDimension = 1;
int hiddenLayers = 1;
double learningRate = 0.001;
double initParamsStdDev = 0.08;
INetwork nn = NetworkBuilder.MakeFeedForward(inputDimension,
hiddenDimension,
hiddenLayers,
outputDimension,
data.GetModelOutputUnitToUse(),
data.GetModelOutputUnitToUse(),
initParamsStdDev, rng);
int reportEveryNthEpoch = 10;
int trainingEpochs = 100000;
Trainer.train<NeuralNetwork>(trainingEpochs, learningRate, nn, data, reportEveryNthEpoch, rng);
Console.WriteLine("Training Completed.");
Console.WriteLine("Test: 1,1");
Matrix input = new Matrix(new double[] {1, 1});
Graph g = new Graph(false);
Matrix output = nn.Activate(input, g);
Console.WriteLine("Test: 1,1. Output:" + output.W[0]);
Matrix input1 = new Matrix(new double[] { 0, 1 });
Graph g1 = new Graph(false);
Matrix output1 = nn.Activate(input1, g1);
Console.WriteLine("Test: 0,1. Output:" + output1.W[0]);
Console.WriteLine("done.");
}
public Matrix Activate(Matrix input, Graph g)
{
//input gate
Matrix sum0 = g.Mul(_wix, input);
Matrix sum1 = g.Mul(_wih, _hiddenContext);
Matrix inputGate = g.Nonlin(_inputGateActivation, g.Add(g.Add(sum0, sum1), _inputBias));
//forget gate
Matrix sum2 = g.Mul(_wfx, input);
Matrix sum3 = g.Mul(_wfh, _hiddenContext);
Matrix forgetGate = g.Nonlin(_forgetGateActivation, g.Add(g.Add(sum2, sum3), _forgetBias));
//output gate
Matrix sum4 = g.Mul(_wox, input);
Matrix sum5 = g.Mul(_woh, _hiddenContext);
Matrix outputGate = g.Nonlin(_outputGateActivation, g.Add(g.Add(sum4, sum5), _outputBias));
//write operation on cells
Matrix sum6 = g.Mul(_wcx, input);
Matrix sum7 = g.Mul(_wch, _hiddenContext);
Matrix cellInput = g.Nonlin(_cellInputActivation, g.Add(g.Add(sum6, sum7), _cellWriteBias));
//compute new cell activation
Matrix retainCell = g.Elmul(forgetGate, _cellContext);
Matrix writeCell = g.Elmul(inputGate, cellInput);
Matrix cellAct = g.Add(retainCell, writeCell);
//compute hidden state as gated, saturated cell activations
Matrix output = g.Elmul(outputGate, g.Nonlin(_cellOutputActivation, cellAct));
//rollover activations for next iteration
_hiddenContext = output;
_cellContext = cellAct;
return output;
}
public Matrix Activate(Matrix input, Graph g)
{
Matrix sum0 = g.Mul(_hmix, input);
Matrix sum1 = g.Mul(_hHmix, _context);
Matrix actMix = g.Nonlin(_fMix, g.Add(g.Add(sum0, sum1), _bmix));
Matrix sum2 = g.Mul(_hreset, input);
Matrix sum3 = g.Mul(_hHreset, _context);
Matrix actReset = g.Nonlin(_fReset, g.Add(g.Add(sum2, sum3), _breset));
Matrix sum4 = g.Mul(_hnew, input);
Matrix gatedContext = g.Elmul(actReset, _context);
Matrix sum5 = g.Mul(_hHnew, gatedContext);
Matrix actNewPlusGatedContext = g.Nonlin(_fNew, g.Add(g.Add(sum4, sum5), _bnew));
Matrix memvals = g.Elmul(actMix, _context);
Matrix newvals = g.Elmul(g.OneMinus(actMix), actNewPlusGatedContext);
Matrix output = g.Add(memvals, newvals);
//rollover activations for next iteration
_context = output;
return output;
}
public Matrix Activate(Matrix input, Graph g)
{
Matrix returnObj = g.Mul(_w, input);
return returnObj;
}
