public ConvolutionLayer(INonlinearity func, int count, int h = 3, int w = 3)
{
Filters = new NNValue[count];
for (int i = 0; i < count; i++)
{
Filters[i] = new NNValue(h, w);
}
Function = func;
fs = new FilterStruct()
{
FilterW = w, FilterCount = count, FilterH = h
};
}
public static int PickIndexFromRandomVector(NNValue probs, Random r)
{
double mass = 1.0;
for (int i = 0; i < probs.DataInTensor.Length; i++)
{
double prob = probs.DataInTensor[i] / mass;
if (r.NextDouble() < prob)
{
return(i);
}
mass -= probs.DataInTensor[i];
}
throw new Exception("no target index selected");
}
public static NNValue ReShape(NNValue value, Shape newShape)
{
if (value.Len != newShape.Len)
{
throw new Exception("Преобразование невозможно, не совпадают объемы");
}
NNValue valRes = value.Clone();
valRes.H = newShape.H;
valRes.W = newShape.W;
valRes.D = newShape.D;
return(valRes);
}
/// <summary>
/// Обратный проход(производные)
/// </summary>
/// <param name="x"></param>
/// <returns></returns>
public NNValue Backward(NNValue x)
{
NNValue valueMatrix = new NNValue(x.H, x.W, x.D);
int len = x.Len;
for (int i = 0; i < len; i++)
{
double z = x[i];
z *= z;
z += 1;
valueMatrix.DataInTensor[i] = 1.0 / z;
}
return(valueMatrix);
}
public double Measure(NNValue actualOutput, NNValue targetOutput)
{
var crossentropy = 0.0;
for (int i = 0; i < actualOutput.DataInTensor.Length; i++)
{
crossentropy += targetOutput.DataInTensor[i] * Math.Log(actualOutput.DataInTensor[i] + 1e-15);
}
if (double.IsNaN(crossentropy))
{
int q = 1;
}
return(-crossentropy);
}
private void Shuffle(List <NNValue> inputs, List <NNValue> outputs)
{
var count = inputs.Count;
for (int i = count - 1; i > 0; i--)
{
var j = random.Next(0, i + 1);
NNValue t1 = inputs[i];
NNValue t2 = outputs[i];
inputs[i] = inputs[j];
outputs[i] = outputs[j];
inputs[j] = t1;
outputs[j] = t2;
}
}
public GruLayer(int inputDimension, int outputDimension, Random rng)
{
InputShape = new Shape(inputDimension);
OutputShape = new Shape(outputDimension);
double initParamsStdDev = 1.0 / Math.Sqrt(outputDimension);
_hmix = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rng);
_hHmix = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rng);
_bmix = new NNValue(outputDimension);
_hnew = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rng);
_hHnew = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rng);
_bnew = new NNValue(outputDimension);
_hreset = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rng);
_hHreset = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rng);
_breset = new NNValue(outputDimension);
}
/// <summary>
/// Один проход для минипакетного обучения
/// </summary>
/// <param name="learningRate">Скорость обучения</param>
/// <param name="network">Нейросеть</param>
/// <param name="sequences">Датасет</param>
/// <param name="isTraining">Производится ли обучение</param>
/// <param name="lossFunction">Функция ошибки</param>
/// <returns></returns>
public double PassBatch(double learningRate, INetwork network, List <DataSequence> sequences,
bool isTraining, ILoss lossFunction)
{
double numerLoss = 0;
double denomLoss = 0;
int index, passes = (sequences.Count % BatchSize == 0) ? sequences.Count / BatchSize : sequences.Count / BatchSize + 1;
for (int j = 0; j < passes; j++)
{
GraphCPU g = new GraphCPU(isTraining);
for (int i = 0; i < BatchSize; i++)
{
index = random.Next(sequences.Count);
var seq = sequences[index];
network.ResetState();
foreach (DataStep step in seq.Steps)
{
NNValue output = network.Activate(step.Input, g);
if (step.TargetOutput != null)
{
double loss = lossFunction.Measure(output, step.TargetOutput);
if (Double.IsNaN(loss) || Double.IsInfinity(loss))
{
return(loss);
}
numerLoss += loss;
denomLoss++;
if (isTraining)
{
lossFunction.Backward(output, step.TargetOutput);
}
}
}
}
if (isTraining)
{
g.Backward(); //backprop dw values
UpdateModelParams(network, learningRate); //update params
}
}
return(numerLoss / denomLoss);
}
void Init(Shape inputShape, int outputDimension, Random rng)
{
double initParamsStdDev = 1.0 / Math.Sqrt(outputDimension);
int inputDimension = inputShape.H;
InputShape = new Shape(inputDimension);
OutputShape = new Shape(outputDimension);
_hmix = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rng);
_hHmix = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rng);
_bmix = new NNValue(outputDimension);
_hnew = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rng);
_hHnew = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rng);
_bnew = new NNValue(outputDimension);
_hreset = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rng);
_hHreset = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rng);
_breset = new NNValue(outputDimension);
}
/// <summary>
/// Онлайн обучение
/// </summary>
/// <param name="learningRate"></param>
/// <param name="network"></param>
/// <param name="sequences"></param>
/// <param name="applyTraining"></param>
/// <param name="lossTraining"></param>
/// <returns></returns>
public double Pass(double learningRate, INetwork network, List <DataSequence> sequences,
bool applyTraining, ILoss lossTraining)
{
double numerLoss = 0;
double denomLoss = 0;
foreach (DataSequence seq in sequences)
{
GraphCPU g = new GraphCPU(applyTraining);
network.ResetState();
foreach (DataStep step in seq.Steps)
{
NNValue output = network.Activate(step.Input, g);
if (step.TargetOutput != null)
{
double loss = lossTraining.Measure(output, step.TargetOutput);
if (Double.IsNaN(loss) || Double.IsInfinity(loss))
{
return(loss);
}
numerLoss += loss;
denomLoss++;
if (applyTraining)
{
lossTraining.Backward(output, step.TargetOutput);
}
}
}
if (applyTraining)
{
g.Backward(); //backprop dw values
UpdateModelParams(network, learningRate); //update params
}
}
return(numerLoss / denomLoss);
}
public LstmLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rnd)
{
InputShape = new Shape(inputDimension);
OutputShape = new Shape(outputDimension);
_wix = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_wih = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
_inputBias = new NNValue(outputDimension);
_wfx = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_wfh = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
//set forget bias to 1.0, as described here: http://jmlr.org/proceedings/papers/v37/jozefowicz15.pdf
_forgetBias = NNValue.Ones(outputDimension, 1);
_wox = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_woh = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
_outputBias = new NNValue(outputDimension);
_wcx = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_wch = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
_cellWriteBias = new NNValue(outputDimension);
ResetState(); // Запуск НС
}
/// <summary>
/// Инициализация сети
/// </summary>
void Init(Shape inputShape, int outputDimension, double initParamsStdDev, Random rnd)
{
int inputDimension = inputShape.H;
InputShape = new Shape(inputDimension);
OutputShape = new Shape(outputDimension);
_wix = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_wih = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
_inputBias = new NNValue(outputDimension);
_wfx = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_wfh = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
_forgetBias = NNValue.Ones(outputDimension, 1);
_wox = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_woh = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
_outputBias = new NNValue(outputDimension);
_wcx = NNValue.Random(outputDimension, inputDimension, initParamsStdDev, rnd);
_wch = NNValue.Random(outputDimension, outputDimension, initParamsStdDev, rnd);
_cellWriteBias = new NNValue(outputDimension);
ResetState();
}
public NNValue Forward(NNValue x)
{
NNValue valueMatrix = new NNValue(x.H, x.W, x.D);
int len = x.DataInTensor.Length;
double summ = 0;
for (int i = 0; i < len; i++)
{
valueMatrix.DataInTensor[i] = Math.Exp(x.DataInTensor[i]);
valueMatrix.DataInTensor[i] = valueMatrix.DataInTensor[i] > maxActivate ? maxActivate : valueMatrix.DataInTensor[i];
summ += valueMatrix.DataInTensor[i];
}
for (int i = 0; i < len; i++)
{
valueMatrix.DataInTensor[i] /= summ;
}
return(valueMatrix);
}
public double Measure(NNValue actualOutput, NNValue targetOutput)
{
if (actualOutput.DataInTensor.Length != targetOutput.DataInTensor.Length)
{
throw new Exception("mismatch");
}
for (int i = 0; i < targetOutput.DataInTensor.Length; i++)
{
if (targetOutput.DataInTensor[i] >= 0.5 && actualOutput.DataInTensor[i] < 0.5)
{
return(1);
}
if (targetOutput.DataInTensor[i] < 0.5 && actualOutput.DataInTensor[i] >= 0.5)
{
return(1);
}
}
return(0);
}
public void Forward(int[] inp)
{
GraphCPU graph = new GraphCPU(false);
int indOld = 10;
network.ResetState();
for (int i = 0; i < inp.Length; i++)
{
NNValue valueMatrix = new NNValue(DataSetSeq2Seq.GetValue(inp[i], 11));
network.Activate(valueMatrix, graph);
}
for (int i = 0; i < inp.Length; i++)
{
NNValue valueMatrix = new NNValue(DataSetSeq2Seq.GetValue(indOld, 11));
indOld = GetInd(network.Activate(valueMatrix, graph));
Console.Write(indOld);
}
Console.WriteLine();
}
void Init(Shape inputShape, FilterStruct filterStruct, INonlinearity func, Random rnd)
{
InputShape = inputShape;
fs = filterStruct;
int outputH = inputShape.H - filterStruct.FilterH + 1 + _padY,
outputW = inputShape.W - filterStruct.FilterW + 1 + _padX;
OutputShape = new Shape(outputH, outputW, filterStruct.FilterCount);
double initParamsStdDev = 1.0 / Math.Sqrt(OutputShape.Len);
int d = InputShape.D;
Function = func;
Filters = new NNValue[filterStruct.FilterCount];
for (int i = 0; i < filterStruct.FilterCount; i++)
{
Filters[i] = NNValue.Random(filterStruct.FilterH, filterStruct.FilterW, d, initParamsStdDev, rnd);
}
}
/// <summary>
/// Обратный проход(производные)
/// </summary>
/// <param name="x"></param>
/// <returns></returns>
public NNValue Backward(NNValue x)
{
NNValue valueMatrix = new NNValue(x.H, x.W, x.D);
int len = x.Len;
for (int i = 0; i < len; i++)
{
double absX = Math.Atan(x[i]);
if (absX <= 1)
{
valueMatrix[i] = -x[i];
}
else if (absX >= 2)
{
valueMatrix[i] = 0;
}
else
{
valueMatrix[i] = x[i] - 2 * Math.Sign(x[i]);
}
}
return(valueMatrix);
}
public NNValue Activate(NNValue input, IGraph g)
{
NNValue sum0 = g.Mul(_hmix, input);
NNValue sum1 = g.Mul(_hHmix, _context);
NNValue actMix = g.Nonlin(_fMix, g.Add(g.Add(sum0, sum1), _bmix));
NNValue sum2 = g.Mul(_hreset, input);
NNValue sum3 = g.Mul(_hHreset, _context);
NNValue actReset = g.Nonlin(_fReset, g.Add(g.Add(sum2, sum3), _breset));
NNValue sum4 = g.Mul(_hnew, input);
NNValue gatedContext = g.Elmul(actReset, _context);
NNValue sum5 = g.Mul(_hHnew, gatedContext);
NNValue actNewPlusGatedContext = g.Nonlin(_fNew, g.Add(g.Add(sum4, sum5), _bnew));
NNValue memvals = g.Elmul(actMix, _context);
NNValue newvals = g.Elmul(g.OneMinus(actMix), actNewPlusGatedContext);
NNValue output = g.Add(memvals, newvals);
//rollover activations for next iteration
_context = output;
return(output);
}
public NNValue Activate(NNValue input, IGraph g)
{
//input gate
NNValue sum0 = g.Mul(_wix, input);
NNValue sum1 = g.Mul(_wih, _hiddenContext);
NNValue inputGate = g.Nonlin(_inputGateActivation, g.Add(g.Add(sum0, sum1), _inputBias));
//forget gate
NNValue sum2 = g.Mul(_wfx, input);
NNValue sum3 = g.Mul(_wfh, _hiddenContext);
NNValue forgetGate = g.Nonlin(_forgetGateActivation, g.Add(g.Add(sum2, sum3), _forgetBias));
//output gate
NNValue sum4 = g.Mul(_wox, input);
NNValue sum5 = g.Mul(_woh, _hiddenContext);
NNValue outputGate = g.Nonlin(_outputGateActivation, g.Add(g.Add(sum4, sum5), _outputBias));
//write operation on cells
NNValue sum6 = g.Mul(_wcx, input);
NNValue sum7 = g.Mul(_wch, _hiddenContext);
NNValue cellInput = g.Nonlin(_cellInputActivation, g.Add(g.Add(sum6, sum7), _cellWriteBias));
//compute new cell activation
NNValue retainCell = g.Elmul(forgetGate, _cellContext);
NNValue writeCell = g.Elmul(inputGate, cellInput);
NNValue cellAct = g.Add(retainCell, writeCell);
//compute hidden state as gated, saturated cell activations
NNValue output = g.Elmul(outputGate, g.Nonlin(_cellOutputActivation, cellAct));
//rollover activations for next iteration
_hiddenContext = output;
_cellContext = cellAct;
return(output);
}
public DataStep(NNValue input, NNValue targetOutput)
{
Input = input.Clone();
TargetOutput = targetOutput.Clone();
}
public State(Tensor input)
{
Input = new NNValue(input);
}
public NNValue Activate(NNValue input, IGraph g)
{
NNValue res = g.MaxPooling(input, _h, _w);
return(res);
}
public void Backward(NNValue actualOutput, NNValue targetOutput)
{
throw new NotImplementedException("not implemented");
}
public State(Matrix input)
{
Input = new NNValue(input);
}
public void ResetState()
{
_hiddenContext = new NNValue(OutputShape.H);
_cellContext = new NNValue(OutputShape.H);
}
public State(Vector input)
{
Input = new NNValue(input);
}
public DataStep(NNValue input)
{
Input = input.Clone();
}
public DataStep(double[] input)
{
Input = new NNValue(input);
}
public void ResetState()
{
_context = new NNValue(OutputShape.H);
}
public State(params double[] array)
{
Input = new NNValue(array);
}