public void CalculateParams(IMultipleLayer target)
{
mW = NumMath.Array(target.Size, target.ConectionsSize);
vW = NumMath.Array(target.Size, target.ConectionsSize);
mB = NumMath.Array(target.Size);
vB = NumMath.Array(target.Size);
}
private static void Sample(Array <FloatArray> hprev, int seed_ix, int size)
{
var x = new FloatArray(vocab_size);
x[seed_ix] = 1;
var ixes = new List <int>();
for (int t = 0; t < size; t++)
{
var data = net.Output(x, hprev);
var ix = NumMath.Choice(Enumerable.Range(0, vocab_size).ToArray(), 1, data.ps.ToArray()).First();
x = new FloatArray(vocab_size)
{
[ix] = 1
};
ixes.Add(ix);
hprev = data.hs;
}
Write($"----\n { ixes.Select( c => ix_to_char[c] ).Join("") } \n----");
}
public float Loss(Array <float> targets, Array <float> prediction)
{
var dif = targets - prediction;
var sum = NumMath.Sum(dif);
return(.5f * (sum / targets.View.Length));
}
private static void Sample(FloatArray hprev, FloatArray cprev, int seed_ix, int size)
{
var x = new FloatArray(vocab_size);
x[seed_ix] = 1;
var ixes = new List <int>();
for (int t = 0; t < size; t++)
{
var data = net.FeedForward(x, hprev, cprev);
var ix = NumMath.Choice(Enumerable.Range(0, vocab_size).ToArray(), 1, data.y.ToArray()).First();
x = new FloatArray(vocab_size)
{
[ix] = 1
};
ixes.Add(ix);
hprev = data.h;
cprev = data.c;
}
var str = string.Join("", ixes.Select(c => ix_to_char[c]));
Console.WriteLine($"----\n {str} \n----");
}
public void CalculateParams(IMultipleLayer target)
{
AdaDeltaW = NumMath.Array(target.Size, target.ConectionsSize);
AdaDeltaW_v = NumMath.Array(target.Size, target.ConectionsSize);
AdaDeltaB = NumMath.Array(target.Size);
AdaDeltaB_v = NumMath.Array(target.Size);
}
public RecursiveNeuralNetwork(int inputSize, float learningRate, float std)
{
w = NumMath.Random(inputSize, inputSize + inputSize, std);
wScore = NumMath.Random(1, inputSize, std);
b = NumMath.Repeat(inputSize, 1);
this.learningRate = learningRate;
}
public void BackWard(Array <float> values)
{
var DE = _errorFunction.Error(_target.OutputVector, values);
var DO = _activationFunction.Derivate(_target.SumVector);
_target.ErrorVector = DE * DO;
_target.ErrorWeightVector = NumMath.SumLine(_target.ErrorVector.W * _target.KnowlodgeMatrix);
}
public float Loss(Array <float> targets, float[] prediction)
{
using (var p = new Array <float>(prediction))
{
var dif = targets - p;
var sum = NumMath.Sum(dif);
return(.5f * (sum / targets.View.Length));
}
}
/// <summary>
/// Execute Backpropagation through time
/// </summary>
/// <returns></returns>
public (float loss, Array <FloatArray2D> dwy, Array <FloatArray> dby, Array <Array <FloatArray2D> > dwh, Array <FloatArray> dbh, Array <FloatArray> hs) BPTT(int[] inputs, int[] targets, Array <FloatArray> hprev)
{
float loss = 0f;
//Feedforward
var xs = new Array <FloatArray>(inputs.Length); // Inputs
var hs = new Array <Array <FloatArray> >(inputs.Length); // Hidden Result
var ps = new Array <Array <FloatArray> >(inputs.Length); // Softmax probabilit
var tg = new Array <FloatArray>(inputs.Length); // Targets
hs[-1] = hprev;
//Backward
var dwy = new Array <FloatArray2D>(1);
var dby = new Array <FloatArray>(1);
var dwh = new Array <Array <FloatArray2D> >(recurrentUnits);
var dbh = new Array <FloatArray>(recurrentUnits);
var dhnext = new Array <FloatArray>(recurrentUnits);
for (int t = 0; t < inputs.Length; t++)
{
xs[t] = new FloatArray(input_size);
xs[t][inputs[t]] = 1;
tg[t] = new FloatArray(output_size);
tg[t][targets[t]] = 1;
(hs[t], ps[t]) = FeedForward(xs[t], hs[t - 1]);
}
for (int t = inputs.Length - 1; t >= 0; t--)
{
// Sequencial
(var l, var dy) = decoder.ComputeErrorNBackward(tg[t], ps[t]);
(dhnext, _) = encoder.Backward(dy, dhnext, hs[t]);
// Parallel
(var wy, var by) = decoder.ComputeGradient(hs[t][-1]);
(var wh, var bh) = encoder.ComputeGradient(xs[t], hs[t - 1]);
// Parallel
dwy = dwy.Sum(wy);
dby = dby.Sum(by);
dwh = dwh.Sum(wh);
dbh = dbh.Sum(bh);
loss += l;
}
// Parallel
dwy = NumMath.Normalize(-5, 5, dwy);
dby = NumMath.Normalize(-5, 5, dby);
dwh = NumMath.Normalize(-5, 5, dwh);
dbh = NumMath.Normalize(-5, 5, dbh);
return(loss, dwy, dby, dwh, dbh, hs[-1]);
public RecurrentUnity(int input, int hidden, float learning_rate, float std)
{
input_size = input;
hidden_size = hidden;
this.learning_rate = learning_rate;
Wxt = NumMath.Random(hidden_size, input_size, std);
Wtt = NumMath.Random(hidden_size, hidden_size, std);
bt = NumMath.Repeat(hidden_size, 0);
ResetAdagradParams();
}
FloatArray hs, FloatArray cs) BPTT(Array <FloatArray> inputs, FloatArray error, FloatArray hprev, FloatArray cprev)
{
// store states
var z_s = new Array <FloatArray>(inputs.Length);
var f_s = new Array <FloatArray>(inputs.Length);
var i_s = new Array <FloatArray>(inputs.Length);
var c_s_s = new Array <FloatArray>(inputs.Length);
var c_s = new Array <FloatArray>(inputs.Length);
var o_s = new Array <FloatArray>(inputs.Length);
var h_s = new Array <FloatArray>(inputs.Length);
// init timing
h_s[-1] = hprev.Clone();
c_s[-1] = cprev.Clone();
// forward
for (var t = 0; t < inputs.Length; t++)
{
(z_s[t], f_s[t], i_s[t], c_s_s[t], c_s[t], o_s[t], h_s[t]) =
FeedForward(inputs[t], h_s[t - 1], c_s[t - 1]);
}
// gradients
var dWf = NumMath.Array(Wf.W, Wf.H);
var dWi = NumMath.Array(Wi.W, Wi.H);
var dWc = NumMath.Array(Wc.W, Wc.H);
var dWo = NumMath.Array(Wo.W, Wo.H);
var dBf = NumMath.Array(Bf.Length);
var dBi = NumMath.Array(Bi.Length);
var dBc = NumMath.Array(Bc.Length);
var dBo = NumMath.Array(Bo.Length);
var dhnext = error;
var dcnext = NumMath.Array(hidden_size);
// backward
for (var t = inputs.Length - 1; t >= 0; t--)
{
(dhnext, dcnext) = Backward(dhnext, dcnext, c_s[t - 1],
z_s[t], f_s[t], i_s[t], c_s_s[t], c_s[t], o_s[t], h_s[t],
ref dWf, ref dWi, ref dWc, ref dWo, ref dBf, ref dBi, ref dBc, ref dBo);
}
dWf = NumMath.Normalize(-5, 5, dWf / inputs.Length);
dWi = NumMath.Normalize(-5, 5, dWi / inputs.Length);
dWc = NumMath.Normalize(-5, 5, dWc / inputs.Length);
dWo = NumMath.Normalize(-5, 5, dWo / inputs.Length);
dBf = NumMath.Normalize(-5, 5, dBf / inputs.Length);
dBi = NumMath.Normalize(-5, 5, dBi / inputs.Length);
dBc = NumMath.Normalize(-5, 5, dBc / inputs.Length);
dBo = NumMath.Normalize(-5, 5, dBo / inputs.Length);
return(dWf, dWi, dWc, dWo, dBf, dBi, dBc, dBo, h_s[inputs.Length - 1], c_s[inputs.Length - 1]);
public void ResetAdagradParams()
{
mWf = NumMath.Array(hidden_size, input_size + hidden_size);
mWi = NumMath.Array(hidden_size, input_size + hidden_size);
mWc = NumMath.Array(hidden_size, input_size + hidden_size);
mWo = NumMath.Array(hidden_size, input_size + hidden_size);
mBf = NumMath.Array(hidden_size);
mBi = NumMath.Array(hidden_size);
mBc = NumMath.Array(hidden_size);
mBo = NumMath.Array(hidden_size);
}
public static Array2D <T> operator *(ArrayH <T> v0, Array <T> v1)
{
var size = new Index2(v1.View.Length, v0.View.Length);
var output = NumMath.Allocate <T>(size);
ProcessingDevice
.ArrayDevice
.Executor["_V_X_V_M"]
.Launch(size, output.View.View, v1.View.View, v0.View.View);
ProcessingDevice.ArrayDevice.Executor.Wait();
return(output);
}
public static Array2D <T> operator *(ArrayW <T> v0, Array2D <T> m0)
{
var size = new Index2(m0.View.Width, m0.View.Height);
var output = NumMath.Allocate <T>(size);
ProcessingDevice
.ArrayDevice
.Executor["_V_X_M_line_M"]
.Launch(size, output.View.View, m0.View.View, v0._memoryBuffer.View);
ProcessingDevice.ArrayDevice.Executor.Wait();
return(output);
}
public RecursiveNeuralNetworkWithContext(int inputSize, float learningRate, float std)
{
int hSize = 20;
wpl = NumMath.Random(inputSize, inputSize, std);
wpr = NumMath.Random(inputSize, inputSize, std);
wC = NumMath.Random(inputSize, inputSize, std);
wHP = NumMath.Random(hSize, inputSize, std);
wHC = NumMath.Random(hSize, inputSize, std);
wS = NumMath.Random(1, hSize, 1e-10f);
bC = NumMath.Repeat(inputSize, 1f / (float)inputSize);
bH = NumMath.Repeat(hSize, 1f / (float)hSize);
bP = NumMath.Repeat(inputSize, 1f / (float)inputSize);
mwpl = NumMath.Array(inputSize, inputSize);
mwpr = NumMath.Array(inputSize, inputSize);
mwC = NumMath.Array(inputSize, inputSize);
mwHP = NumMath.Array(hSize, inputSize);
mwHC = NumMath.Array(hSize, inputSize);
mwS = NumMath.Array(1, hSize);
mbC = NumMath.Array(inputSize);
mbH = NumMath.Array(hSize);
mbP = NumMath.Array(inputSize);
Adam_m_wpl = NumMath.Array(inputSize, inputSize);
Adam_m_wpr = NumMath.Array(inputSize, inputSize);
Adam_m_wC = NumMath.Array(inputSize, inputSize);
Adam_m_wHP = NumMath.Array(hSize, inputSize);
Adam_m_wHC = NumMath.Array(hSize, inputSize);
Adam_m_ws = NumMath.Array(1, hSize);
Adam_m_bC = NumMath.Array(inputSize);
Adam_m_bH = NumMath.Array(hSize);
Adam_m_bP = NumMath.Array(inputSize);
Adam_v_wpl = NumMath.Array(inputSize, inputSize);
Adam_v_wpr = NumMath.Array(inputSize, inputSize);
Adam_v_wC = NumMath.Array(inputSize, inputSize);
Adam_v_wHP = NumMath.Array(hSize, inputSize);
Adam_v_wHC = NumMath.Array(hSize, inputSize);
Adam_v_ws = NumMath.Array(1, hSize);
Adam_v_bC = NumMath.Array(inputSize);
Adam_v_bH = NumMath.Array(hSize);
Adam_v_bP = NumMath.Array(inputSize);
_learningRate = learningRate;
}
private static void Main(string[] args)
{
//Performance();
ProcessingDevice.Device = DeviceType.CUDA;
var m0 = NumMath.Random(1000, 1000, 1f);
var arr = new Array<FloatArray2D>(200);
var a = m0 * m0;
Parallel.For(0, 200, i =>
{
Console.WriteLine($"Computing {i}");
arr[i] = m0 * m0;
});
}
public RecurrentNeuralNetwork(int input, int output, int hidden, float learning_rate, float std)
{
input_size = input;
output_size = output;
hidden_size = hidden;
this.learning_rate = learning_rate;
Wxt = NumMath.Random(hidden_size, input_size, std);
Wtt = NumMath.Random(hidden_size, hidden_size, std);
Why = NumMath.Random(output_size, hidden_size, std);
bt = NumMath.Repeat(hidden_size, 0);
by = NumMath.Repeat(output_size, 0);
ResetAdagradParams();
}
public RecursiveNeuralUnity(int inputSize, float learningRate, float std)
{
wpl = NumMath.Random(inputSize, inputSize, std);
wpr = NumMath.Random(inputSize, inputSize, std);
wpr = NumMath.Random(inputSize, inputSize, std);
wDeep = NumMath.Random(inputSize, inputSize, std);
b = NumMath.Repeat(inputSize, 1);
mwpl = NumMath.Random(inputSize, inputSize, 0);
mwpr = NumMath.Random(inputSize, inputSize, 0);
mwpr = NumMath.Random(inputSize, inputSize, 0);
mwDeep = NumMath.Random(inputSize, inputSize, 0);
mb = NumMath.Repeat(inputSize, 0);
this.learningRate = learningRate;
}
public LSTM(int input, int hidden, float learning_rate, float std)
{
input_size = input;
hidden_size = hidden;
this.learning_rate = learning_rate;
Wf = NumMath.Random(hidden_size, input_size + hidden_size, std) + .5f;
Wi = NumMath.Random(hidden_size, input_size + hidden_size, std) + .5f;
Wc = NumMath.Random(hidden_size, input_size + hidden_size, std);
Wo = NumMath.Random(hidden_size, input_size + hidden_size, std) + .5f;
Bf = NumMath.Repeat(hidden_size, 0);
Bi = NumMath.Repeat(hidden_size, 0);
Bc = NumMath.Repeat(hidden_size, 0);
Bo = NumMath.Repeat(hidden_size, 0);
ResetAdagradParams();
}
public (FloatArray2D dWxt, FloatArray2D dWtt, FloatArray dbh, FloatArray hs) BPTT(Array <FloatArray> inputs, FloatArray hprev, FloatArray error)
{
var xs = new Array <FloatArray>(inputs.Length);
var ht = new Array <FloatArray>(inputs.Length);
ht[-1] = hprev;
var dWxt = new FloatArray2D(Wxt.W, Wxt.H);
var dWtt = new FloatArray2D(Wtt.W, Wtt.H);
var dhnext = new FloatArray(hidden_size);
var dbt = new FloatArray(bt.Length);
for (var t = 0; t < inputs.Length; t++)
{
xs[t] = inputs[t];
ht[t] = FeedForward(xs[t], ht[t - 1]);
}
dhnext = error;
for (var t = inputs.Length - 1; t >= 0; t--)
{
var dt = dhnext;
// Compute gradient of T (Derivate of Tanh)
var dtraw = (1 - ht[t] * ht[t]) * dt;
dWtt += ht[t - 1].T * dtraw; // Temporal
dWxt += xs[t].T * dtraw; // Input
dbt += dtraw;
// Acc to next Time
dhnext = (dtraw * Wtt).SumColumn();
}
// Normalize
dWxt = NumMath.Normalize(-5, 5, dWxt);
dWtt = NumMath.Normalize(-5, 5, dWtt);
dbt = NumMath.Normalize(-5, 5, dbt);
return(dWxt, dWtt, dbt, ht[inputs.Length - 1]);
public static int GetBestPosition(FloatArray result, bool prob)
{
if (prob)
{
var p = NumMath.Choice(Enumerable.Range(0, result.Length).ToArray(), 1, result.ToArray()).First();
return(p);
}
var max = 0f;
var pos = 0;
for (var i = 0; i < result.Length; i++)
{
if (max < result[i])
{
max = result[i];
pos = i;
}
}
return(pos);
}
public FloatArray Activate(FloatArray sum)
{
return(NumMath.Max(.001f * sum, sum));
}
public Array <float> Activate(Array <float> v)
{
return(NumMath.Max(.01f * v, v));
}
public void FeedForward(Array <float> feed)
{
_target.SumVector = NumMath.SumColumn(feed.H * _target.KnowlodgeMatrix) + _target.BiasVector;
_target.OutputVector = _activationFunction.Activate(_target.SumVector);
}
FloatArray hs, FloatArray cs) BPTT(int[] inputs, int[] targets, FloatArray hprev, FloatArray cprev)
{
// store states
var x_s = new Array <FloatArray>(inputs.Length);
var z_s = new Array <FloatArray>(inputs.Length);
var f_s = new Array <FloatArray>(inputs.Length);
var i_s = new Array <FloatArray>(inputs.Length);
var c_s_s = new Array <FloatArray>(inputs.Length);
var c_s = new Array <FloatArray>(inputs.Length);
var o_s = new Array <FloatArray>(inputs.Length);
var h_s = new Array <FloatArray>(inputs.Length);
var v_s = new Array <FloatArray>(inputs.Length);
var y_s = new Array <FloatArray>(inputs.Length);
var t_g = new Array <FloatArray>(inputs.Length);
// loss
var loss = 0d;
// init timing
h_s[-1] = hprev.Clone();
c_s[-1] = cprev.Clone();
// forward
for (var t = 0; t < inputs.Length; t++)
{
x_s[t] = new FloatArray(input_size);
x_s[t][inputs[t]] = 1;
t_g[t] = new FloatArray(output_size);
t_g[t][targets[t]] = 1;
(z_s[t], f_s[t], i_s[t], c_s_s[t], c_s[t], o_s[t], h_s[t], v_s[t], y_s[t]) =
FeedForward(x_s[t], h_s[t - 1], c_s[t - 1]);
loss += -(t_g[t] * y_s[t].Log()).Sum();
}
// gradients
var dWf = NumMath.Array(Wf.W, Wf.H);
var dWi = NumMath.Array(Wi.W, Wi.H);
var dWc = NumMath.Array(Wc.W, Wc.H);
var dWo = NumMath.Array(Wo.W, Wo.H);
var dWv = NumMath.Array(Wv.W, Wv.H);
var dBf = NumMath.Array(Bf.Length);
var dBi = NumMath.Array(Bi.Length);
var dBc = NumMath.Array(Bc.Length);
var dBo = NumMath.Array(Bo.Length);
var dBv = NumMath.Array(Bv.Length);
var dhnext = NumMath.Array(hidden_size);
var dcnext = NumMath.Array(hidden_size);
// backward
for (var t = inputs.Length - 1; t >= 0; t--)
{
(dhnext, dcnext) = Backward(targets[t], dhnext, dcnext, c_s[t - 1],
z_s[t], f_s[t], i_s[t], c_s_s[t], c_s[t], o_s[t], h_s[t], v_s[t], y_s[t],
ref dWf, ref dWi, ref dWc, ref dWo, ref dWv, ref dBf, ref dBi, ref dBc, ref dBo, ref dBv);
}
dWf = NumMath.Normalize(-5, 5, dWf / inputs.Length);
dWi = NumMath.Normalize(-5, 5, dWi / inputs.Length);
dWc = NumMath.Normalize(-5, 5, dWc / inputs.Length);
dWo = NumMath.Normalize(-5, 5, dWo / inputs.Length);
dWv = NumMath.Normalize(-5, 5, dWv / inputs.Length);
dBf = NumMath.Normalize(-5, 5, dBf / inputs.Length);
dBi = NumMath.Normalize(-5, 5, dBi / inputs.Length);
dBc = NumMath.Normalize(-5, 5, dBc / inputs.Length);
dBo = NumMath.Normalize(-5, 5, dBo / inputs.Length);
dBv = NumMath.Normalize(-5, 5, dBv / inputs.Length);
return(loss, dWf, dWi, dWc, dWo, dWv, dBf, dBi, dBc, dBo, dBv, h_s[inputs.Length - 1], c_s[inputs.Length - 1]);
BPTT(int[] inputs, int[] targets, FloatArray hprev)
{
var loss = 0f;
var xs = new Array <FloatArray>(inputs.Length);
var ht = new Array <FloatArray>(inputs.Length);
var ps = new Array <FloatArray>(inputs.Length);
var tg = new Array <FloatArray>(inputs.Length);
ht[-1] = hprev;
var dWxt = new FloatArray2D(Wxt.W, Wxt.H);
var dWtt = new FloatArray2D(Wtt.W, Wtt.H);
var dWhy = new FloatArray2D(Why.W, Why.H);
var dhnext = new FloatArray(hidden_size);
var dbt = new FloatArray(bt.Length);
var dby = new FloatArray(by.Length);
for (var t = 0; t < inputs.Length; t++)
{
xs[t] = new FloatArray(input_size);
xs[t][inputs[t]] = 1;
tg[t] = new FloatArray(output_size);
tg[t][targets[t]] = 1;
(ps[t], ht[t]) = FeedForward(xs[t], ht[t - 1]);
loss += -(tg[t] * ps[t].Log()).Sum();
}
for (var t = inputs.Length - 1; t >= 0; t--)
{
// output probabilities
var dy = ps[t].Clone();
// derive our first gradient
dy[targets[t]] -= 1;
// backpropagate to
var dt = (Why * dy).SumColumn() + dhnext;
// Compute gradient of T (Derivate of Tanh)
var dtraw = (1 - ht[t] * ht[t]) * dt;
dWtt += ht[t - 1].T * dtraw; // Temporal
dWxt += xs[t].T * dtraw; // Input
dbt += dtraw;
// Acc to next Time
dhnext = (dtraw * Wtt).SumColumn();
// Compute Derivates
dWhy += ht[t].T * dy;
dby += dy;
}
// Normalize
dWxt = NumMath.Normalize(-5, 5, dWxt);
dWtt = NumMath.Normalize(-5, 5, dWtt);
dWhy = NumMath.Normalize(-5, 5, dWhy);
dbt = NumMath.Normalize(-5, 5, dbt);
dby = NumMath.Normalize(-5, 5, dby);
return(loss, dWxt, dWtt, dWhy, dbt, dby, ht[inputs.Length - 1]);
public Array <float> Activate(Array <float> v)
{
return(1f / (1f + NumMath.Exp(-2f * v)));
}
public void CalculateParams(ILayer target)
{
gW = NumMath.Array(target.Size, target.ConectionsSize);
bW = NumMath.Array(target.Size);
}