private void Adam(RNdArray dw, ref RNdArray w, params object[] param)
{
if (adam_t == 0)
{
adam_m = w.Clone(); adam_m.Fill(0);
adam_v = w.Clone(); adam_v.Fill(0);
rho = adam_alpha;
}
adam_t++;
var bt1 = (1 - Math.Pow(adam_beta1, adam_t));
var bt2 = (1 - Math.Pow(adam_beta2, adam_t));
rho = (param.Length >= 1 && Convert.ToSingle(param[0]) >= 0) ? Convert.ToSingle(param[0]) / 1000 : adam_alpha * (float)(Math.Sqrt(bt2) / bt1);
if (rho > adam_alpha)
{
rho = adam_alpha;
}
float dropoput = param.Length >= 2 ? Convert.ToSingle(param[1]) : 1.0f;
var c = w;
var m = adam_m;
var v = adam_v;
var dpo = DropOutFlag(w.TotalLength, dropoput);
if (Program == null)
{
Parallel.For(0, w.TotalLength, i =>
{
var grad = dw.Data[i];
m.Data[i] = adam_beta1 * m.Data[i] + (1 - adam_beta1) * grad;
v.Data[i] = adam_beta2 * v.Data[i] + (1 - adam_beta2) * grad * grad;
if (dpo[i] > 0)
{
var mhat = m.Data[i] / bt1;
var vhat = v.Data[i] / bt2;
c.Data[i] -= (float)(rho * (mhat / (Math.Sqrt(vhat) + adam_ep)));
}
});
}
else
{
Program.Startup();
using (Cloo.ComputeBuffer <float> __w = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, c.Data))
using (Cloo.ComputeBuffer <float> __dw = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, dw.Data))
using (Cloo.ComputeBuffer <float> __m = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, m.Data))
using (Cloo.ComputeBuffer <float> __v = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, v.Data))
using (Cloo.ComputeBuffer <float> __dpo = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, dpo))
{
Program.SetParameter(__w);
Program.SetParameter(__dw);
Program.SetParameter(__m);
Program.SetParameter(__v);
Program.SetParameter(__dpo);
Program.SetParameter(w.Width, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program.SetParameter(w.Height, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program.SetParameter(rho, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_beta1, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_beta2, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_ep, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_t, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.Execute(w.Width, w.Height);
Program.ReadBuffer(__m, ref m.Data);
Program.ReadBuffer(__v, ref v.Data);
Program.ReadBuffer(__w, ref c.Data);
}
}
}
public double Back(R1dArray sigma, R1dArray input, R1dArray u, ref R2dArray w, out R1dArray p, Activator act)
{
p = new R1dArray(input.Width, input.Batch);
var ipt = input << 1;
var du = (RNdArray)(new R1dArray(u.Width, u.Batch));
var s = new R1dArray(sigma.Width, sigma.Batch);
var _p = p << 1;
var _w = w;
if (act != null)
{
act.DeActivation(u, ref du);
u = (R1dArray)du;
}
else
{
u.Fill(1);
}
if (Option_B_01 != null)
{
Option_B_01.Startup();
using (Cloo.ComputeBuffer <float> __sigma = Option_B_01.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, sigma.Data))
using (Cloo.ComputeBuffer <float> __u = Option_B_01.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, u.Data))
using (Cloo.ComputeBuffer <float> __s = Option_B_01.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, s.Data))
{
Option_B_01.SetParameter(__sigma);
Option_B_01.SetParameter(__u);
Option_B_01.SetParameter(__s);
Option_B_01.SetParameter(sigma.Width, Gpgpu.ProgramOption.ValueMode.INT);
Option_B_01.Execute(sigma.Batch, sigma.Width);
Option_B_01.ReadBuffer(__s, ref s.Data);
}
}
else
{
Parallel.For(0, sigma.Batch, b =>
{
Parallel.For(0, sigma.Width, i =>
{
s[i, b] = sigma[i, b] * u[i, b];
});
});
}
if (dw == null)
{
dw = w.Clone();
dw.Fill(0);
}
var _dw = (R2dArray)dw;
if (Option_B_02 != null)
{
Option_B_02.Startup();
using (Cloo.ComputeBuffer <float> __ipt = Option_B_02.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, ipt.Data))
using (Cloo.ComputeBuffer <float> __s = Option_B_02.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, s.Data))
using (Cloo.ComputeBuffer <float> __dw = Option_B_02.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, _dw.Data))
{
Option_B_02.SetParameter(__ipt);
Option_B_02.SetParameter(__s);
Option_B_02.SetParameter(__dw);
Option_B_02.SetParameter(dw.Width, Gpgpu.ProgramOption.ValueMode.INT);
Option_B_02.SetParameter(dw.Height, Gpgpu.ProgramOption.ValueMode.INT);
Option_B_02.SetParameter(sigma.Batch, Gpgpu.ProgramOption.ValueMode.INT);
Option_B_02.Execute(dw.Width, dw.Height);
Option_B_02.ReadBuffer(__dw, ref _dw.Data);
}
}
else
{
Parallel.For(0, dw.Width, i =>
{
Parallel.For(0, dw.Height, j =>
{
for (int b = 0; b < s.Batch; b++)
{
_dw[i, j] += s[j, b] * ipt[i, b];
}
});
});
}
if (Option_B_03 != null)
{
Option_B_03.Startup();
using (Cloo.ComputeBuffer <float> __s = Option_B_03.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, s.Data))
using (Cloo.ComputeBuffer <float> __w = Option_B_03.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, _w.Data))
using (Cloo.ComputeBuffer <float> __p = Option_B_03.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, _p.Data))
{
Option_B_03.SetParameter(__s);
Option_B_03.SetParameter(__w);
Option_B_03.SetParameter(__p);
Option_B_03.SetParameter(w.Width, Gpgpu.ProgramOption.ValueMode.INT);
Option_B_03.SetParameter(w.Height, Gpgpu.ProgramOption.ValueMode.INT);
Option_B_03.Execute(dw.Width, _p.Batch);
Option_B_03.ReadBuffer(__p, ref _p.Data);
}
}
else
{
Parallel.For(0, w.Width, i =>
{
Parallel.For(0, _p.Batch, b =>
{
for (int j = 0; j < _w.Height; j++)
{
_p[i, b] += _w[i, j] * s[j, b];
}
});
});
}
p = _p >> 1;
return(sigma.Power);
}