private void SDG(RNdArray dw, ref RNdArray w, params object[] param)
{
float rho = param.Length >= 1 ? Convert.ToSingle(param[0]) / 1000 : 0.01f;
float dropoput = param.Length >= 2 ? Convert.ToSingle(param[1]) : 1.0f;
var c = w;
var dpo = DropOutFlag(w.TotalLength, dropoput);
if (Program == null)
{
Parallel.For(0, w.TotalLength, i =>
{
if (dpo[i] > 0)
{
c.Data[i] -= rho * dw.Data[i];
}
});
}
else
{
Program.Startup();
using (Cloo.ComputeBuffer <float> __w = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, c.Data))
using (Cloo.ComputeBuffer <float> __dw = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, dw.Data))
using (Cloo.ComputeBuffer <float> __dpo = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, dpo))
{
Program.SetParameter(__w);
Program.SetParameter(__dw);
Program.SetParameter(__dpo);
Program.SetParameter(w.Width, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program.SetParameter(w.Height, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program.SetParameter(rho, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.Execute(w.Width, w.Height);
Program.ReadBuffer(__w, ref c.Data);
}
}
}
private void Adam(RNdArray dw, ref RNdArray w, params object[] param)
{
if (adam_t == 0)
{
adam_m = w.Clone(); adam_m.Fill(0);
adam_v = w.Clone(); adam_v.Fill(0);
rho = adam_alpha;
}
adam_t++;
var bt1 = (1 - Math.Pow(adam_beta1, adam_t));
var bt2 = (1 - Math.Pow(adam_beta2, adam_t));
rho = (param.Length >= 1 && Convert.ToSingle(param[0]) >= 0) ? Convert.ToSingle(param[0]) / 1000 : adam_alpha * (float)(Math.Sqrt(bt2) / bt1);
if (rho > adam_alpha)
{
rho = adam_alpha;
}
float dropoput = param.Length >= 2 ? Convert.ToSingle(param[1]) : 1.0f;
var c = w;
var m = adam_m;
var v = adam_v;
var dpo = DropOutFlag(w.TotalLength, dropoput);
if (Program == null)
{
Parallel.For(0, w.TotalLength, i =>
{
var grad = dw.Data[i];
m.Data[i] = adam_beta1 * m.Data[i] + (1 - adam_beta1) * grad;
v.Data[i] = adam_beta2 * v.Data[i] + (1 - adam_beta2) * grad * grad;
if (dpo[i] > 0)
{
var mhat = m.Data[i] / bt1;
var vhat = v.Data[i] / bt2;
c.Data[i] -= (float)(rho * (mhat / (Math.Sqrt(vhat) + adam_ep)));
}
});
}
else
{
Program.Startup();
using (Cloo.ComputeBuffer <float> __w = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, c.Data))
using (Cloo.ComputeBuffer <float> __dw = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, dw.Data))
using (Cloo.ComputeBuffer <float> __m = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, m.Data))
using (Cloo.ComputeBuffer <float> __v = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, v.Data))
using (Cloo.ComputeBuffer <float> __dpo = Program.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, dpo))
{
Program.SetParameter(__w);
Program.SetParameter(__dw);
Program.SetParameter(__m);
Program.SetParameter(__v);
Program.SetParameter(__dpo);
Program.SetParameter(w.Width, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program.SetParameter(w.Height, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program.SetParameter(rho, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_beta1, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_beta2, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_ep, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.SetParameter(adam_t, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program.Execute(w.Width, w.Height);
Program.ReadBuffer(__m, ref m.Data);
Program.ReadBuffer(__v, ref v.Data);
Program.ReadBuffer(__w, ref c.Data);
}
}
}
private void LReLU(RNdArray u, ref RNdArray v, Direction dir, params object[] param)
{
float alpha = param.Length > 0 ? Convert.ToSingle(param[0]) : 0.01f;
var vt = v;
switch (dir)
{
case Direction.Activation:
if (Program01 == null)
{
Parallel.For(0, u.TotalLength, i =>
{
vt.Data[i] = u.Data[i] > 0 ? u.Data[i] : u.Data[i] * alpha;
});
}
else
{
Program01.Startup();
using (Cloo.ComputeBuffer <float> __u = Program01.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, u.Data))
using (Cloo.ComputeBuffer <float> __vt = Program01.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, vt.Data))
{
Program01.SetParameter(__u);
Program01.SetParameter(__vt);
Program01.SetParameter(u.Batch, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program01.SetParameter(u.Width, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program01.SetParameter(alpha, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program01.Execute(u.Batch, u.Width);
Program01.ReadBuffer(__vt, ref vt.Data);
}
}
break;
case Direction.Deactivation:
if (Program02 == null)
{
Parallel.For(0, u.TotalLength, i =>
{
vt.Data[i] = u.Data[i] > 0 ? 1 : alpha;
});
}
else
{
Program02.Startup();
using (Cloo.ComputeBuffer <float> __u = Program02.ConvertBuffer(Cloo.ComputeMemoryFlags.ReadOnly, u.Data))
using (Cloo.ComputeBuffer <float> __vt = Program02.ConvertBuffer(Cloo.ComputeMemoryFlags.WriteOnly, vt.Data))
{
Program02.SetParameter(__u);
Program02.SetParameter(__vt);
Program02.SetParameter(u.Batch, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program02.SetParameter(u.Width, OpenDM.Gpgpu.ProgramOption.ValueMode.INT);
Program02.SetParameter(alpha, OpenDM.Gpgpu.ProgramOption.ValueMode.FLOAT);
Program02.Execute(u.Batch, u.Width);
Program02.ReadBuffer(__vt, ref vt.Data);
}
}
break;
default:
break;
}
}
