public WeightTensor(int rows, int columns, float c, int deviceId, bool keepCache = true)
{
DeviceId = deviceId;
Rows = rows;
Columns = columns;
var n = rows * columns;
var allocator = TensorAllocator.Allocator(deviceId);
TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TGradient, 0.0f);
if (keepCache)
{
TCache = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TCache, 0.0f);
TLrW = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TLrW, 0.0f);
}
TWeight = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TWeight, c);
}
public WeightTensor(long[] sizes, int deviceId, string name = "", bool isTrainable = false, bool normal = false)
{
Name = name;
DeviceId = deviceId;
IsTrainable = isTrainable;
allocator = TensorAllocator.Allocator(DeviceId);
Sizes = sizes;
if (normal)
{
var n = Rows * Columns;
float[] weight = new float[n];
var scale = (float)Math.Sqrt(1.0 / (Rows * Columns));
if (normal)
{
scale = 0.08f;
}
for (int i = 0; i < n; i++)
{
weight[i] = RandomGenerator.NormalRandom(0.0f, scale);
}
TGradient = new Tensor(allocator, DType.Float32, Sizes);
Ops.Fill(TGradient, 0.0f);
TWeight = Tensor.FromArray(allocator, weight).View(Sizes);
}
}
public WeightTensor(int rows, int columns, int deviceId, bool normal = false)
{
DeviceId = deviceId;
Rows = rows;
Columns = columns;
var n = rows * columns;
float[] weight = new float[n];
var scale = (float)Math.Sqrt(1.0 / (rows * columns));
if (normal)
{
scale = 0.08f;
}
for (int i = 0; i < n; i++)
{
weight[i] = RandomGenerator.NormalRandom(0.0f, scale);
}
var allocator = TensorAllocator.Allocator(deviceId);
TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TGradient, 0.0f);
TCash = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TCash, 0.0f);
TLrW = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TLrW, 0.0f);
TWeight = Tensor.FromArray(allocator, weight).View(Rows, Columns);
}
public WeightTensor(long[] sizes, float c, int deviceId, string name = "", bool isTrainable = false)
{
Name = name;
DeviceId = deviceId;
IsTrainable = isTrainable;
Sizes = sizes;
m_allocator = TensorAllocator.Allocator(DeviceId);
TWeight = new Tensor(m_allocator, DType.Float32, Sizes);
Ops.Fill(TWeight, c);
}
public float UpdateCost(IWeightTensor m, int[] ids)
{
WeightTensor t = m as WeightTensor;
using (Tensor idsTensor = new Tensor(TensorAllocator.Allocator(m_deviceId), DType.Int32, 1, ids.Length))
{
idsTensor.SetElementsAsInt(ids);
using (Tensor costs = Ops.UpdateCost(null, t.TWeight, idsTensor))
{
return(Ops.SumAll(costs));
}
}
}
public void CopyOrAddGradient(WeightTensor src)
{
if (m_TGradient == null)
{
allocator = TensorAllocator.Allocator(DeviceId);
m_TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Copy(m_TGradient, src.TGradient);
}
else
{
Ops.Add(m_TGradient, m_TGradient, src.TGradient);
}
}
public void AddMulGradient(Tensor w, Tensor g)
{
if (m_TGradient == null)
{
allocator = TensorAllocator.Allocator(DeviceId);
m_TGradient = new Tensor(allocator, DType.Float32, w.Sizes);
Ops.Mul(m_TGradient, w, g);
}
else
{
Ops.AddMul(m_TGradient, m_TGradient, w, g);
}
}
public void AddSigmoidGradient(WeightTensor src)
{
if (m_TGradient == null)
{
allocator = TensorAllocator.Allocator(DeviceId);
m_TGradient = new Tensor(allocator, DType.Float32, src.TWeight.Sizes);
Ops.SigmoidD(m_TGradient, src.TWeight, src.TGradient);
}
else
{
Ops.AddSigmoidD(m_TGradient, m_TGradient, src.TWeight, src.TGradient);
}
}
public void AddSoftmaxGradient(WeightTensor src)
{
if (m_TGradient == null)
{
allocator = TensorAllocator.Allocator(DeviceId);
m_TGradient = new Tensor(allocator, DType.Float32, src.TGradient.Sizes);
Ops.SoftmaxGrad(m_TGradient, src.TGradient, src.TWeight, false);
}
else
{
Ops.SoftmaxGrad(m_TGradient, src.TGradient, src.TWeight);
}
}
public void CopyOrAddGradient(Tensor src)
{
if (m_TGradient == null)
{
allocator = TensorAllocator.Allocator(DeviceId);
m_TGradient = new Tensor(allocator, DType.Float32, src.Sizes);
Ops.Copy(m_TGradient, src);
}
else
{
Ops.Add(m_TGradient, m_TGradient, src);
}
}
public WeightTensor(int rows, int columns, int deviceId)
{
DeviceId = deviceId;
Rows = rows;
Columns = columns;
var allocator = TensorAllocator.Allocator(deviceId);
TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TGradient, 0.0f);
TWeight = new Tensor(allocator, DType.Float32, Rows, Columns);
}
public IWeightMatrix SoftmaxM(IWeightMatrix w, bool bp = true)
{
WeightTensor m = w as WeightTensor;
var res = weightTensorFactory.CreateWeightTensor(m.Rows, m.Columns, deviceId, new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, m.Rows, m.Columns), bp);
Tensor tTmp = new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, m.Rows, m.Columns);
var maxval = Ops.Max(null, m.TWeight, 1);
var maxvalM = maxval.Expand(m.Rows, m.Columns);
Ops.ExpSub2(tTmp, m.TWeight, maxvalM);
var sumV = Ops.Sum(null, tTmp, 1);
var sumM = sumV.Expand(m.Rows, m.Columns);
Ops.Div(res.TWeight, tTmp, sumM);
maxval.Dispose();
maxvalM.Dispose();
sumV.Dispose();
sumM.Dispose();
if (this.needs_backprop && bp)
{
Action backward = () =>
{
Ops.Mul(tTmp, res.TGradient, res.TWeight);
Ops.Add(m.TGradient, m.TGradient, tTmp);
var ss = Ops.Sum(null, tTmp, 1);
var ssN = Ops.Neg(null, ss);
var ssM = ssN.Expand(m.Rows, m.Columns);
Ops.AddMul(m.TGradient, m.TGradient, res.TWeight, ssM);
tTmp.Dispose();
ss.Dispose();
ssM.Dispose();
ssN.Dispose();
};
this.backprop.Add(backward);
}
else
{
tTmp.Dispose();
}
return(res);
}
public WeightTensor(long[] sizes, float c, int deviceId, string name = "", bool isTrainable = false)
{
Name = name;
DeviceId = deviceId;
IsTrainable = isTrainable;
allocator = TensorAllocator.Allocator(DeviceId);
Sizes = sizes;
var n = Rows * Columns;
TGradient = new Tensor(allocator, DType.Float32, Sizes);
Ops.Fill(TGradient, 0.0f);
TWeight = new Tensor(allocator, DType.Float32, Sizes);
Ops.Fill(TWeight, c);
}
private Tensor BuildRandomTensor(int rows, int columns, int batchSize, float prob)
{
using (Tensor noise = new Tensor(TensorAllocator.Allocator(m_deviceId), DType.Float32, rows / batchSize, columns))
{
Ops.RandomBernoulli(noise, seedSource, prob);
if (rows / batchSize == 1)
{
return(noise.Expand(rows, columns));
}
else
{
return(noise.RepeatTensor(batchSize, 1));
}
}
}
public WeightTensor(int rows, int columns, Tensor weight, int deviceId, bool graident = true)
{
DeviceId = deviceId;
Rows = rows;
Columns = columns;
TWeight = weight;
if (graident)
{
var allocator = TensorAllocator.Allocator(deviceId);
TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
Ops.Fill(TGradient, 0.0f);
}
}
private Tensor BuildRandomTensor(int rows, int columns, int batchSize, float prob)
{
using (Tensor noise = new Tensor(TensorAllocator.Allocator(m_deviceId), DType.Float32, rows / batchSize, columns))
{
float[] w = TensorSharp.RandomGenerator.BuildRandomBernoulliWeight(new long[] { rows / batchSize, columns }, prob);
noise.SetElementsAsFloat(w);
if (rows / batchSize == 1)
{
return(noise.Expand(rows, columns));
}
else
{
return(noise.RepeatTensor(batchSize, 1));
}
}
}
private Tensor BuildRandomTensor(int rows, int columns, double prob)
{
float[] weights = new float[rows * columns];
for (int i = 0; i < weights.Length; i++)
{
double r = rnd.NextDouble();
if (r < prob)
{
weights[i] = 1.0f;
}
}
Tensor noise = new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, rows, columns);
noise.SetElementsAsFloat(weights);
return(noise);
}
public void AddMulGradient(Tensor w, Tensor g, bool inPlace = false)
{
if (m_TGradient == null)
{
m_allocator = TensorAllocator.Allocator(DeviceId);
m_TGradient = new Tensor(m_allocator, DType.Float32, w.Sizes);
Ops.Mul(m_TGradient, w, g);
}
else
{
if (inPlace)
{
Ops.Mul(m_TGradient, w, g);
}
else
{
Ops.AddMul(m_TGradient, m_TGradient, w, g);
}
}
}
public WeightTensor(long[] sizes, int deviceId, string name = "", bool isTrainable = false, bool normal = false)
{
Name = name;
DeviceId = deviceId;
IsTrainable = isTrainable;
m_allocator = TensorAllocator.Allocator(DeviceId);
Sizes = sizes;
if (normal)
{
var n = Rows * Columns;
float[] weight = new float[n];
var scale = (float)Math.Sqrt(2.0 / Rows);
for (int i = 0; i < n; i++)
{
weight[i] = RandomGenerator.NormalRandom(0.0f, scale);
}
TWeight = Tensor.FromArray(m_allocator, weight).View(Sizes);
}
}
public IWeightMatrix Softmax(IWeightMatrix w, bool bp = true)
{
WeightTensor m = w as WeightTensor;
var res = weightTensorFactory.CreateWeightTensor(m.Rows, m.Columns, deviceId, new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, m.Rows, m.Columns), bp);
Ops.Softmax(res.TWeight, m.TWeight);
if (this.needs_backprop && bp)
{
Action backward = () =>
{
Ops.SoftmaxGrad(m.TGradient, res.TGradient, res.TWeight);
res.Dispose();
};
this.backprop.Add(backward);
}
return(res);
}
