public IWeightMatrix ConcatColumns(params IWeightMatrix[] wl)
{
if (wl.Length == 1)
{
return(wl[0]);
}
List <Tensor> twl = new List <Tensor>();
int sx = 0;
int sy = 0;
foreach (IWeightMatrix item in wl)
{
WeightTensor m = item as WeightTensor;
sx = m.Rows;
sy += m.Columns;
twl.Add(m.TWeight);
}
var res = weightTensorFactory.CreateWeightTensor(sx, sy, deviceId);
Ops.Concat(res.TWeight, 1, twl.ToArray());
if (this.needs_backprop)
{
Action backward = () =>
{
res.ReleaseWeight();
sy = 0;
foreach (IWeightMatrix item in wl)
{
WeightTensor m = item as WeightTensor;
Tensor tTmp = res.TGradient.Narrow(1, sy, m.Columns);
// Ops.Add(m.TGradient, m.TGradient, tTmp);
m.CopyOrAddGradient(tTmp);
sy += m.Columns;
tTmp.Dispose();
}
res.Dispose();
};
this.backprop.Add(backward);
}
return(res);
}
public IWeightTensor ConcatRows(List <IWeightTensor> wl)
{
if (wl.Count == 1)
{
return(wl[0]);
}
List <string> wlNameList = new List <string>();
List <Tensor> twl = new List <Tensor>();
int sx = 0;
int sy = 0;
foreach (IWeightTensor item in wl)
{
WeightTensor m = item as WeightTensor;
sx += m.Rows;
sy = m.Columns;
twl.Add(m.TWeight);
wlNameList.Add(item.Name);
}
var wlName = String.Join("_", wlNameList);
var res = m_weightTensorFactory.CreateWeightTensor(sx, sy, m_deviceId, name: $"{GetHashString(wlName)}.ConcatRows");
VisualizeNodes(wl, res);
Ops.Concat(res.TWeight, 0, twl.ToArray());
if (m_needsBackprop)
{
Action backward = () =>
{
res.ReleaseWeight();
sx = 0;
foreach (IWeightTensor item in wl)
{
WeightTensor m = item as WeightTensor;
using (var tTmp = res.TGradient.Narrow(0, sx, m.Rows))
{
m.CopyOrAddGradient(tTmp);
sx += m.Rows;
}
}
res.Dispose();
};
this.m_backprop.Add(backward);
}
return(res);
}
public IWeightTensor ConcatColumns(params IWeightTensor[] wl)
{
if (wl.Length == 1)
{
return(wl[0]);
}
List <string> srcNameList = new List <string>();
List <Tensor> twl = new List <Tensor>();
int sx = 0;
int sy = 0;
foreach (IWeightTensor item in wl)
{
WeightTensor m = item as WeightTensor;
sx = m.Rows;
sy += m.Columns;
twl.Add(m.TWeight);
srcNameList.Add(item.Name);
}
string srcNames = String.Join("_", srcNameList);
var res = m_weightTensorFactory.CreateWeightTensor(sx, sy, m_deviceId, name: $"{GetHashString(srcNames)}.ConcatColumns");
VisualizeNodes(wl, res);
Ops.Concat(res.TWeight, 1, twl.ToArray());
if (this.m_needsBackprop)
{
Action backward = () =>
{
res.ReleaseWeight();
sy = 0;
foreach (IWeightTensor item in wl)
{
WeightTensor m = item as WeightTensor;
using (Tensor tTmp = res.TGradient.Narrow(1, sy, m.Columns))
{
m.CopyOrAddGradient(tTmp);
sy += m.Columns;
}
}
res.Dispose();
};
this.m_backprop.Add(backward);
}
return(res);
}
public IWeightMatrix View(IWeightMatrix m, int r, int c)
{
WeightTensor t = m as WeightTensor;
var res = weightTensorFactory.CreateWeightTensor(r, c, deviceId);
res.TWeight = t.TWeight.View(r, c);
if (this.needs_backprop)
{
Action backward = () =>
{
var g = res.TGradient.View(m.Rows, m.Columns);
t.CopyOrAddGradient(g);
};
this.backprop.Add(backward);
}
return(res);
}
public IWeightMatrix MulAdd(IWeightMatrix m1, IWeightMatrix m2, IWeightMatrix m3)
{
WeightTensor t1 = m1 as WeightTensor;
WeightTensor t2 = m2 as WeightTensor;
WeightTensor t3 = m3 as WeightTensor;
var n = t1.Rows;
var d = t2.Columns;
WeightTensor res = weightTensorFactory.CreateWeightTensor(n, d, deviceId);
Ops.Addmm(res.TWeight, 1.0f, t3.TWeight, 1.0f, t1.TWeight, t2.TWeight);
if (this.needs_backprop)
{
Action backward = () =>
{
res.ReleaseWeight();
// Ops.Add(t3.TGradient, t3.TGradient, res.TGradient);
t3.CopyOrAddGradient(res);
var tW2 = t2.TWeight.Transpose();
Ops.Addmm(t1.TGradient, 1.0f, t1.TGradient, 1.0f, res.TGradient, tW2);
var tW1 = t1.TWeight.Transpose();
Ops.Addmm(t2.TGradient, 1.0f, t2.TGradient, 1.0f, tW1, res.TGradient);
tW1.Dispose();
tW2.Dispose();
res.Dispose();
};
this.backprop.Add(backward);
}
return(res);
}
public IWeightTensor MulAdd(IWeightTensor m1, IWeightTensor m2, IWeightTensor m3)
{
WeightTensor t1 = m1 as WeightTensor;
WeightTensor t2 = m2 as WeightTensor;
WeightTensor t3 = m3 as WeightTensor;
var n = t1.Rows;
var d = t2.Columns;
WeightTensor res = m_weightTensorFactory.CreateWeightTensor(n, d, m_deviceId, name: $"{GetHashString(m1.Name, m2.Name, m3.Name)}.MulAdd");
VisualizeNodes(new IWeightTensor[] { m1, m2, m3 }, res);
Ops.Addmm(res.TWeight, 1.0f, t3.TWeight, 1.0f, t1.TWeight, t2.TWeight);
if (this.m_needsBackprop)
{
Action backward = () =>
{
res.ReleaseWeight();
t3.CopyOrAddGradient(res);
using (var tW2 = t2.TWeight.Transpose())
{
Ops.Addmm(t1.TGradient, 1.0f, t1.TGradient, 1.0f, res.TGradient, tW2);
}
using (var tW1 = t1.TWeight.Transpose())
{
Ops.Addmm(t2.TGradient, 1.0f, t2.TGradient, 1.0f, tW1, res.TGradient);
}
res.Dispose();
};
this.m_backprop.Add(backward);
}
return(res);
}
public IWeightTensor Transpose(IWeightTensor w)
{
WeightTensor m = w as WeightTensor;
var res = m_weightTensorFactory.CreateWeightTensor(m.Sizes, m_deviceId, name: $"{GetHashString(w.Name)}.Transpose");
VisualizeNodes(w, res);
res.TWeight = m.TWeight.Transpose();
if (m_needsBackprop)
{
Action backward = () =>
{
using (var gT = res.TGradient.Transpose())
{
m.CopyOrAddGradient(gT);
}
res.Dispose();
};
this.m_backprop.Add(backward);
}
return(res);
}
public IWeightMatrix ConcatRowColumn(List <IWeightMatrix> wl1, List <IWeightMatrix> wl2)
{
int sx = wl1[0].Rows * wl1.Count;
int sy = wl1[0].Columns + wl2[0].Columns;
var res = weightTensorFactory.CreateWeightTensor(sx, sy, deviceId);
var resTWC1 = res.TWeight.Narrow(1, 0, wl1[0].Columns);
var resTWC2 = res.TWeight.Narrow(1, wl1[0].Columns, wl2[0].Columns);
for (int i = 0; i < wl1.Count; i++)
{
WeightTensor m1 = wl1[i] as WeightTensor;
WeightTensor m2 = wl2[i] as WeightTensor;
var resTWC1R = resTWC1.Narrow(0, i * m1.Rows, m1.Rows);
Ops.Copy(resTWC1R, m1.TWeight);
var resTWC2R = resTWC2.Narrow(0, i * m2.Rows, m2.Rows);
Ops.Copy(resTWC2R, m2.TWeight);
resTWC1R.Dispose();
resTWC2R.Dispose();
}
resTWC1.Dispose();
resTWC2.Dispose();
if (this.needs_backprop)
{
Action backward = () =>
{
res.ReleaseWeight();
var res1 = res.TGradient.Narrow(1, 0, wl1[0].Columns);
var res2 = res.TGradient.Narrow(1, wl1[0].Columns, wl2[0].Columns);
for (int i = 0; i < wl1.Count; i++)
{
WeightTensor m1 = wl1[i] as WeightTensor;
WeightTensor m2 = wl2[i] as WeightTensor;
var resTGC1R = res1.Narrow(0, i * m1.Rows, m1.Rows);
var resTGC2R = res2.Narrow(0, i * m1.Rows, m1.Rows);
//Ops.Add(m1.TGradient, m1.TGradient, resTGC1R);
//Ops.Add(m2.TGradient, m2.TGradient, resTGC2R);
m1.CopyOrAddGradient(resTGC1R);
m2.CopyOrAddGradient(resTGC2R);
resTGC1R.Dispose();
resTGC2R.Dispose();
}
res1.Dispose();
res2.Dispose();
res.Dispose();
};
this.backprop.Add(backward);
}
return(res);
}
