/// <summary>
/// Allocate an aligned vector with the given alignment (in bytes).
/// The alignment must be a power of two and at least sizeof(Float).
/// </summary>
public CpuAlignedVector(int size, int cbAlign)
{
Contracts.Assert(0 < size);
// cbAlign should be a power of two.
Contracts.Assert(sizeof(Float) <= cbAlign);
Contracts.Assert((cbAlign & (cbAlign - 1)) == 0);
int cfltAlign = cbAlign / sizeof(Float);
int cflt = RoundUp(size, cfltAlign);
_items = new AlignedArray(cflt, cbAlign);
_size = size;
AssertValid();
}
private static unsafe void ZeroItemsU(AlignedArray dst, int c, int[] indices, int cindices)
{
fixed(float *pdst = &dst.Items[0])
fixed(int *pidx = &indices[0])
{
for (int i = 0; i < cindices; ++i)
{
int index = pidx[i];
Contracts.Assert(index >= 0);
Contracts.Assert(index < c);
pdst[index] = 0;
}
}
}
public static void ApplyTanh(AlignedArray src, AlignedArray dst, int c)
{
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(src.Size == dst.Size);
Contracts.Assert(0 < c && c <= dst.Size);
unsafe
{
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
Thunk.ApplyTanhX(Ptr(src, psrc), Ptr(dst, pdst), c);
}
}
public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices)
{
Contracts.Assert(ccol > 0);
Contracts.Assert(ccol <= cfltRow);
if (ccol == cfltRow)
{
ZeroItemsU(dst, dst.Size, indices, indices.Length);
}
else
{
ZeroMatrixItemsCore(dst, dst.Size, ccol, cfltRow, indices, indices.Length);
}
}
public static void AddScale(float a, AlignedArray src, AlignedArray dst, float momentum, AlignedArray delta)
{
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(Compat(delta));
Contracts.Assert(src.Size == dst.Size);
Contracts.Assert(src.Size == delta.Size);
unsafe
{
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
fixed(float *pdel = &delta.Items[0])
Thunk.AddScaleMomX(a, Ptr(src, psrc), Ptr(dst, pdst), momentum, Ptr(delta, pdel), dst.Size);
}
}
public static void ApplyAbsDerivative(AlignedArray input, AlignedArray output, AlignedArray grad, bool drop)
{
Contracts.Assert(Compat(input));
Contracts.Assert(Compat(output));
Contracts.Assert(Compat(grad));
Contracts.Assert(output.Size == input.Size);
Contracts.Assert(output.Size == grad.Size);
unsafe
{
fixed(float *px = &input.Items[0])
fixed(float *py = &output.Items[0])
fixed(float *pg = &grad.Items[0])
Thunk.ApplyAbsDerivativeX(Ptr(input, px), Ptr(output, py), Ptr(grad, pg), grad.Size, drop);
}
}
public static void AddXYTran(float a, AlignedArray x, AlignedArray y, AlignedArray mat, int crow, float decay)
{
Contracts.Assert(Compat(x));
Contracts.Assert(Compat(y));
Contracts.Assert(Compat(mat));
Contracts.Assert(0 < crow && crow <= x.Size);
Contracts.Assert(x.Size * y.Size == mat.Size);
Contracts.Assert(decay >= 0);
unsafe
{
fixed(float *px = &x.Items[0])
fixed(float *py = &y.Items[0])
fixed(float *pmat = &mat.Items[0])
Thunk.AddXYTranX(a, Ptr(x, px), Ptr(y, py), Ptr(mat, pmat), crow, y.Size, decay);
}
}
public static void AddXYTran(float a, AlignedArray x, AlignedArray y, AlignedArray mat, float momentum, AlignedArray delta, int crow)
{
Contracts.Assert(Compat(x));
Contracts.Assert(Compat(y));
Contracts.Assert(Compat(mat));
Contracts.Assert(Compat(delta));
Contracts.Assert(0 < crow && crow <= x.Size);
Contracts.Assert(x.Size * y.Size == mat.Size);
Contracts.Assert(mat.Size == delta.Size);
unsafe
{
fixed(float *px = &x.Items[0])
fixed(float *py = &y.Items[0])
fixed(float *pmat = &mat.Items[0])
fixed(float *pdel = &delta.Items[0])
Thunk.AddXYTranMomX(a, Ptr(x, px), Ptr(y, py), Ptr(mat, pmat), momentum, Ptr(delta, pdel), crow, y.Size);
}
}
public static void MatTimesSrc(AlignedArray mat, int[] rgposSrc, AlignedArray srcValues,
int posMin, int iposMin, int iposLim, AlignedArray dst, int crun)
{
Contracts.AssertValue(rgposSrc);
Contracts.Assert(iposMin >= 0);
Contracts.Assert(iposMin <= iposLim);
Contracts.Assert(iposLim <= rgposSrc.Length);
Contracts.Assert(mat.Size == dst.Size * srcValues.Size);
if (iposMin >= iposLim)
{
dst.ZeroItems();
return;
}
Contracts.AssertNonEmpty(rgposSrc);
Contracts.Assert(crun >= 0);
if (Avx.IsSupported)
{
Contracts.Assert(crun <= dst.Size);
AvxIntrinsics.MatMulP(mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size);
}
else if (Sse.IsSupported)
{
Contracts.Assert(crun <= dst.Size);
SseIntrinsics.MatMulP(mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size);
}
else
{
Contracts.Assert(crun <= dst.Size);
for (int i = 0; i < crun; i++)
{
float dotProduct = 0;
for (int j = iposMin; j < iposLim; j++)
{
int col = rgposSrc[j] - posMin;
dotProduct += mat[i * srcValues.Size + col] * srcValues[col];
}
dst[i] = dotProduct;
}
}
}
public static void MatrixTimesSource(AlignedArray matrix, ReadOnlySpan <int> rgposSrc, AlignedArray sourceValues,
int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride)
{
Contracts.AssertValue(rgposSrc);
Contracts.Assert(iposMin >= 0);
Contracts.Assert(iposMin <= iposLimit);
Contracts.Assert(iposLimit <= rgposSrc.Length);
Contracts.Assert(matrix.Size == destination.Size * sourceValues.Size);
if (iposMin >= iposLimit)
{
destination.ZeroItems();
return;
}
Contracts.AssertNonEmpty(rgposSrc);
Contracts.Assert(stride >= 0);
if (Avx.IsSupported)
{
Contracts.Assert(stride <= destination.Size);
AvxIntrinsics.MatMulP(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size);
}
else if (Sse.IsSupported)
{
Contracts.Assert(stride <= destination.Size);
SseIntrinsics.MatMulP(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size);
}
else
{
Contracts.Assert(stride <= destination.Size);
for (int i = 0; i < stride; i++)
{
float dotProduct = 0;
for (int j = iposMin; j < iposLimit; j++)
{
int col = rgposSrc[j] - posMin;
dotProduct += matrix[i * sourceValues.Size + col] * sourceValues[col];
}
destination[i] = dotProduct;
}
}
}
public static void AddScale(AlignedArray src, AlignedArray dst,
AlignedArray accGrads, AlignedArray accUpdates, float decay, float cond)
{
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(Compat(accGrads));
Contracts.Assert(Compat(accUpdates));
Contracts.Assert(src.Size == dst.Size);
Contracts.Assert(src.Size == accGrads.Size);
Contracts.Assert(src.Size == accUpdates.Size);
unsafe
{
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
fixed(float *pag = &accGrads.Items[0])
fixed(float *pau = &accUpdates.Items[0])
Thunk.AddScaleGradX(Ptr(src, psrc), Ptr(dst, pdst), Ptr(accGrads, pag), Ptr(accUpdates, pau), decay, cond, dst.Size);
}
}
public static void ScaleMaxNorm(bool tran, float maxNorm, AlignedArray mat, int crun, int runLenPhy)
{
// Called only with Avx alignment.
Contracts.Assert(Compat(mat));
unsafe
{
fixed(float *pmat = &mat.Items[0])
{
if (!tran)
{
Thunk.ScaleMaxNormX(maxNorm, Ptr(mat, pmat), crun, runLenPhy);
}
else
{
Thunk.ScaleMaxNormTranU(maxNorm, Ptr(mat, pmat), crun, runLenPhy);
}
}
}
}
public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices)
{
Contracts.Assert(0 < ccol && ccol <= cfltRow);
unsafe
{
fixed(float *pdst = &dst.Items[0])
fixed(int *pi = &indices[0])
{
if (ccol == cfltRow)
{
Thunk.ZeroItemsU(Ptr(dst, pdst), dst.Size, pi, indices.Length);
}
else
{
Thunk.ZeroMatrixItemsCore(Ptr(dst, pdst), dst.Size, ccol, cfltRow, pi, indices.Length);
}
}
}
}
/// <summary>
/// Allocate an aligned matrix with the given alignment (in bytes).
/// </summary>
protected CpuAlignedMatrixBase(int runLen, int runCnt, int cbAlign)
{
Contracts.Assert(0 < runLen);
Contracts.Assert(0 < runCnt);
// cbAlign should be a power of two.
Contracts.Assert(sizeof(Float) <= cbAlign);
Contracts.Assert((cbAlign & (cbAlign - 1)) == 0);
RunLen = runLen;
RunCnt = runCnt;
FloatAlign = cbAlign / sizeof(Float);
Shift = GeneralUtils.CbitLowZero((uint)FloatAlign);
Mask = FloatAlign - 1;
RunLenPhy = RoundUp(runLen, FloatAlign);
RunCntPhy = RoundUp(runCnt, FloatAlign);
Items = new AlignedArray(RunLenPhy * RunCntPhy, cbAlign);
AssertValid();
}
public static void MatTimesSrc(bool add, int[] mprowiv, int[] mprowcol,
int[] mprowrun, int[] runs, float[] coefs,
AlignedArray src, AlignedArray dst, int crow)
{
Contracts.AssertNonEmpty(mprowiv);
Contracts.Assert(mprowiv.Length == crow);
Contracts.AssertNonEmpty(mprowcol);
Contracts.Assert(mprowcol.Length == crow);
Contracts.Assert(mprowrun == null || mprowrun.Length == crow);
Contracts.AssertNonEmpty(runs);
Contracts.AssertNonEmpty(coefs);
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(0 < crow && crow <= dst.Size);
unsafe
{
fixed(int *pmprowiv = &mprowiv[0])
fixed(int *pmprowcol = &mprowcol[0])
fixed(int *pruns = &runs[0])
fixed(float *pcoefs = &coefs[0])
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
{
if (mprowrun == null)
{
Thunk.MatMulCX(add, pmprowiv, pmprowcol, pruns, pcoefs,
Ptr(src, psrc), Ptr(dst, pdst), crow);
}
else
{
fixed(int *pmprowrun = &mprowrun[0])
{
Thunk.MatMulDX(add, pmprowiv, pmprowcol, pmprowrun, pruns, pcoefs,
Ptr(src, psrc), Ptr(dst, pdst), crow);
}
}
}
}
}
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues,
int posMin, int iposMin, int iposLim, AlignedArray dst, int crun)
{
Contracts.Assert(Compat(mat));
Contracts.Assert(Compat(srcValues));
Contracts.Assert(Compat(dst));
Contracts.AssertValue(rgposSrc);
Contracts.Assert(0 <= iposMin && iposMin <= iposLim && iposLim <= rgposSrc.Length);
Contracts.Assert(mat.Size == dst.Size * srcValues.Size);
if (iposMin >= iposLim)
{
if (!add)
{
dst.ZeroItems();
}
return;
}
Contracts.AssertNonEmpty(rgposSrc);
unsafe
{
fixed(float *pdst = &dst.Items[0])
fixed(float *pmat = &mat.Items[0])
fixed(float *psrc = &srcValues.Items[0])
fixed(int *ppossrc = &rgposSrc[0])
{
if (!tran)
{
Contracts.Assert(0 <= crun && crun <= dst.Size);
Thunk.MatMulPX(add, Ptr(mat, pmat), ppossrc, Ptr(srcValues, psrc), posMin, iposMin, iposLim, Ptr(dst, pdst), crun, srcValues.Size);
}
else
{
Contracts.Assert(0 <= crun && crun <= srcValues.Size);
Thunk.MatMulTranPX(add, Ptr(mat, pmat), ppossrc, Ptr(srcValues, psrc), posMin, iposMin, iposLim, Ptr(dst, pdst), dst.Size);
}
}
}
}
public static void MatTranTimesSrc(bool add, int[] mpcoliv, int[] mpcolrow, int[] mpcolrun,
int[] runs, float[] coefs, AlignedArray src, AlignedArray dst, int ccol)
{
Contracts.AssertNonEmpty(mpcoliv);
Contracts.Assert(mpcoliv.Length == ccol);
Contracts.AssertNonEmpty(mpcolrow);
Contracts.Assert(mpcolrow.Length == ccol);
Contracts.AssertNonEmpty(runs);
Contracts.AssertNonEmpty(coefs);
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(mpcolrun == null || mpcolrun.Length == ccol);
Contracts.Assert(0 < ccol && ccol <= src.Size);
unsafe
{
fixed(int *pmpcoliv = &mpcoliv[0])
fixed(int *pmpcolrow = &mpcolrow[0])
fixed(int *pruns = &runs[0])
fixed(float *pcoefs = &coefs[0])
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
{
if (mpcolrun == null)
{
Thunk.MatMulTranCX(add, pmpcoliv, pmpcolrow, pruns, pcoefs,
Ptr(src, psrc), Ptr(dst, pdst), dst.Size, ccol);
}
else
{
fixed(int *pmpcolrun = &mpcolrun[0])
{
Thunk.MatMulTranDX(add, pmpcoliv, pmpcolrow, pmpcolrun, pruns, pcoefs,
Ptr(src, psrc), Ptr(dst, pdst), dst.Size, ccol);
}
}
}
}
}
public static void AddXYTran(AlignedArray x, AlignedArray y, AlignedArray mat, AlignedArray accGrads, AlignedArray accUpdates,
float decay, float cond, int crow)
{
Contracts.Assert(Compat(x));
Contracts.Assert(Compat(y));
Contracts.Assert(Compat(mat));
Contracts.Assert(Compat(accGrads));
Contracts.Assert(Compat(accUpdates));
Contracts.Assert(0 < crow && crow <= x.Size);
Contracts.Assert(x.Size * y.Size == mat.Size);
Contracts.Assert(mat.Size == accGrads.Size);
Contracts.Assert(mat.Size == accUpdates.Size);
unsafe
{
fixed(float *px = &x.Items[0])
fixed(float *py = &y.Items[0])
fixed(float *pmat = &mat.Items[0])
fixed(float *pag = &accGrads.Items[0])
fixed(float *pau = &accUpdates.Items[0])
Thunk.AddXYTranGradX(Ptr(x, px), Ptr(y, py), Ptr(mat, pmat), Ptr(accGrads, pag), Ptr(accUpdates, pau), decay, cond, crow, y.Size);
}
}
public static void MeanBackOfSrc(bool add, int[] mpcolrow, int[] mpcolindices,
int[] indices, AlignedArray src, AlignedArray dst, int ccol)
{
Contracts.AssertNonEmpty(mpcolrow);
Contracts.Assert(mpcolrow.Length == ccol);
Contracts.Assert(mpcolindices == null || mpcolindices.Length == ccol);
Contracts.AssertNonEmpty(indices);
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(0 < ccol && ccol <= src.Size);
unsafe
{
fixed(int *pmpcolrow = &mpcolrow[0])
fixed(int *pmpcolindices = mpcolindices)
fixed(int *pindices = &indices[0])
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
{
// REVIEW: Implement using AVX
Thunk.MeanBackU(add, pmpcolrow, pmpcolindices, pindices, Ptr(src, psrc), Ptr(dst, pdst), dst.Size, ccol);
}
}
}
public static void MaxOfSrc(bool add, int[] mprowcol, int[] mprowindices,
int[] indices, AlignedArray src, AlignedArray dst, int crow)
{
Contracts.AssertNonEmpty(mprowcol);
Contracts.Assert(mprowcol.Length == crow);
Contracts.Assert(mprowindices == null || mprowindices.Length == crow);
Contracts.AssertNonEmpty(indices);
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(0 < crow && crow <= dst.Size);
unsafe
{
fixed(int *pmprowcol = &mprowcol[0])
fixed(int *pmprowindices = mprowindices)
fixed(int *pindices = &indices[0])
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
{
// REVIEW: Implement using AVX
Thunk.MaxU(add, pmprowcol, pmprowindices, pindices, Ptr(src, psrc), Ptr(dst, pdst), crow);
}
}
}
public static void MatrixTimesSource(bool transpose, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(transpose, matrix, source, destination, stride);
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun)
{
Contracts.Assert(mat.Size == dst.Size * src.Size);
Contracts.Assert(crun >= 0);
if (Avx.IsSupported)
{
if (!tran)
{
Contracts.Assert(crun <= dst.Size);
AvxIntrinsics.MatMulX(add, mat, src, dst, crun, src.Size);
}
else
{
Contracts.Assert(crun <= src.Size);
AvxIntrinsics.MatMulTranX(add, mat, src, dst, dst.Size, crun);
}
}
else if (Sse.IsSupported)
{
if (!tran)
{
Contracts.Assert(crun <= dst.Size);
SseIntrinsics.MatMulA(add, mat, src, dst, crun, src.Size);
}
else
{
Contracts.Assert(crun <= src.Size);
SseIntrinsics.MatMulTranA(add, mat, src, dst, dst.Size, crun);
}
}
else
{
if (!tran)
{
Contracts.Assert(crun <= dst.Size);
for (int i = 0; i < crun; i++)
{
float dotProduct = 0;
for (int j = 0; j < src.Size; j++)
{
dotProduct += mat[i * src.Size + j] * src[j];
}
if (add)
{
dst[i] += dotProduct;
}
else
{
dst[i] = dotProduct;
}
}
}
else
{
Contracts.Assert(crun <= src.Size);
for (int i = 0; i < dst.Size; i++)
{
float dotProduct = 0;
for (int j = 0; j < crun; j++)
{
dotProduct += mat[j * src.Size + i] * src[j];
}
if (add)
{
dst[i] += dotProduct;
}
else
{
dst[i] = dotProduct;
}
}
}
}
}
public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices) => SseUtils.ZeroMatrixItems(dst, ccol, cfltRow, indices);
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues,
int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) => SseUtils.MatTimesSrc(tran, add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun);
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) => SseUtils.MatTimesSrc(tran, add, mat, src, dst, crun);
public static void MatrixTimesSource(bool transpose, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride)
{
Contracts.Assert(matrix.Size == destination.Size * source.Size);
Contracts.Assert(stride >= 0);
if (Avx.IsSupported)
{
if (!transpose)
{
Contracts.Assert(stride <= destination.Size);
AvxIntrinsics.MatMul(matrix, source, destination, stride, source.Size);
}
else
{
Contracts.Assert(stride <= source.Size);
AvxIntrinsics.MatMulTran(matrix, source, destination, destination.Size, stride);
}
}
else if (Sse.IsSupported)
{
if (!transpose)
{
Contracts.Assert(stride <= destination.Size);
SseIntrinsics.MatMul(matrix, source, destination, stride, source.Size);
}
else
{
Contracts.Assert(stride <= source.Size);
SseIntrinsics.MatMulTran(matrix, source, destination, destination.Size, stride);
}
}
else
{
if (!transpose)
{
Contracts.Assert(stride <= destination.Size);
for (int i = 0; i < stride; i++)
{
float dotProduct = 0;
for (int j = 0; j < source.Size; j++)
{
dotProduct += matrix[i * source.Size + j] * source[j];
}
destination[i] = dotProduct;
}
}
else
{
Contracts.Assert(stride <= source.Size);
for (int i = 0; i < destination.Size; i++)
{
float dotProduct = 0;
for (int j = 0; j < stride; j++)
{
dotProduct += matrix[j * source.Size + i] * source[j];
}
destination[i] = dotProduct;
}
}
}
}
private static bool Compat(AlignedArray a)
{
Contracts.AssertValue(a);
Contracts.Assert(a.Size > 0);
return(a.CbAlign == Vector128Alignment);
}
public static void MatTimesSrc(AlignedArray mat, ReadOnlySpan <int> rgposSrc, AlignedArray srcValues,
int posMin, int iposMin, int iposLim, AlignedArray dst, int crun)
{
Contracts.Assert(Compat(mat));
Contracts.Assert(Compat(srcValues));
Contracts.Assert(Compat(dst));
Contracts.Assert(0 <= iposMin && iposMin <= iposLim && iposLim <= rgposSrc.Length);
Contracts.Assert(mat.Size == dst.Size * srcValues.Size);
if (iposMin >= iposLim)
{
dst.ZeroItems();
return;
}
Contracts.AssertNonEmpty(rgposSrc);
unsafe
{
fixed(float *pdst = &dst.Items[0])
fixed(float *pmat = &mat.Items[0])
fixed(float *psrc = &srcValues.Items[0])
fixed(int *ppossrc = &rgposSrc[0])
{
Contracts.Assert(0 <= crun && crun <= dst.Size);
Thunk.MatMulP(Ptr(mat, pmat), ppossrc, Ptr(srcValues, psrc), posMin, iposMin, iposLim, Ptr(dst, pdst), crun, srcValues.Size);
}
}
}
public static void MatrixTimesSource(AlignedArray matrix, ReadOnlySpan <int> rgposSrc, AlignedArray sourceValues,
int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride);
public static void MatTimesSrc(bool tran, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun)
{
Contracts.Assert(Compat(mat));
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(mat.Size == dst.Size * src.Size);
unsafe
{
fixed(float *pmat = &mat.Items[0])
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
{
if (!tran)
{
Contracts.Assert(0 <= crun && crun <= dst.Size);
Thunk.MatMul(Ptr(mat, pmat), Ptr(src, psrc), Ptr(dst, pdst), crun, src.Size);
}
else
{
Contracts.Assert(0 <= crun && crun <= src.Size);
Thunk.MatMulTran(Ptr(mat, pmat), Ptr(src, psrc), Ptr(dst, pdst), dst.Size, crun);
}
}
}
}