/// <summary> /// Allocate an aligned vector with the given alignment (in bytes). /// The alignment must be a power of two and at least sizeof(Float). /// </summary> public CpuAlignedVector(int size, int cbAlign) { Contracts.Assert(0 < size); // cbAlign should be a power of two. Contracts.Assert(sizeof(Float) <= cbAlign); Contracts.Assert((cbAlign & (cbAlign - 1)) == 0); int cfltAlign = cbAlign / sizeof(Float); int cflt = RoundUp(size, cfltAlign); _items = new AlignedArray(cflt, cbAlign); _size = size; AssertValid(); }
private static unsafe void ZeroItemsU(AlignedArray dst, int c, int[] indices, int cindices) { fixed(float *pdst = &dst.Items[0]) fixed(int *pidx = &indices[0]) { for (int i = 0; i < cindices; ++i) { int index = pidx[i]; Contracts.Assert(index >= 0); Contracts.Assert(index < c); pdst[index] = 0; } } }
public static void ApplyTanh(AlignedArray src, AlignedArray dst, int c) { Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(src.Size == dst.Size); Contracts.Assert(0 < c && c <= dst.Size); unsafe { fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) Thunk.ApplyTanhX(Ptr(src, psrc), Ptr(dst, pdst), c); } }
public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices) { Contracts.Assert(ccol > 0); Contracts.Assert(ccol <= cfltRow); if (ccol == cfltRow) { ZeroItemsU(dst, dst.Size, indices, indices.Length); } else { ZeroMatrixItemsCore(dst, dst.Size, ccol, cfltRow, indices, indices.Length); } }
public static void AddScale(float a, AlignedArray src, AlignedArray dst, float momentum, AlignedArray delta) { Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(Compat(delta)); Contracts.Assert(src.Size == dst.Size); Contracts.Assert(src.Size == delta.Size); unsafe { fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) fixed(float *pdel = &delta.Items[0]) Thunk.AddScaleMomX(a, Ptr(src, psrc), Ptr(dst, pdst), momentum, Ptr(delta, pdel), dst.Size); } }
public static void ApplyAbsDerivative(AlignedArray input, AlignedArray output, AlignedArray grad, bool drop) { Contracts.Assert(Compat(input)); Contracts.Assert(Compat(output)); Contracts.Assert(Compat(grad)); Contracts.Assert(output.Size == input.Size); Contracts.Assert(output.Size == grad.Size); unsafe { fixed(float *px = &input.Items[0]) fixed(float *py = &output.Items[0]) fixed(float *pg = &grad.Items[0]) Thunk.ApplyAbsDerivativeX(Ptr(input, px), Ptr(output, py), Ptr(grad, pg), grad.Size, drop); } }
public static void AddXYTran(float a, AlignedArray x, AlignedArray y, AlignedArray mat, int crow, float decay) { Contracts.Assert(Compat(x)); Contracts.Assert(Compat(y)); Contracts.Assert(Compat(mat)); Contracts.Assert(0 < crow && crow <= x.Size); Contracts.Assert(x.Size * y.Size == mat.Size); Contracts.Assert(decay >= 0); unsafe { fixed(float *px = &x.Items[0]) fixed(float *py = &y.Items[0]) fixed(float *pmat = &mat.Items[0]) Thunk.AddXYTranX(a, Ptr(x, px), Ptr(y, py), Ptr(mat, pmat), crow, y.Size, decay); } }
public static void AddXYTran(float a, AlignedArray x, AlignedArray y, AlignedArray mat, float momentum, AlignedArray delta, int crow) { Contracts.Assert(Compat(x)); Contracts.Assert(Compat(y)); Contracts.Assert(Compat(mat)); Contracts.Assert(Compat(delta)); Contracts.Assert(0 < crow && crow <= x.Size); Contracts.Assert(x.Size * y.Size == mat.Size); Contracts.Assert(mat.Size == delta.Size); unsafe { fixed(float *px = &x.Items[0]) fixed(float *py = &y.Items[0]) fixed(float *pmat = &mat.Items[0]) fixed(float *pdel = &delta.Items[0]) Thunk.AddXYTranMomX(a, Ptr(x, px), Ptr(y, py), Ptr(mat, pmat), momentum, Ptr(delta, pdel), crow, y.Size); } }
public static void MatTimesSrc(AlignedArray mat, int[] rgposSrc, AlignedArray srcValues, int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) { Contracts.AssertValue(rgposSrc); Contracts.Assert(iposMin >= 0); Contracts.Assert(iposMin <= iposLim); Contracts.Assert(iposLim <= rgposSrc.Length); Contracts.Assert(mat.Size == dst.Size * srcValues.Size); if (iposMin >= iposLim) { dst.ZeroItems(); return; } Contracts.AssertNonEmpty(rgposSrc); Contracts.Assert(crun >= 0); if (Avx.IsSupported) { Contracts.Assert(crun <= dst.Size); AvxIntrinsics.MatMulP(mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size); } else if (Sse.IsSupported) { Contracts.Assert(crun <= dst.Size); SseIntrinsics.MatMulP(mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size); } else { Contracts.Assert(crun <= dst.Size); for (int i = 0; i < crun; i++) { float dotProduct = 0; for (int j = iposMin; j < iposLim; j++) { int col = rgposSrc[j] - posMin; dotProduct += mat[i * srcValues.Size + col] * srcValues[col]; } dst[i] = dotProduct; } } }
public static void MatrixTimesSource(AlignedArray matrix, ReadOnlySpan <int> rgposSrc, AlignedArray sourceValues, int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) { Contracts.AssertValue(rgposSrc); Contracts.Assert(iposMin >= 0); Contracts.Assert(iposMin <= iposLimit); Contracts.Assert(iposLimit <= rgposSrc.Length); Contracts.Assert(matrix.Size == destination.Size * sourceValues.Size); if (iposMin >= iposLimit) { destination.ZeroItems(); return; } Contracts.AssertNonEmpty(rgposSrc); Contracts.Assert(stride >= 0); if (Avx.IsSupported) { Contracts.Assert(stride <= destination.Size); AvxIntrinsics.MatMulP(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size); } else if (Sse.IsSupported) { Contracts.Assert(stride <= destination.Size); SseIntrinsics.MatMulP(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size); } else { Contracts.Assert(stride <= destination.Size); for (int i = 0; i < stride; i++) { float dotProduct = 0; for (int j = iposMin; j < iposLimit; j++) { int col = rgposSrc[j] - posMin; dotProduct += matrix[i * sourceValues.Size + col] * sourceValues[col]; } destination[i] = dotProduct; } } }
public static void AddScale(AlignedArray src, AlignedArray dst, AlignedArray accGrads, AlignedArray accUpdates, float decay, float cond) { Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(Compat(accGrads)); Contracts.Assert(Compat(accUpdates)); Contracts.Assert(src.Size == dst.Size); Contracts.Assert(src.Size == accGrads.Size); Contracts.Assert(src.Size == accUpdates.Size); unsafe { fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) fixed(float *pag = &accGrads.Items[0]) fixed(float *pau = &accUpdates.Items[0]) Thunk.AddScaleGradX(Ptr(src, psrc), Ptr(dst, pdst), Ptr(accGrads, pag), Ptr(accUpdates, pau), decay, cond, dst.Size); } }
public static void ScaleMaxNorm(bool tran, float maxNorm, AlignedArray mat, int crun, int runLenPhy) { // Called only with Avx alignment. Contracts.Assert(Compat(mat)); unsafe { fixed(float *pmat = &mat.Items[0]) { if (!tran) { Thunk.ScaleMaxNormX(maxNorm, Ptr(mat, pmat), crun, runLenPhy); } else { Thunk.ScaleMaxNormTranU(maxNorm, Ptr(mat, pmat), crun, runLenPhy); } } } }
public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices) { Contracts.Assert(0 < ccol && ccol <= cfltRow); unsafe { fixed(float *pdst = &dst.Items[0]) fixed(int *pi = &indices[0]) { if (ccol == cfltRow) { Thunk.ZeroItemsU(Ptr(dst, pdst), dst.Size, pi, indices.Length); } else { Thunk.ZeroMatrixItemsCore(Ptr(dst, pdst), dst.Size, ccol, cfltRow, pi, indices.Length); } } } }
/// <summary> /// Allocate an aligned matrix with the given alignment (in bytes). /// </summary> protected CpuAlignedMatrixBase(int runLen, int runCnt, int cbAlign) { Contracts.Assert(0 < runLen); Contracts.Assert(0 < runCnt); // cbAlign should be a power of two. Contracts.Assert(sizeof(Float) <= cbAlign); Contracts.Assert((cbAlign & (cbAlign - 1)) == 0); RunLen = runLen; RunCnt = runCnt; FloatAlign = cbAlign / sizeof(Float); Shift = GeneralUtils.CbitLowZero((uint)FloatAlign); Mask = FloatAlign - 1; RunLenPhy = RoundUp(runLen, FloatAlign); RunCntPhy = RoundUp(runCnt, FloatAlign); Items = new AlignedArray(RunLenPhy * RunCntPhy, cbAlign); AssertValid(); }
public static void MatTimesSrc(bool add, int[] mprowiv, int[] mprowcol, int[] mprowrun, int[] runs, float[] coefs, AlignedArray src, AlignedArray dst, int crow) { Contracts.AssertNonEmpty(mprowiv); Contracts.Assert(mprowiv.Length == crow); Contracts.AssertNonEmpty(mprowcol); Contracts.Assert(mprowcol.Length == crow); Contracts.Assert(mprowrun == null || mprowrun.Length == crow); Contracts.AssertNonEmpty(runs); Contracts.AssertNonEmpty(coefs); Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(0 < crow && crow <= dst.Size); unsafe { fixed(int *pmprowiv = &mprowiv[0]) fixed(int *pmprowcol = &mprowcol[0]) fixed(int *pruns = &runs[0]) fixed(float *pcoefs = &coefs[0]) fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) { if (mprowrun == null) { Thunk.MatMulCX(add, pmprowiv, pmprowcol, pruns, pcoefs, Ptr(src, psrc), Ptr(dst, pdst), crow); } else { fixed(int *pmprowrun = &mprowrun[0]) { Thunk.MatMulDX(add, pmprowiv, pmprowcol, pmprowrun, pruns, pcoefs, Ptr(src, psrc), Ptr(dst, pdst), crow); } } } } }
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues, int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) { Contracts.Assert(Compat(mat)); Contracts.Assert(Compat(srcValues)); Contracts.Assert(Compat(dst)); Contracts.AssertValue(rgposSrc); Contracts.Assert(0 <= iposMin && iposMin <= iposLim && iposLim <= rgposSrc.Length); Contracts.Assert(mat.Size == dst.Size * srcValues.Size); if (iposMin >= iposLim) { if (!add) { dst.ZeroItems(); } return; } Contracts.AssertNonEmpty(rgposSrc); unsafe { fixed(float *pdst = &dst.Items[0]) fixed(float *pmat = &mat.Items[0]) fixed(float *psrc = &srcValues.Items[0]) fixed(int *ppossrc = &rgposSrc[0]) { if (!tran) { Contracts.Assert(0 <= crun && crun <= dst.Size); Thunk.MatMulPX(add, Ptr(mat, pmat), ppossrc, Ptr(srcValues, psrc), posMin, iposMin, iposLim, Ptr(dst, pdst), crun, srcValues.Size); } else { Contracts.Assert(0 <= crun && crun <= srcValues.Size); Thunk.MatMulTranPX(add, Ptr(mat, pmat), ppossrc, Ptr(srcValues, psrc), posMin, iposMin, iposLim, Ptr(dst, pdst), dst.Size); } } } }
public static void MatTranTimesSrc(bool add, int[] mpcoliv, int[] mpcolrow, int[] mpcolrun, int[] runs, float[] coefs, AlignedArray src, AlignedArray dst, int ccol) { Contracts.AssertNonEmpty(mpcoliv); Contracts.Assert(mpcoliv.Length == ccol); Contracts.AssertNonEmpty(mpcolrow); Contracts.Assert(mpcolrow.Length == ccol); Contracts.AssertNonEmpty(runs); Contracts.AssertNonEmpty(coefs); Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(mpcolrun == null || mpcolrun.Length == ccol); Contracts.Assert(0 < ccol && ccol <= src.Size); unsafe { fixed(int *pmpcoliv = &mpcoliv[0]) fixed(int *pmpcolrow = &mpcolrow[0]) fixed(int *pruns = &runs[0]) fixed(float *pcoefs = &coefs[0]) fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) { if (mpcolrun == null) { Thunk.MatMulTranCX(add, pmpcoliv, pmpcolrow, pruns, pcoefs, Ptr(src, psrc), Ptr(dst, pdst), dst.Size, ccol); } else { fixed(int *pmpcolrun = &mpcolrun[0]) { Thunk.MatMulTranDX(add, pmpcoliv, pmpcolrow, pmpcolrun, pruns, pcoefs, Ptr(src, psrc), Ptr(dst, pdst), dst.Size, ccol); } } } } }
public static void AddXYTran(AlignedArray x, AlignedArray y, AlignedArray mat, AlignedArray accGrads, AlignedArray accUpdates, float decay, float cond, int crow) { Contracts.Assert(Compat(x)); Contracts.Assert(Compat(y)); Contracts.Assert(Compat(mat)); Contracts.Assert(Compat(accGrads)); Contracts.Assert(Compat(accUpdates)); Contracts.Assert(0 < crow && crow <= x.Size); Contracts.Assert(x.Size * y.Size == mat.Size); Contracts.Assert(mat.Size == accGrads.Size); Contracts.Assert(mat.Size == accUpdates.Size); unsafe { fixed(float *px = &x.Items[0]) fixed(float *py = &y.Items[0]) fixed(float *pmat = &mat.Items[0]) fixed(float *pag = &accGrads.Items[0]) fixed(float *pau = &accUpdates.Items[0]) Thunk.AddXYTranGradX(Ptr(x, px), Ptr(y, py), Ptr(mat, pmat), Ptr(accGrads, pag), Ptr(accUpdates, pau), decay, cond, crow, y.Size); } }
public static void MeanBackOfSrc(bool add, int[] mpcolrow, int[] mpcolindices, int[] indices, AlignedArray src, AlignedArray dst, int ccol) { Contracts.AssertNonEmpty(mpcolrow); Contracts.Assert(mpcolrow.Length == ccol); Contracts.Assert(mpcolindices == null || mpcolindices.Length == ccol); Contracts.AssertNonEmpty(indices); Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(0 < ccol && ccol <= src.Size); unsafe { fixed(int *pmpcolrow = &mpcolrow[0]) fixed(int *pmpcolindices = mpcolindices) fixed(int *pindices = &indices[0]) fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) { // REVIEW: Implement using AVX Thunk.MeanBackU(add, pmpcolrow, pmpcolindices, pindices, Ptr(src, psrc), Ptr(dst, pdst), dst.Size, ccol); } } }
public static void MaxOfSrc(bool add, int[] mprowcol, int[] mprowindices, int[] indices, AlignedArray src, AlignedArray dst, int crow) { Contracts.AssertNonEmpty(mprowcol); Contracts.Assert(mprowcol.Length == crow); Contracts.Assert(mprowindices == null || mprowindices.Length == crow); Contracts.AssertNonEmpty(indices); Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(0 < crow && crow <= dst.Size); unsafe { fixed(int *pmprowcol = &mprowcol[0]) fixed(int *pmprowindices = mprowindices) fixed(int *pindices = &indices[0]) fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) { // REVIEW: Implement using AVX Thunk.MaxU(add, pmprowcol, pmprowindices, pindices, Ptr(src, psrc), Ptr(dst, pdst), crow); } } }
public static void MatrixTimesSource(bool transpose, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(transpose, matrix, source, destination, stride);
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) { Contracts.Assert(mat.Size == dst.Size * src.Size); Contracts.Assert(crun >= 0); if (Avx.IsSupported) { if (!tran) { Contracts.Assert(crun <= dst.Size); AvxIntrinsics.MatMulX(add, mat, src, dst, crun, src.Size); } else { Contracts.Assert(crun <= src.Size); AvxIntrinsics.MatMulTranX(add, mat, src, dst, dst.Size, crun); } } else if (Sse.IsSupported) { if (!tran) { Contracts.Assert(crun <= dst.Size); SseIntrinsics.MatMulA(add, mat, src, dst, crun, src.Size); } else { Contracts.Assert(crun <= src.Size); SseIntrinsics.MatMulTranA(add, mat, src, dst, dst.Size, crun); } } else { if (!tran) { Contracts.Assert(crun <= dst.Size); for (int i = 0; i < crun; i++) { float dotProduct = 0; for (int j = 0; j < src.Size; j++) { dotProduct += mat[i * src.Size + j] * src[j]; } if (add) { dst[i] += dotProduct; } else { dst[i] = dotProduct; } } } else { Contracts.Assert(crun <= src.Size); for (int i = 0; i < dst.Size; i++) { float dotProduct = 0; for (int j = 0; j < crun; j++) { dotProduct += mat[j * src.Size + i] * src[j]; } if (add) { dst[i] += dotProduct; } else { dst[i] = dotProduct; } } } } }
public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices) => SseUtils.ZeroMatrixItems(dst, ccol, cfltRow, indices);
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues, int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) => SseUtils.MatTimesSrc(tran, add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun);
public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) => SseUtils.MatTimesSrc(tran, add, mat, src, dst, crun);
public static void MatrixTimesSource(bool transpose, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride) { Contracts.Assert(matrix.Size == destination.Size * source.Size); Contracts.Assert(stride >= 0); if (Avx.IsSupported) { if (!transpose) { Contracts.Assert(stride <= destination.Size); AvxIntrinsics.MatMul(matrix, source, destination, stride, source.Size); } else { Contracts.Assert(stride <= source.Size); AvxIntrinsics.MatMulTran(matrix, source, destination, destination.Size, stride); } } else if (Sse.IsSupported) { if (!transpose) { Contracts.Assert(stride <= destination.Size); SseIntrinsics.MatMul(matrix, source, destination, stride, source.Size); } else { Contracts.Assert(stride <= source.Size); SseIntrinsics.MatMulTran(matrix, source, destination, destination.Size, stride); } } else { if (!transpose) { Contracts.Assert(stride <= destination.Size); for (int i = 0; i < stride; i++) { float dotProduct = 0; for (int j = 0; j < source.Size; j++) { dotProduct += matrix[i * source.Size + j] * source[j]; } destination[i] = dotProduct; } } else { Contracts.Assert(stride <= source.Size); for (int i = 0; i < destination.Size; i++) { float dotProduct = 0; for (int j = 0; j < stride; j++) { dotProduct += matrix[j * source.Size + i] * source[j]; } destination[i] = dotProduct; } } } }
private static bool Compat(AlignedArray a) { Contracts.AssertValue(a); Contracts.Assert(a.Size > 0); return(a.CbAlign == Vector128Alignment); }
public static void MatTimesSrc(AlignedArray mat, ReadOnlySpan <int> rgposSrc, AlignedArray srcValues, int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) { Contracts.Assert(Compat(mat)); Contracts.Assert(Compat(srcValues)); Contracts.Assert(Compat(dst)); Contracts.Assert(0 <= iposMin && iposMin <= iposLim && iposLim <= rgposSrc.Length); Contracts.Assert(mat.Size == dst.Size * srcValues.Size); if (iposMin >= iposLim) { dst.ZeroItems(); return; } Contracts.AssertNonEmpty(rgposSrc); unsafe { fixed(float *pdst = &dst.Items[0]) fixed(float *pmat = &mat.Items[0]) fixed(float *psrc = &srcValues.Items[0]) fixed(int *ppossrc = &rgposSrc[0]) { Contracts.Assert(0 <= crun && crun <= dst.Size); Thunk.MatMulP(Ptr(mat, pmat), ppossrc, Ptr(srcValues, psrc), posMin, iposMin, iposLim, Ptr(dst, pdst), crun, srcValues.Size); } } }
public static void MatrixTimesSource(AlignedArray matrix, ReadOnlySpan <int> rgposSrc, AlignedArray sourceValues, int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride);
public static void MatTimesSrc(bool tran, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) { Contracts.Assert(Compat(mat)); Contracts.Assert(Compat(src)); Contracts.Assert(Compat(dst)); Contracts.Assert(mat.Size == dst.Size * src.Size); unsafe { fixed(float *pmat = &mat.Items[0]) fixed(float *psrc = &src.Items[0]) fixed(float *pdst = &dst.Items[0]) { if (!tran) { Contracts.Assert(0 <= crun && crun <= dst.Size); Thunk.MatMul(Ptr(mat, pmat), Ptr(src, psrc), Ptr(dst, pdst), crun, src.Size); } else { Contracts.Assert(0 <= crun && crun <= src.Size); Thunk.MatMulTran(Ptr(mat, pmat), Ptr(src, psrc), Ptr(dst, pdst), dst.Size, crun); } } } }