/// <summary>
/// Allocate an aligned vector with the given alignment (in bytes).
/// The alignment must be a power of two and at least sizeof(Float).
/// </summary>
public CpuAlignedVector(int size, int cbAlign)
{
Contracts.Assert(0 < size);
// cbAlign should be a power of two.
Contracts.Assert(sizeof(Float) <= cbAlign);
Contracts.Assert((cbAlign & (cbAlign - 1)) == 0);
int cfltAlign = cbAlign / sizeof(Float);
int cflt = RoundUp(size, cfltAlign);
_items = new AlignedArray(cflt, cbAlign);
_size = size;
AssertValid();
}
private static unsafe void ZeroItemsU(AlignedArray destination, int c, int[] indices, int cindices)
{
fixed(float *pdst = &destination.Items[0])
fixed(int *pidx = &indices[0])
{
for (int i = 0; i < cindices; ++i)
{
int index = pidx[i];
Contracts.Assert(index >= 0);
Contracts.Assert(index < c);
pdst[index] = 0;
}
}
}
/// <summary>
/// Sets the matrix items to zero.
/// </summary>
/// <param name="destination">The destination values.</param>
/// <param name="ccol">The stride column.</param>
/// <param name="cfltRow">The row to use.</param>
/// <param name="indices">The indicies.</param>
public static void ZeroMatrixItems(AlignedArray destination, int ccol, int cfltRow, int[] indices)
{
Contracts.Assert(ccol > 0);
Contracts.Assert(ccol <= cfltRow);
if (ccol == cfltRow)
{
ZeroItemsU(destination, destination.Size, indices, indices.Length);
}
else
{
ZeroMatrixItemsCore(destination, destination.Size, ccol, cfltRow, indices, indices.Length);
}
}
/// <summary>
/// Multiplies a matrix times a source.
/// </summary>
/// <param name="matrix">The input matrix.</param>
/// <param name="rgposSrc">The source positions.</param>
/// <param name="sourceValues">The source values.</param>
/// <param name="posMin">The minimum position.</param>
/// <param name="iposMin">The minimum position index.</param>
/// <param name="iposLimit">The position limit.</param>
/// <param name="destination">The destination matrix.</param>
/// <param name="stride">The column stride.</param>
public static void MatrixTimesSource(AlignedArray matrix, ReadOnlySpan <int> rgposSrc, AlignedArray sourceValues,
int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride)
{
Contracts.Assert(iposMin >= 0);
Contracts.Assert(iposMin <= iposLimit);
Contracts.Assert(iposLimit <= rgposSrc.Length);
Contracts.Assert(matrix.Size == destination.Size * sourceValues.Size);
if (iposMin >= iposLimit)
{
destination.ZeroItems();
return;
}
Contracts.AssertNonEmpty(rgposSrc);
Contracts.Assert(stride >= 0);
if (Avx.IsSupported)
{
Contracts.Assert(stride <= destination.Size);
AvxIntrinsics.MatMulP(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size);
}
else if (Sse.IsSupported)
{
Contracts.Assert(stride <= destination.Size);
SseIntrinsics.MatMulP(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size);
}
else
{
Contracts.Assert(stride <= destination.Size);
for (int i = 0; i < stride; i++)
{
float dotProduct = 0;
for (int j = iposMin; j < iposLimit; j++)
{
int col = rgposSrc[j] - posMin;
dotProduct += matrix[i * sourceValues.Size + col] * sourceValues[col];
}
destination[i] = dotProduct;
}
}
}
public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices)
{
Contracts.Assert(0 < ccol && ccol <= cfltRow);
unsafe
{
fixed(float *pdst = &dst.Items[0])
fixed(int *pi = &indices[0])
{
if (ccol == cfltRow)
{
Thunk.ZeroItemsU(Ptr(dst, pdst), dst.Size, pi, indices.Length);
}
else
{
Thunk.ZeroMatrixItemsCore(Ptr(dst, pdst), dst.Size, ccol, cfltRow, pi, indices.Length);
}
}
}
}
/// <summary>
/// Allocate an aligned matrix with the given alignment (in bytes).
/// </summary>
protected CpuAlignedMatrixBase(int runLen, int runCnt, int cbAlign)
{
Contracts.Assert(0 < runLen);
Contracts.Assert(0 < runCnt);
// cbAlign should be a power of two.
Contracts.Assert(sizeof(Float) <= cbAlign);
Contracts.Assert((cbAlign & (cbAlign - 1)) == 0);
RunLen = runLen;
RunCnt = runCnt;
FloatAlign = cbAlign / sizeof(Float);
Shift = GeneralUtils.CbitLowZero((uint)FloatAlign);
Mask = FloatAlign - 1;
RunLenPhy = RoundUp(runLen, FloatAlign);
RunCntPhy = RoundUp(runCnt, FloatAlign);
Items = new AlignedArray(RunLenPhy * RunCntPhy, cbAlign);
AssertValid();
}
/// <summary>
/// Multiplies a matrix times a source.
/// </summary>
/// <param name="transpose"><see langword="true"/> to transpose the matrix; otherwise <see langword="false"/>.</param>
/// <param name="matrix">The input matrix.</param>
/// <param name="source">The source matrix.</param>
/// <param name="destination">The destination matrix.</param>
/// <param name="stride">The column stride.</param>
public static void MatrixTimesSource(bool transpose, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride)
{
Contracts.Assert(matrix.Size == destination.Size * source.Size);
Contracts.Assert(stride >= 0);
if (Avx.IsSupported)
{
if (!transpose)
{
Contracts.Assert(stride <= destination.Size);
AvxIntrinsics.MatMul(matrix, source, destination, stride, source.Size);
}
else
{
Contracts.Assert(stride <= source.Size);
AvxIntrinsics.MatMulTran(matrix, source, destination, destination.Size, stride);
}
}
else if (Sse.IsSupported)
{
if (!transpose)
{
Contracts.Assert(stride <= destination.Size);
SseIntrinsics.MatMul(matrix, source, destination, stride, source.Size);
}
else
{
Contracts.Assert(stride <= source.Size);
SseIntrinsics.MatMulTran(matrix, source, destination, destination.Size, stride);
}
}
else
{
if (!transpose)
{
Contracts.Assert(stride <= destination.Size);
for (int i = 0; i < stride; i++)
{
float dotProduct = 0;
for (int j = 0; j < source.Size; j++)
{
dotProduct += matrix[i * source.Size + j] * source[j];
}
destination[i] = dotProduct;
}
}
else
{
Contracts.Assert(stride <= source.Size);
for (int i = 0; i < destination.Size; i++)
{
float dotProduct = 0;
for (int j = 0; j < stride; j++)
{
dotProduct += matrix[j * destination.Size + i] * source[j];
}
destination[i] = dotProduct;
}
}
}
}
public static void MatrixTimesSource(AlignedArray mat, ReadOnlySpan <int> rgposSrc, AlignedArray srcValues,
int posMin, int iposMin, int iposLim, AlignedArray dst, int crun)
{
Contracts.Assert(Compat(mat));
Contracts.Assert(Compat(srcValues));
Contracts.Assert(Compat(dst));
Contracts.Assert(0 <= iposMin && iposMin <= iposLim && iposLim <= rgposSrc.Length);
Contracts.Assert(mat.Size == dst.Size * srcValues.Size);
if (iposMin >= iposLim)
{
dst.ZeroItems();
return;
}
Contracts.AssertNonEmpty(rgposSrc);
unsafe
{
fixed(float *pdst = &dst.Items[0])
fixed(float *pmat = &mat.Items[0])
fixed(float *psrc = &srcValues.Items[0])
fixed(int *ppossrc = &rgposSrc[0])
{
Contracts.Assert(0 <= crun && crun <= dst.Size);
Thunk.MatMulP(Ptr(mat, pmat), ppossrc, Ptr(srcValues, psrc), posMin, iposMin, iposLim, Ptr(dst, pdst), crun, srcValues.Size);
}
}
}
public static void MatrixTimesSource(bool tran, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun)
{
Contracts.Assert(Compat(mat));
Contracts.Assert(Compat(src));
Contracts.Assert(Compat(dst));
Contracts.Assert(mat.Size == dst.Size * src.Size);
unsafe
{
fixed(float *pmat = &mat.Items[0])
fixed(float *psrc = &src.Items[0])
fixed(float *pdst = &dst.Items[0])
{
if (!tran)
{
Contracts.Assert(0 <= crun && crun <= dst.Size);
Thunk.MatMul(Ptr(mat, pmat), Ptr(src, psrc), Ptr(dst, pdst), crun, src.Size);
}
else
{
Contracts.Assert(0 <= crun && crun <= src.Size);
Thunk.MatMulTran(Ptr(mat, pmat), Ptr(src, psrc), Ptr(dst, pdst), dst.Size, crun);
}
}
}
}
private static bool Compat(AlignedArray a)
{
Contracts.AssertValue(a);
Contracts.Assert(a.Size > 0);
return(a.CbAlign == Vector128Alignment);
}
