public Tensor Gather(Tensor result, Tensor src, int dim, Tensor indices)
{
if (result != null && result.DimensionCount != src.DimensionCount)
{
throw new InvalidOperationException("result and src must have same number of dimensions");
}
if (result != null && dim < 0 && dim >= result.DimensionCount)
{
throw new ArgumentOutOfRangeException("dim");
}
if (indices.DimensionCount != src.DimensionCount)
{
throw new InvalidOperationException("src and indices must have same number of dimensions");
}
if (result != null && !result.IsSameSizeAs(indices))
{
throw new InvalidOperationException("result and indices must be the same size");
}
if (result != null && !TensorResultBuilder.ArrayEqualExcept(src.Sizes, result.Sizes, dim))
{
throw new InvalidOperationException("result and src must be the same size except in dimension dim");
}
var writeTarget = TensorResultBuilder.GetWriteTarget(result, indices.Allocator, src.ElementType, false, indices.Sizes);
NativeWrapper.InvokeTypeMatch(gather_func, writeTarget, src, dim, indices);
return(writeTarget);
}
public Tensor ScatterFill(Tensor result, float value, int dim, Tensor indices)
{
if (result == null)
{
throw new ArgumentNullException("result");
}
if (dim < 0 && dim >= result.DimensionCount)
{
throw new ArgumentOutOfRangeException("dim");
}
if (indices.DimensionCount != result.DimensionCount)
{
throw new InvalidOperationException("result and indices must have same number of dimensions");
}
if (!TensorResultBuilder.ArrayEqualExcept(indices.Sizes, result.Sizes, dim))
{
throw new InvalidOperationException("result and indices must be the same size except in dimension dim");
}
var writeTarget = result;
NativeWrapper.InvokeTypeMatch(scatterFill_func, writeTarget, value, dim, indices);
return(writeTarget);
}
public Tensor LayerNormGrad(Tensor result, Tensor gradGamma_, Tensor gradBeta_, Tensor adj_, Tensor y_, Tensor x_, Tensor gamma_, Tensor beta_, float eps)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, adj_, false, adj_.Sizes);
NativeWrapper.InvokeTypeMatch(layerNormGrad_func, writeTarget, gradGamma_, gradBeta_, adj_, y_, x_, gamma_, beta_, (int)adj_.Sizes[0], (int)adj_.Sizes[1], eps);
return(writeTarget);
}
public static Tensor Mul_M_M(TSCudaContext context, Tensor result, Tensor lhs, Tensor rhs)
{
if (lhs.ElementType != rhs.ElementType || (result != null && result.ElementType != lhs.ElementType))
{
throw new InvalidOperationException("All tensors must have the same element type");
}
CudaHelpers.ThrowIfDifferentDevices(result, lhs, rhs);
if (result != null && !(result.Storage is CudaStorage))
{
throw new ArgumentException("result must be a CUDA tensor", "result");
}
if (!(lhs.Storage is CudaStorage))
{
throw new ArgumentException("lhs must be a CUDA tensor", "lhs");
}
if (!(rhs.Storage is CudaStorage))
{
throw new ArgumentException("rhs must be a CUDA tensor", "rhs");
}
var writeTarget = TensorResultBuilder.GetWriteTarget(result, lhs, false, lhs.Sizes[0], rhs.Sizes[1]);
Gemm(context, 1, lhs, rhs, 0, writeTarget);
return(writeTarget);
}
public static Tensor Invoke(CudaCode kernels, string funcName, Tensor result, Tensor src)
{
try
{
TSCudaContext context = CudaHelpers.TSContextForTensor(src);
CudaContext cudaContext = context.CudaContextForTensor(src);
cudaContext.SetCurrent();
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, src, false, src.Sizes);
long elementCount = writeTarget.ElementCount();
byte[] ptx = kernels.GetPtx(context.Compiler);
if (result == src)
{
ApplyOpInvoke.Invoke(context, cudaContext, ptx, "t1_" + funcName, writeTarget, elementCount);
}
else
{
ApplyOpInvoke.Invoke(context, cudaContext, ptx, "t2_" + funcName, writeTarget, src, elementCount);
}
return(writeTarget);
}
catch (Exception e)
{
Logger.WriteLine($"Error = '{e.Message}', Call stack = '{e.StackTrace}'");
throw;
}
}
/// <summary>
/// Muls the m m.
/// </summary>
/// <param name="result">The result.</param>
/// <param name="lhs">The LHS.</param>
/// <param name="rhs">The RHS.</param>
/// <returns>Tensor.</returns>
/// <exception cref="InvalidOperationException">All tensors must have the same element type</exception>
/// <exception cref="ArgumentException">
/// result must be a CPU tensor - result
/// or
/// lhs must be a CPU tensor - lhs
/// or
/// rhs must be a CPU tensor - rhs
/// </exception>
public static NDArray Mul_M_M(NDArray result, NDArray lhs, NDArray rhs)
{
if (lhs.ElementType != rhs.ElementType || (result != null && result.ElementType != lhs.ElementType))
{
throw new InvalidOperationException("All tensors must have the same element type");
}
if (result != null && !(result.Storage is CpuStorage))
{
throw new ArgumentException("result must be a CPU tensor", "result");
}
if (!(lhs.Storage is CpuStorage))
{
throw new ArgumentException("lhs must be a CPU tensor", "lhs");
}
if (!(rhs.Storage is CpuStorage))
{
throw new ArgumentException("rhs must be a CPU tensor", "rhs");
}
var writeTarget = TensorResultBuilder.GetWriteTarget(result, lhs, false, lhs.Shape[0], rhs.Shape[1]);
Gemm(1, lhs, rhs, 0, writeTarget);
return(writeTarget);
}
public Tensor SumAll(Tensor result, Tensor src)
{
var writeTarget = TensorResultBuilder.GetWriteTarget(result, src, false, 1);
NativeWrapper.InvokeTypeMatch(sumall_func, writeTarget, src);
return(writeTarget);
}
public static Tensor Mul_M_M(Tensor result, Tensor lhs, Tensor rhs)
{
// ReSharper disable once ArrangeRedundantParentheses
if (lhs.ElementType != rhs.ElementType || (result != null && result.ElementType != lhs.ElementType))
{
throw new InvalidOperationException("All tensors must have the same element type");
}
if (result != null && !(result.Storage is CpuStorage))
{
throw new ArgumentException("result must be a CPU tensor", nameof(result));
}
if (!(lhs.Storage is CpuStorage))
{
throw new ArgumentException("lhs must be a CPU tensor", nameof(lhs));
}
if (!(rhs.Storage is CpuStorage))
{
throw new ArgumentException("rhs must be a CPU tensor", nameof(rhs));
}
var writeTarget = TensorResultBuilder.GetWriteTarget(result, lhs, false, lhs.Sizes[0], rhs.Sizes[1]);
Gemm(1, lhs, rhs, 0, writeTarget);
return(writeTarget);
}
public void AddLayerNormGrad(Tensor result1, Tensor result2, Tensor gradGamma_, Tensor gradBeta_, Tensor adj_, Tensor y_, Tensor x1_, Tensor x2_, Tensor gamma_, Tensor beta_, float eps)
{
Tensor writeTarget1 = TensorResultBuilder.GetWriteTarget(result1, adj_, false, adj_.Sizes);
Tensor writeTarget2 = TensorResultBuilder.GetWriteTarget(result2, adj_, false, adj_.Sizes);
NativeWrapper.InvokeTypeMatch(addlayerNormGrad_func, writeTarget1, writeTarget2, gradGamma_, gradBeta_, adj_, y_, x1_, x2_, gamma_, beta_, (int)adj_.Sizes[0], (int)adj_.Sizes[1], eps);
}
public Tensor NormAll(Tensor result, Tensor src, float value)
{
var writeTarget = TensorResultBuilder.GetWriteTarget(result, src, false, 1);
NativeWrapper.InvokeTypeMatch(normall_func, writeTarget, src, value);
return(writeTarget);
}
public Tensor LayerNorm(Tensor result, Tensor src, Tensor gamma_, Tensor beta_, float eps)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, src, true, src.Sizes);
TensorApplyCPU.LayerNorm(writeTarget, src, gamma_, beta_, eps, (int)src.Sizes[0], (int)src.Sizes[1]);
return(writeTarget);
}
public Tensor AddLayerNorm(Tensor result, Tensor src1, Tensor src2, Tensor gamma_, Tensor beta_, float eps)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, src1, false, src1.Sizes);
NativeWrapper.InvokeTypeMatch(addlayerNorm_func, writeTarget, src1, src2, gamma_, beta_, eps, (int)src1.Sizes[0], (int)src1.Sizes[1]);
return(writeTarget);
}
public NDArray Scatter(NDArray result, NDArray src, int dim, NDArray indices)
{
if (result == null)
{
throw new ArgumentNullException("result");
}
if (result.DimensionCount != src.DimensionCount)
{
throw new InvalidOperationException("result and src must have same number of dimensions");
}
if (dim < 0 && dim >= result.DimensionCount)
{
throw new ArgumentOutOfRangeException("dim");
}
if (indices.DimensionCount != src.DimensionCount)
{
throw new InvalidOperationException("src and indices must have same number of dimensions");
}
if (!src.IsSameSizeAs(indices))
{
throw new InvalidOperationException("src and indices must be the same size");
}
if (!TensorResultBuilder.ArrayEqualExcept(src.Shape, result.Shape, dim))
{
throw new InvalidOperationException("result and src must be the same size except in dimension dim");
}
var writeTarget = result;
NativeWrapper.InvokeTypeMatch(scatter_func, writeTarget, src, dim, indices);
return(writeTarget);
}
public Tensor AddMulV(Tensor result, Tensor x, Tensor y, float z)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, x, false, x.Sizes);
TensorApplyCPU.AddMulV(writeTarget, x, y, z);
return(writeTarget);
}
public Tensor Scatter(Tensor result, Tensor src, int dim, Tensor indices)
{
TSCudaContext context = CudaHelpers.TSContextForTensor(src);
CudaContext cudaContext = context.CudaContextForTensor(src);
if (result == null)
{
throw new ArgumentNullException("result");
}
if (result.DimensionCount != src.DimensionCount)
{
throw new InvalidOperationException("result and src must have same number of dimensions");
}
if (dim < 0 && dim >= result.DimensionCount)
{
throw new ArgumentOutOfRangeException("dim");
}
if (indices.DimensionCount != src.DimensionCount)
{
throw new InvalidOperationException("src and indices must have same number of dimensions");
}
if (!src.IsSameSizeAs(indices))
{
throw new InvalidOperationException("src and indices must be the same size");
}
if (!TensorResultBuilder.ArrayEqualExcept(src.Sizes, result.Sizes, dim))
{
throw new InvalidOperationException("result and src must be the same size except in dimension dim");
}
Tensor writeTarget = result;
long nElement = indices.ElementCount();
dim3 block = ApplyUtils.GetApplyBlock();
dim3 grid = ApplyUtils.GetApplyGrid(context.DeviceInfoForContext(cudaContext), nElement);
if (ApplyUtils.CanUse32BitIndexMath(writeTarget) &&
ApplyUtils.CanUse32BitIndexMath(src) &&
ApplyUtils.CanUse32BitIndexMath(indices))
{
int dims = indices.DimensionCount <= 3 ? indices.DimensionCount : -1;
string kernelName = MakeKernelName(ScatterBaseName, true, dims);
Invoke(context, cudaContext, kernelName, grid, block, 0, CUstream.NullStream, true,
writeTarget, src, indices, dim, (int)nElement);
}
else
{
string kernelName = MakeKernelName(ScatterBaseName, false, -1);
Invoke(context, cudaContext, kernelName, grid, block, 0, CUstream.NullStream, false,
writeTarget, src, indices, dim, nElement);
}
return(writeTarget);
}
public Tensor Pow(Tensor result, Tensor src, float value)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, src, false, src.Sizes);
TensorApplyCPU.Pow(writeTarget, src, value);
return(writeTarget);
}
public Tensor Mul(Tensor result, Tensor lhs, Tensor rhs)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, lhs, false, lhs.Sizes);
TensorApplyCPU.Mul(writeTarget, lhs, rhs);
return(writeTarget);
}
public Tensor ReluD(Tensor result, Tensor w, Tensor g)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, w, false, w.Sizes);
TensorApplyCPU.ReluD(result, w, g);
return(writeTarget);
}
public Tensor AddTanhD(Tensor result, Tensor srcX, Tensor srcY, Tensor srcZ)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, srcX, false, srcX.Sizes);
TensorApplyCPU.AddTanhD(writeTarget, srcX, srcY, srcZ);
return(writeTarget);
}
public Tensor MulMulAdd(Tensor result, Tensor srcX, Tensor srcY, Tensor srcZ, Tensor srcW)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, srcX, false, srcX.Sizes);
TensorApplyCPU.MulMulAdd(writeTarget, srcX, srcY, srcZ, srcW);
return(writeTarget);
}
public Tensor SigmoidD(Tensor result, Tensor resW, Tensor resG)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, resW, false, resW.Sizes);
TensorApplyCPU.SigmoidD(writeTarget, resW, resG);
return(writeTarget);
}
public Tensor Sigmoid(Tensor result, Tensor src)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, src, false, src.Sizes);
TensorApplyCPU.Sigmoid(writeTarget, src);
return(writeTarget);
}
public Tensor Sub(Tensor result, float lhs, Tensor rhs)
{
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, rhs, false, rhs.Sizes);
TensorApplyCPU.RSub(writeTarget, lhs, rhs);
return(writeTarget);
}
public void AddLayerNormGrad(Tensor out1Grad, Tensor out2Grad, Tensor alphaGrad, Tensor betaGrad, Tensor inGrad, Tensor y, Tensor x1, Tensor x2, Tensor alpha, Tensor beta, float eps = 1e-9f)
{
TSCudaContext context = CudaHelpers.TSContextForTensor(inGrad);
Tensor writeTarget1 = TensorResultBuilder.GetWriteTarget(out1Grad, inGrad, false, inGrad.Sizes);
Tensor writeTarget2 = TensorResultBuilder.GetWriteTarget(out2Grad, inGrad, false, inGrad.Sizes);
AddLayerNormGrad(context, writeTarget1, writeTarget2, alphaGrad, betaGrad, inGrad, y, x1, x2, alpha, beta, eps);
}
public Tensor Addmm(Tensor result, float beta, Tensor src, float alpha, Tensor m1, Tensor m2)
{
//Console.WriteLine($"src0 = {src.Sizes[0]}, src1 = {src.Sizes[1]}, m1_0 = {m1.Sizes[0]}, m1_1 = {m1.Sizes[1]}, m2_0 = {m2.Sizes[0]}, m2_1 = {m2.Sizes[1]}");
// ReSharper disable once ArrangeRedundantParentheses
if (src.ElementType != m1.ElementType || src.ElementType != m2.ElementType || (result != null && result.ElementType != src.ElementType))
{
throw new InvalidOperationException("All tensors must have the same element type");
}
if (result != null && !(result.Storage is CpuStorage))
{
throw new ArgumentException("result must be a CPU tensor", nameof(result));
}
if (!(m1.Storage is CpuStorage))
{
throw new ArgumentException("m1 must be a CPU tensor", nameof(m1));
}
if (!(m2.Storage is CpuStorage))
{
throw new ArgumentException("m2 must be a CPU tensor", nameof(m2));
}
if (src.DimensionCount != 2)
{
throw new ArgumentException("src must be a matrix", nameof(src));
}
if (m1.DimensionCount != 2)
{
throw new ArgumentException("m1 must be a matrix", nameof(m1));
}
if (m2.DimensionCount != 2)
{
throw new ArgumentException("m2 must be a matrix", nameof(m2));
}
if (src.Sizes[0] != m1.Sizes[0] || src.Sizes[1] != m2.Sizes[1] || m1.Sizes[1] != m2.Sizes[0])
{
throw new InvalidOperationException("Size mismatch");
}
var writeTarget = TensorResultBuilder.GetWriteTarget(result, src, true, src.Sizes);
if (writeTarget != src)
{
Ops.Copy(writeTarget, src);
}
MatrixMultiplication.Gemm(alpha, m1, m2, beta, writeTarget);
return(writeTarget);
}
public Tensor Softmax(Tensor result, Tensor src)
{
TSCudaContext context = CudaHelpers.TSContextForTensor(src);
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, src, true, src.Sizes);
Softmax(context, writeTarget, src);
return(writeTarget);
}
public Tensor AddLayerNorm(Tensor result, Tensor src1, Tensor src2, Tensor alpha, Tensor beta, float eps = 1e-9f)
{
TSCudaContext context = CudaHelpers.TSContextForTensor(src1);
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(result, src1, false, src1.Sizes);
AddLayerNorm(context, writeTarget, src1, src2, alpha, beta, eps);
return(writeTarget);
}
public Tensor LayerNormGrad(Tensor outGrad, Tensor alphaGrad, Tensor betaGrad, Tensor inGrad, Tensor y, Tensor x, Tensor alpha, Tensor beta, float eps = 1e-9f)
{
TSCudaContext context = CudaHelpers.TSContextForTensor(inGrad);
Tensor writeTarget = TensorResultBuilder.GetWriteTarget(outGrad, inGrad, false, inGrad.Sizes);
LayerNormGrad(context, writeTarget, alphaGrad, betaGrad, inGrad, y, x, alpha, beta, eps);
return(writeTarget);
}
public Tensor BuildSrcTgtMask(Tensor originalSrcLengths, Tensor originalTgtLengths, int batchSize, int paddedSrcLength, int paddedTgtLength)
{
var context = CudaHelpers.TSContextForTensor(originalSrcLengths);
var writeTarget = TensorResultBuilder.GetWriteTarget(null, originalSrcLengths.Allocator, DType.Float32, true, new long[] { batchSize, paddedTgtLength, paddedSrcLength });
this.BuildSrcTgtMask(context, writeTarget, originalSrcLengths, originalTgtLengths, batchSize);
return(writeTarget);
}
public Tensor LayerNorm(Tensor result, Tensor src, Tensor alpha, Tensor beta, float eps = 1e-9f)
{
var context = CudaHelpers.TSContextForTensor(src);
var writeTarget = TensorResultBuilder.GetWriteTarget(result, src, false, src.Sizes);
this.LayerNorm(context, writeTarget, src, alpha, beta, eps);
return(writeTarget);
}
