/// <summary>
/// Clips gradient norm of an iterable of parameters.
/// The norm is computed over all gradients together, as if they were concatenated into a single vector.
/// Gradients are modified in-place.
/// </summary>
/// <param name="tensors"></param>
/// <param name="max_norm"></param>
/// <param name="norm_type"></param>
/// <returns></returns>
public static double clip_grad_norm_(IList <Tensor> tensors, double max_norm, double norm_type = 2.0)
{
using (var parray = new PinnedArray <IntPtr>()) {
IntPtr tensorsRef = parray.CreateArray(tensors.Select(p => p.Handle).ToArray());
return(THSTensor_clip_grad_norm_(tensorsRef, parray.Array.Length, max_norm, norm_type));
}
}
/// <summary>
/// Get the parameters that the optimizer is handling.
/// </summary>
public virtual IEnumerable <Parameter> parameters()
{
IntPtr[] ptrArray;
using (var pa = new PinnedArray <IntPtr>()) {
THSNN_Optimizer_getParameters(handle, pa.CreateArray);
torch.CheckForErrors();
ptrArray = pa.Array;
}
return(ptrArray.Select(x => new Parameter(x)));
}
/// <summary>
/// Stack tensors in sequence depthwise (along third axis).
/// </summary>
/// <param name="tensors"></param>
/// <returns></returns>
/// <remarks>This is equivalent to concatenation along the third axis after 1-D and 2-D tensors have been reshaped by torch.atleast_3d().</remarks>
public static Tensor dstack(IList <Tensor> tensors)
{
using (var parray = new PinnedArray <IntPtr>()) {
IntPtr tensorsRef = parray.CreateArray(tensors.Select(p => p.Handle).ToArray());
var res = THSTensor_dstack(tensorsRef, parray.Array.Length);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new Tensor(res));
}
}
/// <summary>
/// Concatenates a sequence of tensors along a new dimension.
/// </summary>
/// <returns></returns>
/// <remarks>All tensors need to be of the same size.</remarks>
public static Tensor stack(IEnumerable <Tensor> tensors, long dimension = 0)
{
using (var parray = new PinnedArray <IntPtr>()) {
IntPtr tensorsRef = parray.CreateArray(tensors.Select(p => p.Handle).ToArray());
var res = THSTensor_stack(tensorsRef, parray.Array.Length, dimension);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new Tensor(res));
}
}
/// <summary>
/// Implements stochastic gradient descent (optionally with momentum).
/// </summary>
/// <param name="parameters">Prameters to optimize</param>
/// <param name="learningRate">Learning rate</param>
/// <param name="momentum">Momentum factor (default: 0)</param>
/// <param name="dampening">Dampening for momentum (default: 0)</param>
/// <param name="weight_decay">Weight decay (L2 penalty) (default: 0)</param>
/// <param name="nesterov">Enables Nesterov momentum (default: False)</param>
/// <returns></returns>
public static SGDOptimizer SGD(IEnumerable <Tensor> parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false)
{
var parray = new PinnedArray <IntPtr>();
IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray());
var res = THSNN_SGD_ctor(paramsRef, parray.Array.Length, learningRate, momentum, dampening, weight_decay, nesterov);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new SGDOptimizer(res, learningRate));
}
/// <summary>
/// Implements Adagrad algorithm.
///
/// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization.
/// </summary>
/// <param name="parameters">Prameters to optimize</param>
/// <param name="learningRate">learning rate (default: 1e-2)</param>
/// <param name="lr_decay">learning rate decay (default: 0)</param>
/// <param name="weight_decay">weight decay (L2 penalty) (default: 0)</param>
/// <param name="initial_accumulator_value"></param>
/// <param name="eps">Term added to the denominator to improve numerical stability (default: 1e-10)</param>
/// <returns></returns>
public static AdagradOptimizer Adagrad(IEnumerable <Tensor> parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10)
{
var parray = new PinnedArray <IntPtr>();
IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray());
var res = THSNN_Adagrad_ctor(paramsRef, parray.Array.Length, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new AdagradOptimizer(res, learningRate));
}
/// <summary>
/// Implements Adam algorithm.
///
/// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization.
/// </summary>
/// <param name="parameters">Prameters to optimize</param>
/// <param name="learningRate">learning rate (default: 1e-3)</param>
/// <param name="beta1">Coefficient used for computing running averages of gradient and its square (default: 0.9)</param>
/// <param name="beta2">Coefficient used for computing running averages of gradient and its square (default: 0.999)</param>
/// <param name="eps">Term added to the denominator to improve numerical stability (default: 1e-8)</param>
/// <param name="weight_decay">Weight decay (L2 penalty) (default: 0)</param>
/// <param name="amsgrad">Whether to use the AMSGrad variant of this algorithm. (default: False)</param>
/// <returns></returns>
public static AdamWOptimizer AdamW(IEnumerable <Tensor> parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false)
{
var parray = new PinnedArray <IntPtr>();
IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray());
var res = THSNN_AdamW_ctor(paramsRef, parray.Array.Length, learningRate, beta1, beta2, eps, weight_decay, amsgrad);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new AdamWOptimizer(res, learningRate));
}
/// <summary>
/// Implements RMSprop algorithm.
///
/// Proposed by G.Hinton in his course.
/// </summary>
/// <param name="parameters">Prameters to optimize</param>
/// <param name="learningRate">Learning rate (default: 1e-2)</param>
/// <param name="alpha">Smoothing constant (default: 0.99)</param>
/// <param name="eps">Term added to the denominator to improve numerical stability (default: 1e-8)</param>
/// <param name="weight_decay">Weight decay (L2 penalty) (default: 0)</param>
/// <param name="momentum">Momentum factor (default: 0)</param>
/// <param name="centered">if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance</param>
/// <returns></returns>
public static RMSPropOptimizer RMSProp(IEnumerable <Tensor> parameters, double learningRate = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false)
{
var parray = new PinnedArray <IntPtr>();
IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray());
var res = THSNN_RMSprop_ctor(paramsRef, parray.Array.Length, learningRate, alpha, eps, weight_decay, momentum, centered);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new RMSPropOptimizer(res, learningRate));
}
public static TorchTensor multi_dot(IList <TorchTensor> tensors)
{
if (tensors.Count == 0)
{
throw new ArgumentException(nameof(tensors));
}
if (tensors.Count == 1)
{
return(tensors[0]);
}
using (var parray = new PinnedArray <IntPtr>()) {
IntPtr tensorsRef = parray.CreateArray(tensors.Select(p => p.Handle).ToArray());
return(new TorchTensor(THSLinalg_multi_dot(tensorsRef, parray.Array.Length)));
}
}
/// <summary>
/// Concatenates the given sequence of seq tensors in the given dimension.
/// </summary>
/// <param name="tensors"></param>
/// <param name="dimension"></param>
/// <returns></returns>
/// <remarks> All tensors must either have the same shape (except in the concatenating dimension) or be empty.</remarks>
public static Tensor cat(IList <Tensor> tensors, long dimension)
{
if (tensors.Count == 0)
{
throw new ArgumentException(nameof(tensors));
}
if (tensors.Count == 1)
{
return(tensors[0]);
}
using (var parray = new PinnedArray <IntPtr>()) {
IntPtr tensorsRef = parray.CreateArray(tensors.Select(p => p.Handle).ToArray());
return(new Tensor(THSTensor_cat(tensorsRef, parray.Array.Length, dimension)));
}
}
public static LBFGSOptimizer LBFGS(IEnumerable <Tensor> parameters, double learningRate = 0.01, long max_iter = 20, long?max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100)
{
if (!max_eval.HasValue)
{
max_eval = 5 * max_iter / 4;
}
var parray = new PinnedArray <IntPtr>();
IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray());
var res = THSNN_LBFGS_ctor(paramsRef, parray.Array.Length, learningRate, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new LBFGSOptimizer(res, learningRate));
}
public static IList <Tensor> grad(IList <Tensor> outputs, IList <Tensor> inputs, IList <Tensor> grad_outputs = null, bool retain_graph = false, bool create_graph = false, bool allow_unused = false)
{
IntPtr[] result;
using (var outs = new PinnedArray <IntPtr>())
using (var ins = new PinnedArray <IntPtr>())
using (var grads = new PinnedArray <IntPtr>())
using (var results = new PinnedArray <IntPtr>()) {
IntPtr outsRef = outs.CreateArray(outputs.Select(p => p.Handle).ToArray());
IntPtr insRef = ins.CreateArray(inputs.Select(p => p.Handle).ToArray());
IntPtr gradsRef = grad_outputs == null ? IntPtr.Zero : grads.CreateArray(grad_outputs.Select(p => p.Handle).ToArray());
long gradsLength = grad_outputs == null ? 0 : grads.Array.Length;
THSAutograd_grad(outsRef, outs.Array.Length, insRef, ins.Array.Length, gradsRef, gradsLength, retain_graph, create_graph, allow_unused, results.CreateArray);
torch.CheckForErrors();
result = results.Array;
}
return(result.Select(x => new Tensor(x)).ToList());
}
public static Tensor multi_dot(IList <Tensor> tensors)
{
if (tensors.Count == 0)
{
throw new ArgumentException(nameof(tensors));
}
if (tensors.Count == 1)
{
return(tensors[0]);
}
using (var parray = new PinnedArray <IntPtr>()) {
IntPtr tensorsRef = parray.CreateArray(tensors.Select(p => p.Handle).ToArray());
var res = THSLinalg_multi_dot(tensorsRef, parray.Array.Length);
if (res == IntPtr.Zero)
{
torch.CheckForErrors();
}
return(new Tensor(res));
}
}
public static IList <Tensor> broadcast_tensors(params Tensor[] tensors)
{
if (tensors.Length == 0)
{
throw new ArgumentException(nameof(tensors));
}
if (tensors.Length == 1)
{
return(tensors);
}
IntPtr[] ptrArray;
using (var pa = new PinnedArray <IntPtr>())
using (var parray = new PinnedArray <IntPtr>()) {
IntPtr tensorsRef = parray.CreateArray(tensors.Select(p => p.Handle).ToArray());
THSTensor_broadcast_tensors(tensorsRef, tensors.Length, pa.CreateArray);
torch.CheckForErrors();
ptrArray = pa.Array;
}
return(ptrArray.Select(x => new Tensor(x)).ToList());
}
