/// <summary>
/// Create a <see cref="OnnxScoringEstimator"/>, which applies a pre-trained Onnx model to the <paramref name="inputColumnName"/> column.
/// Please refer to <see cref="OnnxScoringEstimator"/> to learn more about the necessary dependencies,
/// and how to run it on a GPU.
/// </summary>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="outputColumnName">The output column resulting from the transformation.</param>
/// <param name="inputColumnName">The input column.</param>
/// <param name="modelFile">The path of the file containing the ONNX model.</param>
/// <param name="gpuDeviceId">Optional GPU device ID to run execution on, <see langword="null" /> to run on CPU.</param>
/// <param name="fallbackToCpu">If GPU error, raise exception or fallback to CPU.</param>
/// <remarks>
/// If the gpuDeviceId value is <see langword="null" /> the <see cref="P:MLContext.GpuDeviceId"/> value will be used if it is not <see langword="null" />.
/// </remarks>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static OnnxScoringEstimator ApplyOnnxModel(this TransformsCatalog catalog,
string outputColumnName,
string inputColumnName,
string modelFile,
int?gpuDeviceId = null,
bool fallbackToCpu = false)
{
var(env, gpuDeviceIdToUse, fallbackToCpuToUse) = GetGpuDeviceId(catalog, gpuDeviceId, fallbackToCpu);
return(new OnnxScoringEstimator(env, new[] { outputColumnName }, new[] { inputColumnName }, modelFile, gpuDeviceIdToUse, fallbackToCpuToUse));
}
/// <summary>Initializes a new instance of <see cref="PrincipalComponentAnalyzer"/>.</summary>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
/// <param name="exampleWeightColumnName">The name of the example weight column (optional).</param>
/// <param name="rank">The number of principal components.</param>
/// <param name="overSampling">Oversampling parameter for randomized PrincipalComponentAnalysis training.</param>
/// <param name="ensureZeroMean">If enabled, data is centered to be zero mean.</param>
/// <param name="seed">The seed for random number generation.</param>
public static PrincipalComponentAnalyzer ProjectToPrincipalComponents(this TransformsCatalog catalog,
string outputColumnName,
string inputColumnName = null,
string exampleWeightColumnName = null,
int rank = PrincipalComponentAnalyzer.Defaults.Rank,
int overSampling = PrincipalComponentAnalyzer.Defaults.Oversampling,
bool ensureZeroMean = PrincipalComponentAnalyzer.Defaults.EnsureZeroMean,
int?seed = null)
=> new PrincipalComponentAnalyzer(CatalogUtils.GetEnvironment(catalog),
outputColumnName, inputColumnName, exampleWeightColumnName, rank, overSampling, ensureZeroMean, seed);
/// <summary>
/// Create a <see cref="NormalizingEstimator"/>, which normalizes by assigning the data into bins based on correlation with the <paramref name="labelColumnName"/> column.
/// </summary>
/// <param name="catalog">The transform catalog</param>
/// <param name="columns">The pairs of input and output columns.
/// The input columns must be of data type <see cref="System.Single"/>, <see cref="System.Double"/> or a known-sized vector of those types.
/// The data type for the output column will be the same as the associated input column.</param>
/// <param name="labelColumnName">Name of the label column for supervised binning.</param>
/// <param name="maximumExampleCount">Maximum number of examples used to train the normalizer.</param>
/// <param name="fixZero">Whether to map zero to zero, preserving sparsity.</param>
/// <param name="maximumBinCount">Maximum number of bins (power of 2 recommended).</param>
/// <param name="mininimumExamplesPerBin">Minimum number of examples per bin.</param>
public static NormalizingEstimator NormalizeSupervisedBinning(this TransformsCatalog catalog, InputOutputColumnPair[] columns,
string labelColumnName = DefaultColumnNames.Label,
long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
bool fixZero = NormalizingEstimator.Defaults.EnsureZeroUntouched,
int maximumBinCount = NormalizingEstimator.Defaults.MaximumBinCount,
int mininimumExamplesPerBin = NormalizingEstimator.Defaults.MininimumBinSize) =>
new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog),
columns.Select(column =>
new NormalizingEstimator.SupervisedBinningColumOptions(
column.OutputColumnName, column.InputColumnName, labelColumnName, maximumExampleCount, fixZero, maximumBinCount, mininimumExamplesPerBin)).ToArray());
/// <summary>
/// Create a <see cref="NormalizingEstimator"/>, which normalizes by assigning the data into bins with equal density.
/// </summary>
/// <param name="catalog">The transform catalog</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
/// The data type on this column is the same as the input column.</param>
/// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.
/// The data type on this column should be <see cref="System.Single"/>, <see cref="System.Double"/> or a known-sized vector of those types.</param>
/// <param name="maximumExampleCount">Maximum number of examples used to train the normalizer.</param>
/// <param name="fixZero">Whether to map zero to zero, preserving sparsity.</param>
/// <param name="maximumBinCount">Maximum number of bins (power of 2 recommended).</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static NormalizingEstimator NormalizeBinning(this TransformsCatalog catalog,
string outputColumnName, string inputColumnName = null,
long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
bool fixZero = NormalizingEstimator.Defaults.EnsureZeroUntouched,
int maximumBinCount = NormalizingEstimator.Defaults.MaximumBinCount)
{
var columnOptions = new NormalizingEstimator.BinningColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, fixZero, maximumBinCount);
return(new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions));
}
/// <include file='doc.xml' path='doc/members/member[@name="ImagePixelExtractingEstimator"]/*' />
/// <param name="catalog">The transform's catalog.</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
/// <param name="colors">What colors to extract.</param>
/// <param name="order">In which order to extract colors from pixel.</param>
/// <param name="interleave">Whether to interleave the pixels colors, meaning keep them in the <paramref name="order"/> order, or leave them in the plannar form:
/// all the values for one color for all pixels, then all the values for another color and so on.</param>
/// <param name="offset">Offset pixel's color value by this amount. Applied to color value first.</param>
/// <param name="scale">Scale pixel's color value by this amount. Applied to color value second.</param>
/// <param name="asFloat">Output array as float array. If false, output as byte array and ignores <paramref name="offset"/> and <paramref name="scale"/>.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]></format>
/// </example>
public static ImagePixelExtractingEstimator ExtractPixels(this TransformsCatalog catalog,
string outputColumnName,
string inputColumnName = null,
ImagePixelExtractingEstimator.ColorBits colors = ImagePixelExtractingEstimator.Defaults.Colors,
ImagePixelExtractingEstimator.ColorsOrder order = ImagePixelExtractingEstimator.Defaults.Order,
bool interleave = false,
float offset = ImagePixelExtractingEstimator.Defaults.Offset,
float scale = ImagePixelExtractingEstimator.Defaults.Scale,
bool asFloat = ImagePixelExtractingEstimator.Defaults.Convert)
=> new ImagePixelExtractingEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, inputColumnName, colors, order, interleave, offset, scale, asFloat);
/// <summary>
/// Converts vectors of pixels into <see cref="ImageType"/> representation.
/// </summary>
/// <param name="catalog">The transforms' catalog.</param>
/// <param name="imageHeight">The height of the output images.</param>
/// <param name="imageWidth">The width of the output images.</param>
/// <param name="outputColumnName"> Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName"> Name of column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
/// <param name="colorsPresent">Specifies which <see cref="ImagePixelExtractingEstimator.ColorBits"/> are in present the input pixel vectors. The order of colors is specified in <paramref name="orderOfColors"/>.</param>
/// <param name="orderOfColors">The order in which colors are presented in the input vector.</param>
/// <param name="interleavedColors">Whether the pixels are interleaved, meaning whether they are in <paramref name="orderOfColors"/> order, or separated in the planar form:
/// all the values for one color for all pixels, then all the values for another color and so on.</param>
/// <param name="scaleImage">The values are scaled by this value before being converted to pixels. Applied to vector value before <paramref name="offsetImage"/>.</param>
/// <param name="offsetImage">The offset is subtracted before converting the values to pixels. Applied to vector value after <paramref name="scaleImage"/>.</param>
/// <param name="defaultAlpha">Default value for alpha color, would be overriden if <paramref name="colorsPresent"/> contains <see cref="ImagePixelExtractingEstimator.ColorBits.Alpha"/>.</param>
/// <param name="defaultRed">Default value for red color, would be overriden if <paramref name="colorsPresent"/> contains <see cref="ImagePixelExtractingEstimator.ColorBits.Red"/>.</param>
/// <param name="defaultGreen">Default value for grenn color, would be overriden if <paramref name="colorsPresent"/> contains <see cref="ImagePixelExtractingEstimator.ColorBits.Green"/>.</param>
/// <param name="defaultBlue">Default value for blue color, would be overriden if <paramref name="colorsPresent"/> contains <see cref="ImagePixelExtractingEstimator.ColorBits.Blue"/>.</param>
public static VectorToImageConvertingEstimator ConvertToImage(this TransformsCatalog catalog, int imageHeight, int imageWidth, string outputColumnName, string inputColumnName = null,
ImagePixelExtractingEstimator.ColorBits colorsPresent = ImagePixelExtractingEstimator.Defaults.Colors,
ImagePixelExtractingEstimator.ColorsOrder orderOfColors = ImagePixelExtractingEstimator.Defaults.Order,
bool interleavedColors = ImagePixelExtractingEstimator.Defaults.Interleave,
float scaleImage = VectorToImageConvertingEstimator.Defaults.Scale,
float offsetImage = VectorToImageConvertingEstimator.Defaults.Offset,
int defaultAlpha = VectorToImageConvertingEstimator.Defaults.DefaultAlpha,
int defaultRed = VectorToImageConvertingEstimator.Defaults.DefaultRed,
int defaultGreen = VectorToImageConvertingEstimator.Defaults.DefaultGreen,
int defaultBlue = VectorToImageConvertingEstimator.Defaults.DefaultBlue)
=> new VectorToImageConvertingEstimator(CatalogUtils.GetEnvironment(catalog), imageHeight, imageWidth, outputColumnName, inputColumnName, colorsPresent, orderOfColors, interleavedColors, scaleImage, offsetImage);
/// <summary>
/// Create a <see cref="NormalizingEstimator"/>, which normalizes using statistics that are robust to outliers by centering the data around 0 (removing the median) and scales
/// the data according to the quantile range (defaults to the interquartile range).
/// </summary>
/// <param name="catalog">The transform catalog</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
/// The data type on this column is the same as the input column.</param>
/// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.
/// The data type on this column should be <see cref="System.Single"/>, <see cref="System.Double"/> or a known-sized vector of those types.</param>
/// <param name="maximumExampleCount">Maximum number of examples used to train the normalizer.</param>
/// <param name="centerData">Whether to center the data around 0 be removing the median. Defaults to true.</param>
/// <param name="quantileMin">Quantile min used to scale the data. Defaults to 25.</param>
/// <param name="quantileMax">Quantile max used to scale the data. Defaults to 75.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static NormalizingEstimator NormalizeRobustScaling(this TransformsCatalog catalog,
string outputColumnName, string inputColumnName = null,
long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
bool centerData = NormalizingEstimator.Defaults.CenterData,
uint quantileMin = NormalizingEstimator.Defaults.QuantileMin,
uint quantileMax = NormalizingEstimator.Defaults.QuantileMax)
{
var columnOptions = new NormalizingEstimator.RobustScalingColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, centerData, quantileMin, quantileMax);
return(new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions));
}
/// <summary>
/// Create a <see cref="NormalizingEstimator"/>, which normalizes based on the computed mean and variance of the logarithm of the data.
/// </summary>
/// <param name="catalog">The transform catalog</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
/// The data type on this column is the same as the input column.</param>
/// <param name="fixZero">Whether to map zero to zero, preserving sparsity.</param>
/// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.
/// The data type on this column should be <see cref="System.Single"/>, <see cref="System.Double"/> or a known-sized vector of those types.</param>
/// <param name="maximumExampleCount">Maximum number of examples used to train the normalizer.</param>
/// <param name="useCdf">Whether to use CDF as the output.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatalog catalog,
string outputColumnName,
bool fixZero,
string inputColumnName = null,
long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf)
{
var columnOptions = new NormalizingEstimator.LogMeanVarianceColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, useCdf, fixZero);
return(new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions));
}
private static (Runtime.IHostEnvironment, int?, bool) GetGpuDeviceId(TransformsCatalog catalog, int?gpuDeviceId, bool fallbackToCpu)
{
Runtime.IHostEnvironment env = CatalogUtils.GetEnvironment(catalog);
if (gpuDeviceId == null && env is Runtime.IHostEnvironmentInternal localEnvironment && localEnvironment.GpuDeviceId != null)
{
gpuDeviceId = localEnvironment.GpuDeviceId;
fallbackToCpu = localEnvironment.FallbackToCpu;
}
return(env, gpuDeviceId, fallbackToCpu);
}
/// <summary>
/// Create a <see cref="OnnxScoringEstimator"/>, which applies a pre-trained Onnx model to the <paramref name="inputColumnNames"/> columns.
/// Please refer to <see cref="OnnxScoringEstimator"/> to learn more about the necessary dependencies,
/// and how to run it on a GPU.
/// </summary>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="outputColumnNames">The output columns resulting from the transformation.</param>
/// <param name="inputColumnNames">The input columns.</param>
/// <param name="modelFile">The path of the file containing the ONNX model.</param>
/// <param name="shapeDictionary">ONNX shapes to be used over those loaded from <paramref name="modelFile"/>.
/// For keys use names as stated in the ONNX model, e.g. "input". Stating the shapes with this parameter
/// is particularly useful for working with variable dimension inputs and outputs.
/// </param>
/// <param name="gpuDeviceId">Optional GPU device ID to run execution on, <see langword="null" /> to run on CPU.</param>
/// <param name="fallbackToCpu">If GPU error, raise exception or fallback to CPU.</param>
/// <remarks>
/// If the gpuDeviceId value is <see langword="null" /> the <see cref="P:MLContext.GpuDeviceId"/> value will be used if it is not <see langword="null" />.
/// </remarks>
public static OnnxScoringEstimator ApplyOnnxModel(this TransformsCatalog catalog,
string[] outputColumnNames,
string[] inputColumnNames,
string modelFile,
IDictionary <string, int[]> shapeDictionary,
int?gpuDeviceId = null,
bool fallbackToCpu = false)
{
var(env, gpuDeviceIdToUse, fallbackToCpuToUse) = GetGpuDeviceId(catalog, gpuDeviceId, fallbackToCpu);
return(new OnnxScoringEstimator(env, outputColumnNames, inputColumnNames, modelFile, gpuDeviceIdToUse, fallbackToCpuToUse, shapeDictionary: shapeDictionary));
}
/// <summary>
/// Create a <see cref="ColumnCopyingEstimator"/>, which copies the data from the column specified in <see cref="InputOutputColumnPair.InputColumnName" />
/// to a new column: <see cref="InputOutputColumnPair.OutputColumnName" /> and replaces missing values in it according to <paramref name="replacementMode"/>.
/// </summary>
/// <remarks>This transform can operate over several columns.</remarks>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="columns">The pairs of input and output columns. This estimator operates over scalar or vector of floats or doubles.</param>
/// <param name="replacementMode">The type of replacement to use as specified in <see cref="MissingValueReplacingEstimator.ReplacementMode"/></param>
/// <param name="imputeBySlot">If <see langword="true"/>, per-slot imputation of replacement is performed.
/// Otherwise, replacement value is imputed for the entire vector column. This setting is ignored for scalars and variable vectors,
/// where imputation is always for the entire column.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]></format>
/// </example>
public static MissingValueReplacingEstimator ReplaceMissingValues(this TransformsCatalog catalog,
InputOutputColumnPair[] columns,
MissingValueReplacingEstimator.ReplacementMode replacementMode = MissingValueReplacingEstimator.Defaults.Mode,
bool imputeBySlot = MissingValueReplacingEstimator.Defaults.ImputeBySlot)
{
var env = CatalogUtils.GetEnvironment(catalog);
env.CheckValue(columns, nameof(columns));
var columnOptions = columns.Select(x => new MissingValueReplacingEstimator.ColumnOptions(x.OutputColumnName, x.InputColumnName, replacementMode, imputeBySlot)).ToArray();
return(new MissingValueReplacingEstimator(env, columnOptions));
}
/// <summary>
/// The values are assigned into bins based on correlation with the <paramref name="labelColumnName"/> column.
/// </summary>
/// <param name="catalog">The transform catalog</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
/// <param name="labelColumnName">Name of the label column for supervised binning.</param>
/// <param name="maximumExampleCount">Maximum number of examples used to train the normalizer.</param>
/// <param name="fixZero">Whether to map zero to zero, preserving sparsity.</param>
/// <param name="maximumBinCount">Maximum number of bins (power of 2 recommended).</param>
/// <param name="mininimumExamplesPerBin">Minimum number of examples per bin.</param>
public static NormalizingEstimator NormalizeSupervisedBinning(this TransformsCatalog catalog,
string outputColumnName, string inputColumnName = null,
string labelColumnName = DefaultColumnNames.Label,
long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
bool fixZero = NormalizingEstimator.Defaults.EnsureZeroUntouched,
int maximumBinCount = NormalizingEstimator.Defaults.MaximumBinCount,
int mininimumExamplesPerBin = NormalizingEstimator.Defaults.MininimumBinSize)
{
var columnOptions = new NormalizingEstimator.SupervisedBinningColumOptions(outputColumnName, inputColumnName, labelColumnName, maximumExampleCount, fixZero, maximumBinCount, mininimumExamplesPerBin);
return(new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions));
}
/// <summary>
/// Create the ML context.
/// </summary>
/// <param name="seed">Random seed. Set to <c>null</c> for a non-deterministic environment.</param>
/// <param name="conc">Concurrency level. Set to 1 to run single-threaded. Set to 0 to pick automatically.</param>
public MLContext(int?seed = null, int conc = 0)
{
_env = new LocalEnvironment(seed, conc, MakeCompositionContainer);
_env.AddListener(ProcessMessage);
BinaryClassification = new BinaryClassificationContext(_env);
MulticlassClassification = new MulticlassClassificationContext(_env);
Regression = new RegressionContext(_env);
Clustering = new ClusteringContext(_env);
Ranking = new RankingContext(_env);
Transforms = new TransformsCatalog(_env);
Model = new ModelOperationsCatalog(_env);
Data = new DataOperations(_env);
}
/// <summary>
/// Create <see cref="SsaChangePointEstimator"/>, which predicts change points in time series
/// using <a href="https://en.wikipedia.org/wiki/Singular_spectrum_analysis">Singular Spectrum Analysis (SSA)</a>.
/// </summary>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
/// The column data is a vector of <see cref="System.Double"/>. The vector contains 4 elements: alert (non-zero value means a change point), raw score, p-Value and martingale score.</param>
/// <param name="inputColumnName">Name of column to transform. The column data must be <see cref="System.Single"/>.
/// If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
/// <param name="confidence">The confidence for change point detection in the range [0, 100].</param>
/// <param name="trainingWindowSize">The number of points from the beginning of the sequence used for training.</param>
/// <param name="changeHistoryLength">The size of the sliding window for computing the p-value.</param>
/// <param name="seasonalityWindowSize">An upper bound on the largest relevant seasonality in the input time-series.</param>
/// <param name="errorFunction">The function used to compute the error between the expected and the observed value.</param>
/// <param name="martingale">The martingale used for scoring.</param>
/// <param name="eps">The epsilon parameter for the Power martingale.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static SsaChangePointEstimator DetectChangePointBySsa(this TransformsCatalog catalog, string outputColumnName, string inputColumnName,
int confidence, int changeHistoryLength, int trainingWindowSize, int seasonalityWindowSize, ErrorFunction errorFunction = ErrorFunction.SignedDifference,
MartingaleType martingale = MartingaleType.Power, double eps = 0.1)
=> new SsaChangePointEstimator(CatalogUtils.GetEnvironment(catalog), new SsaChangePointDetector.Options
{
Name = outputColumnName,
Source = inputColumnName ?? outputColumnName,
Confidence = confidence,
ChangeHistoryLength = changeHistoryLength,
TrainingWindowSize = trainingWindowSize,
SeasonalWindowSize = seasonalityWindowSize,
Martingale = martingale,
PowerMartingaleEpsilon = eps,
ErrorFunction = errorFunction
});
/// <summary>
/// Create the ML context.
/// </summary>
/// <param name="seed">Random seed. Set to <c>null</c> for a non-deterministic environment.</param>
public MLContext(int?seed = null)
{
_env = new LocalEnvironment(seed);
_env.AddListener(ProcessMessage);
BinaryClassification = new BinaryClassificationCatalog(_env);
MulticlassClassification = new MulticlassClassificationCatalog(_env);
Regression = new RegressionCatalog(_env);
Clustering = new ClusteringCatalog(_env);
Ranking = new RankingCatalog(_env);
AnomalyDetection = new AnomalyDetectionCatalog(_env);
Transforms = new TransformsCatalog(_env);
Model = new ModelOperationsCatalog(_env);
Data = new DataOperationsCatalog(_env);
}
/// <summary>
/// Feature Contribution Calculation computes model-specific contribution scores for each feature.
/// Note that this functionality is not supported by all the models. See <see cref="FeatureContributionCalculatingTransformer"/> for a list of the suported models.
/// </summary>
/// <param name="catalog">The model explainability operations catalog.</param>
/// <param name="predictionTransformer">A <see cref="ISingleFeaturePredictionTransformer{TModel}"/> that supports Feature Contribution Calculation,
/// and which will also be used for scoring.</param>
/// <param name="numberOfPositiveContributions">The number of positive contributions to report, sorted from highest magnitude to lowest magnitude.
/// Note that if there are fewer features with positive contributions than <paramref name="numberOfPositiveContributions"/>, the rest will be returned as zeros.</param>
/// <param name="numberOfNegativeContributions">The number of negative contributions to report, sorted from highest magnitude to lowest magnitude.
/// Note that if there are fewer features with negative contributions than <paramref name="numberOfNegativeContributions"/>, the rest will be returned as zeros.</param>
/// <param name="normalize">Whether the feature contributions should be normalized to the [-1, 1] interval.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static FeatureContributionCalculatingEstimator CalculateFeatureContribution <TModelParameters, TCalibrator>(this TransformsCatalog catalog,
ISingleFeaturePredictionTransformer <CalibratedModelParametersBase <TModelParameters, TCalibrator> > predictionTransformer,
int numberOfPositiveContributions = FeatureContributionDefaults.NumberOfPositiveContributions,
int numberOfNegativeContributions = FeatureContributionDefaults.NumberOfNegativeContributions,
bool normalize = FeatureContributionDefaults.Normalize)
where TModelParameters : class, ICalculateFeatureContribution
where TCalibrator : class, ICalibrator
=> new FeatureContributionCalculatingEstimator(CatalogUtils.GetEnvironment(catalog), predictionTransformer.Model.SubModel, numberOfPositiveContributions, numberOfNegativeContributions, predictionTransformer.FeatureColumnName, normalize);
/// <summary>
/// Scores a dataset using a pre-traiend TensorFlow model specified via <paramref name="tensorFlowModel"/>.
/// </summary>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="tensorFlowModel">The pre-trained TensorFlow model.</param>
/// <param name="inputs"> The names of the model inputs.</param>
/// <param name="outputs">The names of the requested model outputs.</param>
public static TensorFlowEstimator ScoreTensorFlowModel(this TransformsCatalog catalog,
TensorFlowModelInfo tensorFlowModel,
string[] inputs,
string[] outputs)
=> new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), tensorFlowModel, inputs, outputs);
/// <summary>
/// Create a <see cref="OnnxScoringEstimator"/>, which applies a pre-trained Onnx model to the input column.
/// Input/output columns are determined based on the input/output columns of the provided ONNX model.
/// Please refer to <see cref="OnnxScoringEstimator"/> to learn more about the necessary dependencies,
/// and how to run it on a GPU.
/// </summary>
/// <remarks>
/// The name/type of input columns must exactly match name/type of the ONNX model inputs.
/// The name/type of the produced output columns will match name/type of the ONNX model outputs.
/// </remarks>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="modelFile">The path of the file containing the ONNX model.</param>
/// <param name="gpuDeviceId">Optional GPU device ID to run execution on, <see langword="null" /> to run on CPU.</param>
/// <param name="fallbackToCpu">If GPU error, raise exception or fallback to CPU.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static OnnxScoringEstimator ApplyOnnxModel(this TransformsCatalog catalog,
string modelFile,
int?gpuDeviceId = null,
bool fallbackToCpu = false)
=> new OnnxScoringEstimator(CatalogUtils.GetEnvironment(catalog), modelFile, gpuDeviceId, fallbackToCpu);
/// <summary>
/// Feature Contribution Calculation computes model-specific contribution scores for each feature.
/// Note that this functionality is not supported by all the models. See <see cref="FeatureContributionCalculatingTransformer"/> for a list of the suported models.
/// </summary>
/// <param name="catalog">The model explainability operations catalog.</param>
/// <param name="predictionTransformer">A <see cref="ISingleFeaturePredictionTransformer{TModel}"/> that supports Feature Contribution Calculation,
/// and which will also be used for scoring.</param>
/// <param name="numberOfPositiveContributions">The number of positive contributions to report, sorted from highest magnitude to lowest magnitude.
/// Note that if there are fewer features with positive contributions than <paramref name="numberOfPositiveContributions"/>, the rest will be returned as zeros.</param>
/// <param name="numberOfNegativeContributions">The number of negative contributions to report, sorted from highest magnitude to lowest magnitude.
/// Note that if there are fewer features with negative contributions than <paramref name="numberOfNegativeContributions"/>, the rest will be returned as zeros.</param>
/// <param name="normalize">Whether the feature contributions should be normalized to the [-1, 1] interval.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static FeatureContributionCalculatingEstimator CalculateFeatureContribution(this TransformsCatalog catalog,
ISingleFeaturePredictionTransformer <ICalculateFeatureContribution> predictionTransformer,
int numberOfPositiveContributions = FeatureContributionDefaults.NumberOfPositiveContributions,
int numberOfNegativeContributions = FeatureContributionDefaults.NumberOfNegativeContributions,
bool normalize = FeatureContributionDefaults.Normalize)
=> new FeatureContributionCalculatingEstimator(CatalogUtils.GetEnvironment(catalog), predictionTransformer.Model, numberOfPositiveContributions, numberOfNegativeContributions, predictionTransformer.FeatureColumnName, normalize);
/// <summary> /// Converts the images to grayscale. /// </summary> /// <param name="catalog">The transform's catalog.</param> /// <param name="columns">The name of the columns containing the image paths(first item of the tuple), and the name of the resulting output column (second item of the tuple).</param> public static ImageGrayscalingEstimator ConvertToGrayscale(this TransformsCatalog catalog, params (string input, string output)[] columns)
/// <summary>
/// Create <see cref="DnnImageFeaturizerEstimator"/>, which applies one of the pre-trained DNN models in
/// <see cref="DnnImageModelSelector"/> to featurize an image.
/// </summary>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="outputColumnName">The name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="modelFactory">An extension method on the <see cref="DnnImageModelSelector"/> that creates a chain of two
/// <see cref="OnnxScoringEstimator"/> (one for preprocessing and one with a pretrained image DNN) with specific models
/// included in a package together with that extension method.</param>
/// <param name="inputColumnName">Name of column to transform.
/// If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static DnnImageFeaturizerEstimator DnnFeaturizeImage(this TransformsCatalog catalog,
string outputColumnName,
Func <DnnImageFeaturizerInput, EstimatorChain <ColumnCopyingTransformer> > modelFactory,
string inputColumnName = null)
=> new DnnImageFeaturizerEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, modelFactory, inputColumnName);
/// <include file='doc.xml' path='doc/members/member[@name="ImagePixelExtractingEstimator"]/*' /> /// <param name="catalog">The transform's catalog.</param> /// <param name="columnOptions">The <see cref="ImagePixelExtractingEstimator.ColumnOptions"/> describing how the transform handles each image pixel extraction output input column pair.</param> public static ImagePixelExtractingEstimator ExtractPixels(this TransformsCatalog catalog, params ImagePixelExtractingEstimator.ColumnOptions[] columnOptions) => new ImagePixelExtractingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions);
/// <summary> /// Loads the images from the <see cref="ImageLoadingTransformer.ImageFolder" /> into memory. /// </summary> /// <remarks> /// The image get loaded in memory as a <see cref="System.Drawing.Bitmap" /> type. /// Loading is the first step of almost every pipeline that does image processing, and further analysis on images. /// The images to load need to be in the formats supported by <see cref = "System.Drawing.Bitmap" />. /// For end-to-end image processing pipelines, and scenarios in your applications, see the /// <a href="https://github.com/dotnet/machinelearning-samples/tree/master/samples/csharp/getting-started"> examples in the machinelearning-samples github repository.</a> /// <seealso cref = "ImageEstimatorsCatalog" /> /// </remarks> /// <param name="catalog">The transform's catalog.</param> /// <param name="imageFolder">The images folder.</param> /// <param name="columns">Specifies the names of the input columns for the transformation, and their respective output column names.</param> /// <example> /// <format type="text/markdown"> /// <] /// ]]></format> /// </example> public static ImageLoadingEstimator LoadImages(this TransformsCatalog catalog, string imageFolder, params ColumnOptions[] columns) => new ImageLoadingEstimator(CatalogUtils.GetEnvironment(catalog), imageFolder, ColumnOptions.ConvertToValueTuples(columns));
/// <include file='doc.xml' path='doc/members/member[@name="ImageGrayscalingEstimator"]/*' /> /// <param name="catalog">The transform's catalog.</param> /// <param name="columns">Specifies the names of the input columns for the transformation, and their respective output column names.</param> /// <example> /// <format type="text/markdown"> /// <] /// ]]></format> /// </example> public static ImageGrayscalingEstimator ConvertToGrayscale(this TransformsCatalog catalog, params ColumnOptions[] columns) => new ImageGrayscalingEstimator(CatalogUtils.GetEnvironment(catalog), ColumnOptions.ConvertToValueTuples(columns));
/// <summary>
/// Scores or retrains (based on setting of the <see cref="TensorFlowTransform.Arguments.ReTrain"/>) a pre-traiend TensorFlow model specified via <paramref name="tensorFlowModel"/>.
/// </summary>
/// <param name="catalog">The transform's catalog.</param>
/// <param name="args">The <see cref="TensorFlowTransform.Arguments"/> specifying the inputs and the settings of the <see cref="TensorFlowEstimator"/>.</param>
/// <param name="tensorFlowModel">The pre-trained TensorFlow model.</param>
public static TensorFlowEstimator TensorFlow(this TransformsCatalog catalog,
TensorFlowTransform.Arguments args,
TensorFlowModelInfo tensorFlowModel)
=> new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), args, tensorFlowModel);
/// <summary>
/// Normalize (rescale) the column according to the specified <paramref name="mode"/>.
/// </summary>
/// <param name="catalog">The transform catalog</param>
/// <param name="inputName">The column name</param>
/// <param name="outputName">The column name</param>
/// <param name="mode">The <see cref="NormalizingEstimator.NormalizerMode"/> used to map the old values in the new scale. </param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static NormalizingEstimator Normalize(this TransformsCatalog catalog,
string inputName,
string outputName = null,
NormalizingEstimator.NormalizerMode mode = NormalizingEstimator.NormalizerMode.MinMax)
=> new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), inputName, outputName, mode);
/// <summary>
/// Initializes a new instance of <see cref="MissingValueIndicatorEstimator"/>
/// </summary>
/// <param name="catalog">The categorical transform's catalog.</param>
/// <param name="columns">The names of the input columns of the transformation and the corresponding names for the output columns.</param>
public static MissingValueIndicatorEstimator IndicateMissingValues(this TransformsCatalog catalog,
params (string inputColumn, string outputColumn)[] columns)
/// <summary> /// Resizes the images to a new width and height. /// </summary> /// <remarks> /// In image processing pipelines, often machine learning practitioner make use of<a href= "https://blogs.msdn.microsoft.com/mlserver/2017/04/12/image-featurization-with-a-pre-trained-deep-neural-network-model/"> /// pre - trained DNN featurizers</a> to extract features for usage in the machine learning algorithms. /// Those pre-trained models have a defined width and height for their input images, so often, after getting loaded, the images will need to get resized before /// further processing. /// The new width and height, as well as other properties of resizing, like type of scaling (uniform, or non-uniform), and whether to pad the image, /// or just crop it can be specified separately for each column loaded, through the <see cref="ImageResizingEstimator.ColumnOptions"/>. /// <seealso cref = "ImageEstimatorsCatalog" /> /// <seealso cref= "ImageLoadingEstimator" /> /// </remarks > /// <param name="catalog">The transform's catalog.</param> /// <param name="columnOptions">The <see cref="ImageResizingEstimator.ColumnOptions"/> describing how the transform handles each image resize column.</param> /// <example> /// <format type="text/markdown"> /// <] /// ]]></format> /// </example> public static ImageResizingEstimator ResizeImages(this TransformsCatalog catalog, params ImageResizingEstimator.ColumnOptions[] columnOptions) => new ImageResizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions);
/// <summary>
/// Normalize (rescale) several columns according to the specified <paramref name="mode"/>.
/// </summary>
/// <param name="catalog">The transform catalog</param>
/// <param name="mode">The <see cref="NormalizingEstimator.NormalizerMode"/> used to map the old values to the new ones. </param>
/// <param name="columns">The pairs of input and output columns.</param>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]>
/// </format>
/// </example>
public static NormalizingEstimator Normalize(this TransformsCatalog catalog,
NormalizingEstimator.NormalizerMode mode,
params (string input, string output)[] columns)
/// <summary> /// Converts vectors of pixels into <see cref="ImageType"/> representation. /// </summary> /// <param name="catalog">The transform's catalog.</param> /// <param name="columnOptions">The <see cref="VectorToImageConvertingEstimator.ColumnOptions"/> describing how the transform handles each vector to image conversion column pair.</param> public static VectorToImageConvertingEstimator ConvertToImage(this TransformsCatalog catalog, params VectorToImageConvertingEstimator.ColumnOptions[] columnOptions) => new VectorToImageConvertingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions);
