/// <summary>
/// Predict a target using a linear binary classification model trained with the AveragedPerceptron trainer, and a custom loss.
/// </summary>
/// <param name="catalog">The binary classification catalog trainer object.</param>
/// <param name="label">The label, or dependent variable.</param>
/// <param name="features">The features, or independent variables.</param>
/// <param name="lossFunction">The custom loss.</param>
/// <param name="weights">The optional example weights.</param>
/// <param name="options">Advanced arguments to the algorithm.</param>
/// <param name="onFit">A delegate that is called every time the
/// <see cref="Estimator{TInShape, TOutShape, TTransformer}.Fit(DataView{TInShape})"/> method is called on the
/// <see cref="Estimator{TInShape, TOutShape, TTransformer}"/> instance created out of this. This delegate will receive
/// the linear model that was trained, as well as the calibrator on top of that model. Note that this action cannot change the
/// result in any way; it is only a way for the caller to be informed about what was learnt.</param>
/// <returns>The set of output columns including in order the predicted binary classification score (which will range
/// from negative to positive infinity), and the predicted label.</returns>
/// <seealso cref="AveragedPerceptronTrainer"/>.
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]></format>
/// </example>
public static (Scalar <float> score, Scalar <bool> predictedLabel) AveragedPerceptron(
this BinaryClassificationCatalog.BinaryClassificationTrainers catalog,
Scalar <bool> label,
Vector <float> features,
Scalar <float> weights,
IClassificationLoss lossFunction,
AveragedPerceptronTrainer.Options options,
Action <LinearBinaryModelParameters> onFit = null
)
{
Contracts.CheckValue(label, nameof(label));
Contracts.CheckValue(features, nameof(features));
Contracts.CheckValueOrNull(weights);
Contracts.CheckValueOrNull(options);
Contracts.CheckValueOrNull(onFit);
bool hasProbs = lossFunction is LogLoss;
var rec = new TrainerEstimatorReconciler.BinaryClassifierNoCalibration(
(env, labelName, featuresName, weightsName) =>
{
options.LabelColumn = labelName;
options.FeatureColumn = featuresName;
options.InitialWeights = weightsName;
var trainer = new AveragedPerceptronTrainer(env, options);
if (onFit != null)
{
return(trainer.WithOnFitDelegate(trans => onFit(trans.Model)));
}
else
{
return(trainer);
}
}, label, features, weights, hasProbs);
return(rec.Output);
}
/// <summary>
/// Predict a target using a field-aware factorization machine.
/// </summary>
/// <param name="ctx">The binary classifier context trainer object.</param>
/// <param name="label">The label, or dependent variable.</param>
/// <param name="features">The features, or independent variables.</param>
/// <param name="learningRate">Initial learning rate.</param>
/// <param name="numIterations">Number of training iterations.</param>
/// <param name="numLatentDimensions">Latent space dimensions.</param>
/// <param name="advancedSettings">A delegate to set more settings.</param>
/// <param name="onFit">A delegate that is called every time the
/// <see cref="Estimator{TTupleInShape, TTupleOutShape, TTransformer}.Fit(DataView{TTupleInShape})"/> method is called on the
/// <see cref="Estimator{TTupleInShape, TTupleOutShape, TTransformer}"/> instance created out of this. This delegate will receive
/// the model that was trained. Note that this action cannot change the result in any way; it is only a way for the caller to
/// be informed about what was learnt.</param>
/// <returns>The predicted output.</returns>
public static (Scalar <float> score, Scalar <bool> predictedLabel) FieldAwareFactorizationMachine(this BinaryClassificationContext.BinaryClassificationTrainers ctx,
Scalar <bool> label, Vector <float>[] features,
float learningRate = 0.1f,
int numIterations = 5,
int numLatentDimensions = 20,
Action <FieldAwareFactorizationMachineTrainer.Arguments> advancedSettings = null,
Action <FieldAwareFactorizationMachinePredictor> onFit = null)
{
Contracts.CheckValue(label, nameof(label));
Contracts.CheckNonEmpty(features, nameof(features));
Contracts.CheckParam(learningRate > 0, nameof(learningRate), "Must be positive");
Contracts.CheckParam(numIterations > 0, nameof(numIterations), "Must be positive");
Contracts.CheckParam(numLatentDimensions > 0, nameof(numLatentDimensions), "Must be positive");
Contracts.CheckValueOrNull(advancedSettings);
Contracts.CheckValueOrNull(onFit);
var rec = new CustomReconciler((env, labelCol, featureCols) =>
{
var trainer = new FieldAwareFactorizationMachineTrainer(env, labelCol, featureCols, advancedSettings:
args =>
{
advancedSettings?.Invoke(args);
args.LearningRate = learningRate;
args.Iters = numIterations;
args.LatentDim = numLatentDimensions;
});
if (onFit != null)
{
return(trainer.WithOnFitDelegate(trans => onFit(trans.Model)));
}
else
{
return(trainer);
}
}, label, features);
return(rec.Output);
}
public SchemaImpl(IExceptionContext ectx, TreeEnsembleFeaturizerBindableMapper parent,
ColumnType treeValueColType, ColumnType leafIdColType, ColumnType pathIdColType)
{
Contracts.CheckValueOrNull(ectx);
_ectx = ectx;
_ectx.AssertValue(parent);
_ectx.AssertValue(treeValueColType);
_ectx.AssertValue(leafIdColType);
_ectx.AssertValue(pathIdColType);
_parent = parent;
_names = new string[3];
_names[TreeIdx] = OutputColumnNames.Trees;
_names[LeafIdx] = OutputColumnNames.Leaves;
_names[PathIdx] = OutputColumnNames.Paths;
_types = new ColumnType[3];
_types[TreeIdx] = treeValueColType;
_types[LeafIdx] = leafIdColType;
_types[PathIdx] = pathIdColType;
}
/// <summary>
/// Given a schema and a bunch of column names, create the BoundSchema object. Any or all of the column
/// names may be null or whitespace, in which case they are ignored. Any columns that are specified but not
/// valid columns of the schema are also ignored.
/// </summary>
public static RoleMappedSchema CreateRoleMappedSchemaOpt(ISchema schema, string feature, string group, IEnumerable <KeyValuePair <ColumnRole, string> > custom = null)
{
Contracts.CheckValueOrNull(feature);
Contracts.CheckValueOrNull(custom);
var list = new List <KeyValuePair <ColumnRole, string> >();
if (!string.IsNullOrWhiteSpace(feature))
{
list.Add(ColumnRole.Feature.Bind(feature));
}
if (!string.IsNullOrWhiteSpace(group))
{
list.Add(ColumnRole.Group.Bind(group));
}
if (custom != null)
{
list.AddRange(custom);
}
return(RoleMappedSchema.CreateOpt(schema, list));
}
/// <summary>
/// Searches for the given column name in the schema. This method applies a
/// common policy that throws an exception if the column is not found
/// and the column name was explicitly specified. If the column is not found
/// and the column name was not explicitly specified, it returns null.
/// </summary>
public static string FindColumnOrNull(IExceptionContext ectx, Schema schema, Optional <string> value)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(schema, nameof(schema));
ectx.CheckValue(value, nameof(value));
if (value == "")
{
return(null);
}
int col;
if (!schema.TryGetColumnIndex(value, out col))
{
if (value.IsExplicit)
{
throw ectx.Except("Column '{0}' not found", value);
}
return(null);
}
return(value);
}
/// <summary>
/// Initializes a new instance of <see cref="MultiFileSource"/>.
/// In case of usage from Maml, the paths would be wildcard concatenated in the first string of <paramref name="paths"/>.
/// </summary>
/// <param name="paths">The paths of the files to load.</param>
/// <remarks>
/// The provided <paramref name="paths"/> can utilize wildcards to load all source files. For example:
/// paths = "Data/*" includes all files in directory Data
/// paths = "DataFolder/.../*" includes all files in all subdirectories inside directory Data.
/// paths = "Data1/*", "Data2/*" includes all files in directories Data1 and Data2
/// </remarks>
public MultiFileSource(params string[] paths)
{
Contracts.CheckValueOrNull(paths);
// calling the ctor passing null, creates an array of 1, null element
// The types using MFS know how to account for an empty path
// if the paths array is empty, therefore keeping that behavior.
if (paths == null || (paths.Length == 1 && paths[0] == null))
{
_paths = new string[0];
return;
}
List <string> concatenated = new List <string>();
if (paths != null)
{
foreach (string path in paths)
{
foreach (string rPath in StreamUtils.ExpandWildCards(path))
{
concatenated.Add(rPath);
}
}
}
else
{
concatenated = null;
}
if (concatenated != null && concatenated.Count > 0)
{
_paths = concatenated.ToArray();
}
else
{
_paths = paths;
}
}
public static JObject BuildEntryPointManifest(IExceptionContext ectx, ModuleCatalog.EntryPointInfo entryPointInfo, ModuleCatalog catalog)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(entryPointInfo, nameof(entryPointInfo));
ectx.CheckValue(catalog, nameof(catalog));
var result = new JObject();
result[FieldNames.Name] = entryPointInfo.Name;
result[FieldNames.Desc] = entryPointInfo.Description;
result[FieldNames.FriendlyName] = entryPointInfo.FriendlyName;
result[FieldNames.ShortName] = entryPointInfo.ShortName;
// There supposed to be 2 parameters, env and input.
result[FieldNames.Inputs] = BuildInputManifest(ectx, entryPointInfo.InputType, catalog);
result[FieldNames.Outputs] = BuildOutputManifest(ectx, entryPointInfo.OutputType, catalog);
if (entryPointInfo.InputKinds != null)
{
var jInputKinds = new JArray();
foreach (var kind in entryPointInfo.InputKinds)
{
jInputKinds.Add(kind.Name);
}
result[FieldNames.InputKind] = jInputKinds;
}
if (entryPointInfo.OutputKinds != null)
{
var jOutputKinds = new JArray();
foreach (var kind in entryPointInfo.OutputKinds)
{
jOutputKinds.Add(kind.Name);
}
result[FieldNames.OutputKind] = jOutputKinds;
}
return(result);
}
public InputBuilder(IExceptionContext ectx, Type inputType, ComponentCatalog catalog)
{
Contracts.CheckValue(ectx, nameof(ectx));
_ectx = ectx;
_ectx.CheckValue(inputType, nameof(inputType));
_ectx.CheckValue(catalog, nameof(catalog));
_type = inputType;
_catalog = catalog;
var fields = new List <FieldInfo>();
var attrs = new List <Attributes>();
foreach (var fieldInfo in _type.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
{
var attr = (ArgumentAttribute)fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault();
if (attr == null || attr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
_ectx.Check(!fieldInfo.IsStatic && !fieldInfo.IsInitOnly && !fieldInfo.IsLiteral);
var rangeAttr = fieldInfo.GetCustomAttributes(typeof(TlcModule.RangeAttribute), false).FirstOrDefault()
as TlcModule.RangeAttribute;
Contracts.CheckValueOrNull(rangeAttr);
var optional = fieldInfo.GetCustomAttributes(typeof(TlcModule.OptionalInputAttribute), false).Any();
fields.Add(fieldInfo);
attrs.Add(new Attributes(attr, rangeAttr, optional));
}
_ectx.Assert(fields.Count == attrs.Count);
_instance = Activator.CreateInstance(inputType);
_fields = fields.ToArray();
_attrs = attrs.ToArray();
_wasSet = new bool[_fields.Length];
}
/// <summary>
/// Predict a target using a linear binary classification model trained with the SDCA trainer, and log-loss.
/// </summary>
/// <param name="ctx">The binary classification context trainer object.</param>
/// <param name="label">The label, or dependent variable.</param>
/// <param name="features">The features, or independent variables.</param>
/// <param name="weights">The optional example weights.</param>
/// <param name="l2Const">The L2 regularization hyperparameter.</param>
/// <param name="l1Threshold">The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.</param>
/// <param name="maxIterations">The maximum number of passes to perform over the data.</param>
/// <param name="advancedSettings">A delegate to set more settings.
/// The settings here will override the ones provided in the direct method signature,
/// if both are present and have different values.
/// The columns names, however need to be provided directly, not through the <paramref name="advancedSettings"/>.</param>
/// <param name="onFit">A delegate that is called every time the
/// <see cref="Estimator{TInShape, TOutShape, TTransformer}.Fit(DataView{TInShape})"/> method is called on the
/// <see cref="Estimator{TInShape, TOutShape, TTransformer}"/> instance created out of this. This delegate will receive
/// the linear model that was trained, as well as the calibrator on top of that model. Note that this action cannot change the
/// result in any way; it is only a way for the caller to be informed about what was learnt.</param>
/// <returns>The set of output columns including in order the predicted binary classification score (which will range
/// from negative to positive infinity), the calibrated prediction (from 0 to 1), and the predicted label.</returns>
/// <example>
/// <format type="text/markdown">
/// <]
/// ]]></format>
/// </example>
public static (Scalar <float> score, Scalar <float> probability, Scalar <bool> predictedLabel) Sdca(
this BinaryClassificationContext.BinaryClassificationTrainers ctx,
Scalar <bool> label, Vector <float> features, Scalar <float> weights = null,
float?l2Const = null,
float?l1Threshold = null,
int?maxIterations = null,
Action <SdcaBinaryTrainer.Arguments> advancedSettings = null,
Action <LinearBinaryModelParameters, ParameterMixingCalibratedPredictor> onFit = null)
{
Contracts.CheckValue(label, nameof(label));
Contracts.CheckValue(features, nameof(features));
Contracts.CheckValueOrNull(weights);
Contracts.CheckParam(!(l2Const < 0), nameof(l2Const), "Must not be negative, if specified.");
Contracts.CheckParam(!(l1Threshold < 0), nameof(l1Threshold), "Must not be negative, if specified.");
Contracts.CheckParam(!(maxIterations < 1), nameof(maxIterations), "Must be positive if specified");
Contracts.CheckValueOrNull(advancedSettings);
Contracts.CheckValueOrNull(onFit);
var rec = new TrainerEstimatorReconciler.BinaryClassifier(
(env, labelName, featuresName, weightsName) =>
{
var trainer = new SdcaBinaryTrainer(env, labelName, featuresName, weightsName, loss: new LogLoss(), l2Const, l1Threshold, maxIterations, advancedSettings);
if (onFit != null)
{
return(trainer.WithOnFitDelegate(trans =>
{
// Under the default log-loss we assume a calibrated predictor.
var model = trans.Model;
var cali = (ParameterMixingCalibratedPredictor)model;
var pred = (LinearBinaryModelParameters)cali.SubPredictor;
onFit(pred, cali);
}));
}
return(trainer);
}, label, features, weights);
return(rec.Output);
}
public static bool TryGet(string resourceKey, out string resourceValue, string locale = null)
{
Contracts.CheckValue(resourceKey, "action");
Contracts.CheckValueOrNull(locale, "locale");
if (locale == null)
{
locale = CurrentLocaleInfo.CurrentUILanguageName;
Contracts.CheckNonEmpty(locale, "currentLocale");
}
Dictionary <string, string> strings;
if (!Strings.TryGetValue(locale, out strings))
{
Dictionary <string, ErrorResource> errorResources;
LoadFromResource(locale, ResourceNamePrefix, typeof(TypeFromThisAssembly), ResourceFileName, ResourceFormat.Resw, out strings, out errorResources);
Strings[locale] = strings;
ErrorResources[locale] = errorResources;
}
return(strings.TryGetValue(resourceKey, out resourceValue) || (ExternalStringResources?.TryGet(resourceKey, out resourceValue, locale) ?? false));
}
/// <summary>
/// Compare equality with the given system string value. Returns false if "this" is NA.
/// </summary>
public bool EqualsStr(string s)
{
Contracts.CheckValueOrNull(s);
// Note that "NA" doesn't match any string.
if (s == null)
{
return(Length == 0);
}
if (s.Length != Length)
{
return(false);
}
for (int i = 0; i < Length; i++)
{
if (s[i] != _outerBuffer[_ichMin + i])
{
return(false);
}
}
return(true);
}
/// <summary>
/// Try to map a System.Type to a corresponding DataKind value.
/// </summary>
public static bool TryGetDataKind(this Type type, out DataKind kind)
{
Contracts.CheckValueOrNull(type);
// REVIEW: Make this more efficient. Should we have a global dictionary?
if (type == typeof(sbyte))
{
kind = DataKind.I1;
}
else if (type == typeof(byte))
{
kind = DataKind.U1;
}
else if (type == typeof(short))
{
kind = DataKind.I2;
}
else if (type == typeof(ushort))
{
kind = DataKind.U2;
}
else if (type == typeof(int))
{
kind = DataKind.I4;
}
else if (type == typeof(uint))
{
kind = DataKind.U4;
}
else if (type == typeof(long))
{
kind = DataKind.I8;
}
else if (type == typeof(ulong))
{
kind = DataKind.U8;
}
else if (type == typeof(Single))
{
kind = DataKind.R4;
}
else if (type == typeof(Double))
{
kind = DataKind.R8;
}
else if (type == typeof(ReadOnlyMemory <char>) || type == typeof(string))
{
kind = DataKind.TX;
}
else if (type == typeof(bool))
{
kind = DataKind.BL;
}
else if (type == typeof(TimeSpan))
{
kind = DataKind.TS;
}
else if (type == typeof(DateTime))
{
kind = DataKind.DT;
}
else if (type == typeof(DateTimeOffset))
{
kind = DataKind.DZ;
}
else if (type == typeof(RowId))
{
kind = DataKind.UG;
}
else
{
kind = default(DataKind);
return(false);
}
return(true);
}
/// <summary>
/// Wraps a set of row columns as a row.
/// </summary>
/// <param name="counted">The counted object that the output row will wrap for its own implementation of
/// <see cref="ICounted"/>, or if null, the output row will yield default values for those implementations,
/// that is, a totally static row</param>
/// <param name="columns">A set of row columns</param>
/// <returns>A row with items derived from <paramref name="columns"/></returns>
public static IRow GetRow(ICounted counted, params IColumn[] columns)
{
Contracts.CheckValueOrNull(counted);
Contracts.CheckValue(columns, nameof(columns));
return(new RowColumnRow(counted, columns));
}
/// <summary>
/// Builds a JSON representation of all entry points and components of the <paramref name="catalog"/>.
/// </summary>
/// <param name="ectx">The exception context to use</param>
/// <param name="catalog">The module catalog</param>
public static JObject BuildAllManifests(IExceptionContext ectx, ModuleCatalog catalog)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(catalog, nameof(catalog));
var jEntryPoints = new JArray();
var entryPointInfos = catalog.AllEntryPoints().ToArray();
foreach (var entryPointInfo in entryPointInfos.OrderBy(x => x.Name))
{
jEntryPoints.Add(BuildEntryPointManifest(ectx, entryPointInfo, catalog));
}
var jKinds = new JArray();
foreach (var kind in catalog.GetAllComponentKinds())
{
var jKind = new JObject();
jKind[FieldNames.Kind] = kind;
var jComponents = new JArray();
foreach (var component in catalog.GetAllComponents(kind))
{
jComponents.Add(BuildComponentManifest(ectx, component, catalog));
}
jKind[FieldNames.Components] = jComponents;
jKinds.Add(jKind);
}
var jepKinds = new JArray();
var kinds = new List <Type>();
foreach (var entryPointInfo in entryPointInfos)
{
if (entryPointInfo.InputKinds != null)
{
kinds.AddRange(entryPointInfo.InputKinds);
}
if (entryPointInfo.OutputKinds != null)
{
kinds.AddRange(entryPointInfo.OutputKinds);
}
}
foreach (var epKind in kinds.Distinct().OrderBy(k => k.Name))
{
var jepKind = new JObject();
jepKind[FieldNames.Kind] = epKind.Name;
var jepKindFields = new JArray();
var propertyInfos = epKind.GetProperties().AsEnumerable();
propertyInfos = epKind.GetInterfaces().Aggregate(propertyInfos, (current, face) => current.Union(face.GetProperties()));
foreach (var fieldInfo in propertyInfos)
{
var jField = new JObject();
jField[FieldNames.Name] = fieldInfo.Name;
var type = CSharpGeneratorUtils.ExtractOptionalOrNullableType(fieldInfo.PropertyType);
// Dive inside Var.
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>))
{
type = type.GetGenericArguments()[0];
}
var typeEnum = TlcModule.GetDataType(type);
jField[FieldNames.Type] = typeEnum.ToString();
jepKindFields.Add(jField);
}
jepKind[FieldNames.Settings] = jepKindFields;
jepKinds.Add(jepKind);
}
var jResult = new JObject();
jResult[FieldNames.TopEntryPoints] = jEntryPoints;
jResult[FieldNames.TopComponents] = jKinds;
jResult[FieldNames.TopEntryPointKinds] = jepKinds;
return(jResult);
}
/// <summary>
/// Splits this instance on the left-most occurrence of an element of separators character array and
/// produces the left and right <see cref="DvText"/> values. If this instance does not contain any of the
/// characters in separators, thiss return false and initializes <paramref name="left"/> to this instance
/// and <paramref name="right"/> to the default <see cref="DvText"/> value.
/// </summary>
public bool SplitOne(char[] separators, out DvText left, out DvText right)
{
Contracts.CheckValueOrNull(separators);
if (!HasChars || separators == null || separators.Length == 0)
{
left = this;
right = default(DvText);
return(false);
}
string text = _outerBuffer;
int ichMin = _ichMin;
int ichLim = IchLim;
int ichCur = ichMin;
if (separators.Length == 1)
{
// Note: This duplicates code of the other SplitOne, but doing so improves perf because this is
// used so heavily in instances parsing.
char chSep = separators[0];
for (; ; ichCur++)
{
Contracts.Assert(ichMin <= ichCur && ichCur <= ichLim);
if (ichCur >= ichLim)
{
left = this;
right = default(DvText);
return(false);
}
if (text[ichCur] == chSep)
{
break;
}
}
}
else
{
for (; ; ichCur++)
{
Contracts.Assert(ichMin <= ichCur && ichCur <= ichLim);
if (ichCur >= ichLim)
{
left = this;
right = default(DvText);
return(false);
}
// REVIEW: Can this be faster?
if (ContainsChar(text[ichCur], separators))
{
break;
}
}
}
// Note that we don't use any fields of "this" here in case one
// of the out parameters is the same as "this".
left = new DvText(text, _ichMin, ichCur);
right = new DvText(text, ichCur + 1, ichLim);
return(true);
}
public override bool TryGetColumnIndex(string name, out int col)
{
Contracts.CheckValueOrNull(name);
col = 0;
return(name == MetadataUtils.Const.ScoreValueKind.PredictedLabel);
}
internal IntellisenseResult(IIntellisenseData data, List <IntellisenseSuggestion> suggestions, Exception exception = null)
{
Contracts.AssertValue(suggestions);
_script = data.Script;
Contracts.CheckValue(_script, "script");
ReplacementStartIndex = data.ReplacementStartIndex;
Contracts.CheckParam(0 <= data.ReplacementStartIndex, "replacementStartIndex");
ReplacementLength = data.ReplacementLength;
Contracts.CheckParam(0 <= data.ReplacementLength, "replacementLength");
var argIndex = data.ArgIndex;
var argCount = data.ArgCount;
Contracts.CheckParam(0 <= argIndex, "argIndex");
Contracts.CheckParam(0 <= argCount, "argCount");
Contracts.Check(argIndex <= argCount, "argIndex out of bounds.");
var func = data.CurFunc;
Contracts.CheckValueOrNull(func);
_suggestions = suggestions;
_functionSignatures = new List <SignatureInformation>();
_functionOverloads = new List <IntellisenseSuggestion>();
CurrentFunctionOverloadIndex = -1;
_currentArgumentIndex = argIndex;
Exception = exception;
if (func == null)
{
IsFunctionScope = false;
}
else
{
IsFunctionScope = true;
int highlightStart = -1;
int highlightEnd = -1;
int minMatchingArgCount = int.MaxValue;
foreach (var signature in func.GetSignatures(argCount))
{
int signatureIndex = 0;
string argumentSeparator = string.Empty;
string highlightedFuncParamDescription = string.Empty;
string listSep = TexlLexer.LocalizedInstance.LocalizedPunctuatorListSeparator + " ";
StringBuilder funcDisplayString = new StringBuilder(func.Name);
funcDisplayString.Append('(');
var parameters = new List <ParameterInformation>();
while (signatureIndex < signature.Count())
{
Contracts.AssertValue(signature[signatureIndex]);
funcDisplayString.Append(argumentSeparator);
// We need to change the highlight information if the argument should be highlighted, but
// otherwise we still want to collect parameter information
var unalteredParamName = signature[signatureIndex]();
var invariantParamName = signature[signatureIndex]("en-US");
(var paramName, var parameterHighlightStart, var parameterHighlightEnd, var funcParamDescription) = GetParameterHighlightAndDescription(data, unalteredParamName, invariantParamName, funcDisplayString);
parameters.Add(new ParameterInformation()
{
Documentation = funcParamDescription,
Label = paramName
});
if (ArgNeedsHighlight(func, argCount, argIndex, signature.Count(), signatureIndex))
{
(highlightStart, highlightEnd, highlightedFuncParamDescription) = (parameterHighlightStart, parameterHighlightEnd, funcParamDescription);
}
// For variadic function, we want to generate FuncName(arg1,arg1,...,arg1,...) as description.
if (func.SignatureConstraint != null && argCount > func.SignatureConstraint.RepeatTopLength && canParamOmit(func, argCount, argIndex, signature.Count(), signatureIndex))
{
funcDisplayString.Append("...");
signatureIndex += func.SignatureConstraint.RepeatSpan;
}
else
{
funcDisplayString.Append(signature[signatureIndex]());
signatureIndex++;
}
argumentSeparator = listSep;
}
if (func.MaxArity > func.MinArity && func.MaxArity > argCount)
{
funcDisplayString.Append(argumentSeparator + "...");
}
funcDisplayString.Append(')');
var signatureInformation = new SignatureInformation()
{
Documentation = func.Description,
Label = CreateFunctionSignature(func.Name, parameters),
Parameters = parameters.ToArray()
};
_functionSignatures.Add(signatureInformation);
_functionOverloads.Add(new IntellisenseSuggestion(new UIString(funcDisplayString.ToString(), highlightStart, highlightEnd), SuggestionKind.Function, SuggestionIconKind.Function, func.ReturnType, signatureIndex, func.Description, func.Name, highlightedFuncParamDescription));
if ((signatureIndex >= argCount || (func.SignatureConstraint != null && argCount > func.SignatureConstraint.RepeatTopLength)) && minMatchingArgCount > signatureIndex)
{
// _functionOverloads has at least one item at this point.
CurrentFunctionOverloadIndex = _functionOverloads.Count - 1;
minMatchingArgCount = signatureIndex;
}
}
// Handling of case where the function does not take any arguments.
if (_functionOverloads.Count == 0 && func.MinArity == 0)
{
var signatureInformation = new SignatureInformation()
{
Documentation = func.Description,
Label = CreateFunctionSignature(func.Name),
Parameters = new ParameterInformation[0]
};
_functionSignatures.Add(signatureInformation);
_functionOverloads.Add(new IntellisenseSuggestion(new UIString(func.Name + "()", 0, func.Name.Length + 1), SuggestionKind.Function, SuggestionIconKind.Function, func.ReturnType, string.Empty, 0, func.Description, func.Name));
CurrentFunctionOverloadIndex = 0;
}
}
Contracts.Assert(_functionSignatures.Count == _functionOverloads.Count);
}
/// <summary>
/// Try to map a System.Type to a corresponding DataKind value.
/// </summary>
public static bool TryGetDataKind(this Type type, out DataKind kind)
{
Contracts.CheckValueOrNull(type);
// REVIEW: Make this more efficient. Should we have a global dictionary?
if (type == typeof(DvInt1) || type == typeof(sbyte) || type == typeof(sbyte?))
{
kind = DataKind.I1;
}
else if (type == typeof(byte) || type == typeof(byte?))
{
kind = DataKind.U1;
}
else if (type == typeof(DvInt2) || type == typeof(short) || type == typeof(short?))
{
kind = DataKind.I2;
}
else if (type == typeof(ushort) || type == typeof(ushort?))
{
kind = DataKind.U2;
}
else if (type == typeof(DvInt4) || type == typeof(int) || type == typeof(int?))
{
kind = DataKind.I4;
}
else if (type == typeof(uint) || type == typeof(uint?))
{
kind = DataKind.U4;
}
else if (type == typeof(DvInt8) || type == typeof(long) || type == typeof(long?))
{
kind = DataKind.I8;
}
else if (type == typeof(ulong) || type == typeof(ulong?))
{
kind = DataKind.U8;
}
else if (type == typeof(Single) || type == typeof(Single?))
{
kind = DataKind.R4;
}
else if (type == typeof(Double) || type == typeof(Double?))
{
kind = DataKind.R8;
}
else if (type == typeof(DvText))
{
kind = DataKind.TX;
}
else if (type == typeof(DvBool) || type == typeof(bool) || type == typeof(bool?))
{
kind = DataKind.BL;
}
else if (type == typeof(DvTimeSpan))
{
kind = DataKind.TS;
}
else if (type == typeof(DvDateTime))
{
kind = DataKind.DT;
}
else if (type == typeof(DvDateTimeZone))
{
kind = DataKind.DZ;
}
else if (type == typeof(UInt128))
{
kind = DataKind.UG;
}
else
{
kind = default(DataKind);
return(false);
}
return(true);
}
/// <summary>
/// Load an image from an input column that holds the paths to images.
/// </summary>
/// <param name="path">The scalar text column that holds paths to the images</param>
/// <param name="relativeTo">If specified, paths are considered to be relative to this directory.
/// However, since the transform can be persisted across machines, it is generally considered more
/// safe for users to simply always make their input paths absolute.</param>
/// <returns>The loaded images</returns>
/// <seealso cref="ImageLoaderEstimator"/>
public static Scalar <UnknownSizeBitmap> LoadAsImage(this Scalar <string> path, string relativeTo = null)
{
Contracts.CheckValue(path, nameof(path));
Contracts.CheckValueOrNull(relativeTo);
return(new ImageLoaderEstimator.OutPipelineColumn(path, relativeTo));
}
/// <summary>
/// Load an image from an input column that holds the paths to images.
/// </summary>
/// <param name="path">The scalar text column that holds paths to the images</param>
/// <param name="relativeTo">If specified, paths are considered to be relative to this directory.
/// However, since the transform can be persisted across machines, it is generally considered more
/// safe for users to simply always make their input paths absolute.</param>
/// <returns>The loaded images</returns>
/// <seealso cref="ImageLoadingEstimator"/>
public static Custom <UnknownSizeBitmap> LoadAsImage(this Scalar <string> path, string relativeTo = null)
{
Contracts.CheckValue(path, nameof(path));
Contracts.CheckValueOrNull(relativeTo);
return(new ImageLoadingStaticExtensions.OutPipelineColumn(path, relativeTo));
}
/// <summary>
/// This only knows about column zero, the Score column. Derived classes should handle all others.
/// </summary>
public virtual bool TryGetColumnIndex(string name, out int col)
{
Contracts.CheckValueOrNull(name);
col = 0;
return(name == MetadataUtils.Const.ScoreValueKind.Score);
}
public GetterImpl(string name, IRow meta, ColumnType type, ValueGetter <T> getter)
: base(name, meta, type)
{
Contracts.CheckValueOrNull(getter);
_getter = getter;
}
public IEnumerable <DvText> Split(char[] separators)
{
Contracts.CheckValueOrNull(separators);
if (!HasChars)
{
yield break;
}
if (separators == null || separators.Length == 0)
{
yield return(this);
yield break;
}
string text = _outerBuffer;
int ichLim = IchLim;
if (separators.Length == 1)
{
char chSep = separators[0];
for (int ichCur = _ichMin; ;)
{
int ichMin = ichCur;
for (; ; ichCur++)
{
Contracts.Assert(ichCur <= ichLim);
if (ichCur >= ichLim)
{
yield return(new DvText(text, ichMin, ichCur));
yield break;
}
if (text[ichCur] == chSep)
{
break;
}
}
yield return(new DvText(text, ichMin, ichCur));
// Skip the separator.
ichCur++;
}
}
else
{
for (int ichCur = _ichMin; ;)
{
int ichMin = ichCur;
for (; ; ichCur++)
{
Contracts.Assert(ichCur <= ichLim);
if (ichCur >= ichLim)
{
yield return(new DvText(text, ichMin, ichCur));
yield break;
}
// REVIEW: Can this be faster?
if (ContainsChar(text[ichCur], separators))
{
break;
}
}
yield return(new DvText(text, ichMin, ichCur));
// Skip the separator.
ichCur++;
}
}
}
public static RepositoryReader Open(Stream stream, IExceptionContext ectx = null, bool useFileSystem = true)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(stream, nameof(stream));
return(new RepositoryReader(stream, ectx, useFileSystem));
}
