protected GetterInfo(string kind, ColumnType type)
{
Contracts.CheckNonWhiteSpace(kind, nameof(kind), "Invalid metadata kind");
Contracts.CheckValue(type, nameof(type));
Kind = kind;
Type = type;
}
public ColumnMetadataInfo(string name)
{
Contracts.CheckNonWhiteSpace(name, nameof(name));
Name = name;
_infos = new Dictionary <string, MetadataInfo>();
}
internal ColumnInfoBase(ModelLoadContext ctx, string input, string output, bool normKindSerialized)
{
Contracts.AssertValue(ctx);
Contracts.CheckNonWhiteSpace(input, nameof(input));
Contracts.CheckNonWhiteSpace(output, nameof(output));
Input = input;
Output = output;
// *** Binary format ***
// byte: SubtractMean
// byte: NormKind
// Float: Scale
SubtractMean = ctx.Reader.ReadBoolByte();
byte normKindVal = ctx.Reader.ReadByte();
Contracts.CheckDecode(Enum.IsDefined(typeof(LpNormalizingEstimatorBase.NormalizerKind), normKindVal));
NormKind = (LpNormalizingEstimatorBase.NormalizerKind)normKindVal;
// Note: In early versions, a bool option (useStd) to whether to normalize by StdDev rather than
// L2 norm was used. normKind was added in version=verVectorNormalizerSupported.
// normKind was defined in a way such that the serialized boolean (0: use StdDev, 1: use L2) is
// still valid.
Contracts.CheckDecode(normKindSerialized ||
(NormKind == LpNormalizingEstimatorBase.NormalizerKind.L2Norm || NormKind == LpNormalizingEstimatorBase.NormalizerKind.StdDev));
Scale = ctx.Reader.ReadFloat();
Contracts.CheckDecode(0 < Scale && Scale < float.PositiveInfinity);
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="name">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of column to transform. If set to <see langword="null"/>, the value of the <paramref name="name"/> will be used as source.</param>
/// <param name="bag">Whether to combine multiple indicator vectors into a single bag vector instead of concatenating them. This is only relevant when the input column is a vector.</param>
public ColumnInfo(string name, string inputColumnName = null, bool bag = KeyToVectorMappingEstimator.Defaults.Bag)
{
Contracts.CheckNonWhiteSpace(name, nameof(name));
Name = name;
InputColumnName = inputColumnName ?? name;
Bag = bag;
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="input">Name of input column.</param>
/// <param name="output">Name of the column resulting from the transformation of <paramref name="input"/>. Null means <paramref name="input"/> is replaced.</param>
/// <param name="bag">Whether to combine multiple indicator vectors into a single bag vector instead of concatenating them. This is only relevant when the input column is a vector.</param>
public ColumnInfo(string input, string output = null, bool bag = KeyToVectorMappingEstimator.Defaults.Bag)
{
Contracts.CheckNonWhiteSpace(input, nameof(input));
Input = input;
Output = output ?? input;
Bag = bag;
}
public Bindings(ArgumentsBase args, bool keep, ISchema schemaInput)
{
Contracts.AssertValue(args);
Contracts.AssertNonEmpty(args.Columns);
Contracts.AssertValue(schemaInput);
Keep = keep;
Input = schemaInput;
Names = new HashSet <string>();
for (int i = 0; i < args.Columns.Length; i++)
{
var name = args.Columns[i];
Contracts.CheckNonWhiteSpace(name, nameof(args.Columns));
// REVIEW: Should this just be a warning?
if (!Names.Add(name))
{
throw Contracts.ExceptUserArg(nameof(args.Columns), "Column '{0}' specified multiple times", name);
}
}
BuildMap(out ColMap, out NameToCol);
AsSchema = Schema.Create(this);
}
/// <summary>
/// Opens a stream writer for the specified file using the specified encoding and buffer size.
/// If the file exists, it can be either overwritten or appended to.
/// If the file does not exist, a new file is created.
/// </summary>
/// <param name="path">The complete file path to write to.</param>
/// <param name="append">
/// true to append data to the file; false to overwrite the file.
/// If the specified file does not exist, this parameter has no effect and a new file is created.
/// </param>
/// <param name="encoding">The character encoding to use.</param>
/// <param name="bufferSize">The buffer size, in bytes.</param>
/// <returns>The stream writer to write to the specified file.</returns>
protected static StreamWriter OpenWriter(string path, bool append = false, Encoding encoding = null, int bufferSize = 1024)
{
Contracts.CheckNonWhiteSpace(path, nameof(path));
#if CORECLR
return(Utils.OpenWriter(File.Open(path, append ? FileMode.Append : FileMode.OpenOrCreate), encoding, bufferSize, false));
#else
return(new StreamWriter(path, append));
#endif
}
/// <summary>
/// Opens a stream reader for the specified file name, with the specified character encoding, byte order mark detection option, and buffer size.
/// </summary>
/// <param name="path">The complete file path to be read.</param>
/// <returns>The stream reader to read from the specified file.</returns>
protected static StreamReader OpenReader(string path)
{
Contracts.CheckNonWhiteSpace(path, nameof(path));
#if CORECLR
return(new StreamReader(File.Open(path, FileMode.Open, FileAccess.Read)));
#else
return(new StreamReader(path));
#endif
}
/// <summary>
/// Generic facilities for calling a function.
/// </summary>
/// <param name="func">The function to call</param>
/// <param name="prms">The parameters for the function</param>
/// <returns></returns>
public static JObject Call(string func, params JToken[] prms)
{
Contracts.CheckNonWhiteSpace(func, nameof(func));
Contracts.CheckValue(prms, nameof(prms));
var retval = new JObject();
retval[func] = new JArray(prms);
return(retval);
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="input">Name of input column.</param>
/// <param name="output">Name of the column resulting from the transformation of <paramref name="input"/>. Null means <paramref name="input"/> is replaced.</param>
/// <param name="replacementMode">What to replace the missing value with.</param>
/// <param name="imputeBySlot">If true, per-slot imputation of replacement is performed.
/// Otherwise, replacement value is imputed for the entire vector column. This setting is ignored for scalars and variable vectors,
/// where imputation is always for the entire column.</param>
public ColumnInfo(string input, string output = null, ReplacementMode replacementMode = MissingValueReplacingEstimator.Defaults.ReplacementMode,
bool imputeBySlot = MissingValueReplacingEstimator.Defaults.ImputeBySlot)
{
Contracts.CheckNonWhiteSpace(input, nameof(input));
Input = input;
Output = output ?? input;
ImputeBySlot = imputeBySlot;
Replacement = replacementMode;
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="name">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of column to transform. If set to <see langword="null"/>, the value of the <paramref name="name"/> will be used as source.</param>
/// <param name="replacementMode">What to replace the missing value with.</param>
/// <param name="imputeBySlot">If true, per-slot imputation of replacement is performed.
/// Otherwise, replacement value is imputed for the entire vector column. This setting is ignored for scalars and variable vectors,
/// where imputation is always for the entire column.</param>
public ColumnInfo(string name, string inputColumnName = null, ReplacementMode replacementMode = MissingValueReplacingEstimator.Defaults.ReplacementMode,
bool imputeBySlot = MissingValueReplacingEstimator.Defaults.ImputeBySlot)
{
Contracts.CheckNonWhiteSpace(name, nameof(name));
Name = name;
InputColumnName = inputColumnName ?? name;
ImputeBySlot = imputeBySlot;
Replacement = replacementMode;
}
public RepositoryStreamWrapper(RepositoryReader repository, string directory, string filename)
{
Contracts.CheckValue(repository, nameof(repository));
Contracts.CheckNonWhiteSpace(directory, nameof(directory));
Contracts.CheckNonWhiteSpace(filename, nameof(filename));
_repository = repository;
_directory = directory;
_filename = filename;
}
private protected ColumnOptionsBase(string outputColumnName, string inputColumnName,
int maximumNumberOfKeys, KeyOrdinality keyOrdinality, bool addKeyValueAnnotationsAsText)
{
Contracts.CheckNonWhiteSpace(outputColumnName, nameof(outputColumnName));
OutputColumnName = outputColumnName;
InputColumnName = inputColumnName ?? outputColumnName;
KeyOrdinality = keyOrdinality;
MaximumNumberOfKeys = maximumNumberOfKeys;
AddKeyValueAnnotationsAsText = addKeyValueAnnotationsAsText;
}
private protected ColumnOptionsBase(string outputColumnName, string inputColumnName,
int maxNumKeys, SortOrder sort, string[] term, bool textKeyValues)
{
Contracts.CheckNonWhiteSpace(outputColumnName, nameof(outputColumnName));
OutputColumnName = outputColumnName;
InputColumnName = inputColumnName ?? outputColumnName;
Sort = sort;
MaxNumKeys = maxNumKeys;
TermArray = term;
TextKeyValues = textKeyValues;
}
internal ColumnInfoBase(string input, string output, bool substractMean, LpNormalizingEstimatorBase.NormalizerKind normalizerKind, float scale)
{
Contracts.CheckNonWhiteSpace(input, nameof(input));
Contracts.CheckNonWhiteSpace(output, nameof(output));
Input = input;
Output = output;
SubtractMean = substractMean;
Contracts.CheckUserArg(0 < scale && scale < float.PositiveInfinity, nameof(scale), "scale must be a positive finite value");
Scale = scale;
NormKind = normalizerKind;
}
public static Range Parse(string str)
{
Contracts.CheckNonWhiteSpace(str, nameof(str));
var res = new Range();
if (res.TryParse(str))
{
return(res);
}
return(null);
}
internal static Column Parse(string str)
{
Contracts.CheckNonWhiteSpace(str, nameof(str));
var res = new Column();
if (res.TryParse(str))
{
return(res);
}
return(null);
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
/// <param name="maxNumKeys">Maximum number of keys to keep per column when auto-training.</param>
/// <param name="sort">How items should be ordered when vectorized. If <see cref="SortOrder.Occurrence"/> choosen they will be in the order encountered.
/// If <see cref="SortOrder.Value"/>, items are sorted according to their default comparison, for example, text sorting will be case sensitive (for example, 'A' then 'Z' then 'a').</param>
/// <param name="term">List of terms.</param>
/// <param name="textKeyValues">Whether key value metadata should be text, regardless of the actual input type.</param>
public ColumnOptions(string outputColumnName, string inputColumnName = null,
int maxNumKeys = Defaults.MaxNumKeys,
SortOrder sort = Defaults.Sort,
string[] term = null,
bool textKeyValues = false
)
{
Contracts.CheckNonWhiteSpace(outputColumnName, nameof(outputColumnName));
OutputColumnName = outputColumnName;
InputColumnName = inputColumnName ?? outputColumnName;
Sort = sort;
MaxNumKeys = maxNumKeys;
Term = term;
TextKeyValues = textKeyValues;
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="name">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="name"/> will be used as source.</param>
/// <param name="maxNumTerms">Maximum number of terms to keep per column when auto-training.</param>
/// <param name="sort">How items should be ordered when vectorized. If <see cref="SortOrder.Occurrence"/> choosen they will be in the order encountered.
/// If <see cref="SortOrder.Value"/>, items are sorted according to their default comparison, for example, text sorting will be case sensitive (for example, 'A' then 'Z' then 'a').</param>
/// <param name="term">List of terms.</param>
/// <param name="textKeyValues">Whether key value metadata should be text, regardless of the actual input type.</param>
public ColumnInfo(string name, string inputColumnName = null,
int maxNumTerms = ValueToKeyMappingEstimator.Defaults.MaxNumTerms,
SortOrder sort = ValueToKeyMappingEstimator.Defaults.Sort,
string[] term = null,
bool textKeyValues = false
)
{
Contracts.CheckNonWhiteSpace(name, nameof(name));
Name = name;
InputColumnName = inputColumnName ?? name;
Sort = sort;
MaxNumTerms = maxNumTerms;
Term = term;
TextKeyValues = textKeyValues;
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="input">Name of input column.</param>
/// <param name="output">Name of the column resulting from the transformation of <paramref name="input"/>. Null means <paramref name="input"/> is replaced.</param>
/// <param name="maxNumTerms">Maximum number of terms to keep per column when auto-training.</param>
/// <param name="sort">How items should be ordered when vectorized. If <see cref="SortOrder.Occurrence"/> choosen they will be in the order encountered.
/// If <see cref="SortOrder.Value"/>, items are sorted according to their default comparison, for example, text sorting will be case sensitive (for example, 'A' then 'Z' then 'a').</param>
/// <param name="term">List of terms.</param>
/// <param name="textKeyValues">Whether key value metadata should be text, regardless of the actual input type.</param>
public ColumnInfo(string input, string output = null,
int maxNumTerms = ValueToKeyMappingEstimator.Defaults.MaxNumTerms,
SortOrder sort = ValueToKeyMappingEstimator.Defaults.Sort,
string[] term = null,
bool textKeyValues = false
)
{
Contracts.CheckNonWhiteSpace(input, nameof(input));
Input = input;
Output = output ?? input;
Sort = sort;
MaxNumTerms = maxNumTerms;
Term = term;
TextKeyValues = textKeyValues;
}
/// <summary>
/// Describes how the transformer handles one input-output column pair.
/// </summary>
/// <param name="name">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="inputColumnName">Name of column to transform. If set to <see langword="null"/>, the value of the <paramref name="name"/> will be used as source.</param>
/// <param name="colors">What colors to extract.</param>
/// <param name="interleave"></param>
/// <param name="scale">Scale color pixel value by this amount.</param>
/// <param name="offset">Offset color pixel value by this amount.</param>
/// <param name="asFloat">Output array as float array. If false, output as byte array.</param>
public ColumnInfo(string name, string inputColumnName = null,
ColorBits colors = Defaults.Colors,
bool interleave = Defaults.Interleave,
float scale = Defaults.Scale,
float offset = Defaults.Offset,
bool asFloat = Defaults.Convert)
{
Contracts.CheckNonWhiteSpace(name, nameof(name));
Name = name;
InputColumnName = inputColumnName ?? name;
Colors = colors;
if ((Colors & ColorBits.Alpha) == ColorBits.Alpha)
{
Planes++;
}
if ((Colors & ColorBits.Red) == ColorBits.Red)
{
Planes++;
}
if ((Colors & ColorBits.Green) == ColorBits.Green)
{
Planes++;
}
if ((Colors & ColorBits.Blue) == ColorBits.Blue)
{
Planes++;
}
Contracts.CheckParam(Planes > 0, nameof(colors), "Need to use at least one color plane");
Interleave = interleave;
AsFloat = asFloat;
if (!AsFloat)
{
Offset = Defaults.Offset;
Scale = Defaults.Scale;
}
else
{
Offset = offset;
Scale = scale;
}
Contracts.CheckParam(FloatUtils.IsFinite(Offset), nameof(offset));
Contracts.CheckParam(FloatUtils.IsFiniteNonZero(Scale), nameof(scale));
}
/// <summary>
/// Parse all column values from the directory path.
/// </summary>
/// <param name="path">The directory path to parse for name/value pairs.</param>
/// <param name="results">The resulting name value pairs.</param>
/// <returns>true if the parsing was successfull.</returns>
private bool TryParseValuesFromPath(string path, out List <string> results)
{
Contracts.CheckNonWhiteSpace(path, nameof(path));
results = null;
try
{
results = _parent._pathParser.ParseValues(path).ToList();
return(true);
}
catch (InvalidOperationException e)
{
Ch.Warning($"Could not parse column values from the path {path}. Ex: {e.Message}");
results = null;
return(false);
}
}
/// <summary>
/// For parsing name and source from a string. This supports "name" and "name:source" and "name:extra:source".
/// For the last form, the out extra parameter is set accordingly. For the other forms, extra is set to null.
/// </summary>
public static bool TryParse(string str, out string name, out string source, out string extra)
{
Contracts.CheckNonWhiteSpace(str, nameof(str));
extra = null;
int ich = str.IndexOf(':');
if (ich < 0)
{
name = str;
source = str;
return(true);
}
if (ich == 0 || ich >= str.Length - 1)
{
name = null;
source = null;
return(false);
}
name = str.Substring(0, ich);
int ichMin = ich + 1;
ich = str.LastIndexOf(':');
Contracts.Assert(ich >= ichMin - 1);
if (ich == ichMin - 1)
{
source = str.Substring(ichMin);
return(true);
}
if (ich == ichMin || ich >= str.Length - 1)
{
name = null;
source = null;
return(false);
}
extra = str.Substring(ichMin, ich - ichMin);
source = str.Substring(ich + 1);
return(true);
}
/// <summary>
/// For parsing name and source from a string. This supports "name" and "name:source".
/// </summary>
public static bool TryParse(string str, out string name, out string source)
{
Contracts.CheckNonWhiteSpace(str, nameof(str));
int ich = str.IndexOf(':');
if (ich < 0)
{
name = str;
source = str;
return(true);
}
if (0 < ich && ich < str.Length - 1)
{
name = str.Substring(0, ich);
source = str.Substring(ich + 1);
return(!source.Contains(":"));
}
name = null;
source = null;
return(false);
}
public void Rename(string newName)
{
Contracts.CheckNonWhiteSpace(newName, nameof(newName));
VariableName = newName;
}
/// <summary>
/// Describes how the transformer handles one column pair.
/// </summary>
/// <param name="input">Name of input column.</param>
/// <param name="output">Name of the column resulting from the transformation of <paramref name="input"/>. Null means <paramref name="input"/> is replaced.</param>
public ColumnInfo(string input, string output = null)
{
Contracts.CheckNonWhiteSpace(input, nameof(input));
Input = input;
Output = output ?? input;
}
public static JObject FuncRef(string func)
{
Contracts.CheckNonWhiteSpace(func, nameof(func));
return(((JObject)null).AddReturn("fcn", func));
}
public static JObject Cell(string name)
{
Contracts.CheckNonWhiteSpace(name, nameof(name));
return(((JObject)null).AddReturn("cell", name));
}
/// <summary>
/// Opens a stream reader for the specified file name, with the specified character encoding, byte order mark detection option, and buffer size.
/// </summary>
/// <param name="path">The complete file path to be read.</param>
/// <returns>The stream reader to read from the specified file.</returns>
protected static StreamReader OpenReader(string path)
{
Contracts.CheckNonWhiteSpace(path, nameof(path));
return(new StreamReader(File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)));
}
/// <summary>
/// Start building metadata for a column that passes through metadata of the given kind from
/// a source column.
/// </summary>
public Builder BuildMetadata(int index, Schema schemaSrc, int indexSrc, string kindSrc)
{
Contracts.CheckValue(schemaSrc, nameof(schemaSrc));
Contracts.CheckNonWhiteSpace(kindSrc, nameof(kindSrc));
return(new Builder(this, index, schemaSrc, indexSrc, (k, i) => k == kindSrc));
}