コード例 #1
0
 public ColumnInfo(string input, string output, OneHotEncodingTransformer.OutputKind outputKind = Defaults.OutKind,
                   int maxNumTerms = ValueToKeyMappingEstimator.Defaults.MaxNumTerms, TermTransform.SortOrder sort = ValueToKeyMappingEstimator.Defaults.Sort,
                   string[] term   = null)
     : base(input, output, maxNumTerms, sort, term, true)
 {
     OutputKind = outputKind;
 }
コード例 #2
0
 /// <summary>
 /// Convert a text column into hash-based one-hot encoded vector.
 /// </summary>
 /// <param name="catalog">The transform catalog</param>
 /// <param name="inputColumn">The input column</param>
 /// <param name="outputColumn">The output column. If <c>null</c>, <paramref name="inputColumn"/> is used.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
 /// <param name="invertHash">During hashing we constuct mappings between original values and the produced hash values.
 /// Text representation of original values are stored in the slot names of the  metadata for the new column.Hashing, as such, can map many initial values to one.
 /// <paramref name="invertHash"/> specifies the upper bound of the number of distinct input values mapping to a hash that should be retained.
 /// <value>0</value> does not retain any input values. <value>-1</value> retains all input values mapping to each hash.</param>
 /// <param name="outputKind">The conversion mode.</param>
 /// <returns></returns>
 public static OneHotHashEncodingEstimator OneHotHashEncoding(this TransformsCatalog.CategoricalTransforms catalog,
                                                              string inputColumn,
                                                              string outputColumn = null,
                                                              int hashBits        = OneHotHashEncodingEstimator.Defaults.HashBits,
                                                              int invertHash      = OneHotHashEncodingEstimator.Defaults.InvertHash,
                                                              OneHotEncodingTransformer.OutputKind outputKind = OneHotEncodingTransformer.OutputKind.Ind)
 => new OneHotHashEncodingEstimator(CatalogUtils.GetEnvironment(catalog), inputColumn, outputColumn, hashBits, invertHash, outputKind);
コード例 #3
0
 /// <summary>
 /// Instantiates a new instance of <see cref="OneHotHashEncodingEstimator"/>.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
 /// <param name="inputColumnName">Name of the column to transform.
 /// If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
 /// <param name="invertHash">During hashing we constuct mappings between original values and the produced hash values.
 /// Text representation of original values are stored in the slot names of the  metadata for the new column.Hashing, as such, can map many initial values to one.
 /// <paramref name="invertHash"/> specifies the upper bound of the number of distinct input values mapping to a hash that should be retained.
 /// <value>0</value> does not retain any input values. <value>-1</value> retains all input values mapping to each hash.</param>
 /// <param name="outputKind">The type of output expected.</param>
 internal OneHotHashEncodingEstimator(IHostEnvironment env,
                                      string outputColumnName,
                                      string inputColumnName = null,
                                      int hashBits           = Defaults.HashBits,
                                      int invertHash         = Defaults.InvertHash,
                                      OneHotEncodingTransformer.OutputKind outputKind = Defaults.OutputKind)
     : this(env, new ColumnInfo(outputColumnName, inputColumnName ?? outputColumnName, outputKind, hashBits, invertHash : invertHash))
 {
 }
コード例 #4
0
 /// A helper method to create <see cref="OneHotHashEncodingEstimator"/> for public facing API.
 /// <param name="env">Host Environment.</param>
 /// <param name="inputColumn">Name of the input column.</param>
 /// <param name="outputColumn">Name of the output column. If this is null '<paramref name="inputColumn"/>' will be used.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
 /// <param name="invertHash">During hashing we constuct mappings between original values and the produced hash values.
 /// Text representation of original values are stored in the slot names of the  metadata for the new column.Hashing, as such, can map many initial values to one.
 /// <paramref name="invertHash"/> specifies the upper bound of the number of distinct input values mapping to a hash that should be retained.
 /// <value>0</value> does not retain any input values. <value>-1</value> retains all input values mapping to each hash.</param>
 /// <param name="outputKind">The type of output expected.</param>
 public OneHotHashEncodingEstimator(IHostEnvironment env,
                                    string inputColumn,
                                    string outputColumn,
                                    int hashBits   = OneHotHashEncodingEstimator.Defaults.HashBits,
                                    int invertHash = OneHotHashEncodingEstimator.Defaults.InvertHash,
                                    OneHotEncodingTransformer.OutputKind outputKind = Defaults.OutputKind)
     : this(env, new ColumnInfo(inputColumn, outputColumn ?? inputColumn, outputKind, hashBits, invertHash : invertHash))
 {
 }
コード例 #5
0
 /// <summary>
 /// Describes how the transformer handles one column pair.
 /// </summary>
 /// <param name="name">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
 /// <param name="inputColumnName">Name of column to transform. If set to <see langword="null"/>, the value of the <paramref name="name"/> will be used as source.</param>
 /// <param name="outputKind">Kind of output: bag, indicator vector etc.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 /// <param name="seed">Hashing seed.</param>
 /// <param name="ordered">Whether the position of each term should be included in the hash.</param>
 /// <param name="invertHash">During hashing we constuct mappings between original values and the produced hash values.
 /// Text representation of original values are stored in the slot names of the  metadata for the new column.Hashing, as such, can map many initial values to one.
 /// <paramref name="invertHash"/> specifies the upper bound of the number of distinct input values mapping to a hash that should be retained.
 /// <value>0</value> does not retain any input values. <value>-1</value> retains all input values mapping to each hash.</param>
 public ColumnInfo(string name, string inputColumnName             = null,
                   OneHotEncodingTransformer.OutputKind outputKind = Defaults.OutputKind,
                   int hashBits   = Defaults.HashBits,
                   uint seed      = Defaults.Seed,
                   bool ordered   = Defaults.Ordered,
                   int invertHash = Defaults.InvertHash)
 {
     HashInfo   = new HashingEstimator.ColumnInfo(name, inputColumnName ?? name, hashBits, seed, ordered, invertHash);
     OutputKind = outputKind;
 }
コード例 #6
0
 /// <summary>
 /// A helper method to create <see cref="OneHotHashEncodingTransformer"/>.
 /// </summary>
 /// <param name="env">Host Environment.</param>
 /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
 /// <param name="name">Name of the output column.</param>
 /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 30, inclusive.</param>
 /// <param name="invertHash">During hashing we constuct mappings between original values and the produced hash values.
 /// Text representation of original values are stored in the slot names of the  metadata for the new column.Hashing, as such, can map many initial values to one.
 /// <paramref name="invertHash"/> specifies the upper bound of the number of distinct input values mapping to a hash that should be retained.
 /// <value>0</value> does not retain any input values. <value>-1</value> retains all input values mapping to each hash.</param>
 /// <param name="outputKind">The type of output expected.</param>
 private static IDataView Create(IHostEnvironment env,
                                 IDataView input,
                                 string name,
                                 string source  = null,
                                 int hashBits   = OneHotHashEncodingEstimator.Defaults.HashBits,
                                 int invertHash = OneHotHashEncodingEstimator.Defaults.InvertHash,
                                 OneHotEncodingTransformer.OutputKind outputKind = OneHotHashEncodingEstimator.Defaults.OutputKind)
 {
     return(new OneHotHashEncodingEstimator(env, name, source, hashBits, invertHash, outputKind).Fit(input).Transform(input) as IDataView);
 }
コード例 #7
0
 /// <summary>
 /// Describes how the transformer handles one column pair.
 /// </summary>
 /// <param name="input">Name of input column.</param>
 /// <param name="output">Name of output column.</param>
 /// <param name="outputKind">Kind of output: bag, indicator vector etc.</param>
 /// <param name="hashBits">Number of bits to hash into. Must be between 1 and 31, inclusive.</param>
 /// <param name="seed">Hashing seed.</param>
 /// <param name="ordered">Whether the position of each term should be included in the hash.</param>
 /// <param name="invertHash">During hashing we constuct mappings between original values and the produced hash values.
 /// Text representation of original values are stored in the slot names of the  metadata for the new column.Hashing, as such, can map many initial values to one.
 /// <paramref name="invertHash"/> specifies the upper bound of the number of distinct input values mapping to a hash that should be retained.
 /// <value>0</value> does not retain any input values. <value>-1</value> retains all input values mapping to each hash.</param>
 public ColumnInfo(string input, string output,
                   OneHotEncodingTransformer.OutputKind outputKind = Defaults.OutputKind,
                   int hashBits   = Defaults.HashBits,
                   uint seed      = Defaults.Seed,
                   bool ordered   = Defaults.Ordered,
                   int invertHash = Defaults.InvertHash)
 {
     HashInfo   = new HashingTransformer.ColumnInfo(input, output, hashBits, seed, ordered, invertHash);
     OutputKind = outputKind;
 }
コード例 #8
0
        public OneHotEncodingEstimator(IHostEnvironment env, ColumnInfo[] columns,
                                       string file = null, string termsColumn = null,
                                       IComponentFactory <IMultiStreamSource, IDataLoader> loaderFactory = null)
        {
            Contracts.CheckValue(env, nameof(env));
            _host = env.Register(nameof(OneHotEncodingEstimator));
            _term = new ValueToKeyMappingEstimator(_host, columns, file, termsColumn, loaderFactory);
            var binaryCols = new List <(string input, string output)>();
            var cols       = new List <(string input, string output, bool bag)>();

            for (int i = 0; i < columns.Length; i++)
            {
                var column = columns[i];
                OneHotEncodingTransformer.OutputKind kind = columns[i].OutputKind;
                switch (kind)
                {
                default:
                    throw _host.ExceptUserArg(nameof(column.OutputKind));

                case OneHotEncodingTransformer.OutputKind.Key:
                    continue;

                case OneHotEncodingTransformer.OutputKind.Bin:
                    binaryCols.Add((column.Output, column.Output));
                    break;

                case OneHotEncodingTransformer.OutputKind.Ind:
                    cols.Add((column.Output, column.Output, false));
                    break;

                case OneHotEncodingTransformer.OutputKind.Bag:
                    cols.Add((column.Output, column.Output, true));
                    break;
                }
            }
            IEstimator <ITransformer> toBinVector = null;
            IEstimator <ITransformer> toVector    = null;

            if (binaryCols.Count > 0)
            {
                toBinVector = new KeyToBinaryVectorMappingEstimator(_host, binaryCols.Select(x => new KeyToBinaryVectorMappingTransformer.ColumnInfo(x.input, x.output)).ToArray());
            }
            if (cols.Count > 0)
            {
                toVector = new KeyToVectorMappingEstimator(_host, cols.Select(x => new KeyToVectorMappingTransformer.ColumnInfo(x.input, x.output, x.bag)).ToArray());
            }

            if (toBinVector != null && toVector != null)
            {
                _toSomething = toVector.Append(toBinVector);
            }
            else
            {
                if (toBinVector != null)
                {
                    _toSomething = toBinVector;
                }
                else
                {
                    _toSomething = toVector;
                }
            }
        }
コード例 #9
0
 /// Initializes an instance of the <see cref="OneHotEncodingEstimator"/>.
 /// <param name="env">Host Environment.</param>
 /// <param name="inputColumn">Name of the column to be transformed.</param>
 /// <param name="outputColumn">Name of the output column. If this is <c>null</c>, <paramref name="inputColumn"/> is used.</param>
 /// <param name="outputKind">The type of output expected.</param>
 public OneHotEncodingEstimator(IHostEnvironment env, string inputColumn,
                                string outputColumn = null, OneHotEncodingTransformer.OutputKind outputKind = Defaults.OutKind)
     : this(env, new[] { new ColumnInfo(inputColumn, outputColumn ?? inputColumn, outputKind) })
 {
 }
コード例 #10
0
        internal OneHotHashEncodingEstimator(IHostEnvironment env, params ColumnInfo[] columns)
        {
            Contracts.CheckValue(env, nameof(env));
            _host = env.Register(nameof(ValueToKeyMappingEstimator));
            _hash = new HashingEstimator(_host, columns.Select(x => x.HashInfo).ToArray());
            using (var ch = _host.Start(nameof(OneHotHashEncodingEstimator)))
            {
                var binaryCols = new List <(string outputColumnName, string inputColumnName)>();
                var cols       = new List <(string outputColumnName, string inputColumnName, bool bag)>();
                for (int i = 0; i < columns.Length; i++)
                {
                    var column = columns[i];
                    OneHotEncodingTransformer.OutputKind kind = columns[i].OutputKind;
                    switch (kind)
                    {
                    default:
                        throw _host.ExceptUserArg(nameof(column.OutputKind));

                    case OneHotEncodingTransformer.OutputKind.Key:
                        continue;

                    case OneHotEncodingTransformer.OutputKind.Bin:
                        if ((column.HashInfo.InvertHash) != 0)
                        {
                            ch.Warning("Invert hashing is being used with binary encoding.");
                        }
                        binaryCols.Add((column.HashInfo.Name, column.HashInfo.Name));
                        break;

                    case OneHotEncodingTransformer.OutputKind.Ind:
                        cols.Add((column.HashInfo.Name, column.HashInfo.Name, false));
                        break;

                    case OneHotEncodingTransformer.OutputKind.Bag:
                        cols.Add((column.HashInfo.Name, column.HashInfo.Name, true));
                        break;
                    }
                }
                IEstimator <ITransformer> toBinVector = null;
                IEstimator <ITransformer> toVector    = null;
                if (binaryCols.Count > 0)
                {
                    toBinVector = new KeyToBinaryVectorMappingEstimator(_host, binaryCols.Select(x => (x.outputColumnName, x.inputColumnName)).ToArray());
                }
                if (cols.Count > 0)
                {
                    toVector = new KeyToVectorMappingEstimator(_host, cols.Select(x => new KeyToVectorMappingEstimator.ColumnInfo(x.outputColumnName, x.inputColumnName, x.bag)).ToArray());
                }

                if (toBinVector != null && toVector != null)
                {
                    _toSomething = toVector.Append(toBinVector);
                }
                else
                {
                    if (toBinVector != null)
                    {
                        _toSomething = toBinVector;
                    }
                    else
                    {
                        _toSomething = toVector;
                    }
                }
            }
        }
コード例 #11
0
 /// <summary>
 /// Convert a text column into one-hot encoded vector.
 /// </summary>
 /// <param name="catalog">The transform catalog</param>
 /// <param name="inputColumn">The input column</param>
 /// <param name="outputColumn">The output column. If <c>null</c>, <paramref name="inputColumn"/> is used.</param>
 /// <param name="outputKind">The conversion mode.</param>
 /// <returns></returns>
 public static OneHotEncodingEstimator OneHotEncoding(this TransformsCatalog.CategoricalTransforms catalog,
                                                      string inputColumn,
                                                      string outputColumn = null,
                                                      OneHotEncodingTransformer.OutputKind outputKind = OneHotEncodingTransformer.OutputKind.Ind)
 => new OneHotEncodingEstimator(CatalogUtils.GetEnvironment(catalog), inputColumn, outputColumn, outputKind);