public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null) { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.Column)); SetMetadata(); }
// Public constructor corresponding to SignatureDataTransform. public ImageGrayscaleTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, t => t is ImageType ? null : "Expected Image type") { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.Column)); Metadata.Seal(); }
// Public constructor corresponding to SignatureDataTransform. public ImageLoaderTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestIsText) { Host.AssertNonEmpty(Infos); _imageFolder = args.ImageFolder; Host.Assert(Infos.Length == Utils.Size(args.Column)); _type = new ImageType(); Metadata.Seal(); }
public LpNormNormalizerTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestIsFloatVector) { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.Column)); _exes = new ColInfoEx[Infos.Length]; for (int i = 0; i < _exes.Length; i++) { _exes[i] = new ColInfoEx(args.Column[i], args); } SetMetadata(); }
public ConvertTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null) { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.Column)); _exes = new ColInfoEx[Infos.Length]; for (int i = 0; i < _exes.Length; i++) { DataKind kind; KeyRange range; var col = args.Column[i]; if (col.ResultType != null) { kind = col.ResultType.Value; range = !string.IsNullOrEmpty(col.Range) ? KeyRange.Parse(col.Range) : col.KeyRange; } else if (col.KeyRange != null || !string.IsNullOrEmpty(col.Range)) { kind = Infos[i].TypeSrc.IsKey ? Infos[i].TypeSrc.RawKind : DataKind.U4; range = col.KeyRange ?? KeyRange.Parse(col.Range); } else if (args.ResultType != null) { kind = args.ResultType.Value; range = !string.IsNullOrEmpty(args.Range) ? KeyRange.Parse(args.Range) : args.KeyRange; } else if (args.KeyRange != null || !string.IsNullOrEmpty(args.Range)) { kind = Infos[i].TypeSrc.IsKey ? Infos[i].TypeSrc.RawKind : DataKind.U4; range = args.KeyRange ?? KeyRange.Parse(args.Range); } else { kind = DataKind.Num; range = null; } Host.CheckUserArg(Enum.IsDefined(typeof(DataKind), kind), nameof(args.ResultType)); PrimitiveType itemType; if (!TryCreateEx(Host, Infos[i], kind, range, out itemType, out _exes[i])) { throw Host.ExceptUserArg(nameof(args.Column), "source column '{0}' with item type '{1}' is not compatible with destination type '{2}'", input.Schema.GetColumnName(Infos[i].Source), Infos[i].TypeSrc.ItemType, itemType); } } SetMetadata(); }
public MulticlassConvertTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, "args").column, input, null) { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.column)); _exes = new ColInfoEx[Infos.Length]; for (int i = 0; i < _exes.Length; i++) { DataKind kind; KeyCount range; var col = args.column[i]; if (col.ResultType != null) { kind = col.ResultType.Value; range = col.KeyCount; } else if (col.KeyCount != null) { kind = Infos[i].TypeSrc.IsKey() ? Infos[i].TypeSrc.RawKind() : DataKind.UInt32; range = col.KeyCount; } else if (args.resultType != null) { kind = args.resultType.Value; range = args.keyCount; } else if (args.keyCount != null) { kind = Infos[i].TypeSrc.IsKey() ? Infos[i].TypeSrc.RawKind() : DataKind.UInt32; range = args.keyCount; } else { kind = DataKind.Single; // or UInt32 ??? range = null; } Host.CheckUserArg(Enum.IsDefined(typeof(DataKind), kind), "resultType"); PrimitiveDataViewType itemType; if (!TryCreateEx(Host, Infos[i], kind, range, out itemType, out _exes[i])) { throw Host.ExceptUserArg("source", "Source column '{0}' with item type '{1}' is not compatible with destination type '{2}'", input.Schema[Infos[i].Source].Name, Infos[i].TypeSrc.ItemType(), itemType); } } SetMetadata(); }
// Public constructor corresponding to SignatureDataTransform. public ImageResizerTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, t => t is ImageType ? null : "Expected Image type") { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.Column)); _exes = new ColInfoEx[Infos.Length]; for (int i = 0; i < _exes.Length; i++) { var item = args.Column[i]; _exes[i] = new ColInfoEx( item.ImageWidth ?? args.ImageWidth, item.ImageHeight ?? args.ImageHeight, item.Resizing ?? args.Resizing, item.CropAnchor ?? args.CropAnchor); } Metadata.Seal(); }
/// <summary> /// Public constructor corresponding to SignatureDataTransform. /// </summary> public TermLookupTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestIsText) { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.Column)); Host.CheckUserArg(!string.IsNullOrWhiteSpace(args.DataFile), nameof(args.DataFile), "must specify dataFile"); Host.CheckUserArg(string.IsNullOrEmpty(args.TermColumn) == string.IsNullOrEmpty(args.ValueColumn), nameof(args.TermColumn), "Either both term and value column should be specified, or neither."); using (var ch = Host.Start("Training")) { _bytes = GetBytes(Host, Infos, args); _ldr = GetLoader(Host, _bytes); _valueMap = Train(ch, _ldr); SetMetadata(); } }
/// <summary> /// Public constructor corresponding to SignatureDataTransform. /// </summary> public LpNormNormalizerTransform(IHostEnvironment env, GcnArguments args, IDataView input) : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestIsFloatVector) { Host.AssertNonEmpty(Infos); Host.Assert(Infos.Length == Utils.Size(args.Column)); _exes = new ColInfoEx[Infos.Length]; for (int i = 0; i < _exes.Length; i++) { _exes[i] = new ColInfoEx(args.Column[i], args); } // REVIEW: for now check only global (default) values. Move to Bindings/ColInfoEx? if (!args.SubMean && args.UseStdDev) { using (var ch = Host.Start("Argument validation")) { ch.Warning("subMean parameter is false while useStd is true. It is advisable to set subMean to true in case useStd is set to true."); } } SetMetadata(); }
/// <summary> /// Public constructor corresponding to SignatureDataTransform. /// </summary> public DropSlotsTransform(IHostEnvironment env, Arguments args, IDataView input) : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null) { Host.CheckNonEmpty(args.Column, nameof(args.Column)); var size = Infos.Length; _exes = new ColInfoEx[size]; for (int i = 0; i < size; i++) { var col = args.Column[i]; int[] slotsMin; int[] slotsMax; GetSlotsMinMax(col, out slotsMin, out slotsMax); SlotDropper slotDropper = new SlotDropper(Infos[i].TypeSrc.ValueCount, slotsMin, slotsMax); bool suppressed; ColumnType typeDst; int[] categoricalRanges; ComputeType(Source.Schema, slotsMin, slotsMax, i, slotDropper, out suppressed, out typeDst, out categoricalRanges); _exes[i] = new ColInfoEx(slotDropper, suppressed, typeDst, categoricalRanges); } Metadata.Seal(); }
internal PredictionEngine(IHostEnvironment env, IDataView dataPipe, bool ignoreMissingColumns, SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null) : this(env, new TransformWrapper(env, env.CheckRef(dataPipe, nameof(dataPipe))), ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition) { }
public MissingValueDroppingTransformer(IHostEnvironment env, Arguments args, IDataView input) : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType) { Host.CheckNonEmpty(args.Column, nameof(args.Column)); _isNAs = InitIsNAAndMetadata(); }
public HashTransform(IHostEnvironment env, Arguments args, IDataView input) : base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType) { if (args.HashBits < NumBitsMin || args.HashBits >= NumBitsLim) { throw Host.ExceptUserArg(nameof(args.HashBits), "hashBits should be between {0} and {1} inclusive", NumBitsMin, NumBitsLim - 1); } _exes = new ColInfoEx[Infos.Length]; List <int> invertIinfos = null; List <int> invertHashMaxCounts = null; for (int iinfo = 0; iinfo < Infos.Length; iinfo++) { _exes[iinfo] = new ColInfoEx(args, args.Column[iinfo]); int invertHashMaxCount = GetAndVerifyInvertHashMaxCount(args, args.Column[iinfo], _exes[iinfo]); if (invertHashMaxCount > 0) { Utils.Add(ref invertIinfos, iinfo); Utils.Add(ref invertHashMaxCounts, invertHashMaxCount); } } _types = InitColumnTypes(); if (Utils.Size(invertIinfos) > 0) { // Build the invert hashes for all columns for which it was requested. var srcs = new HashSet <int>(invertIinfos.Select(i => Infos[i].Source)); using (IRowCursor srcCursor = input.GetRowCursor(srcs.Contains)) { using (var ch = Host.Start("Invert hash building")) { InvertHashHelper[] helpers = new InvertHashHelper[invertIinfos.Count]; Action disposer = null; for (int i = 0; i < helpers.Length; ++i) { int iinfo = invertIinfos[i]; Host.Assert(_types[iinfo].ItemType.KeyCount > 0); var dstGetter = GetGetterCore(ch, srcCursor, iinfo, out disposer); Host.Assert(disposer == null); var ex = _exes[iinfo]; var maxCount = invertHashMaxCounts[i]; helpers[i] = InvertHashHelper.Create(srcCursor, Infos[iinfo], ex, maxCount, dstGetter); } while (srcCursor.MoveNext()) { for (int i = 0; i < helpers.Length; ++i) { helpers[i].Process(); } } _keyValues = new VBuffer <DvText> [_exes.Length]; _kvTypes = new ColumnType[_exes.Length]; for (int i = 0; i < helpers.Length; ++i) { _keyValues[invertIinfos[i]] = helpers[i].GetKeyValuesMetadata(); Host.Assert(_keyValues[invertIinfos[i]].Length == _types[invertIinfos[i]].ItemType.KeyCount); _kvTypes[invertIinfos[i]] = new VectorType(TextType.Instance, _keyValues[invertIinfos[i]].Length); } ch.Done(); } } } SetMetadata(); }