public CopyColumnsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.Column));
SetMetadata();
}
// Public constructor corresponding to SignatureDataTransform.
public ImageGrayscaleTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, t => t is ImageType ? null : "Expected Image type")
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.Column));
Metadata.Seal();
}
// Public constructor corresponding to SignatureDataTransform.
public ImageLoaderTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestIsText)
{
Host.AssertNonEmpty(Infos);
_imageFolder = args.ImageFolder;
Host.Assert(Infos.Length == Utils.Size(args.Column));
_type = new ImageType();
Metadata.Seal();
}
public LpNormNormalizerTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column,
input, TestIsFloatVector)
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.Column));
_exes = new ColInfoEx[Infos.Length];
for (int i = 0; i < _exes.Length; i++)
{
_exes[i] = new ColInfoEx(args.Column[i], args);
}
SetMetadata();
}
public ConvertTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column,
input, null)
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.Column));
_exes = new ColInfoEx[Infos.Length];
for (int i = 0; i < _exes.Length; i++)
{
DataKind kind;
KeyRange range;
var col = args.Column[i];
if (col.ResultType != null)
{
kind = col.ResultType.Value;
range = !string.IsNullOrEmpty(col.Range) ? KeyRange.Parse(col.Range) : col.KeyRange;
}
else if (col.KeyRange != null || !string.IsNullOrEmpty(col.Range))
{
kind = Infos[i].TypeSrc.IsKey ? Infos[i].TypeSrc.RawKind : DataKind.U4;
range = col.KeyRange ?? KeyRange.Parse(col.Range);
}
else if (args.ResultType != null)
{
kind = args.ResultType.Value;
range = !string.IsNullOrEmpty(args.Range) ? KeyRange.Parse(args.Range) : args.KeyRange;
}
else if (args.KeyRange != null || !string.IsNullOrEmpty(args.Range))
{
kind = Infos[i].TypeSrc.IsKey ? Infos[i].TypeSrc.RawKind : DataKind.U4;
range = args.KeyRange ?? KeyRange.Parse(args.Range);
}
else
{
kind = DataKind.Num;
range = null;
}
Host.CheckUserArg(Enum.IsDefined(typeof(DataKind), kind), nameof(args.ResultType));
PrimitiveType itemType;
if (!TryCreateEx(Host, Infos[i], kind, range, out itemType, out _exes[i]))
{
throw Host.ExceptUserArg(nameof(args.Column),
"source column '{0}' with item type '{1}' is not compatible with destination type '{2}'",
input.Schema.GetColumnName(Infos[i].Source), Infos[i].TypeSrc.ItemType, itemType);
}
}
SetMetadata();
}
public MulticlassConvertTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, "args").column, input, null)
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.column));
_exes = new ColInfoEx[Infos.Length];
for (int i = 0; i < _exes.Length; i++)
{
DataKind kind;
KeyCount range;
var col = args.column[i];
if (col.ResultType != null)
{
kind = col.ResultType.Value;
range = col.KeyCount;
}
else if (col.KeyCount != null)
{
kind = Infos[i].TypeSrc.IsKey() ? Infos[i].TypeSrc.RawKind() : DataKind.UInt32;
range = col.KeyCount;
}
else if (args.resultType != null)
{
kind = args.resultType.Value;
range = args.keyCount;
}
else if (args.keyCount != null)
{
kind = Infos[i].TypeSrc.IsKey() ? Infos[i].TypeSrc.RawKind() : DataKind.UInt32;
range = args.keyCount;
}
else
{
kind = DataKind.Single; // or UInt32 ???
range = null;
}
Host.CheckUserArg(Enum.IsDefined(typeof(DataKind), kind), "resultType");
PrimitiveDataViewType itemType;
if (!TryCreateEx(Host, Infos[i], kind, range, out itemType, out _exes[i]))
{
throw Host.ExceptUserArg("source",
"Source column '{0}' with item type '{1}' is not compatible with destination type '{2}'",
input.Schema[Infos[i].Source].Name, Infos[i].TypeSrc.ItemType(), itemType);
}
}
SetMetadata();
}
// Public constructor corresponding to SignatureDataTransform.
public ImageResizerTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, t => t is ImageType ? null : "Expected Image type")
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.Column));
_exes = new ColInfoEx[Infos.Length];
for (int i = 0; i < _exes.Length; i++)
{
var item = args.Column[i];
_exes[i] = new ColInfoEx(
item.ImageWidth ?? args.ImageWidth,
item.ImageHeight ?? args.ImageHeight,
item.Resizing ?? args.Resizing,
item.CropAnchor ?? args.CropAnchor);
}
Metadata.Seal();
}
/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
public TermLookupTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column,
input, TestIsText)
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.Column));
Host.CheckUserArg(!string.IsNullOrWhiteSpace(args.DataFile), nameof(args.DataFile), "must specify dataFile");
Host.CheckUserArg(string.IsNullOrEmpty(args.TermColumn) == string.IsNullOrEmpty(args.ValueColumn), nameof(args.TermColumn),
"Either both term and value column should be specified, or neither.");
using (var ch = Host.Start("Training"))
{
_bytes = GetBytes(Host, Infos, args);
_ldr = GetLoader(Host, _bytes);
_valueMap = Train(ch, _ldr);
SetMetadata();
}
}
/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
public LpNormNormalizerTransform(IHostEnvironment env, GcnArguments args, IDataView input)
: base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column,
input, TestIsFloatVector)
{
Host.AssertNonEmpty(Infos);
Host.Assert(Infos.Length == Utils.Size(args.Column));
_exes = new ColInfoEx[Infos.Length];
for (int i = 0; i < _exes.Length; i++)
{
_exes[i] = new ColInfoEx(args.Column[i], args);
}
// REVIEW: for now check only global (default) values. Move to Bindings/ColInfoEx?
if (!args.SubMean && args.UseStdDev)
{
using (var ch = Host.Start("Argument validation"))
{
ch.Warning("subMean parameter is false while useStd is true. It is advisable to set subMean to true in case useStd is set to true.");
}
}
SetMetadata();
}
/// <summary>
/// Public constructor corresponding to SignatureDataTransform.
/// </summary>
public DropSlotsTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, null)
{
Host.CheckNonEmpty(args.Column, nameof(args.Column));
var size = Infos.Length;
_exes = new ColInfoEx[size];
for (int i = 0; i < size; i++)
{
var col = args.Column[i];
int[] slotsMin;
int[] slotsMax;
GetSlotsMinMax(col, out slotsMin, out slotsMax);
SlotDropper slotDropper = new SlotDropper(Infos[i].TypeSrc.ValueCount, slotsMin, slotsMax);
bool suppressed;
ColumnType typeDst;
int[] categoricalRanges;
ComputeType(Source.Schema, slotsMin, slotsMax, i, slotDropper, out suppressed, out typeDst, out categoricalRanges);
_exes[i] = new ColInfoEx(slotDropper, suppressed, typeDst, categoricalRanges);
}
Metadata.Seal();
}
internal PredictionEngine(IHostEnvironment env, IDataView dataPipe, bool ignoreMissingColumns,
SchemaDefinition inputSchemaDefinition = null, SchemaDefinition outputSchemaDefinition = null)
: this(env, new TransformWrapper(env, env.CheckRef(dataPipe, nameof(dataPipe))), ignoreMissingColumns, inputSchemaDefinition, outputSchemaDefinition)
{
}
public MissingValueDroppingTransformer(IHostEnvironment env, Arguments args, IDataView input)
: base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestType)
{
Host.CheckNonEmpty(args.Column, nameof(args.Column));
_isNAs = InitIsNAAndMetadata();
}
public HashTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(Contracts.CheckRef(env, nameof(env)), RegistrationName, env.CheckRef(args, nameof(args)).Column,
input, TestType)
{
if (args.HashBits < NumBitsMin || args.HashBits >= NumBitsLim)
{
throw Host.ExceptUserArg(nameof(args.HashBits), "hashBits should be between {0} and {1} inclusive", NumBitsMin, NumBitsLim - 1);
}
_exes = new ColInfoEx[Infos.Length];
List <int> invertIinfos = null;
List <int> invertHashMaxCounts = null;
for (int iinfo = 0; iinfo < Infos.Length; iinfo++)
{
_exes[iinfo] = new ColInfoEx(args, args.Column[iinfo]);
int invertHashMaxCount = GetAndVerifyInvertHashMaxCount(args, args.Column[iinfo], _exes[iinfo]);
if (invertHashMaxCount > 0)
{
Utils.Add(ref invertIinfos, iinfo);
Utils.Add(ref invertHashMaxCounts, invertHashMaxCount);
}
}
_types = InitColumnTypes();
if (Utils.Size(invertIinfos) > 0)
{
// Build the invert hashes for all columns for which it was requested.
var srcs = new HashSet <int>(invertIinfos.Select(i => Infos[i].Source));
using (IRowCursor srcCursor = input.GetRowCursor(srcs.Contains))
{
using (var ch = Host.Start("Invert hash building"))
{
InvertHashHelper[] helpers = new InvertHashHelper[invertIinfos.Count];
Action disposer = null;
for (int i = 0; i < helpers.Length; ++i)
{
int iinfo = invertIinfos[i];
Host.Assert(_types[iinfo].ItemType.KeyCount > 0);
var dstGetter = GetGetterCore(ch, srcCursor, iinfo, out disposer);
Host.Assert(disposer == null);
var ex = _exes[iinfo];
var maxCount = invertHashMaxCounts[i];
helpers[i] = InvertHashHelper.Create(srcCursor, Infos[iinfo], ex, maxCount, dstGetter);
}
while (srcCursor.MoveNext())
{
for (int i = 0; i < helpers.Length; ++i)
{
helpers[i].Process();
}
}
_keyValues = new VBuffer <DvText> [_exes.Length];
_kvTypes = new ColumnType[_exes.Length];
for (int i = 0; i < helpers.Length; ++i)
{
_keyValues[invertIinfos[i]] = helpers[i].GetKeyValuesMetadata();
Host.Assert(_keyValues[invertIinfos[i]].Length == _types[invertIinfos[i]].ItemType.KeyCount);
_kvTypes[invertIinfos[i]] = new VectorType(TextType.Instance, _keyValues[invertIinfos[i]].Length);
}
ch.Done();
}
}
}
SetMetadata();
}
