ValueMapper <TIn, TOut> IValueMapper.GetMapper <TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer <Single>));
Host.Check(typeof(TOut) == typeof(VBuffer <Single>));
var combine = Combiner.GetCombiner();
var features = new VBuffer <Single> [_mappers.Length];
var predictions = new VBuffer <Single> [_mappers.Length];
var maps = new ValueMapper <VBuffer <Single>, VBuffer <Single> > [_mappers.Length];
for (int i = 0; i < _mappers.Length; i++)
{
// IsValid method ensures we go this else path only if the OutputType.VectorSize of
// all _mappers is greater than zero
Host.Assert(_mappers[i].OutputType.GetVectorSize() > 0);
maps[i] = _mappers[i].GetMapper <VBuffer <Single>, VBuffer <Single> >();
}
ValueMapper <VBuffer <Single>, VBuffer <Single> > del =
(in VBuffer <Single> src, ref VBuffer <Single> dst) =>
{
if (_inputType.Size > 0)
{
Host.Check(src.Length == _inputType.Size);
}
var tmp = src;
Parallel.For(0, maps.Length, i =>
{
var model = Models[i];
if (model.SelectedFeatures != null)
{
EnsembleUtils.SelectFeatures(in tmp, model.SelectedFeatures, model.Cardinality, ref features[i]);
maps[i](in features[i], ref predictions[i]);
}
public ValueMapper <TIn, TOut> GetMapper <TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer <float>));
Host.Check(typeof(TOut) == typeof(VBuffer <float>));
var maps = new ValueMapper <VBuffer <float>, float, float> [_mappers.Length];
for (int i = 0; i < _mappers.Length; i++)
{
maps[i] = _mappers[i].GetMapper <VBuffer <float>, float, float>();
}
var parallelOptions = Host.ConcurrencyFactor < 1 ? new ParallelOptions() : new ParallelOptions()
{
MaxDegreeOfParallelism = Host.ConcurrencyFactor
};
var buffer = new double[_mappers.Length];
ValueMapper <VBuffer <float>, VBuffer <float> > del =
(in VBuffer <float> src, ref VBuffer <float> dst) =>
{
if (InputType.VectorSize > 0)
{
Host.Check(src.Length == InputType.VectorSize);
}
var tmp = src;
Parallel.For(0, maps.Length, parallelOptions, i =>
{
float score = 0;
float prob = 0;
maps[i](in tmp, ref score, ref prob);
buffer[i] = prob;
});
public static ValueMapper <TLabel, TDest> GetConverter <TLabel, TDest>(out bool identity)
{
var col1 = GetColumnType <TLabel>();
var col2 = GetColumnType <TDest>();
if (typeof(TLabel) == typeof(float))
{
if (typeof(TDest) == typeof(uint))
{
ValueMapper <float, uint> temp = (in float src, ref uint dst) =>
{
if (src < 0)
{
throw Contracts.ExceptValue("Unable to converter {0} '{1}' into {2}.", typeof(float).ToString(), src, typeof(uint));
}
dst = (uint)src;
};
identity = false;
return(temp as ValueMapper <TLabel, TDest>);
}
else if (typeof(TDest) == typeof(int))
{
ValueMapper <float, int> temp = (in float src, ref int dst) =>
{
dst = (int)src;
};
identity = false;
return(temp as ValueMapper <TLabel, TDest>);
}
}
return(Conversions.Instance.GetStandardConversion <TLabel, TDest>(col1, col2, out identity));
}
public override ValueMapper <VBuffer <float>, VBuffer <float> > GetMapper()
{
var maps = new ValueMapper <VBuffer <float>, float> [Predictors.Length];
for (int i = 0; i < Predictors.Length; i++)
{
maps[i] = Predictors[i].GetMapper <VBuffer <float>, float>();
}
var buffer = new float[maps.Length];
return
((in VBuffer <float> src, ref VBuffer <float> dst) =>
{
if (InputType.VectorSize > 0)
{
Contracts.Check(src.Length == InputType.VectorSize);
}
var tmp = src;
Parallel.For(0, maps.Length, i => maps[i](in tmp, ref buffer[i]));
var editor = VBufferEditor.Create(ref dst, maps.Length);
buffer.CopyTo(editor.Values);
dst = editor.Commit();
});
public void Should_Throw_Exception_When_Return_Type_Is_Enumerable_But_Cypher_Value_Is_Not_Enumerable()
{
var result = Assert.Throws <InvalidOperationException>(
() => ValueMapper.MapValue <IEnumerable <string> >(1));
result.Message.Should().Be(
"The cypher value is not a list and cannot be mapped to target type: System.Collections.Generic.IEnumerable`1[System.String]");
}
private void BinBools(ref VBuffer <DvBool> input, ref VBuffer <int> output)
{
if (_boolMapper == null)
{
_boolMapper = CreateVectorMapper <DvBool, int>(BinOneBool);
}
_boolMapper(ref input, ref output);
}
public void Should_Throw_Exception_When_Cypher_Value_Is_Enumerable_But_Return_Type_Is_Not_Enumerable()
{
var result = Assert.Throws <InvalidOperationException>(
() => ValueMapper.MapValue <int>(new[] { 1 }));
result.Message.Should().Be(
"The cypher value is a list and cannot be mapped to target type: System.Int32");
}
ValueMapper<TIn, TOut> IValueMapper.GetMapper<TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer<float>), "Input type does not match.");
Host.Check(typeof(TOut) == typeof(float), "Output type does not match.");
ValueMapper<VBuffer<float>, float> del = Map;
return (ValueMapper<TIn, TOut>)(Delegate)del;
}
ValueMapper <TIn, TOut> IValueMapper.GetMapper <TIn, TOut>()
{
Contracts.Check(typeof(TIn) == typeof(VBuffer <float>));
Contracts.Check(typeof(TOut) == typeof(float));
ValueMapper <VBuffer <float>, float> del = Map;
return((ValueMapper <TIn, TOut>)(Delegate) del);
}
/// <summary>
/// Loads a pipeline saved in zip format.
/// </summary>
protected void Load(Stream fs)
{
var transformPipe = ModelFileUtils.LoadPipeline(_env, fs, new MultiFileSource(null), true);
var pred = _env.LoadPredictorOrNull(fs);
IDataView root;
for (root = transformPipe; root is IDataTransform && !(root is PassThroughTransform); root = ((IDataTransform)root).Source)
{
;
}
if (!(root is PassThroughTransform))
{
var tr = new PassThroughTransform(_env, new PassThroughTransform.Arguments(), root);
transformPipe = ApplyTransformUtils.ApplyAllTransformsToData(_env, transformPipe, tr, root);
}
var stack = new List <IDataView>();
for (root = transformPipe; root is IDataTransform; root = ((IDataTransform)root).Source)
{
stack.Add(root);
}
stack.Reverse();
_transforms = new StepTransform[stack.Count];
for (int i = 0; i < _transforms.Length; ++i)
{
_transforms[i] = new StepTransform()
{
transform = stack[i] as IDataTransform, transformSettings = null
}
}
;
if (pred == null)
{
_predictor = new StepPredictor()
{
predictor = null, roleMapData = null, trainer = null, trainerSettings = null
}
}
;
else
{
#pragma warning disable CS0618
var ipred = pred.GetPredictorObject() as IPredictor;
#pragma warning restore CS0618
_roles = ModelFileUtils.LoadRoleMappingsOrNull(_env, fs).ToList();
var data = new RoleMappedData(transformPipe, _roles);
_predictor = new StepPredictor()
{
predictor = ipred, roleMapData = data, trainer = null, trainerSettings = null
};
}
_fastValueMapper = null;
}
public ValueMapper <TIn, TOut> GetMapper <TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer <float>));
Host.Check(typeof(TOut) == typeof(VBuffer <float>));
ValueMapper <VBuffer <float>, VBuffer <float> > del = Map;
return((ValueMapper <TIn, TOut>)(Delegate) del);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="env">environment</param>
/// <param name="modelStream">stream</param>
/// <param name="output">name of the output column</param>
/// <param name="outputIsFloat">output is a gloat (true) or a vector of floats (false)</param>
/// <param name="conc">number of concurrency threads</param>
/// <param name="features">features name</param>
public ValueMapperPredictionEngineFloat(IHostEnvironment env, Stream modelStream,
string output = "Probability", bool outputIsFloat = true, int conc = 1,
string features = "Features")
{
_env = env;
if (_env == null)
{
throw Contracts.Except("env must not be null");
}
var inputs = new FloatVectorInput[0];
var view = ComponentCreation.CreateStreamingDataView <FloatVectorInput>(_env, inputs);
long modelPosition = modelStream.Position;
_predictor = ComponentCreation.LoadPredictorOrNull(_env, modelStream);
if (_predictor == null)
{
throw _env.Except("Unable to load a model.");
}
modelStream.Seek(modelPosition, SeekOrigin.Begin);
_transforms = ComponentCreation.LoadTransforms(_env, modelStream, view);
if (_transforms == null)
{
throw _env.Except("Unable to load a model.");
}
var data = _env.CreateExamples(_transforms, features);
if (data == null)
{
throw _env.Except("Cannot create rows.");
}
var scorer = _env.CreateDefaultScorer(data, _predictor);
if (scorer == null)
{
throw _env.Except("Cannot create a scorer.");
}
_valueMapper = new ValueMapperFromTransformFloat <VBuffer <float> >(_env,
scorer, features, output, conc: conc);
if (_valueMapper == null)
{
throw _env.Except("Cannot create a mapper.");
}
if (outputIsFloat)
{
_mapper = _valueMapper.GetMapper <VBuffer <float>, float>();
_mapperVector = null;
}
else
{
_mapper = null;
_mapperVector = _valueMapper.GetMapper <VBuffer <float>, VBuffer <float> >();
}
}
public ValueMapper <TIn, TOut> GetMapper <TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer <Single>));
Host.Check(typeof(TOut) == typeof(Single));
var combine = Combiner.GetCombiner();
var predictions = new Single[_mappers.Length];
var buffers = new VBuffer <Single> [_mappers.Length];
var maps = new ValueMapper <VBuffer <Single>, Single> [_mappers.Length];
for (int i = 0; i < _mappers.Length; i++)
{
maps[i] = _mappers[i].GetMapper <VBuffer <Single>, Single>();
}
ValueMapper <VBuffer <Single>, Single> del =
(ref VBuffer <Single> src, ref Single dst) =>
{
if (InputType.VectorSize > 0)
{
Host.Check(src.Length == InputType.VectorSize);
}
var tmp = src;
Parallel.For(0, maps.Length, i =>
{
var model = Models[i];
if (model.SelectedFeatures != null)
{
EnsembleUtils.SelectFeatures(ref tmp, model.SelectedFeatures, model.Cardinality, ref buffers[i]);
maps[i](ref buffers[i], ref predictions[i]);
}
else
{
maps[i](ref tmp, ref predictions[i]);
}
});
combine(ref dst, predictions, Weights);
};
return((ValueMapper <TIn, TOut>)(Delegate) del);
}
public ValueMapper <TIn, TOut, TDist> GetMapper <TIn, TOut, TDist>()
{
Contracts.Check(typeof(TIn) == typeof(VBuffer <Float>));
Contracts.Check(typeof(TOut) == typeof(Float));
Contracts.Check(typeof(TDist) == typeof(Float));
ValueMapper <VBuffer <Float>, Float, Float> del = MapDist;
return((ValueMapper <TIn, TOut, TDist>)(Delegate) del);
}
/// <summary>
/// Creates an instance of <see cref="PredictionFunctionDataFrame"/>.
/// </summary>
/// <param name="env">The host environment.</param>
/// <param name="transformer">The model (transformer) to use for prediction.</param>
/// <param name="inputSchema">Input schema.</param>
/// <param name="conc">Number of threads.</param>
public PredictionFunctionDataFrame(IHostEnvironment env, ITransformer transformer, Schema inputSchema, int conc = 1)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(transformer, nameof(transformer));
var df = new DataFrame(transformer.GetOutputSchema(inputSchema), 0);
var tr = transformer.Transform(df) as IDataTransform;
_fastValueMapperObject = new ValueMapperDataFrameFromTransform(env, tr, conc: conc);
_fastValueMapper = _fastValueMapperObject.GetMapper <DataFrame, DataFrame>();
}
/// <summary>
/// Applies the same function on every value of the column.
/// </summary>
public NumericColumn Apply<TSrc, TDst>(ValueMapper<TSrc, TDst> mapper)
where TDst : IEquatable<TDst>, IComparable<TDst>
{
var maptyped = mapper as ValueMapper<DType, TDst>;
if (maptyped == null)
throw new DataValueError("Unexpected input type for this column.");
var res = new DataColumn<TDst>(Length);
for (int i = 0; i < res.Length; ++i)
maptyped(in Data[i], ref res.Data[i]);
return new NumericColumn(res);
}
private ValueMapper <VBuffer <Single>, Single, Single>[] GetMaps()
{
Host.AssertValue(_mappers);
var maps = new ValueMapper <VBuffer <Single>, Single, Single> [_mappers.Length];
for (int i = 0; i < _mappers.Length; i++)
{
maps[i] = _mappers[i].GetMapper <VBuffer <Single>, Single, Single>();
}
return(maps);
}
private static void EnsureCachedResultValueMapper(ValueMapper <VBuffer <Float>, Float, Float> mapper,
ref long cachedPosition, ValueGetter <VBuffer <Float> > featureGetter, ref VBuffer <Float> features,
ref Float score, ref Float prob, IRow input)
{
Contracts.AssertValue(mapper);
if (cachedPosition != input.Position)
{
featureGetter(ref features);
mapper(ref features, ref score, ref prob);
cachedPosition = input.Position;
}
}
public Impl(IHostEnvironment env, string name, IDataView input,
int colSrc, InPredicate <T2> pred, ValueMapper <T1, T2> conv = null)
: base(env, name, input)
{
Host.AssertValue(pred);
Host.Assert(conv != null | typeof(T1) == typeof(T2));
Host.Assert(0 <= colSrc & colSrc < Source.Schema.Count);
_colSrc = colSrc;
_pred = pred;
_conv = conv;
}
public override ISet <Guid> Create()
{
var table = new Table("Guids", new [] { "Value" }, new Dictionary <string, SqlType>()
{
{ "Value", new SqlType(DbType.Guid) }
}, "test");
_serverFixture.DropTable(table.Schema, table.Name);
var mapper = new ValueMapper <Guid>("Value");
return(new SqlSet <Guid>(_serverFixture.DatabaseDriver, _serverFixture.ConnectionString, table, mapper));
}
private static bool ReadField(IDataReader reader, IDictionary <string, int> ordinalMapping, FieldInfo field, out object value)
{
try
{
var fieldName = (field.Alias ?? field.Name).ToLower(System.Globalization.CultureInfo.InvariantCulture);
if (ordinalMapping.TryGetValue(fieldName, out var ordinal))
{
var runtimeType = field.RunTimeType;
object dbValue = null;
try
{
dbValue = reader.GetValue(ordinal);
}
catch (FormatException) when(runtimeType == typeof(string))
{
dbValue = reader.GetString(ordinal);
}
catch (FormatException) when(runtimeType == typeof(DateTime) || runtimeType == typeof(DateTime?))
{
// HACK if date time is not in ISO8601 then System.Data.Sqlite
// will fail when calling GetValue because it is attempting to convert it to a datetime
// the solution is to attempt to read it as a string and let the conversion in ProcessValue
// convert the string to a DateTime
dbValue = reader.GetString(ordinal);
}
try
{
value = ValueMapper.ProcessValue(runtimeType, dbValue);
}
catch (FormatException) when(runtimeType == typeof(Guid) || runtimeType == typeof(Guid?))
{
var bytes = new byte[16];
reader.GetBytes(ordinal, 0, bytes, 0, 16);
value = ValueMapper.ProcessValue(runtimeType, bytes);
}
return(true);
}
else
{
value = null;
return(false);
}
}
catch (Exception e)
{
throw new ORMException("Error in ReadData: field info = " + field.ToString(), e);
}
}
protected void CreateFastValueMapper(int conc)
{
IDataTransform tr = _transforms.Last().transform;
if (_predictor != null && _predictor.predictor != null)
{
var roles = new RoleMappedData(tr, _roles ?? new List <KeyValuePair <RoleMappedSchema.ColumnRole, string> >());
tr = PredictorHelper.Predict(_env, _predictor.predictor, roles);
}
_fastValueMapperObject = new ValueMapperDataFrameFromTransform(_env, tr, conc: conc);
_fastValueMapper = _fastValueMapperObject.GetMapper <DataFrame, DataFrame>();
}
public ValueMapper <TSrc, VBuffer <Float> > GetWhatTheFeatureMapper <TSrc, TDstContributions>(int top, int bottom, bool normalize)
{
Contracts.Check(typeof(TSrc) == typeof(VBuffer <Float>));
Contracts.Check(typeof(TDstContributions) == typeof(VBuffer <Float>));
ValueMapper <VBuffer <Float>, VBuffer <Float> > del =
(ref VBuffer <Float> src, ref VBuffer <Float> dstContributions) =>
{
GetFeatureContributions(ref src, ref dstContributions, top, bottom, normalize);
};
return((ValueMapper <TSrc, VBuffer <Float> >)(Delegate) del);
}
private void OnRemoteControlMessageReceived(IMessage message)
{
var rcMessage = (RemoteControlMessage)message;
if (rcMessage.Key == (int)RemoteControlKey.Throttle)
{
_messageBroker.Publish(new ServoMessage
{
Id = 0,
PulseWidth = (int)ValueMapper.Map(rcMessage.Value, -1, 1, 500, 2500)
});
}
}
public void Should_Convert_Null_LocalDate_To_DateTime_Default()
{
MapperConfig.RegisterLocalDateToDateTimeConverter();
LocalDate localDate = null;
var map = new Dictionary <string, object>
{
{ nameof(EntityWithClrTypes.DateValue), localDate }
};
var entity = ValueMapper.MapValue <EntityWithClrTypes>(map);
entity.DateValue.Should().Be(default);
public ValueMapper <TIn, TOut> GetMapper <TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer <Float>));
Host.Check(typeof(TOut) == typeof(VBuffer <Float>));
var maps = new ValueMapper <VBuffer <Float>, Float, Float> [_mappers.Length];
for (int i = 0; i < _mappers.Length; i++)
{
maps[i] = _mappers[i].GetMapper <VBuffer <Float>, Float, Float>();
}
var buffer = new Double[_predictors.Length];
ValueMapper <VBuffer <Float>, VBuffer <Float> > del =
(ref VBuffer <Float> src, ref VBuffer <Float> dst) =>
{
if (InputType.VectorSize > 0)
{
Host.Check(src.Length == InputType.VectorSize);
}
var values = dst.Values;
var tmp = src;
Parallel.For(0, maps.Length, i =>
{
Float score = 0;
Float prob = 0;
maps[i](ref tmp, ref score, ref prob);
buffer[i] = prob;
});
ReconcilePredictions(buffer);
ComputeProbabilities(buffer, ref values);
dst = new VBuffer <Float>(_numClasses, values, dst.Indices);
};
return((ValueMapper <TIn, TOut>)(Delegate) del);
}
public override INumberMap <Guid> Create()
{
var table = TableBuilder
.WithName("GuidNumbers")
.WithKeyColumnFromType <Guid>("Key")
.WithColumnFromType <long>("Counter")
.Build();
_serverFixture.DropTable(table.Schema, table.Name);
var keyMapper = new ValueMapper <Guid>("Key");
return(new SqlNumberMap <Guid>(_serverFixture.DatabaseDriver, _serverFixture.ConnectionString, table, keyMapper, "Counter"));
}
public ValueMapper <TIn, TOut> GetMapper <TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer <Float>));
Host.Check(typeof(TOut) == typeof(Float));
ValueMapper <VBuffer <Float>, Float> del =
(ref VBuffer <Float> src, ref Float dst) =>
{
Host.Check(src.Length == _dimension);
dst = Score(ref src);
};
return((ValueMapper <TIn, TOut>)(Delegate) del);
}
void _CreateMapper(IDataScorerTransform scorer)
{
_mapperBinaryClassification = null;
var schema = scorer.Schema;
int i1, i2, i3;
i1 = SchemaHelper.GetColumnIndex(schema, "PredictedLabel");
i2 = SchemaHelper.GetColumnIndex(schema, "Score");
i3 = SchemaHelper.GetColumnIndex(schema, "Probability");
var map = new ValueMapperFromTransform <TRowValue, PredictionTypeForBinaryClassification>(_env, scorer);
_mapperBinaryClassification = map.GetMapper <TRowValue, PredictionTypeForBinaryClassification>();
_valueMapper = map;
}
public ValueMapper <TIn, TOut> GetMapper <TIn, TOut>()
{
Host.Check(typeof(TIn) == typeof(VBuffer <Single>));
Host.Check(typeof(TOut) == typeof(VBuffer <Single>));
var combine = Combiner.GetCombiner();
var features = new VBuffer <Single> [_mappers.Length];
var predictions = new VBuffer <Single> [_mappers.Length];
var maps = new ValueMapper <VBuffer <Single>, VBuffer <Single> > [_mappers.Length];
for (int i = 0; i < _mappers.Length; i++)
{
// IsValid method ensures we go this else path only if the OutputType.VectorSize of
// all _mappers is greater than zero
Host.Assert(_mappers[i].OutputType.VectorSize > 0);
maps[i] = _mappers[i].GetMapper <VBuffer <Single>, VBuffer <Single> >();
}
ValueMapper <VBuffer <Single>, VBuffer <Single> > del =
(ref VBuffer <Single> src, ref VBuffer <Single> dst) =>
{
if (_inputType.VectorSize > 0)
{
Host.Check(src.Length == _inputType.VectorSize);
}
var tmp = src;
Parallel.For(0, maps.Length, i =>
{
var model = Models[i];
if (model.SelectedFeatures != null)
{
EnsembleUtils.SelectFeatures(ref tmp, model.SelectedFeatures, model.Cardinality, ref features[i]);
maps[i](ref features[i], ref predictions[i]);
}
else
{
maps[i](ref tmp, ref predictions[i]);
}
// individual maps delegates will return always the same VBuffer length
Host.Check(predictions[i].Length == _mappers[i].OutputType.VectorSize);
});
combine(ref dst, predictions, Weights);
};
return((ValueMapper <TIn, TOut>)(Delegate) del);
}
private bool DoParseArgumentsCore(string[] args, object options)
{
var hadError = false;
var optionMap = OptionMap.Create(options, _settings);
optionMap.SetDefaults();
var valueMapper = new ValueMapper(options, _settings.ParsingCulture);
var arguments = new StringArrayEnumerator(args);
while (arguments.MoveNext())
{
var argument = arguments.Current;
if (string.IsNullOrEmpty(argument))
{
continue;
}
var parser = ArgumentParser.Create(argument, _settings.IgnoreUnknownArguments);
if (parser != null)
{
var result = parser.Parse(arguments, optionMap, options);
if ((result & PresentParserState.Failure) == PresentParserState.Failure)
{
SetParserStateIfNeeded(options, parser.PostParsingState);
hadError = true;
continue;
}
if ((result & PresentParserState.MoveOnNextElement) == PresentParserState.MoveOnNextElement)
{
arguments.MoveNext();
}
}
else if (valueMapper.CanReceiveValues)
{
if (!valueMapper.MapValueItem(argument))
{
hadError = true;
}
}
}
hadError |= !optionMap.EnforceRules();
return !hadError;
}
