internal ImplRaw(TScalarPredictor[] predictors)
{
Contracts.CheckNonEmpty(predictors, nameof(predictors));
Predictors = new IValueMapper[predictors.Length];
ColumnType inputType = null;
for (int i = 0; i < predictors.Length; i++)
{
var vm = predictors[i] as IValueMapper;
Contracts.Check(IsValid(vm, ref inputType), "Predictor doesn't implement the expected interface");
Predictors[i] = vm;
}
CanSavePfa = Predictors.All(m => (m as ISingleCanSavePfa)?.CanSavePfa == true);
Contracts.AssertValue(inputType);
InputType = inputType;
}
public static List <TObject> GetValueItemsList <TObject>(IValueMapper <TObject> mapper, DataReaderAdapter dataReader)
{
if (mapper == null)
{
throw new ArgumentNullException("mapper");
}
List <TObject> result = new List <TObject>();
if (dataReader != null)
{
while (dataReader.Read())
{
result.Add(mapper.GetValue(dataReader));
}
}
return(result);
}
private void InitializeMappers(out IValueMapper[] mappers, out VectorType inputType, out VectorType outputType)
{
Host.AssertNonEmpty(Models);
mappers = new IValueMapper[Models.Length];
inputType = null;
outputType = null;
for (int i = 0; i < Models.Length; i++)
{
var vm = Models[i].Predictor as IValueMapper;
if (!IsValid(vm, out VectorType vmInputType, out VectorType vmOutputType))
{
throw Host.Except("Predictor does not implement expected interface");
}
if (vmInputType.Size > 0)
{
if (inputType == null)
{
inputType = vmInputType;
}
else if (vmInputType.Size != inputType.Size)
{
throw Host.Except("Predictor input type mismatch");
}
}
if (outputType == null || vmOutputType.Size > outputType.Size)
{
outputType = vmOutputType;
}
mappers[i] = vm;
}
Host.AssertValue(outputType);
if (inputType == null)
{
inputType = new VectorType(NumberDataViewType.Single);
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="env">environment like ConsoleEnvironment</param>
/// <param name="mapper">IValueMapper</param>
/// <param name="source">source to replace</param>
/// <param name="inputColumn">name of the input column (the last one sharing the same type)</param>
/// <param name="outputColumn">name of the output column</param>
public TransformFromValueMapper(IHostEnvironment env, IValueMapper mapper, IDataView source,
string inputColumn, string outputColumn = "output")
{
Contracts.AssertValue(env);
Contracts.AssertValue(mapper);
Contracts.AssertValue(source);
_host = env;
if (string.IsNullOrEmpty(inputColumn))
{
var inputType = mapper.InputType;
for (int i = source.Schema.ColumnCount - 1; i >= 0; --i)
{
var ty = source.Schema.GetColumnType(i);
if (ty.SameSizeAndItemType(inputType))
{
inputColumn = source.Schema.GetColumnName(i);
break;
}
}
}
_source = source;
_mapper = mapper;
int index;
if (!_source.Schema.TryGetColumnIndex(inputColumn, out index))
{
throw env.Except("Unable to find column '{0}' in input schema.", inputColumn);
}
_inputColumn = inputColumn;
if (_source.Schema.TryGetColumnIndex(outputColumn, out index))
{
throw env.Except("Column '{0}' already present in input schema.", outputColumn);
}
_outputColumn = outputColumn;
_schema = Schema.Create(new ExtendedSchema(source.Schema, new[] { outputColumn }, new[] { mapper.OutputType }));
_transform = CreateMemoryTransform();
}
static object MapDataReaderToObject(IDataContext dataContext, IDataReader dataReader, MappingData data)
{
var source = data.MappingSchema.CreateDataReaderMapper(dataReader);
var initContext = new InitContext
{
MappingSchema = data.MappingSchema,
DataSource = source,
SourceObject = dataReader,
ObjectMapper = data.ObjectMapper
};
var destObject = dataContext.CreateInstance(initContext) ?? data.ObjectMapper.CreateInstance(initContext);
if (initContext.StopMapping)
{
return(destObject);
}
var smDest = destObject as ISupportMapping;
if (smDest != null)
{
smDest.BeginMapping(initContext);
if (initContext.StopMapping)
{
return(destObject);
}
}
if (data.ValueMappers == null)
{
var mappers = new IValueMapper[data.Index.Length];
for (var i = 0; i < data.Index.Length; i++)
{
var n = data.Index[i];
if (n < 0)
{
continue;
}
if (!data.ObjectMapper.SupportsTypedValues(i))
{
mappers[i] = data.MappingSchema.DefaultValueMapper;
continue;
}
var sourceType = source.GetFieldType(n);
var destType = data.ObjectMapper.GetFieldType(i);
if (sourceType == null)
{
sourceType = typeof(object);
}
if (destType == null)
{
destType = typeof(object);
}
IValueMapper t;
if (sourceType == destType)
{
lock (data.MappingSchema.SameTypeMappers)
if (!data.MappingSchema.SameTypeMappers.TryGetValue(sourceType, out t))
{
data.MappingSchema.SameTypeMappers.Add(sourceType, t = data.MappingSchema.GetValueMapper(sourceType, destType));
}
}
else
{
var key = new KeyValuePair <Type, Type>(sourceType, destType);
lock (data.MappingSchema.DifferentTypeMappers)
if (!data.MappingSchema.DifferentTypeMappers.TryGetValue(key, out t))
{
data.MappingSchema.DifferentTypeMappers.Add(key, t = data.MappingSchema.GetValueMapper(sourceType, destType));
}
}
mappers[i] = t;
}
data.ValueMappers = mappers;
}
var dest = data.ObjectMapper;
var idx = data.Index;
var ms = data.ValueMappers;
for (var i = 0; i < idx.Length; i++)
{
var n = idx[i];
if (n >= 0)
{
ms[i].Map(source, dataReader, n, dest, destObject, i);
}
}
if (smDest != null)
{
smDest.EndMapping(initContext);
}
return(destObject);
}
/// <summary>
/// Initializes a new instance of the <see cref="PropertyMapper{TSource, TTarget}"/> class.
/// </summary>
/// <param name="sourceProperty">Source property.</param>
/// <param name="targetProperty">Target property.</param>
/// <param name="valueMapper">Value mapper.</param>
public PropertyMapper(IProperty <TSource> sourceProperty, IProperty <TTarget> targetProperty, IValueMapper <TSource, TTarget> valueMapper)
{
SourceProperty = sourceProperty.AssertArgumentNotNull(nameof(sourceProperty));
TargetProperty = targetProperty.AssertArgumentNotNull(nameof(targetProperty));
ValueMapper = valueMapper.AssertArgumentNotNull(nameof(valueMapper));
}
public IKTable <K, VR> MapValues <VR>(IValueMapper <V, VR> mapper, Materialized <K, VR, IKeyValueStore <Bytes, byte[]> > materialized, string named = null) => this.MapValues(WithKey(mapper), materialized, named);
public async Task <IEnumerable <TResult> > Query <TResult>(QueryRequest request, IValueMapper <TResult> valueMapper)
{
var response = await PerformDbAction(request, (client, r) => client.QueryAsync(r));
if (LogCapacityForRequest(request.ReturnConsumedCapacity))
{
logWriter.LogInformation($"Consumed DynamoDB capacity:\n{JsonConvert.SerializeObject(response.ConsumedCapacity)}");
}
return(response.Items.Select(valueMapper.FromDbItem));
}
public AdviceCallExpressionBuilder(IValueMapper valueMapper)
{
_valueMapper = valueMapper;
}
protected static WrappedValueMapperWithKey <K, V, VR> WithKey <VR>(IValueMapper <V, VR> valueMapper)
{
valueMapper = valueMapper ?? throw new ArgumentNullException(nameof(valueMapper));
return(new WrappedValueMapperWithKey <K, V, VR>((readOnlyKey, value) => valueMapper.Apply(value)));
}
private static void AddSameType(Type type, IValueMapper mapper)
{
_mappers.Add(new KeyValue(type, type), mapper);
}
public JoinPointExpressionBuilder(IInvocationTypeProvider invocationTypeProvider, IValueMapper valueMapper)
{
_invocationTypeProvider = invocationTypeProvider;
_valueMapper = valueMapper;
}
public IKStream <K, VR> FlatMapValues <VR>(IValueMapper <V, IEnumerable <VR> > mapper, string named = null) => this.FlatMapValues(WithKey(mapper), named);
public Task <TResult> GetItem <TResult>(GetItemRequest request, IValueMapper <TResult> valueMapper)
{
return(GetItem(request, valueMapper, default(TResult)));
}
public virtual ValueMapper <VBuffer <float>, VBuffer <float> > GetMapper()
{
Contracts.Assert(_mappers != null);
Contracts.Assert(_mappers.Length == 1);
var mapper = _mappers[0].GetMapper <VBuffer <float>, float>();
VBuffer <float> inputs = new VBuffer <float>();
float[] labelClasses = _classes.Values.Select(c => _labelConverter(c)).ToArray();
int maxClass = _dstIndices == null ? _classes.Length : _dstIndices.Max() + 1;
bool labelKey = _labelKey;
IValueMapper imapperFinal = _reclassificationPredictor == null ? null : _reclassificationPredictor as IValueMapper;
var mapperFinal = _reclassificationPredictor == null ? null : imapperFinal.GetMapper <VBuffer <float>, VBuffer <float> >();
var tempOut = new VBuffer <float>();
if (_singleColumn)
{
if (labelKey)
{
for (int i = 0; i < labelClasses.Length; ++i)
{
--labelClasses[i];
}
}
// Only one new column added which contain the label.
return
((in VBuffer <float> src, ref VBuffer <float> dst) =>
{
if (dst.Count != maxClass)
{
dst = new VBuffer <float>(maxClass, _classes.Length, new float[_classes.Length], _dstIndices);
}
var values = inputs.Values;
var indices = inputs.Indices;
if (src.IsDense)
{
if (values == null || values.Length <= src.Count)
{
values = new float[src.Length + 1];
}
Array.Copy(src.Values, values, src.Length);
Contracts.Assert(src.Indices == null);
inputs = new VBuffer <float>(src.Length + 1, values);
}
else
{
if (src.Indices == null)
{
if (src.Values == null)
{
// All missing.
for (int i = 0; i < _classes.Length; ++i)
{
dst.Values[i] = 0;
}
return;
}
else
{
throw Contracts.Except("Inconsistency in input vector. Sparse vector with null indices.");
}
}
int nb = src.Count + 1;
if (values == null || values.Length < nb)
{
values = new float[nb];
}
if (indices == null || indices.Length < nb)
{
indices = new int[nb];
}
Array.Copy(src.Values, values, src.Count);
Array.Copy(src.Indices, indices, src.Count);
indices[src.Count] = src.Length;
inputs = new VBuffer <float>(src.Length + 1, src.Count + 1, values, indices);
}
for (int i = 0; i < _classes.Length; ++i)
{
inputs.Values[src.Count] = labelClasses[i];
mapper(in inputs, ref dst.Values[i]);
}
// Only if probabilities
// Normalize(dst.Values, _classes.Length);
#region debug
#if (DEBUG)
var dense = dst.DenseValues().ToArray();
if (dense.Length != maxClass)
{
throw Contracts.Except("Different dimension {0} != {1}-{2}", maxClass, dst.Length, dst.Count);
}
#endif
#endregion
if (mapperFinal != null)
{
mapperFinal(in dst, ref tempOut);
dst = tempOut;
}
});
private bool IsValid(IValueMapper mapper)
{
return(mapper != null &&
mapper.InputType.IsVector && mapper.InputType.ItemType == NumberType.Float &&
mapper.OutputType == NumberType.Float);
}
/// <summary>
/// Create an enumerator over the value ranges from this <see cref="Trie2"/>.
/// Values from the <see cref="Trie2"/> are passed through a caller-supplied remapping function,
/// and it is the remapped values that determine the ranges that
/// will be produced by the iterator.
/// <para/>
/// Note that this method was named iterator(ValueMapper) in ICU4J.
/// </summary>
/// <param name="mapper">Provides a function to remap values obtained from the <see cref="Trie2"/>.</param>
/// <returns>An enumerator.</returns>
public virtual IEnumerator <Trie2Range> GetEnumerator(IValueMapper mapper)
{
return(new Trie2Enumerator(this, mapper));
}
/// <summary>
/// Create an enumerator over the <see cref="Trie2"/> values for the 1024=0x400 code points
/// corresponding to a given <paramref name="lead"/> surrogate.
/// <para/>
/// For example, for the lead surrogate U+D87E it will enumerate the values
/// for [U+2F800..U+2FC00[.
/// <para/>
/// Note that this method was named iteratorForLeadSurrogate(char, ValueMapper) in ICU4J.
/// </summary>
/// <remarks>
/// Used by data builder code that sets special lead surrogate code unit values
/// for optimized UTF-16 string processing.
/// <para/>
/// Do not modify the <see cref="Trie2"/> during the iteration.
/// <para/>
/// Except for the limited code point range, this functions just like <see cref="Trie2.GetEnumerator()"/>.
/// </remarks>
public virtual IEnumerator <Trie2Range> GetEnumeratorForLeadSurrogate(char lead, IValueMapper mapper)
{
return(new Trie2Enumerator(this, lead, mapper));
}
public IKTable <K, VR> MapValues <VR>(IValueMapper <V, VR> mapper, string named = null) => this.MapValues(WithKey(mapper), named);
private static void AddSameType(Type type, IValueMapper mapper)
{
_mappers.Add(new KeyValue(type, type), mapper);
}
protected object MapDataReaderToObject(Type destObjectType, IDataReader dataReader, int slotNumber)
{
var slot = _mapperSlots[slotNumber];
var source = MappingSchema.CreateDataReaderMapper(dataReader);
var dest = slot.ObjectMapper ?? (slot.ObjectMapper = MappingSchema.GetObjectMapper(destObjectType));
var initContext = new InitContext
{
MappingSchema = MappingSchema,
DataSource = source,
SourceObject = dataReader,
ObjectMapper = dest
};
var destObject = dest.CreateInstance(initContext);
if (initContext.StopMapping)
{
return(destObject);
}
var smDest = destObject as ISupportMapping;
if (smDest != null)
{
smDest.BeginMapping(initContext);
if (initContext.StopMapping)
{
return(destObject);
}
}
var index = slot.Index;
if (slot.ValueMappers == null)
{
var mappers = new IValueMapper[index.Length];
for (var i = 0; i < index.Length; i++)
{
var n = index[i];
if (n < 0)
{
continue;
}
if (!dest.SupportsTypedValues(i))
{
mappers[i] = MappingSchema.DefaultValueMapper;
continue;
}
var sourceType = source.GetFieldType(n);
var destType = dest.GetFieldType(i);
if (sourceType == null)
{
sourceType = typeof(object);
}
if (destType == null)
{
destType = typeof(object);
}
if (sourceType == destType)
{
var t = (IValueMapper)MappingSchema.SameTypeMappers[sourceType];
if (t == null)
{
lock (MappingSchema.SameTypeMappers.SyncRoot)
{
t = (IValueMapper)MappingSchema.SameTypeMappers[sourceType];
if (t == null)
{
MappingSchema.SameTypeMappers[sourceType] = t = MappingSchema.GetValueMapper(sourceType, destType);
}
}
}
mappers[i] = t;
}
else
{
var key = new KeyValuePair <Type, Type>(sourceType, destType);
var t = (IValueMapper)MappingSchema.DifferentTypeMappers[key];
if (t == null)
{
lock (MappingSchema.DifferentTypeMappers.SyncRoot)
{
t = (IValueMapper)MappingSchema.DifferentTypeMappers[key];
if (t == null)
{
MappingSchema.DifferentTypeMappers[key] = t = MappingSchema.GetValueMapper(sourceType, destType);
}
}
}
mappers[i] = t;
}
}
slot.ValueMappers = mappers;
}
var ms = slot.ValueMappers;
for (var i = 0; i < index.Length; i++)
{
var n = index[i];
if (n >= 0)
{
ms[i].Map(source, dataReader, n, dest, destObject, i);
}
}
if (smDest != null)
{
smDest.EndMapping(initContext);
}
return(destObject);
}
