public static bool IsValidRangeFilterColumnType(IExceptionContext ectx, DataViewType type)
{
ectx.CheckValue(type, nameof(type));
return(type == NumberDataViewType.Single || type == NumberDataViewType.Double || type.GetKeyCount() > 0);
}
public static IEnumerable <KeyValuePair <ColumnRole, string> > CheckAndGenerateCustomColumns(IExceptionContext ectx, KeyValuePair <string, string>[] customColumnArg)
{
Contracts.CheckValueOrNull(ectx);
if (customColumnArg == null)
{
return(Enumerable.Empty <KeyValuePair <ColumnRole, string> >());
}
foreach (var kindName in customColumnArg)
{
ectx.CheckUserArg(!string.IsNullOrWhiteSpace(kindName.Value), nameof(TrainCommand.Arguments.CustomColumn), "Names for columns with custom kind must not be empty");
if (string.IsNullOrWhiteSpace(kindName.Key))
{
throw ectx.ExceptUserArg(nameof(TrainCommand.Arguments.CustomColumn), "Custom column with name '{0}' needs a kind. Use col[<Kind>]={0}", kindName.Value);
}
}
return(customColumnArg.Select(kindName => new ColumnRole(kindName.Key).Bind(kindName.Value)));
}
private static JToken BuildValueToken(IExceptionContext ectx, object value, Type valueType, ComponentCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValueOrNull(value);
ectx.AssertValue(valueType);
ectx.AssertValue(catalog);
if (value == null)
{
return(null);
}
// Dive inside Nullable.
if (valueType.IsGenericType && valueType.GetGenericTypeDefinition() == typeof(Nullable <>))
{
valueType = valueType.GetGenericArguments()[0];
}
// Dive inside Optional.
if (valueType.IsGenericType && valueType.GetGenericTypeDefinition() == typeof(Optional <>))
{
valueType = valueType.GetGenericArguments()[0];
value = ((Optional)value).GetValue();
}
var dataType = TlcModule.GetDataType(valueType);
switch (dataType)
{
case TlcModule.DataKind.Bool:
case TlcModule.DataKind.Int:
case TlcModule.DataKind.UInt:
case TlcModule.DataKind.Float:
case TlcModule.DataKind.String:
return(new JValue(value));
case TlcModule.DataKind.Char:
return(new JValue(value.ToString()));
case TlcModule.DataKind.Array:
var valArray = value as Array;
var ja = new JArray();
foreach (var item in valArray)
{
ja.Add(BuildValueToken(ectx, item, item.GetType(), catalog));
}
return(ja);
case TlcModule.DataKind.Enum:
return(value.ToString());
case TlcModule.DataKind.Dictionary:
// REVIEW: need to figure out how to represent these.
throw ectx.ExceptNotSupp("Dictionary and component default values are not supported");
case TlcModule.DataKind.Component:
var factory = value as IComponentFactory;
ectx.AssertValue(factory);
return(BuildComponentToken(ectx, factory, catalog));
default:
throw ectx.ExceptNotSupp("Encountered a default value for unsupported type {0}", dataType);
}
}
private RepositoryWriter(Stream stream, IExceptionContext ectx, bool useFileSystem = true)
: base(useFileSystem, ectx)
{
_archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true);
_closed = new Queue <KeyValuePair <string, Stream> >();
}
// REVIEW: include Subcomponent array for testing once it is supported
//[Argument(ArgumentType.Multiple)]
//public SubComponent[] sub4 = new SubComponent[] { new SubComponent("sub4", "settings4"), new SubComponent("sub5", "settings5") };
/// <summary>
/// ToString is overrided by CmdParser.GetSettings which is of primary for this test
/// </summary>
/// <returns></returns>
public string ToString(IExceptionContext ectx)
{
return(CmdParser.GetSettings(ectx, this, new SimpleArg(), SettingsFlags.None));
}
/// <summary>
/// Builds a JSON representation of all entry points and components of the <paramref name="catalog"/>.
/// </summary>
/// <param name="ectx">The exception context to use</param>
/// <param name="catalog">The module catalog</param>
public static JObject BuildAllManifests(IExceptionContext ectx, ComponentCatalog catalog)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(catalog, nameof(catalog));
var jEntryPoints = new JArray();
var entryPointInfos = catalog.AllEntryPoints().ToArray();
foreach (var entryPointInfo in entryPointInfos.OrderBy(x => x.Name))
{
jEntryPoints.Add(BuildEntryPointManifest(ectx, entryPointInfo, catalog));
}
var jKinds = new JArray();
foreach (var kind in catalog.GetAllComponentKinds())
{
var jKind = new JObject();
jKind[FieldNames.Kind] = kind;
var jComponents = new JArray();
foreach (var component in catalog.GetAllComponents(kind))
{
jComponents.Add(BuildComponentManifest(ectx, component, catalog));
}
jKind[FieldNames.Components] = jComponents;
jKinds.Add(jKind);
}
var jepKinds = new JArray();
var kinds = new List <Type>();
foreach (var entryPointInfo in entryPointInfos)
{
if (entryPointInfo.InputKinds != null)
{
kinds.AddRange(entryPointInfo.InputKinds);
}
if (entryPointInfo.OutputKinds != null)
{
kinds.AddRange(entryPointInfo.OutputKinds);
}
}
foreach (var epKind in kinds.Distinct().OrderBy(k => k.Name))
{
var jepKind = new JObject();
jepKind[FieldNames.Kind] = epKind.Name;
var jepKindFields = new JArray();
var propertyInfos = epKind.GetProperties().AsEnumerable();
propertyInfos = epKind.GetInterfaces().Aggregate(propertyInfos, (current, face) => current.Union(face.GetProperties()));
foreach (var fieldInfo in propertyInfos)
{
var jField = new JObject();
jField[FieldNames.Name] = fieldInfo.Name;
var type = ExtractOptionalOrNullableType(fieldInfo.PropertyType);
// Dive inside Var.
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>))
{
type = type.GetGenericArguments()[0];
}
var typeEnum = TlcModule.GetDataType(type);
jField[FieldNames.Type] = typeEnum.ToString();
jepKindFields.Add(jField);
}
jepKind[FieldNames.Settings] = jepKindFields;
jepKinds.Add(jepKind);
}
var jResult = new JObject();
jResult[FieldNames.TopEntryPoints] = jEntryPoints;
jResult[FieldNames.TopComponents] = jKinds;
jResult[FieldNames.TopEntryPointKinds] = jepKinds;
return(jResult);
}
// This converts in place.
private static void FillValues(IExceptionContext ectx, ref VBuffer <Float> buffer)
{
int size = buffer.Length;
ectx.Check(0 <= size & size < int.MaxValue / 2);
int count = buffer.Count;
var values = buffer.Values;
var indices = buffer.Indices;
int iivDst = 0;
if (count >= size)
{
// Currently, it's dense. We always produce sparse.
ectx.Assert(Utils.Size(values) >= size);
if (Utils.Size(indices) < size)
{
indices = new int[size];
}
for (int ivSrc = 0; ivSrc < count; ivSrc++)
{
ectx.Assert(iivDst <= ivSrc);
var val = values[ivSrc];
if (val == 0)
{
continue;
}
if (Float.IsNaN(val))
{
values[iivDst] = 1;
indices[iivDst] = 2 * ivSrc + 1;
}
else
{
values[iivDst] = val;
indices[iivDst] = 2 * ivSrc;
}
iivDst++;
}
}
else
{
// Currently, it's sparse.
ectx.Assert(Utils.Size(values) >= count);
ectx.Assert(Utils.Size(indices) >= count);
int ivPrev = -1;
for (int iivSrc = 0; iivSrc < count; iivSrc++)
{
ectx.Assert(iivDst <= iivSrc);
var val = values[iivSrc];
if (val == 0)
{
continue;
}
int iv = indices[iivSrc];
ectx.Assert(ivPrev < iv & iv < size);
ivPrev = iv;
if (Float.IsNaN(val))
{
values[iivDst] = 1;
indices[iivDst] = 2 * iv + 1;
}
else
{
values[iivDst] = val;
indices[iivDst] = 2 * iv;
}
iivDst++;
}
}
ectx.Assert(0 <= iivDst & iivDst <= count);
buffer = new VBuffer <Float>(size * 2, iivDst, values, indices);
}
private static void DbgFailEmpty(IExceptionContext ctx = null)
{
DbgFailCore("Non-empty assertion failure", ctx);
}
private protected virtual Func <DataViewSchema, IRowToRowMapper> TransformerChecker(IExceptionContext ectx, ITransformer transformer)
{
ectx.CheckValue(transformer, nameof(transformer));
ectx.CheckParam(transformer.IsRowToRowMapper, nameof(transformer), "Must be a row to row mapper");
return(transformer.GetRowToRowMapper);
}
/// <summary>
/// Returns a standard exception for responding to an invalid call to GetAnnotation.
/// </summary>
public static Exception ExceptGetAnnotation(this IExceptionContext ctx) => ctx.Except("Invalid call to GetAnnotation");
public SimpleSchema(IExceptionContext ectx, params KeyValuePair <string, ColumnType>[] columns)
: base(ectx, columns)
{
_keyValueGetters = new MetadataUtils.MetadataGetter <VBuffer <ReadOnlyMemory <char> > > [ColumnCount];
}
public int FeedTrain(IExceptionContext ectx, in VBuffer <Double> input)
private static void DbgFailEmpty(IExceptionContext ctx, string msg)
{
DbgFailCore(string.Format(CultureInfo.InvariantCulture, "Non-empty assertion failure: {0}", msg), ctx);
}
/// <summary> /// 在Api执行中异常时触发 /// </summary> /// <param name="filterContext">上下文</param> protected abstract void OnException(IExceptionContext filterContext);
/// <summary>
/// Returns a standard exception for responding to an invalid call to GetMetadata.
/// </summary>
public static Exception ExceptGetMetadata(this IExceptionContext ctx)
{
return(ctx.Except("Invalid call to GetMetadata"));
}
public ExceptionVM(ExceptionInfo info, IExceptionContext context) {
this.info = info;
this.context = context;
}
private static object ParseJsonValue(IExceptionContext ectx, Type type, Attributes attributes, JToken value, ComponentCatalog catalog)
{
Contracts.AssertValue(ectx);
ectx.AssertValue(type);
ectx.AssertValueOrNull(value);
ectx.AssertValue(catalog);
if (value == null)
{
return(null);
}
if (value is JValue val && val.Value == null)
{
return(null);
}
if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Optional <>) || type.GetGenericTypeDefinition() == typeof(Nullable <>)))
{
if (type.GetGenericTypeDefinition() == typeof(Optional <>) && value.HasValues)
{
value = value.Values().FirstOrDefault();
}
type = type.GetGenericArguments()[0];
}
if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Var <>)))
{
string varName = value.Value <string>();
ectx.Check(VariableBinding.IsBindingToken(value), "Variable name expected.");
var variable = Activator.CreateInstance(type) as IVarSerializationHelper;
var varBinding = VariableBinding.Create(ectx, varName);
variable.VarName = varBinding.VariableName;
return(variable);
}
if (type == typeof(JArray) && value is JArray)
{
return(value);
}
TlcModule.DataKind dt = TlcModule.GetDataType(type);
try
{
switch (dt)
{
case TlcModule.DataKind.Bool:
return(value.Value <bool>());
case TlcModule.DataKind.String:
return(value.Value <string>());
case TlcModule.DataKind.Char:
return(value.Value <char>());
case TlcModule.DataKind.Enum:
if (!Enum.IsDefined(type, value.Value <string>()))
{
throw ectx.Except($"Requested value '{value.Value<string>()}' is not a member of the Enum type '{type.Name}'");
}
return(Enum.Parse(type, value.Value <string>()));
case TlcModule.DataKind.Float:
if (type == typeof(double))
{
return(value.Value <double>());
}
else if (type == typeof(float))
{
return(value.Value <float>());
}
else
{
ectx.Assert(false);
throw ectx.ExceptNotSupp();
}
case TlcModule.DataKind.Array:
var ja = value as JArray;
ectx.Check(ja != null, "Expected array value");
Func <IExceptionContext, JArray, Attributes, ComponentCatalog, object> makeArray = MakeArray <int>;
return(Utils.MarshalInvoke(makeArray, type.GetElementType(), ectx, ja, attributes, catalog));
case TlcModule.DataKind.Int:
if (type == typeof(long))
{
return(value.Value <long>());
}
if (type == typeof(int))
{
return(value.Value <int>());
}
ectx.Assert(false);
throw ectx.ExceptNotSupp();
case TlcModule.DataKind.UInt:
if (type == typeof(ulong))
{
return(value.Value <ulong>());
}
if (type == typeof(uint))
{
return(value.Value <uint>());
}
ectx.Assert(false);
throw ectx.ExceptNotSupp();
case TlcModule.DataKind.Dictionary:
ectx.Check(value is JObject, "Expected object value");
Func <IExceptionContext, JObject, Attributes, ComponentCatalog, object> makeDict = MakeDictionary <int>;
return(Utils.MarshalInvoke(makeDict, type.GetGenericArguments()[1], ectx, (JObject)value, attributes, catalog));
case TlcModule.DataKind.Component:
var jo = value as JObject;
ectx.Check(jo != null, "Expected object value");
// REVIEW: consider accepting strings alone.
var jName = jo[FieldNames.Name];
ectx.Check(jName != null, "Field '" + FieldNames.Name + "' is required for component.");
ectx.Check(jName is JValue, "Expected '" + FieldNames.Name + "' field to be a string.");
var name = jName.Value <string>();
ectx.Check(jo[FieldNames.Settings] == null || jo[FieldNames.Settings] is JObject,
"Expected '" + FieldNames.Settings + "' field to be an object");
return(GetComponentJson(ectx, type, name, jo[FieldNames.Settings] as JObject, catalog));
default:
var settings = value as JObject;
ectx.Check(settings != null, "Expected object value");
var inputBuilder = new InputBuilder(ectx, type, catalog);
if (inputBuilder._fields.Length == 0)
{
throw ectx.Except($"Unsupported input type: {dt}");
}
if (settings != null)
{
foreach (var pair in settings)
{
if (!inputBuilder.TrySetValueJson(pair.Key, pair.Value))
{
throw ectx.Except($"Unexpected value for component '{type}', field '{pair.Key}': '{pair.Value}'");
}
}
}
var missing = inputBuilder.GetMissingValues().ToArray();
if (missing.Length > 0)
{
throw ectx.Except($"The following required inputs were not provided for component '{type}': {string.Join(", ", missing)}");
}
return(inputBuilder.GetInstance());
}
}
catch (FormatException ex)
{
if (ex.IsMarked())
{
throw;
}
throw ectx.Except(ex, $"Failed to parse JSON value '{value}' as {type}");
}
}
public LdaState(IExceptionContext ectx, ModelLoadContext ctx)
: this()
{
ectx.AssertValue(ctx);
// *** Binary format ***
// <ColInfoEx>
// int: vocabnum
// long: memblocksize
// long: aliasMemBlockSize
// (serializing term by term, for one term)
// int: term_id, int: topic_num, KeyValuePair<int, int>[]: termTopicVector
InfoEx = new ColInfoEx(ectx, ctx);
_numVocab = ctx.Reader.ReadInt32();
ectx.CheckDecode(_numVocab > 0);
long memBlockSize = ctx.Reader.ReadInt64();
ectx.CheckDecode(memBlockSize > 0);
long aliasMemBlockSize = ctx.Reader.ReadInt64();
ectx.CheckDecode(aliasMemBlockSize > 0);
_ldaTrainer = new LdaSingleBox(
InfoEx.NumTopic,
_numVocab, /* Need to set number of vocabulary here */
InfoEx.AlphaSum,
InfoEx.Beta,
InfoEx.NumIter,
InfoEx.LikelihoodInterval,
InfoEx.NumThread,
InfoEx.MHStep,
InfoEx.NumSummaryTermPerTopic,
false,
InfoEx.NumMaxDocToken);
_ldaTrainer.AllocateModelMemory(_numVocab, InfoEx.NumTopic, memBlockSize, aliasMemBlockSize);
for (int i = 0; i < _numVocab; i++)
{
int termID = ctx.Reader.ReadInt32();
ectx.CheckDecode(termID >= 0);
int termTopicNum = ctx.Reader.ReadInt32();
ectx.CheckDecode(termTopicNum >= 0);
int[] topicId = new int[termTopicNum];
int[] topicProb = new int[termTopicNum];
for (int j = 0; j < termTopicNum; j++)
{
topicId[j] = ctx.Reader.ReadInt32();
topicProb[j] = ctx.Reader.ReadInt32();
}
//set the topic into _ldaTrainer inner topic table
_ldaTrainer.SetModel(termID, topicId, topicProb, termTopicNum);
}
//do the preparation
if (!_predictionPreparationDone)
{
_ldaTrainer.InitializeBeforeTest();
_predictionPreparationDone = true;
}
}
public abstract void Train(IExceptionContext ectx, RowCursor cursor, int colTerm, int colValue);
private static JArray BuildInputManifest(IExceptionContext ectx, Type inputType, ComponentCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(inputType);
ectx.AssertValue(catalog);
// Instantiate a value of the input, to pull defaults out of.
var defaults = Activator.CreateInstance(inputType);
var inputs = new List <KeyValuePair <Double, JObject> >();
foreach (var fieldInfo in inputType.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
{
var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute;
if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
var jo = new JObject();
jo[FieldNames.Name] = inputAttr.Name ?? fieldInfo.Name;
jo[FieldNames.Type] = BuildTypeToken(ectx, fieldInfo, fieldInfo.FieldType, catalog);
jo[FieldNames.Desc] = inputAttr.HelpText;
if (inputAttr.Aliases != null)
{
jo[FieldNames.Aliases] = new JArray(inputAttr.Aliases);
}
jo[FieldNames.Required] = inputAttr.IsRequired;
jo[FieldNames.SortOrder] = inputAttr.SortOrder;
jo[FieldNames.IsNullable] = fieldInfo.FieldType.IsGenericType && (fieldInfo.FieldType.GetGenericTypeDefinition() == typeof(Nullable <>));
var defaultValue = fieldInfo.GetValue(defaults);
var dataType = TlcModule.GetDataType(fieldInfo.FieldType);
if (!inputAttr.IsRequired || (dataType != TlcModule.DataKind.Unknown && defaultValue != null))
{
jo[FieldNames.Default] = BuildValueToken(ectx, defaultValue, fieldInfo.FieldType, catalog);
}
if (fieldInfo.FieldType.IsGenericType &&
fieldInfo.FieldType.GetGenericTypeDefinition() == typeof(Optional <>))
{
var val = fieldInfo.GetValue(defaults) as Optional;
if (val == null && !inputAttr.IsRequired)
{
throw ectx.Except("Field '{0}' is an Optional<> type but is null by default, instead of set to a constructed implicit default.", fieldInfo.Name);
}
if (val != null && val.IsExplicit)
{
throw ectx.Except("Field '{0}' is an Optional<> type with a non-implicit default value.", fieldInfo.Name);
}
}
var rangeAttr = fieldInfo.GetCustomAttributes(typeof(TlcModule.RangeAttribute), false).FirstOrDefault() as TlcModule.RangeAttribute;
if (rangeAttr != null)
{
if (!TlcModule.IsNumericKind(TlcModule.GetDataType(fieldInfo.FieldType)))
{
throw ectx.Except("Field '{0}' has a range but is of a non-numeric type.", fieldInfo.Name);
}
if (!rangeAttr.Type.Equals(fieldInfo.FieldType))
{
throw ectx.Except("Field '{0}' has a range attribute that uses a type which is not equal to the field's FieldType.", fieldInfo.Name);
}
var jRange = new JObject();
if (rangeAttr.Sup != null)
{
jRange[FieldNames.Range.Sup] = JToken.FromObject(rangeAttr.Sup);
}
if (rangeAttr.Inf != null)
{
jRange[FieldNames.Range.Inf] = JToken.FromObject(rangeAttr.Inf);
}
if (rangeAttr.Max != null)
{
jRange[FieldNames.Range.Max] = JToken.FromObject(rangeAttr.Max);
}
if (rangeAttr.Min != null)
{
jRange[FieldNames.Range.Min] = JToken.FromObject(rangeAttr.Min);
}
jo[FieldNames.Range.Type] = jRange;
}
// Handle deprecated/obsolete attributes, passing along the message to the manifest.
if (fieldInfo.GetCustomAttributes(typeof(ObsoleteAttribute), false).FirstOrDefault() is ObsoleteAttribute obsAttr)
{
var jParam = new JObject
{
[FieldNames.Deprecated.Message] = JToken.FromObject(obsAttr.Message),
};
jo[FieldNames.Deprecated.ToString()] = jParam;
}
if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableLongParamAttribute), false).FirstOrDefault() is TlcModule.SweepableLongParamAttribute slpAttr)
{
var jParam = new JObject
{
[FieldNames.SweepableLongParam.RangeType] = JToken.FromObject("Long"),
[FieldNames.SweepableLongParam.Min] = JToken.FromObject(slpAttr.Min),
[FieldNames.SweepableLongParam.Max] = JToken.FromObject(slpAttr.Max)
};
if (slpAttr.StepSize != null)
{
jParam[FieldNames.SweepableLongParam.StepSize] = JToken.FromObject(slpAttr.StepSize);
}
if (slpAttr.NumSteps != null)
{
jParam[FieldNames.SweepableLongParam.NumSteps] = JToken.FromObject(slpAttr.NumSteps);
}
if (slpAttr.IsLogScale)
{
jParam[FieldNames.SweepableLongParam.IsLogScale] = JToken.FromObject(true);
}
jo[FieldNames.SweepableLongParam.ToString()] = jParam;
}
if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableFloatParamAttribute), false).FirstOrDefault() is TlcModule.SweepableFloatParamAttribute sfpAttr)
{
var jParam = new JObject
{
[FieldNames.SweepableFloatParam.RangeType] = JToken.FromObject("Float"),
[FieldNames.SweepableFloatParam.Min] = JToken.FromObject(sfpAttr.Min),
[FieldNames.SweepableFloatParam.Max] = JToken.FromObject(sfpAttr.Max)
};
if (sfpAttr.StepSize != null)
{
jParam[FieldNames.SweepableFloatParam.StepSize] = JToken.FromObject(sfpAttr.StepSize);
}
if (sfpAttr.NumSteps != null)
{
jParam[FieldNames.SweepableFloatParam.NumSteps] = JToken.FromObject(sfpAttr.NumSteps);
}
if (sfpAttr.IsLogScale)
{
jParam[FieldNames.SweepableFloatParam.IsLogScale] = JToken.FromObject(true);
}
jo[FieldNames.SweepableFloatParam.ToString()] = jParam;
}
if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableDiscreteParamAttribute), false).FirstOrDefault() is TlcModule.SweepableDiscreteParamAttribute sdpAttr)
{
var jParam = new JObject
{
[FieldNames.SweepableDiscreteParam.RangeType] = JToken.FromObject("Discrete"),
[FieldNames.SweepableDiscreteParam.Options] = JToken.FromObject(sdpAttr.Options)
};
jo[FieldNames.SweepableDiscreteParam.ToString()] = jParam;
}
inputs.Add(new KeyValuePair <Double, JObject>(inputAttr.SortOrder, jo));
}
return(new JArray(inputs.OrderBy(x => x.Key).Select(x => x.Value).ToArray()));
}
private static void FillValues(IExceptionContext ectx, ref VBuffer <Float> src, ref VBuffer <Float> dst, Float divisor, Float scale, Float offset = 0)
{
int count = src.Count;
int length = src.Length;
ectx.Assert(Utils.Size(src.Values) >= count);
ectx.Assert(divisor >= 0);
if (count == 0)
{
dst = new VBuffer <Float>(length, 0, dst.Values, dst.Indices);
return;
}
ectx.Assert(count > 0);
ectx.Assert(length > 0);
Float normScale = scale;
if (divisor > 0)
{
normScale /= divisor;
}
// Don't normalize small values.
if (normScale < MinScale)
{
normScale = 1;
}
if (offset == 0)
{
var dstValues = dst.Values;
if (Utils.Size(dstValues) < count)
{
dstValues = new Float[count];
}
var dstIndices = dst.Indices;
if (!src.IsDense)
{
if (Utils.Size(dstIndices) < count)
{
dstIndices = new int[count];
}
Array.Copy(src.Indices, dstIndices, count);
}
SseUtils.Scale(normScale, src.Values, dstValues, count);
dst = new VBuffer <Float>(length, count, dstValues, dstIndices);
return;
}
// Subtracting the mean requires a dense representation.
src.CopyToDense(ref dst);
if (normScale != 1)
{
SseUtils.ScaleAdd(normScale, -offset, dst.Values, length);
}
else
{
SseUtils.Add(-offset, dst.Values, length);
}
}
private static JToken BuildTypeToken(IExceptionContext ectx, FieldInfo fieldInfo, Type type, ComponentCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(type);
ectx.AssertValue(catalog);
// REVIEW: Allows newly introduced types to not break the manifest bulding process.
// Where possible, these types should be replaced by component kinds.
if (type == typeof(CommonInputs.IEvaluatorInput) ||
type == typeof(CommonOutputs.IEvaluatorOutput))
{
var jo = new JObject();
var typeString = $"{type}".Replace("Microsoft.ML.EntryPoints.", "");
jo[FieldNames.Kind] = "EntryPoint";
jo[FieldNames.ItemType] = typeString;
return(jo);
}
type = ExtractOptionalOrNullableType(type);
// Dive inside Var.
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>))
{
type = type.GetGenericArguments()[0];
}
var typeEnum = TlcModule.GetDataType(type);
switch (typeEnum)
{
case TlcModule.DataKind.Unknown:
var jo = new JObject();
if (type == typeof(JArray))
{
jo[FieldNames.Kind] = TlcModule.DataKind.Array.ToString();
jo[FieldNames.ItemType] = "Node";
return(jo);
}
if (type == typeof(JObject))
{
return("Bindings");
}
var fields = BuildInputManifest(ectx, type, catalog);
if (fields.Count == 0)
{
throw ectx.Except("Unexpected parameter type: {0}", type);
}
jo[FieldNames.Kind] = "Struct";
jo[FieldNames.Fields] = fields;
return(jo);
case TlcModule.DataKind.Float:
case TlcModule.DataKind.Int:
case TlcModule.DataKind.UInt:
case TlcModule.DataKind.Char:
case TlcModule.DataKind.String:
case TlcModule.DataKind.Bool:
case TlcModule.DataKind.DataView:
case TlcModule.DataKind.TransformModel:
case TlcModule.DataKind.PredictorModel:
case TlcModule.DataKind.FileHandle:
return(typeEnum.ToString());
case TlcModule.DataKind.Enum:
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
var values = Enum.GetNames(type).Where(n => type.GetField(n).GetCustomAttribute <HideEnumValueAttribute>() == null);
jo[FieldNames.Values] = new JArray(values);
return(jo);
case TlcModule.DataKind.Array:
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
jo[FieldNames.ItemType] = BuildTypeToken(ectx, fieldInfo, type.GetElementType(), catalog);
return(jo);
case TlcModule.DataKind.Dictionary:
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
jo[FieldNames.ItemType] = BuildTypeToken(ectx, fieldInfo, type.GetGenericArguments()[1], catalog);
return(jo);
case TlcModule.DataKind.Component:
string kind;
if (!catalog.TryGetComponentKind(type, out kind))
{
throw ectx.Except("Field '{0}' is a component of unknown kind", fieldInfo.Name);
}
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
jo[FieldNames.ComponentKind] = kind;
return(jo);
default:
ectx.Assert(false);
throw ectx.ExceptNotSupp();
}
}
/// <summary>
/// 异常时
/// </summary>
/// <param name="filterContext"></param>
protected sealed override void OnException(IExceptionContext filterContext)
{
}
/// <summary>
/// Build a token for component default value. This will look up the component in the catalog, and if it finds an entry, it will
/// build a JSON structure that would be parsed into the default value.
///
/// This is an inherently fragile setup in case when the factory is not trivial, but it will work well for 'property bag' factories
/// that we are currently using.
/// </summary>
private static JToken BuildComponentToken(IExceptionContext ectx, IComponentFactory value, ComponentCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(value);
ectx.AssertValue(catalog);
var type = value.GetType();
ComponentCatalog.ComponentInfo componentInfo;
if (!catalog.TryFindComponent(type, out componentInfo))
{
// The default component is not in the catalog. This is, technically, allowed, but it means that there's no JSON representation
// for the default value. We will emit the one the won't parse back.
return(new JValue("(custom component)"));
}
ectx.Assert(componentInfo.ArgumentType == type);
// Try to invoke default ctor for the factory to obtain defaults.
object defaults;
try
{
defaults = Activator.CreateInstance(type);
}
catch (MissingMemberException ex)
{
// There was no default constructor found.
// This should never happen, since ComponentCatalog would error out if there is no default ctor.
ectx.Assert(false);
throw ectx.Except(ex, "Couldn't find default constructor");
}
var jResult = new JObject();
var jSettings = new JObject();
jResult[FieldNames.Name] = componentInfo.Name;
// Iterate over all fields of the factory object, and compare the values with the defaults.
// If the value differs, insert it into the settings object.
bool anyValue = false;
foreach (var fieldInfo in type.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
{
var attr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault()
as ArgumentAttribute;
if (attr == null || attr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
ectx.Assert(!fieldInfo.IsStatic && !fieldInfo.IsInitOnly && !fieldInfo.IsLiteral);
bool needValue = false;
object actualValue = fieldInfo.GetValue(value);
if (attr.IsRequired)
{
needValue = true;
}
else
{
object defaultValue = fieldInfo.GetValue(defaults);
needValue = !Equals(actualValue, defaultValue);
}
if (!needValue)
{
continue;
}
jSettings[attr.Name ?? fieldInfo.Name] = BuildValueToken(ectx, actualValue, fieldInfo.FieldType, catalog);
anyValue = true;
}
if (anyValue)
{
jResult[FieldNames.Settings] = jSettings;
}
return(jResult);
}
public BoundMapper(IExceptionContext ectx, TreeEnsembleFeaturizerBindableMapper owner,
RoleMappedSchema schema)
{
Contracts.AssertValue(ectx);
ectx.AssertValue(owner);
ectx.AssertValue(schema);
ectx.Assert(schema.Feature.HasValue);
_ectx = ectx;
_owner = owner;
InputRoleMappedSchema = schema;
// A vector containing the output of each tree on a given example.
var treeValueType = new VectorType(NumberType.Float, owner._ensemble.TrainedEnsemble.NumTrees);
// An indicator vector with length = the total number of leaves in the ensemble, indicating which leaf the example
// ends up in all the trees in the ensemble.
var leafIdType = new VectorType(NumberType.Float, owner._totalLeafCount);
// An indicator vector with length = the total number of nodes in the ensemble, indicating the nodes on
// the paths of the example in all the trees in the ensemble.
// The total number of nodes in a binary tree is equal to the number of internal nodes + the number of leaf nodes,
// and it is also equal to the number of children of internal nodes (which is 2 * the number of internal nodes)
// plus one (since the root node is not a child of any node). So we have #internal + #leaf = 2*(#internal) + 1,
// which means that #internal = #leaf - 1.
// Therefore, the number of internal nodes in the ensemble is #leaf - #trees.
var pathIdType = new VectorType(NumberType.Float, owner._totalLeafCount - owner._ensemble.TrainedEnsemble.NumTrees);
// Start creating output schema with types derived above.
var schemaBuilder = new SchemaBuilder();
// Metadata of tree values.
var treeIdMetadataBuilder = new MetadataBuilder();
treeIdMetadataBuilder.Add(MetadataUtils.Kinds.SlotNames, MetadataUtils.GetNamesType(treeValueType.Size),
(ValueGetter <VBuffer <ReadOnlyMemory <char> > >)owner.GetTreeSlotNames);
// Add the column of trees' output values
schemaBuilder.AddColumn(OutputColumnNames.Trees, treeValueType, treeIdMetadataBuilder.GetMetadata());
// Metadata of leaf IDs.
var leafIdMetadataBuilder = new MetadataBuilder();
leafIdMetadataBuilder.Add(MetadataUtils.Kinds.SlotNames, MetadataUtils.GetNamesType(leafIdType.Size),
(ValueGetter <VBuffer <ReadOnlyMemory <char> > >)owner.GetLeafSlotNames);
leafIdMetadataBuilder.Add(MetadataUtils.Kinds.IsNormalized, BoolType.Instance, (ref bool value) => value = true);
// Add the column of leaves' IDs where the input example reaches.
schemaBuilder.AddColumn(OutputColumnNames.Leaves, leafIdType, leafIdMetadataBuilder.GetMetadata());
// Metadata of path IDs.
var pathIdMetadataBuilder = new MetadataBuilder();
pathIdMetadataBuilder.Add(MetadataUtils.Kinds.SlotNames, MetadataUtils.GetNamesType(pathIdType.Size),
(ValueGetter <VBuffer <ReadOnlyMemory <char> > >)owner.GetPathSlotNames);
pathIdMetadataBuilder.Add(MetadataUtils.Kinds.IsNormalized, BoolType.Instance, (ref bool value) => value = true);
// Add the column of encoded paths which the input example passes.
schemaBuilder.AddColumn(OutputColumnNames.Paths, pathIdType, pathIdMetadataBuilder.GetMetadata());
OutputSchema = schemaBuilder.GetSchema();
// Tree values must be the first output column.
Contracts.Assert(OutputSchema[OutputColumnNames.Trees].Index == TreeValuesColumnId);
// leaf IDs must be the second output column.
Contracts.Assert(OutputSchema[OutputColumnNames.Leaves].Index == LeafIdsColumnId);
// Path IDs must be the third output column.
Contracts.Assert(OutputSchema[OutputColumnNames.Paths].Index == PathIdsColumnId);
}
public static RepositoryReader Open(Stream stream, IExceptionContext ectx = null, bool useFileSystem = true)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(stream, nameof(stream));
return(new RepositoryReader(stream, ectx, useFileSystem));
}
private protected override Func <DataViewSchema, IRowToRowMapper> TransformerChecker(IExceptionContext ectx, ITransformer transformer)
{
ectx.CheckValue(transformer, nameof(transformer));
ectx.CheckParam(IsRowToRowMapper(transformer), nameof(transformer), "Must be a row to row mapper or " + nameof(IStatefulTransformer));
InputTransformer = transformer;
return(GetRowToRowMapper);
}
internal static Exception ExceptGetMetadata(this IExceptionContext ctx) => ctx.Except("Invalid call to GetMetadata");
/// <summary>
/// 在Api执行中异常时触发
/// </summary>
/// <param name="filterContext">上下文</param>
void IFilter.OnException(IExceptionContext filterContext)
{
this.OnException(filterContext);
}
/// <summary> /// 在Api执行中异常时触发 /// </summary> /// <param name="filterContext">上下文</param> protected abstract void OnException(IExceptionContext filterContext);
public abstract ValueGetter <T> GetGetter <T>(IExceptionContext ctx);
/// <summary>
/// 在Api执行中异常时触发
/// </summary>
/// <param name="filterContext">上下文</param>
void IFilter.OnException(IExceptionContext filterContext)
{
this.OnException(filterContext);
}
public override ValueGetter <T> GetGetter <T>(IExceptionContext ctx)
{
ctx.Check(typeof(T) == typeof(VBuffer <TValue>));
return((ValueGetter <T>)_getter);
}
/// <summary>
/// 在Api执行中异常时触发
/// </summary>
/// <param name="filterContext">上下文</param>
protected sealed override void OnException(IExceptionContext filterContext)
{
this.OnException(filterContext as ExceptionContext);
}
public ExceptionVM(ExceptionInfo info, IExceptionContext context) {
ExceptionInfo = info;
Context = context;
}
