private void CheckNotDisposed()
{
if (IsDisposed)
{
throw _ectx.Except("SimpleFileHandle has already been disposed");
}
}
public void AddInputVariable(VariableBinding binding, Type type)
{
_ectx.AssertValue(binding);
_ectx.AssertValue(type);
if (binding is ArrayIndexVariableBinding)
{
type = Utils.MarshalInvoke(MakeArray <int>, type);
}
else if (binding is DictionaryKeyVariableBinding)
{
type = Utils.MarshalInvoke(MakeDictionary <int>, type);
}
EntryPointVariable v;
if (!_vars.TryGetValue(binding.VariableName, out v))
{
v = new EntryPointVariable(_ectx, binding.VariableName, type);
_vars[binding.VariableName] = v;
}
else if (v.Type != type)
{
throw _ectx.Except($"Variable '{v.Name}' is used as {v.Type} and as {type}");
}
v.MarkUsage(true);
}
private bool ScanForComponents(IExceptionContext ectx, Type nestedType)
{
var attr = nestedType.GetTypeInfo().GetCustomAttributes(typeof(TlcModule.ComponentAttribute), true).FirstOrDefault()
as TlcModule.ComponentAttribute;
if (attr == null)
{
return(false);
}
bool found = false;
foreach (var faceType in nestedType.GetInterfaces())
{
var faceAttr = faceType.GetTypeInfo().GetCustomAttributes(typeof(TlcModule.ComponentKindAttribute), false).FirstOrDefault()
as TlcModule.ComponentKindAttribute;
if (faceAttr == null)
{
continue;
}
if (!typeof(IComponentFactory).IsAssignableFrom(faceType))
{
throw ectx.Except("Component signature '{0}' doesn't inherit from '{1}'", faceType, typeof(IComponentFactory));
}
try
{
// In order to populate from JSON, we need to invoke the parameterless ctor. Testing that this is possible.
Activator.CreateInstance(nestedType);
}
catch (MissingMemberException ex)
{
throw ectx.Except(ex, "Component type '{0}' doesn't have a default constructor", faceType);
}
var info = new ComponentInfo(ectx, faceType, faceAttr.Kind, nestedType, attr);
var names = (info.Aliases ?? new string[0]).Concat(new[] { info.Name }).Distinct();
_components.Add(info);
foreach (var alias in names)
{
var tag = $"{info.Kind}:{alias}";
if (_componentMap.ContainsKey(tag))
{
// Duplicate component name. We need to show a warning here.
// REVIEW: we will be able to do this once catalog becomes a part of env.
continue;
}
_componentMap[tag] = info;
}
}
return(found);
}
/// <summary>
/// Performs checks on an EntryPoint input class equivalent to the checks that are done
/// when parsing a JSON EntryPoint graph.
///
/// Call this method from EntryPoint methods to ensure that range and required checks are performed
/// in a consistent manner when EntryPoints are created directly from code.
/// </summary>
public static void CheckInputArgs(IExceptionContext ectx, object args)
{
foreach (var fieldInfo in args.GetType().GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
{
var attr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault()
as ArgumentAttribute;
if (attr == null || attr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
var fieldVal = fieldInfo.GetValue(args);
var fieldType = fieldInfo.FieldType;
// Optionals are either left in their Implicit constructed state or
// a new Explicit optional is constructed. They should never be set
// to null.
if (fieldType.IsGenericType && fieldType.GetGenericTypeDefinition() == typeof(Optional <>) && fieldVal == null)
{
throw ectx.Except("Field '{0}' is Optional<> and set to null instead of an explicit value.", fieldInfo.Name);
}
if (attr.IsRequired)
{
bool equalToDefault;
if (fieldType.IsGenericType && fieldType.GetGenericTypeDefinition() == typeof(Optional <>))
{
equalToDefault = !((Optional)fieldVal).IsExplicit;
}
else
{
equalToDefault = fieldType.IsValueType ? Activator.CreateInstance(fieldType).Equals(fieldVal) : fieldVal == null;
}
if (equalToDefault)
{
throw ectx.Except("Field '{0}' is required but is not set.", fieldInfo.Name);
}
}
var rangeAttr = fieldInfo.GetCustomAttributes(typeof(TlcModule.RangeAttribute), false).FirstOrDefault()
as TlcModule.RangeAttribute;
if (rangeAttr != null && fieldVal != null && !rangeAttr.IsValueWithinRange(fieldVal))
{
throw ectx.Except("Field '{0}' is set to a value that falls outside the range bounds.", fieldInfo.Name);
}
}
}
internal static Session LoadTFSessionByModelFilePath(IExceptionContext ectx, string modelFile, bool metaGraph = false)
{
if (string.IsNullOrEmpty(modelFile))
{
throw ectx.Except($"TensorFlow exception triggered while loading model from '{modelFile}'");
}
Graph graph;
try
{
if (metaGraph)
{
graph = LoadMetaGraph(modelFile);
}
else
{
graph = new Graph();
graph.Import(modelFile, "");
}
}
catch (Exception ex)
{
#pragma warning disable MSML_NoMessagesForLoadContext
throw ectx.ExceptDecode(ex, "Tensorflow exception triggered while loading model.");
#pragma warning restore MSML_NoMessagesForLoadContext
}
return(new Session(graph));
}
public static VariableBinding Create(IExceptionContext ectx, string jsonString)
{
Contracts.AssertValue(ectx);
ectx.AssertNonWhiteSpace(jsonString);
var match = _variableRegex.Match(jsonString);
if (!match.Success)
{
throw ectx.Except($"Unable to parse variable string '{jsonString}'");
}
if (match.Groups["NumericAccessor"].Success)
{
return(new ArrayIndexVariableBinding(
match.Groups["Name"].Value,
int.Parse(match.Groups["NumericAccessor"].Value)));
}
if (match.Groups["StringAccessor"].Success)
{
return(new DictionaryKeyVariableBinding(
match.Groups["Name"].Value,
match.Groups["StringAccessor"].Value));
}
return(new SimpleVariableBinding(match.Groups["Name"].Value));
}
/// <summary>
/// If an exception was ever set through <see cref="Set"/>, raise it as an appropriate
/// inner exception. This should only be called just prior to dispose, when the workers
/// have already finished. If there is no stored exception, this will do nothing. Note
/// that this does not "expire" the exception, that is, if you were to call this again,
/// it would throw the same exception.
/// </summary>
public void ThrowIfSet(IExceptionContext ectx)
{
if (_ex != null)
{
throw ectx.Except(_ex, "Exception thrown in {0}", _component);
}
}
public ColumnInfo(IExceptionContext ectx, string[] inputColumnNames, DataViewType[] inputTypes, string expression, string outputColumnName, int vectorInputColumn, LambdaNode node, int[] perm)
{
ectx.AssertNonEmpty(inputTypes);
ectx.Assert(Utils.Size(inputTypes) == Utils.Size(inputColumnNames));
ectx.AssertNonWhiteSpace(expression);
ectx.AssertNonWhiteSpace(outputColumnName);
ectx.AssertValue(node);
InputColumnNames = inputColumnNames;
OutputColumnName = outputColumnName;
OutputColumnItemType = node.ResultType as PrimitiveDataViewType;
ectx.AssertValue(OutputColumnItemType);
VectorInputColumn = vectorInputColumn;
Perm = perm;
Expression = expression;
InputKinds = new InternalDataKind[inputTypes.Length];
for (int i = 0; i < inputTypes.Length; i++)
{
InputKinds[i] = inputTypes[i].GetRawKind();
}
Del = LambdaCompiler.Compile(out var errors, node);
if (Utils.Size(errors) > 0)
{
throw ectx.Except($"generating code failed: {errors[0].GetMessage()}");
}
}
internal ComponentInfo(IExceptionContext ectx, Type interfaceType, string kind, Type argumentType, TlcModule.ComponentAttribute attribute)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(interfaceType);
ectx.AssertNonEmpty(kind);
ectx.AssertValue(argumentType);
ectx.AssertValue(attribute);
Name = attribute.Name;
Description = attribute.Desc;
if (string.IsNullOrWhiteSpace(attribute.FriendlyName))
{
FriendlyName = Name;
}
else
{
FriendlyName = attribute.FriendlyName;
}
Kind = kind;
if (!IsValidName(Kind))
{
throw ectx.Except("Invalid component kind: '{0}'", Kind);
}
Aliases = attribute.Aliases;
if (!IsValidName(Name))
{
throw ectx.Except("Component name '{0}' is not valid.", Name);
}
if (Aliases != null && Aliases.Any(x => !IsValidName(x)))
{
throw ectx.Except("Component '{0}' has an invalid alias '{1}'", Name, Aliases.First(x => !IsValidName(x)));
}
if (!typeof(IComponentFactory).IsAssignableFrom(argumentType))
{
throw ectx.Except("Component '{0}' must inherit from IComponentFactory", argumentType);
}
ArgumentType = argumentType;
InterfaceType = interfaceType;
}
/// <summary>
/// Fetches the value of the column by name, in the given row.
/// Used by the evaluators to retrieve the metrics from the results IDataView.
/// </summary>
public static T Fetch <T>(IExceptionContext ectx, DataViewRow row, string name)
{
if (!row.Schema.TryGetColumnIndex(name, out int col))
{
throw ectx.Except($"Could not find column '{name}'");
}
T val = default;
row.GetGetter <T>(col)(ref val);
return(val);
}
public void Log(ProgressChannel source, ProgressEvent.EventKind kind, ProgressEntry entry)
{
_ectx.AssertValue(source);
_ectx.AssertValueOrNull(entry);
if (kind == ProgressEvent.EventKind.Start)
{
_ectx.Assert(entry == null);
lock (_lock)
{
// Figure out an appropriate name.
int i = 1;
var name = source.Name;
string nameCandidate = name;
while (!_namesUsed.Add(nameCandidate))
{
i++;
nameCandidate = string.Format("{0} #{1}", name, i);
}
var newInfo = new CalculationInfo(++_index, nameCandidate, source);
_infos.Add(newInfo);
_pendingEvents.Enqueue(new ProgressEvent(newInfo.Index, newInfo.Name, newInfo.StartTime, ProgressEvent.EventKind.Start));
return;
}
}
// Not a start event, so we won't modify the _infos.
CalculationInfo info;
lock (_lock)
{
info = _infos.FirstOrDefault(x => x.Channel == source);
if (info == null)
{
throw _ectx.Except("Event sent after the calculation lifetime expired.");
}
}
switch (kind)
{
case ProgressEvent.EventKind.Stop:
_ectx.Assert(entry == null);
info.IsFinished = true;
_pendingEvents.Enqueue(new ProgressEvent(info.Index, info.Name, info.StartTime, ProgressEvent.EventKind.Stop));
break;
default:
_ectx.Assert(entry != null);
_ectx.Assert(kind == ProgressEvent.EventKind.Progress);
_ectx.Assert(!info.IsFinished);
_pendingEvents.Enqueue(new ProgressEvent(info.Index, info.Name, info.StartTime, entry));
break;
}
}
private static IComponentFactory GetComponentJson(IExceptionContext ectx, Type signatureType, string name, JObject settings, ComponentCatalog catalog)
{
Contracts.AssertValue(ectx);
ectx.AssertValue(signatureType);
ectx.AssertNonEmpty(name);
ectx.AssertValueOrNull(settings);
ectx.AssertValue(catalog);
if (!catalog.TryGetComponentKind(signatureType, out string kind))
{
throw ectx.Except($"Component type '{signatureType}' is not a valid signature type.");
}
if (!catalog.TryFindComponent(kind, name, out ComponentCatalog.ComponentInfo component))
{
var available = catalog.GetAllComponents(kind).Select(x => $"'{x.Name}'");
throw ectx.Except($"Component '{name}' of kind '{kind}' is not found. Available components are: {string.Join(", ", available)}");
}
var inputBuilder = new InputBuilder(ectx, component.ArgumentType, catalog);
if (settings != null)
{
foreach (var pair in settings)
{
if (!inputBuilder.TrySetValueJson(pair.Key, pair.Value))
{
throw ectx.Except($"Unexpected value for component '{name}', field '{pair.Key}': '{pair.Value}'");
}
}
}
var missing = inputBuilder.GetMissingValues().ToArray();
if (missing.Length > 0)
{
throw ectx.Except($"The following required inputs were not provided for component '{name}': {string.Join(", ", missing)}");
}
return(inputBuilder.GetInstance() as IComponentFactory);
}
/// <summary>
/// Ensures that the given value can be assigned to an entry point field with
/// type <paramref name="type"/>. This method will wrap the value in the option
/// type if needed and throw an exception if the value isn't assignable.
/// </summary>
/// <param name="ectx">The exception context.</param>
/// <param name="type">Type type of the field this value is to be assigned to.</param>
/// <param name="value">The value, typically originates from either ParseJsonValue, or is an external, user-provided object.</param>
private static object GetFieldAssignableValue(IExceptionContext ectx, Type type, object value)
{
Contracts.AssertValue(ectx);
ectx.AssertValue(type);
ectx.AssertValueOrNull(value);
// If 'type' is optional, make 'value' into an optional (this is the case of optional input).
value = MakeOptionalIfNeeded(ectx, value, type);
if (value != null && !type.IsInstanceOfType(value))
{
throw ectx.Except($"Unexpected value type: {value.GetType()}");
}
return(value);
}
private static T Fetch <T>(IExceptionContext ectx, DataViewRow row, string name)
{
var column = row.Schema.GetColumnOrNull(name);
if (!column.HasValue)
{
throw ectx.Except($"Could not find column '{name}'");
}
T val = default;
row.GetGetter <T>(column.Value)(ref val);
return(val);
}
internal EntryPointInfo(IExceptionContext ectx, MethodInfo method,
TlcModule.EntryPointAttribute attribute, ObsoleteAttribute obsoleteAttribute)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(method);
ectx.AssertValue(attribute);
Name = attribute.Name ?? string.Join(".", method.DeclaringType.Name, method.Name);
Description = attribute.Desc;
Method = method;
ShortName = attribute.ShortName;
FriendlyName = attribute.UserName;
XmlInclude = attribute.XmlInclude;
ObsoleteAttribute = obsoleteAttribute;
// There are supposed to be 2 parameters, env and input for non-macro nodes.
// Macro nodes have a 3rd parameter, the entry point node.
var parameters = method.GetParameters();
if (parameters.Length != 2 && parameters.Length != 3)
{
throw ectx.Except("Method '{0}' has {1} parameters, but must have 2 or 3", method.Name, parameters.Length);
}
if (parameters[0].ParameterType != typeof(IHostEnvironment))
{
throw ectx.Except("Method '{0}', 1st parameter is {1}, but must be IHostEnvironment", method.Name, parameters[0].ParameterType);
}
InputType = parameters[1].ParameterType;
var outputType = method.ReturnType;
if (!outputType.IsClass)
{
throw ectx.Except("Method '{0}' returns {1}, but must return a class", method.Name, outputType);
}
OutputType = outputType;
InputKinds = FindEntryPointKinds(InputType);
OutputKinds = FindEntryPointKinds(OutputType);
}
private static void CheckRange(long src, uint dst, IExceptionContext ectx = null)
{
if (src < 0 || src >= uint.MaxValue)
{
if (ectx != null)
{
ectx.Except("Value long {0} cannot be converted into uint.", src);
}
else
{
Contracts.Except("Value long {0} cannot be converted into uint.", src);
}
}
}
int GetColumnIndex(IExceptionContext ch, string name)
{
if (string.IsNullOrEmpty(name))
{
return(-1);
}
int index;
if (!_input.Schema.TryGetColumnIndex(name, out index))
{
throw ch.Except("Unable to find column '{0}'.", name);
}
return(index);
}
private static void CheckRange(float src, ulong dst, IExceptionContext ectx = null)
{
if (src < 0)
{
if (ectx != null)
{
ectx.Except("Value long {0} cannot be converted into ulong.", src);
}
else
{
Contracts.Except("Value long {0} cannot be converted into ulong.", src);
}
}
}
/// <summary>
/// Searches for the given column name in the schema. This method applies a
/// common policy that throws an exception if the column is not found
/// and the column name was explicitly specified. If the column is not found
/// and the column name was not explicitly specified, it returns null.
/// </summary>
public static string FindColumnOrNull(IExceptionContext ectx, DataViewSchema schema, Optional <string> value)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(schema, nameof(schema));
ectx.CheckValue(value, nameof(value));
if (value == "")
{
return(null);
}
if (schema.GetColumnOrNull(value) == null)
{
if (value.IsExplicit)
{
throw ectx.Except("Column '{0}' not found", value);
}
return(null);
}
return(value);
}
public static string FindColumn(IExceptionContext ectx, Schema schema, Optional <string> value)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(schema, nameof(schema));
ectx.CheckValue(value, nameof(value));
if (string.IsNullOrEmpty(value?.Value))
{
return(null);
}
if (!schema.TryGetColumnIndex(value, out int col))
{
if (value.IsExplicit)
{
throw ectx.Except("Column '{0}' not found", value);
}
return(null);
}
return(value);
}
internal static TFSession LoadTFSession(IExceptionContext ectx, byte[] modelBytes, string modelFile = null)
{
var graph = new TFGraph();
try
{
graph.Import(modelBytes, "");
}
catch (Exception ex)
{
if (!string.IsNullOrEmpty(modelFile))
{
throw ectx.Except($"TensorFlow exception triggered while loading model from '{modelFile}'");
}
#pragma warning disable MSML_NoMessagesForLoadContext
throw ectx.ExceptDecode(ex, "Tensorflow exception triggered while loading model.");
#pragma warning restore MSML_NoMessagesForLoadContext
}
return(new TFSession(graph));
}
public static ComponentCatalog.LoadableClassInfo CheckTrainer <TSig>(IExceptionContext ectx, SubComponent <ITrainer, TSig> trainer, string dataFile)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckUserArg(trainer.IsGood(), nameof(TrainCommand.Arguments.Trainer), "A trainer is required.");
var info = ComponentCatalog.GetLoadableClassInfo <TSig>(trainer.Kind);
if (info == null)
{
throw ectx.ExceptUserArg(nameof(TrainCommand.Arguments.Trainer), "Unknown trainer: '{0}'", trainer.Kind);
}
if (!typeof(ITrainer).IsAssignableFrom(info.Type))
{
throw ectx.Except("Loadable class '{0}' does not implement 'ITrainer'", info.LoadNames[0]);
}
if (string.IsNullOrWhiteSpace(dataFile))
{
throw ectx.ExceptUserArg(nameof(TrainCommand.Arguments.DataFile), "Data file must be defined.");
}
return(info);
}
/// <summary>
/// Returns a standard exception for responding to an invalid call to GetAnnotation.
/// </summary>
public static Exception ExceptGetAnnotation(this IExceptionContext ctx) => ctx.Except("Invalid call to GetAnnotation");
internal static Exception ExceptGetMetadata(this IExceptionContext ctx) => ctx.Except("Invalid call to GetMetadata");
private static JArray BuildInputManifest(IExceptionContext ectx, Type inputType, ModuleCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(inputType);
ectx.AssertValue(catalog);
// Instantiate a value of the input, to pull defaults out of.
var defaults = Activator.CreateInstance(inputType);
var inputs = new List <KeyValuePair <Double, JObject> >();
foreach (var fieldInfo in inputType.GetFields())
{
var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute;
if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
var jo = new JObject();
jo[FieldNames.Name] = inputAttr.Name ?? fieldInfo.Name;
jo[FieldNames.Type] = BuildTypeToken(ectx, fieldInfo, fieldInfo.FieldType, catalog);
jo[FieldNames.Desc] = inputAttr.HelpText;
if (inputAttr.Aliases != null)
{
jo[FieldNames.Aliases] = new JArray(inputAttr.Aliases);
}
jo[FieldNames.Required] = inputAttr.IsRequired;
jo[FieldNames.SortOrder] = inputAttr.SortOrder;
jo[FieldNames.IsNullable] = fieldInfo.FieldType.IsGenericType && (fieldInfo.FieldType.GetGenericTypeDefinition() == typeof(Nullable <>));
var defaultValue = fieldInfo.GetValue(defaults);
var dataType = TlcModule.GetDataType(fieldInfo.FieldType);
if (!inputAttr.IsRequired || (dataType != TlcModule.DataKind.Unknown && defaultValue != null))
{
jo[FieldNames.Default] = BuildValueToken(ectx, defaultValue, fieldInfo.FieldType, catalog);
}
if (fieldInfo.FieldType.IsGenericType &&
fieldInfo.FieldType.GetGenericTypeDefinition() == typeof(Optional <>))
{
var val = fieldInfo.GetValue(defaults) as Optional;
if (val == null && !inputAttr.IsRequired)
{
throw ectx.Except("Field '{0}' is an Optional<> type but is null by default, instead of set to a constructed implicit default.", fieldInfo.Name);
}
if (val != null && val.IsExplicit)
{
throw ectx.Except("Field '{0}' is an Optional<> type with a non-implicit default value.", fieldInfo.Name);
}
}
var rangeAttr = fieldInfo.GetCustomAttributes(typeof(TlcModule.RangeAttribute), false).FirstOrDefault() as TlcModule.RangeAttribute;
if (rangeAttr != null)
{
if (!TlcModule.IsNumericKind(TlcModule.GetDataType(fieldInfo.FieldType)))
{
throw ectx.Except("Field '{0}' has a range but is of a non-numeric type.", fieldInfo.Name);
}
if (!rangeAttr.Type.Equals(fieldInfo.FieldType))
{
throw ectx.Except("Field '{0}' has a range attribute that uses a type which is not equal to the field's FieldType.", fieldInfo.Name);
}
var jRange = new JObject();
if (rangeAttr.Sup != null)
{
jRange[FieldNames.Range.Sup] = JToken.FromObject(rangeAttr.Sup);
}
if (rangeAttr.Inf != null)
{
jRange[FieldNames.Range.Inf] = JToken.FromObject(rangeAttr.Inf);
}
if (rangeAttr.Max != null)
{
jRange[FieldNames.Range.Max] = JToken.FromObject(rangeAttr.Max);
}
if (rangeAttr.Min != null)
{
jRange[FieldNames.Range.Min] = JToken.FromObject(rangeAttr.Min);
}
jo[FieldNames.Range.Type] = jRange;
}
// Handle deprecated/obsolete attributes, passing along the message to the manifest.
if (fieldInfo.GetCustomAttributes(typeof(ObsoleteAttribute), false).FirstOrDefault() is ObsoleteAttribute obsAttr)
{
var jParam = new JObject
{
[FieldNames.Deprecated.Message] = JToken.FromObject(obsAttr.Message),
};
jo[FieldNames.Deprecated.ToString()] = jParam;
}
if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableLongParamAttribute), false).FirstOrDefault() is TlcModule.SweepableLongParamAttribute slpAttr)
{
var jParam = new JObject
{
[FieldNames.SweepableLongParam.RangeType] = JToken.FromObject("Long"),
[FieldNames.SweepableLongParam.Min] = JToken.FromObject(slpAttr.Min),
[FieldNames.SweepableLongParam.Max] = JToken.FromObject(slpAttr.Max)
};
if (slpAttr.StepSize != null)
{
jParam[FieldNames.SweepableLongParam.StepSize] = JToken.FromObject(slpAttr.StepSize);
}
if (slpAttr.NumSteps != null)
{
jParam[FieldNames.SweepableLongParam.NumSteps] = JToken.FromObject(slpAttr.NumSteps);
}
if (slpAttr.IsLogScale)
{
jParam[FieldNames.SweepableLongParam.IsLogScale] = JToken.FromObject(true);
}
jo[FieldNames.SweepableLongParam.ToString()] = jParam;
}
if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableFloatParamAttribute), false).FirstOrDefault() is TlcModule.SweepableFloatParamAttribute sfpAttr)
{
var jParam = new JObject
{
[FieldNames.SweepableFloatParam.RangeType] = JToken.FromObject("Float"),
[FieldNames.SweepableFloatParam.Min] = JToken.FromObject(sfpAttr.Min),
[FieldNames.SweepableFloatParam.Max] = JToken.FromObject(sfpAttr.Max)
};
if (sfpAttr.StepSize != null)
{
jParam[FieldNames.SweepableFloatParam.StepSize] = JToken.FromObject(sfpAttr.StepSize);
}
if (sfpAttr.NumSteps != null)
{
jParam[FieldNames.SweepableFloatParam.NumSteps] = JToken.FromObject(sfpAttr.NumSteps);
}
if (sfpAttr.IsLogScale)
{
jParam[FieldNames.SweepableFloatParam.IsLogScale] = JToken.FromObject(true);
}
jo[FieldNames.SweepableFloatParam.ToString()] = jParam;
}
if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableDiscreteParamAttribute), false).FirstOrDefault() is TlcModule.SweepableDiscreteParamAttribute sdpAttr)
{
var jParam = new JObject
{
[FieldNames.SweepableDiscreteParam.RangeType] = JToken.FromObject("Discrete"),
[FieldNames.SweepableDiscreteParam.Options] = JToken.FromObject(sdpAttr.Options)
};
jo[FieldNames.SweepableDiscreteParam.ToString()] = jParam;
}
inputs.Add(new KeyValuePair <Double, JObject>(inputAttr.SortOrder, jo));
}
return(new JArray(inputs.OrderBy(x => x.Key).Select(x => x.Value).ToArray()));
}
private static JToken BuildTypeToken(IExceptionContext ectx, FieldInfo fieldInfo, Type type, ModuleCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(type);
ectx.AssertValue(catalog);
// REVIEW: Allows newly introduced types to not break the manifest bulding process.
// Where possible, these types should be replaced by component kinds.
if (type == typeof(CommonInputs.IEvaluatorInput) ||
type == typeof(CommonOutputs.IEvaluatorOutput))
{
var jo = new JObject();
var typeString = $"{type}".Replace("Microsoft.ML.Runtime.EntryPoints.", "");
jo[FieldNames.Kind] = "EntryPoint";
jo[FieldNames.ItemType] = typeString;
return(jo);
}
type = CSharpGeneratorUtils.ExtractOptionalOrNullableType(type);
// Dive inside Var.
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>))
{
type = type.GetGenericArguments()[0];
}
var typeEnum = TlcModule.GetDataType(type);
switch (typeEnum)
{
case TlcModule.DataKind.Unknown:
var jo = new JObject();
if (type == typeof(JArray))
{
jo[FieldNames.Kind] = TlcModule.DataKind.Array.ToString();
jo[FieldNames.ItemType] = "Node";
return(jo);
}
if (type == typeof(JObject))
{
return("Bindings");
}
var fields = BuildInputManifest(ectx, type, catalog);
if (fields.Count == 0)
{
throw ectx.Except("Unexpected parameter type: {0}", type);
}
jo[FieldNames.Kind] = "Struct";
jo[FieldNames.Fields] = fields;
return(jo);
case TlcModule.DataKind.Float:
case TlcModule.DataKind.Int:
case TlcModule.DataKind.UInt:
case TlcModule.DataKind.Char:
case TlcModule.DataKind.String:
case TlcModule.DataKind.Bool:
case TlcModule.DataKind.DataView:
case TlcModule.DataKind.TransformModel:
case TlcModule.DataKind.PredictorModel:
case TlcModule.DataKind.FileHandle:
return(typeEnum.ToString());
case TlcModule.DataKind.Enum:
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
var values = Enum.GetNames(type);
jo[FieldNames.Values] = new JArray(values);
return(jo);
case TlcModule.DataKind.Array:
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
jo[FieldNames.ItemType] = BuildTypeToken(ectx, fieldInfo, type.GetElementType(), catalog);
return(jo);
case TlcModule.DataKind.Dictionary:
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
jo[FieldNames.ItemType] = BuildTypeToken(ectx, fieldInfo, type.GetGenericArguments()[1], catalog);
return(jo);
case TlcModule.DataKind.Component:
string kind;
if (!catalog.TryGetComponentKind(type, out kind))
{
throw ectx.Except("Field '{0}' is a component of unknown kind", fieldInfo.Name);
}
jo = new JObject();
jo[FieldNames.Kind] = typeEnum.ToString();
jo[FieldNames.ComponentKind] = kind;
return(jo);
case TlcModule.DataKind.State:
jo = new JObject();
var typeString = $"{type}".Replace("Microsoft.ML.Runtime.Interfaces.", "");
jo[FieldNames.Kind] = "C# Object";
jo[FieldNames.ItemType] = typeString;
return(jo);
default:
ectx.Assert(false);
throw ectx.ExceptNotSupp();
}
}
/// <summary>
/// Build a token for component default value. This will look up the component in the catalog, and if it finds an entry, it will
/// build a JSON structure that would be parsed into the default value.
///
/// This is an inherently fragile setup in case when the factory is not trivial, but it will work well for 'property bag' factories
/// that we are currently using.
/// </summary>
private static JToken BuildComponentToken(IExceptionContext ectx, IComponentFactory value, ModuleCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(value);
ectx.AssertValue(catalog);
var type = value.GetType();
ModuleCatalog.ComponentInfo componentInfo;
if (!catalog.TryFindComponent(type, out componentInfo))
{
// The default component is not in the catalog. This is, technically, allowed, but it means that there's no JSON representation
// for the default value. We will emit the one the won't parse back.
return(new JValue("(custom component)"));
}
ectx.Assert(componentInfo.ArgumentType == type);
// Try to invoke default ctor for the factory to obtain defaults.
object defaults;
try
{
defaults = Activator.CreateInstance(type);
}
catch (MissingMemberException ex)
{
// There was no default constructor found.
// This should never happen, since ModuleCatalog would error out if there is no default ctor.
ectx.Assert(false);
throw ectx.Except(ex, "Couldn't find default constructor");
}
var jResult = new JObject();
var jSettings = new JObject();
jResult[FieldNames.Name] = componentInfo.Name;
// Iterate over all fields of the factory object, and compare the values with the defaults.
// If the value differs, insert it into the settings object.
bool anyValue = false;
foreach (var fieldInfo in type.GetFields())
{
var attr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault()
as ArgumentAttribute;
if (attr == null || attr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
{
continue;
}
ectx.Assert(!fieldInfo.IsStatic && !fieldInfo.IsInitOnly && !fieldInfo.IsLiteral);
bool needValue = false;
object actualValue = fieldInfo.GetValue(value);
if (attr.IsRequired)
{
needValue = true;
}
else
{
object defaultValue = fieldInfo.GetValue(defaults);
needValue = !Equals(actualValue, defaultValue);
}
if (!needValue)
{
continue;
}
jSettings[attr.Name ?? fieldInfo.Name] = BuildValueToken(ectx, actualValue, fieldInfo.FieldType, catalog);
anyValue = true;
}
if (anyValue)
{
jResult[FieldNames.Settings] = jSettings;
}
return(jResult);
}
/// <summary>
/// Returns a standard exception for responding to an invalid call to GetMetadata.
/// </summary>
public static Exception ExceptGetMetadata(this IExceptionContext ctx)
{
return(ctx.Except("Invalid call to GetMetadata"));
}
private static object ParseJsonValue(IExceptionContext ectx, Type type, Attributes attributes, JToken value, ComponentCatalog catalog)
{
Contracts.AssertValue(ectx);
ectx.AssertValue(type);
ectx.AssertValueOrNull(value);
ectx.AssertValue(catalog);
if (value == null)
{
return(null);
}
if (value is JValue val && val.Value == null)
{
return(null);
}
if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Optional <>) || type.GetGenericTypeDefinition() == typeof(Nullable <>)))
{
if (type.GetGenericTypeDefinition() == typeof(Optional <>) && value.HasValues)
{
value = value.Values().FirstOrDefault();
}
type = type.GetGenericArguments()[0];
}
if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Var <>)))
{
string varName = value.Value <string>();
ectx.Check(VariableBinding.IsBindingToken(value), "Variable name expected.");
var variable = Activator.CreateInstance(type) as IVarSerializationHelper;
var varBinding = VariableBinding.Create(ectx, varName);
variable.VarName = varBinding.VariableName;
return(variable);
}
if (type == typeof(JArray) && value is JArray)
{
return(value);
}
TlcModule.DataKind dt = TlcModule.GetDataType(type);
try
{
switch (dt)
{
case TlcModule.DataKind.Bool:
return(value.Value <bool>());
case TlcModule.DataKind.String:
return(value.Value <string>());
case TlcModule.DataKind.Char:
return(value.Value <char>());
case TlcModule.DataKind.Enum:
if (!Enum.IsDefined(type, value.Value <string>()))
{
throw ectx.Except($"Requested value '{value.Value<string>()}' is not a member of the Enum type '{type.Name}'");
}
return(Enum.Parse(type, value.Value <string>()));
case TlcModule.DataKind.Float:
if (type == typeof(double))
{
return(value.Value <double>());
}
else if (type == typeof(float))
{
return(value.Value <float>());
}
else
{
ectx.Assert(false);
throw ectx.ExceptNotSupp();
}
case TlcModule.DataKind.Array:
var ja = value as JArray;
ectx.Check(ja != null, "Expected array value");
Func <IExceptionContext, JArray, Attributes, ComponentCatalog, object> makeArray = MakeArray <int>;
return(Utils.MarshalInvoke(makeArray, type.GetElementType(), ectx, ja, attributes, catalog));
case TlcModule.DataKind.Int:
if (type == typeof(long))
{
return(value.Value <long>());
}
if (type == typeof(int))
{
return(value.Value <int>());
}
ectx.Assert(false);
throw ectx.ExceptNotSupp();
case TlcModule.DataKind.UInt:
if (type == typeof(ulong))
{
return(value.Value <ulong>());
}
if (type == typeof(uint))
{
return(value.Value <uint>());
}
ectx.Assert(false);
throw ectx.ExceptNotSupp();
case TlcModule.DataKind.Dictionary:
ectx.Check(value is JObject, "Expected object value");
Func <IExceptionContext, JObject, Attributes, ComponentCatalog, object> makeDict = MakeDictionary <int>;
return(Utils.MarshalInvoke(makeDict, type.GetGenericArguments()[1], ectx, (JObject)value, attributes, catalog));
case TlcModule.DataKind.Component:
var jo = value as JObject;
ectx.Check(jo != null, "Expected object value");
// REVIEW: consider accepting strings alone.
var jName = jo[FieldNames.Name];
ectx.Check(jName != null, "Field '" + FieldNames.Name + "' is required for component.");
ectx.Check(jName is JValue, "Expected '" + FieldNames.Name + "' field to be a string.");
var name = jName.Value <string>();
ectx.Check(jo[FieldNames.Settings] == null || jo[FieldNames.Settings] is JObject,
"Expected '" + FieldNames.Settings + "' field to be an object");
return(GetComponentJson(ectx, type, name, jo[FieldNames.Settings] as JObject, catalog));
default:
var settings = value as JObject;
ectx.Check(settings != null, "Expected object value");
var inputBuilder = new InputBuilder(ectx, type, catalog);
if (inputBuilder._fields.Length == 0)
{
throw ectx.Except($"Unsupported input type: {dt}");
}
if (settings != null)
{
foreach (var pair in settings)
{
if (!inputBuilder.TrySetValueJson(pair.Key, pair.Value))
{
throw ectx.Except($"Unexpected value for component '{type}', field '{pair.Key}': '{pair.Value}'");
}
}
}
var missing = inputBuilder.GetMissingValues().ToArray();
if (missing.Length > 0)
{
throw ectx.Except($"The following required inputs were not provided for component '{type}': {string.Join(", ", missing)}");
}
return(inputBuilder.GetInstance());
}
}
catch (FormatException ex)
{
if (ex.IsMarked())
{
throw;
}
throw ectx.Except(ex, $"Failed to parse JSON value '{value}' as {type}");
}
}
