private static IServiceLocator BuildLocator() { var catalog = new ComponentCatalog(typeof(IFoo).Assembly); var container = new CompositionContainer(catalog); return new Microsoft.Mef.CommonServiceLocator.MefServiceLocator(container); }
public void ExternalComponentCatalog_IsNotDisposed() { var disposed = false; var server = new ReleasableComponent { Handler = () => disposed = true }; Assert.IsFalse(disposed); var serverSetup = new IpcBinaryServerProtocolSetup("CleanupTest2"); using (var catalog = new ComponentCatalog()) using (var host = new ZyanComponentHost("SampleServer2", serverSetup, new InProcSessionManager(), catalog)) { host.RegisterComponent<ISampleComponent, ReleasableComponent>( server, s => ((ReleasableComponent)s).Release()); Assert.IsFalse(disposed); server.Release(); Assert.IsTrue(disposed); } }
protected virtual string EnumName(CmdParser.ArgInfo.Arg arg, Type sigType) { return(Capitalize(arg.LongName) + ComponentCatalog.SignatureToString(sigType)); }
private void ShowComponents(IndentingTextWriter writer) { Type typeSig; Type typeRes; string kind; if (_kind == null) { // Show commands. typeSig = typeof(SignatureCommand); typeRes = typeof(ICommand); kind = "Command"; writer.WriteLine("Available commands:"); } else { kind = _kind.ToLowerInvariant(); var sigs = ComponentCatalog.GetAllSignatureTypes(); typeSig = sigs.FirstOrDefault(t => ComponentCatalog.SignatureToString(t).ToLowerInvariant() == kind); if (typeSig == null) { typeSig = sigs.FirstOrDefault(t => ComponentCatalog.SignatureToString(t).StartsWithInvariantCultureIgnoreCase(kind)); if (typeSig == null) { writer.WriteLine("Couldn't find kind '{0}'", kind); ListKinds(writer); return; } } typeRes = typeof(object); writer.WriteLine("Available components for kind '{0}':", ComponentCatalog.SignatureToString(typeSig)); } var infos = ComponentCatalog.GetAllDerivedClasses(typeRes, typeSig) .Where(x => !x.IsHidden) .OrderBy(x => x.LoadNames[0].ToLowerInvariant()); using (writer.Nest()) { var components = new List <Component>(); foreach (var info in infos) { _env.Assert(info.LoadNames.Count > 0); writer.Write("{0}", info.LoadNames[0]); if (!string.IsNullOrWhiteSpace(info.UserName)) { writer.Write(": {0}", info.UserName); } writer.WriteLine(); using (writer.Nest()) ShowAliases(writer, info.LoadNames); components.Add(new Component(kind, info, info.CreateArguments())); } if (components.Count > 0) { Serialize(components); } } }
private EventServer() { _catalog = new ComponentCatalog(); _catalog.RegisterComponent<IEventComponentSingleton, EventComponentSingleton>(ActivationType.Singleton); _catalog.RegisterComponent<IEventComponentSingleCall, EventComponentSingleCall>(ActivationType.SingleCall); _catalog.RegisterComponent<ICallbackComponentSingleton, CallbackComponentSingleton>(ActivationType.Singleton); _catalog.RegisterComponent<ICallbackComponentSingleCall, CallbackComponentSingleCall>(ActivationType.SingleCall); _catalog.RegisterComponent<IRequestResponseCallbackSingleCall, RequestResponseCallbackSingleCall>(ActivationType.SingleCall); _catalog.RegisterComponent<ITimerTriggeredEvent, TimerTriggeredEvent>(ActivationType.Singleton); // Setting compression threshold to 1 byte means that all messages will be compressed. // This setting should not be used in production code because smaller packets will grow in size. // By default, Zyan only compresses messages larger than 64 kilobytes. var tcpBinaryProtocol = new TcpBinaryServerProtocolSetup(8082); tcpBinaryProtocol.AddServerSinkBeforeFormatter(new CompressionServerChannelSinkProvider(1, CompressionMethod.LZF)); _tcpBinaryHost = new ZyanComponentHost("TcpBinaryEventTest", tcpBinaryProtocol, _catalog); var ipcBinaryProtocol = new IpcBinaryServerProtocolSetup("IpcTestServer"); ipcBinaryProtocol.AddServerSinkBeforeFormatter(new CompressionServerChannelSinkProvider(1, CompressionMethod.DeflateStream)); _ipcBinaryHost = new ZyanComponentHost("IpcBinaryEventTest", ipcBinaryProtocol, _catalog); var tcpCustomProtocol = new TcpCustomServerProtocolSetup(8083, new NullAuthenticationProvider(), true) { CompressionThreshold = 1, CompressionMethod = CompressionMethod.DeflateStream }; _tcpCustomHost = new ZyanComponentHost("TcpCustomEventTest", tcpCustomProtocol, _catalog); var tcpDuplexProtocol = new TcpDuplexServerProtocolSetup(8084, new NullAuthenticationProvider(), true) { CompressionThreshold = 1, CompressionMethod = CompressionMethod.DeflateStream }; tcpDuplexProtocol.AddChannelSetting("bindTo", "127.0.0.1"); _tcpDuplexHost = new ZyanComponentHost("TcpDuplexEventTest", tcpDuplexProtocol, _catalog); var httpCustomProtocol = new HttpCustomServerProtocolSetup(8085, new NullAuthenticationProvider(), true) { CompressionThreshold = 1, CompressionMethod = CompressionMethod.LZF }; _httpCustomHost = new ZyanComponentHost("HttpCustomEventTest", httpCustomProtocol, _catalog); var nullChannelProtocol = new NullServerProtocolSetup(1234); _nullChannelHost = new ZyanComponentHost("NullEventTest", nullChannelProtocol, _catalog); // use legacy blocking events mode because we check the handlers synchronously ZyanComponentHost.LegacyBlockingEvents = true; }
private void GenerateInputFields(IndentedTextWriter writer, Type inputType, ComponentCatalog catalog, string rootNameSpace) { var defaults = Activator.CreateInstance(inputType); foreach (var fieldInfo in inputType.GetFields()) { var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute; if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly) { continue; } if (fieldInfo.FieldType == typeof(JObject)) { continue; } CSharpGeneratorUtils.GenerateSummary(writer, inputAttr.HelpText); if (fieldInfo.FieldType == typeof(JArray)) { writer.WriteLine($"public Experiment {CSharpGeneratorUtils.Capitalize(inputAttr.Name ?? fieldInfo.Name)} {{ get; set; }}"); writer.WriteLineNoTabs(); continue; } var inputTypeString = CSharpGeneratorUtils.GetInputType(catalog, fieldInfo.FieldType, _generatedClasses, rootNameSpace); if (CSharpGeneratorUtils.IsComponent(fieldInfo.FieldType)) { writer.WriteLine("[JsonConverter(typeof(ComponentSerializer))]"); } if (CSharpGeneratorUtils.Capitalize(inputAttr.Name ?? fieldInfo.Name) != (inputAttr.Name ?? fieldInfo.Name)) { writer.WriteLine($"[JsonProperty(\"{inputAttr.Name ?? fieldInfo.Name}\")]"); } // For range attributes on properties if (fieldInfo.GetCustomAttributes(typeof(TlcModule.RangeAttribute), false).FirstOrDefault() is TlcModule.RangeAttribute ranAttr) { writer.WriteLine(ranAttr.ToString()); } // For obsolete/deprecated attributes if (fieldInfo.GetCustomAttributes(typeof(ObsoleteAttribute), false).FirstOrDefault() is ObsoleteAttribute obsAttr) { writer.WriteLine($"[System.Obsolete(\"{obsAttr.Message}\")]"); } // For sweepable ranges on properties if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableParamAttribute), false).FirstOrDefault() is TlcModule.SweepableParamAttribute sweepableParamAttr) { if (string.IsNullOrEmpty(sweepableParamAttr.Name)) { sweepableParamAttr.Name = fieldInfo.Name; } writer.WriteLine(sweepableParamAttr.ToString()); } var line = $"public {inputTypeString} {CSharpGeneratorUtils.Capitalize(inputAttr.Name ?? fieldInfo.Name)} {{ get; set; }}"; var defaultValue = CSharpGeneratorUtils.GetValue(catalog, fieldInfo.FieldType, fieldInfo.GetValue(defaults), _generatedClasses, rootNameSpace); if (defaultValue != null) { line += $" = {defaultValue};"; } writer.WriteLine(line); writer.WriteLineNoTabs(); } }
private void GenerateColumnAddMethods(IndentedTextWriter writer, Type inputType, ComponentCatalog catalog, string className, out Type columnType) { columnType = null; foreach (var fieldInfo in inputType.GetFields()) { var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute; if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly) { continue; } var type = CSharpGeneratorUtils.ExtractOptionalOrNullableType(fieldInfo.FieldType); var isArray = type.IsArray; if (isArray) { type = type.GetElementType(); } if (type == typeof(JArray) || type == typeof(JObject)) { continue; } if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>)) { continue; } var typeEnum = TlcModule.GetDataType(type); if (typeEnum != TlcModule.DataKind.Unknown) { continue; } if (type.IsSubclassOf(typeof(OneToOneColumn))) { columnType = GenerateOneToOneColumn(writer, className, columnType, fieldInfo, inputAttr, type, isArray); } else if (type.IsSubclassOf(typeof(ManyToOneColumn))) { columnType = GenerateManyToOneColumn(writer, className, columnType, fieldInfo, inputAttr, type, isArray); } } }
private void GenerateInputOutput(IndentedTextWriter writer, ComponentCatalog.EntryPointInfo entryPointInfo, ComponentCatalog catalog) { var classAndMethod = CSharpGeneratorUtils.GetEntryPointMetadata(entryPointInfo); writer.WriteLine($"namespace Legacy.{classAndMethod.Namespace}"); writer.WriteLine("{"); writer.Indent(); GenerateInput(writer, entryPointInfo, catalog); writer.Outdent(); writer.WriteLine("}"); writer.WriteLineNoTabs(); }
/// <summary> /// Returns all kinds of models. /// </summary> public static IEnumerable <ComponentDescription> EnumerateComponents(string kind) { if (kind == "argument") { foreach (var comp in EnumerateComponentsParameter(false)) { yield return(comp); } } else if (kind == "command") { foreach (var comp in EnumerateComponentsParameter(true)) { yield return(comp); } } else { var kinds = GetAllKinds(); if (!string.IsNullOrEmpty(kind) && !kinds.Where(c => c == kind).Any()) { throw new ArgumentException($"Unable to find kind '{kind}' in\n{string.Join("\n", kinds)}."); } using (var env = new ConsoleEnvironment()) { ComponentHelper.AddStandardComponents(env); var sigs = env.ComponentCatalog.GetAllSignatureTypes(); var typeRes = typeof(object); Type[] typeSigs; if (string.IsNullOrEmpty(kind)) { typeSigs = sigs.ToArray(); } else { typeSigs = new[] { sigs.FirstOrDefault(t => ComponentCatalog.SignatureToString(t).ToLowerInvariant() == kind) } }; foreach (var typeSig in typeSigs) { var infos = env.ComponentCatalog.GetAllDerivedClasses(typeRes, typeSig) .Where(x => !x.IsHidden) .OrderBy(x => x.LoadNames[0].ToLowerInvariant()); foreach (var info in infos) { var args = info.CreateArguments(); if (args == null) { yield return(new ComponentDescription(info, args, null, null)); } else { var asse = args.GetType().Assembly; var parsedArgs = CmdParser.GetArgInfo(args.GetType(), args).Args; var arguments = new List <ComponentDescription.Argument>(); foreach (var arg in parsedArgs) { var a = new ComponentDescription.Argument() { Name = arg.LongName, ShortName = arg.ShortNames == null || !arg.ShortNames.Any() ? null : arg.ShortNames.First(), DefaultValue = arg.DefaultValue == null ? null : arg.DefaultValue.ToString(), Help = arg.HelpText, Arg = arg, }; arguments.Add(a); } var cmp = new ComponentDescription(info, args, asse, arguments); yield return(cmp); } } } } } } #endregion }
private static object MakeDictionary <T>(IExceptionContext ectx, JObject jDict, Attributes attributes, ComponentCatalog catalog) { Contracts.AssertValue(ectx); ectx.AssertValue(jDict); var dict = new Dictionary <string, T>(); foreach (var pair in jDict) { dict[pair.Key] = (T)GetFieldAssignableValue(ectx, typeof(T), ParseJsonValue(ectx, typeof(T), attributes, pair.Value, catalog)); } return(dict); }
private static object MakeArray <T>(IExceptionContext ectx, JArray jArray, Attributes attributes, ComponentCatalog catalog) { Contracts.AssertValue(ectx); ectx.AssertValue(jArray); T[] array = new T[jArray.Count]; for (int i = 0; i < array.Length; i++) { array[i] = (T)GetFieldAssignableValue(ectx, typeof(T), ParseJsonValue(ectx, typeof(T), attributes, jArray[i], catalog)); } return(array); }
private static IComponentFactory GetComponentJson(IExceptionContext ectx, Type signatureType, string name, JObject settings, ComponentCatalog catalog) { Contracts.AssertValue(ectx); ectx.AssertValue(signatureType); ectx.AssertNonEmpty(name); ectx.AssertValueOrNull(settings); ectx.AssertValue(catalog); if (!catalog.TryGetComponentKind(signatureType, out string kind)) { throw ectx.Except($"Component type '{signatureType}' is not a valid signature type."); } if (!catalog.TryFindComponent(kind, name, out ComponentCatalog.ComponentInfo component)) { var available = catalog.GetAllComponents(kind).Select(x => $"'{x.Name}'"); throw ectx.Except($"Component '{name}' of kind '{kind}' is not found. Available components are: {string.Join(", ", available)}"); } var inputBuilder = new InputBuilder(ectx, component.ArgumentType, catalog); if (settings != null) { foreach (var pair in settings) { if (!inputBuilder.TrySetValueJson(pair.Key, pair.Value)) { throw ectx.Except($"Unexpected value for component '{name}', field '{pair.Key}': '{pair.Value}'"); } } } var missing = inputBuilder.GetMissingValues().ToArray(); if (missing.Length > 0) { throw ectx.Except($"The following required inputs were not provided for component '{name}': {string.Join(", ", missing)}"); } return(inputBuilder.GetInstance() as IComponentFactory); }
private static object ParseJsonValue(IExceptionContext ectx, Type type, Attributes attributes, JToken value, ComponentCatalog catalog) { Contracts.AssertValue(ectx); ectx.AssertValue(type); ectx.AssertValueOrNull(value); ectx.AssertValue(catalog); if (value == null) { return(null); } if (value is JValue val && val.Value == null) { return(null); } if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Optional <>) || type.GetGenericTypeDefinition() == typeof(Nullable <>))) { if (type.GetGenericTypeDefinition() == typeof(Optional <>) && value.HasValues) { value = value.Values().FirstOrDefault(); } type = type.GetGenericArguments()[0]; } if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Var <>))) { string varName = value.Value <string>(); ectx.Check(VariableBinding.IsBindingToken(value), "Variable name expected."); var variable = Activator.CreateInstance(type) as IVarSerializationHelper; var varBinding = VariableBinding.Create(ectx, varName); variable.VarName = varBinding.VariableName; return(variable); } if (type == typeof(JArray) && value is JArray) { return(value); } TlcModule.DataKind dt = TlcModule.GetDataType(type); try { switch (dt) { case TlcModule.DataKind.Bool: return(value.Value <bool>()); case TlcModule.DataKind.String: return(value.Value <string>()); case TlcModule.DataKind.Char: return(value.Value <char>()); case TlcModule.DataKind.Enum: if (!Enum.IsDefined(type, value.Value <string>())) { throw ectx.Except($"Requested value '{value.Value<string>()}' is not a member of the Enum type '{type.Name}'"); } return(Enum.Parse(type, value.Value <string>())); case TlcModule.DataKind.Float: if (type == typeof(double)) { return(value.Value <double>()); } else if (type == typeof(float)) { return(value.Value <float>()); } else { ectx.Assert(false); throw ectx.ExceptNotSupp(); } case TlcModule.DataKind.Array: var ja = value as JArray; ectx.Check(ja != null, "Expected array value"); Func <IExceptionContext, JArray, Attributes, ComponentCatalog, object> makeArray = MakeArray <int>; return(Utils.MarshalInvoke(makeArray, type.GetElementType(), ectx, ja, attributes, catalog)); case TlcModule.DataKind.Int: if (type == typeof(long)) { return(value.Value <long>()); } if (type == typeof(int)) { return(value.Value <int>()); } ectx.Assert(false); throw ectx.ExceptNotSupp(); case TlcModule.DataKind.UInt: if (type == typeof(ulong)) { return(value.Value <ulong>()); } if (type == typeof(uint)) { return(value.Value <uint>()); } ectx.Assert(false); throw ectx.ExceptNotSupp(); case TlcModule.DataKind.Dictionary: ectx.Check(value is JObject, "Expected object value"); Func <IExceptionContext, JObject, Attributes, ComponentCatalog, object> makeDict = MakeDictionary <int>; return(Utils.MarshalInvoke(makeDict, type.GetGenericArguments()[1], ectx, (JObject)value, attributes, catalog)); case TlcModule.DataKind.Component: var jo = value as JObject; ectx.Check(jo != null, "Expected object value"); // REVIEW: consider accepting strings alone. var jName = jo[FieldNames.Name]; ectx.Check(jName != null, "Field '" + FieldNames.Name + "' is required for component."); ectx.Check(jName is JValue, "Expected '" + FieldNames.Name + "' field to be a string."); var name = jName.Value <string>(); ectx.Check(jo[FieldNames.Settings] == null || jo[FieldNames.Settings] is JObject, "Expected '" + FieldNames.Settings + "' field to be an object"); return(GetComponentJson(ectx, type, name, jo[FieldNames.Settings] as JObject, catalog)); default: var settings = value as JObject; ectx.Check(settings != null, "Expected object value"); var inputBuilder = new InputBuilder(ectx, type, catalog); if (inputBuilder._fields.Length == 0) { throw ectx.Except($"Unsupported input type: {dt}"); } if (settings != null) { foreach (var pair in settings) { if (!inputBuilder.TrySetValueJson(pair.Key, pair.Value)) { throw ectx.Except($"Unexpected value for component '{type}', field '{pair.Key}': '{pair.Value}'"); } } } var missing = inputBuilder.GetMissingValues().ToArray(); if (missing.Length > 0) { throw ectx.Except($"The following required inputs were not provided for component '{type}': {string.Join(", ", missing)}"); } return(inputBuilder.GetInstance()); } } catch (FormatException ex) { if (ex.IsMarked()) { throw; } throw ectx.Except(ex, $"Failed to parse JSON value '{value}' as {type}"); } }
protected void GenerateImplBody(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, string argSuffix) { if (Exclude.Contains(arg.LongName)) { return; } if (arg.IsSubComponentItemType) { // We need to create a tree with all the subcomponents, unless the subcomponent is a Trainer. Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>)); var types = arg.ItemType.GetGenericArguments(); var baseType = types[0]; var sigType = types[1]; if (IsTrainer(sigType)) { if (arg.IsCollection) { w.WriteLine("args{0}.{1} = new[] {{ new SubComponent<{2}, {3}>({4}.Item1, {4}.Item2) }};", argSuffix, arg.LongName, GetCSharpTypeName(baseType), GetCSharpTypeName(sigType), arg.LongName + argSuffix); } else { w.WriteLine("args{0}.{1} = new SubComponent<{2}, {3}>({4}.Item1, {4}.Item2);", argSuffix, arg.LongName, GetCSharpTypeName(baseType), GetCSharpTypeName(sigType), arg.LongName + argSuffix); } return; } if (sigType == typeof(SignatureDataLoader)) { return; } var typeName = EnumName(arg, sigType); w.WriteLine("switch ({0})", arg.LongName + argSuffix); w.WriteLine("{"); using (w.Nest()) { if (arg.NullName != null) { w.WriteLine("case {0}.None:", typeName); using (w.Nest()) { w.WriteLine("args{0}.{1} = null;", argSuffix, arg.LongName); w.WriteLine("break;"); } } var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType); foreach (var info in infos) { w.WriteLine("case {0}.{1}:", typeName, info.LoadNames[0]); using (w.Nest()) { if (info.ArgType != null) { var newArgSuffix = argSuffix + info.LoadNames[0]; w.WriteLine("var args{0} = new {1}();", newArgSuffix, GetCSharpTypeName(info.ArgType)); w.WriteLine("var defs{0} = new {1}();", newArgSuffix, GetCSharpTypeName(info.ArgType)); var args = info.CreateArguments(); if (args != null) { var argInfo = CmdParser.GetArgInfo(args.GetType(), args); foreach (var a in argInfo.Args) { GenerateImplBody(w, a, newArgSuffix); } } w.WriteLine( "args{0}.{1} = new {2}(\"{3}\", CmdParser.GetSettings(args{4}, defs{4}));", argSuffix, arg.LongName, GetCSharpTypeName(arg.ItemType), info.LoadNames[0], newArgSuffix); } else { w.WriteLine("args{0}.{1} = new {2}(\"{3}\");", argSuffix, arg.LongName, GetCSharpTypeName(arg.ItemType), info.LoadNames[0]); } w.WriteLine("break;"); } } } w.WriteLine("}"); } else if (arg.IsCollection) { if (IsColumnType(arg)) { w.WriteLine("args{0}.{1} = {1}.Select({2}.Parse).ToArray();", argSuffix, arg.LongName, GetCSharpTypeName(arg.ItemType)); } else if (IsStringColumnType(arg)) { w.WriteLine("args{0}.{1} = {2};", argSuffix, arg.LongName, arg.LongName + argSuffix); } else { w.WriteLine("args{0}.{1} = new[] {{ {2} }};", argSuffix, arg.LongName, arg.LongName + argSuffix); } } else { w.WriteLine("args{0}.{1} = {2};", argSuffix, arg.LongName, arg.LongName + argSuffix); } }
/// <summary> /// Generate enums for subcomponents. Uses ReflectionUtils to filter only the subcomponents that match the base type and the signature. /// </summary> /// <param name="w"></param> /// <param name="arg"></param> /// <param name="seen"></param> protected void GenerateEnums(IndentingTextWriter w, CmdParser.ArgInfo.Arg arg, HashSet <Tuple <Type, Type> > seen) { if (Exclude.Contains(arg.LongName)) { return; } var moreEnums = new List <CmdParser.ArgInfo.Arg>(); if (arg.IsHidden || !arg.IsSubComponentItemType) { return; } Contracts.Assert(arg.ItemType.GetGenericTypeDefinition() == typeof(SubComponent <,>)); var types = arg.ItemType.GetGenericArguments(); var baseType = types[0]; var sigType = types[1]; var key = new Tuple <Type, Type>(baseType, sigType); if (seen.Contains(key) || IsTrainer(sigType) || sigType == typeof(SignatureDataLoader)) { return; } seen.Add(key); var typeName = EnumName(arg, sigType); w.WriteLine("/// <summary> Available choices for {0} </summary>", sigType); w.WriteLine("public enum {0}", typeName); w.Write("{"); using (w.Nest()) { var pre = ""; if (arg.NullName != null) { w.WriteLine(); GenerateEnumValue(w, null); pre = ","; } var infos = ComponentCatalog.GetAllDerivedClasses(baseType, sigType); foreach (var info in infos) { w.WriteLine(pre); if (pre != "") { w.WriteLine(); } pre = ","; GenerateEnumValue(w, info); var args = info.CreateArguments(); if (args == null) { continue; } var argInfo = CmdParser.GetArgInfo(args.GetType(), args); moreEnums.AddRange(argInfo.Args); } w.WriteLine(); } w.WriteLine("}"); w.WriteLine(); foreach (var argument in moreEnums) { GenerateEnums(w, argument, seen); } }
public void SingletonComponentRegisteredWithComponentInstance_IsNotDisposed() { // this component instance is externally-owned var disposed = false; using (var immortalServer = new DisposableComponent { Handler = () => disposed = true }) using (var cat = new ComponentCatalog()) { cat.RegisterComponent<ISampleComponent, DisposableComponent>(immortalServer); Assert.IsFalse(disposed); var instance = cat.GetComponent<ISampleComponent>(); AssertEx.IsInstanceOf<DisposableComponent>(instance); var reg = cat.GetRegistration(typeof(ISampleComponent)); cat.CleanUpComponentInstance(reg, instance); Assert.IsFalse(disposed); immortalServer.Dispose(); Assert.IsTrue(disposed); } }
public void SingletonComponentRegisteredWithFactoryMethod_IsDisposed() { var disposed = false; var cat = new ComponentCatalog(); cat.RegisterComponent<ISampleComponent>(() => new DisposableComponent { Handler = () => disposed = true }, ActivationType.Singleton); Assert.IsFalse(disposed); var instance = cat.GetComponent<ISampleComponent>(); AssertEx.IsInstanceOf<DisposableComponent>(instance); var reg = cat.GetRegistration(typeof(ISampleComponent)); cat.CleanUpComponentInstance(reg, instance); Assert.IsTrue(disposed); }
public static string GetValue(ComponentCatalog catalog, Type fieldType, object fieldValue, GeneratedClasses generatedClasses, string rootNameSpace) { if (fieldType.IsGenericType && fieldType.GetGenericTypeDefinition() == typeof(Var <>)) { return($"new Var<{GetCSharpTypeName(fieldType.GetGenericTypeArgumentsEx()[0])}>()"); } if (fieldType.IsArray && Var <int> .CheckType(fieldType.GetElementType())) { return($"new ArrayVar<{GetCSharpTypeName(fieldType.GetElementType())}>()"); } if (fieldType.IsGenericType && fieldType.GetGenericTypeDefinition() == typeof(Dictionary <,>) && fieldType.GetGenericTypeArgumentsEx()[0] == typeof(string)) { return($"new DictionaryVar<{GetCSharpTypeName(fieldType.GetGenericTypeArgumentsEx()[1])}>()"); } if (Var <int> .CheckType(fieldType)) { return($"new Var<{GetCSharpTypeName(fieldType)}>()"); } if (fieldValue == null) { return(null); } if (!fieldType.IsInterface) { try { var defaultFieldValue = Activator.CreateInstance(fieldType); if (defaultFieldValue == fieldValue) { return(null); } } catch (MissingMethodException) { // No parameterless constructor, ignore. } } var typeEnum = TlcModule.GetDataType(fieldType); fieldType = ExtractOptionalOrNullableType(fieldType, out bool isNullable, out bool isOptional); switch (typeEnum) { case TlcModule.DataKind.Array: var arr = fieldValue as Array; if (arr != null && arr.GetLength(0) > 0) { return($"{{ {string.Join(", ", arr.Cast<object>().Select(item => GetValue(catalog, fieldType.GetElementType(), item, generatedClasses, rootNameSpace)))} }}"); } return(null); case TlcModule.DataKind.String: var strval = fieldValue as string; if (strval != null) { return(Quote(strval)); } return(null); case TlcModule.DataKind.Float: if (fieldValue is double d) { if (double.IsPositiveInfinity(d)) { return("double.PositiveInfinity"); } if (double.IsNegativeInfinity(d)) { return("double.NegativeInfinity"); } if (d != 0) { return(d.ToString("R") + "d"); } } else if (fieldValue is float f) { if (float.IsPositiveInfinity(f)) { return("float.PositiveInfinity"); } if (float.IsNegativeInfinity(f)) { return("float.NegativeInfinity"); } if (f != 0) { return(f.ToString("R") + "f"); } } return(null); case TlcModule.DataKind.Int: if (fieldValue is int i) { if (i != 0) { return(i.ToString()); } } else if (fieldValue is long l) { if (l != 0) { return(l.ToString()); } } return(null); case TlcModule.DataKind.Bool: return((bool)fieldValue ? "true" : "false"); case TlcModule.DataKind.Enum: string enumAsString = fieldValue.ToString(); if (fieldType.GetField(enumAsString).GetCustomAttribute <HideEnumValueAttribute>() != null) { // The default value for the enum has the hiding attribute on it. We will search for // alternate names. Regrettably I see no way beyond a manual scan. string unhiddenName = Enum.GetNames(fieldType).Zip(Enum.GetValues(fieldType).Cast <object>(), (name, val) => (name, val)) .Where(pair => pair.val.Equals(fieldValue)) .Where(pair => fieldType.GetField(pair.name).GetCustomAttribute <HideEnumValueAttribute>() == null) .Select(pair => pair.name).FirstOrDefault(); enumAsString = unhiddenName ?? throw Contracts.Except($"Could not find unhidden alternative for '{fieldValue}' in type '{fieldType}'"); } if (generatedClasses.IsGenerated(fieldType.FullName)) { return(generatedClasses.GetApiName(fieldType, rootNameSpace) + "." + enumAsString); } else { return(generatedClasses.GetApiName(fieldType, "") + "." + enumAsString); } case TlcModule.DataKind.Char: return($"'{GetCharAsString((char)fieldValue)}'"); case TlcModule.DataKind.Component: var type = fieldValue.GetType(); ComponentCatalog.ComponentInfo componentInfo; if (!catalog.TryFindComponent(fieldType, type, out componentInfo)) { return(null); } object defaultComponent = null; try { defaultComponent = Activator.CreateInstance(componentInfo.ArgumentType); } catch (MissingMethodException) { // No parameterless constructor, ignore. } var propertyBag = new List <string>(); if (defaultComponent != null) { foreach (var fieldInfo in componentInfo.ArgumentType.GetFields()) { var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute; if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly) { continue; } if (fieldInfo.FieldType == typeof(JArray) || fieldInfo.FieldType == typeof(JObject)) { continue; } var propertyValue = GetValue(catalog, fieldInfo.FieldType, fieldInfo.GetValue(fieldValue), generatedClasses, rootNameSpace); var defaultPropertyValue = GetValue(catalog, fieldInfo.FieldType, fieldInfo.GetValue(defaultComponent), generatedClasses, rootNameSpace); if (propertyValue != defaultPropertyValue) { propertyBag.Add($"{Capitalize(inputAttr.Name ?? fieldInfo.Name)} = {propertyValue}"); } } } var properties = propertyBag.Count > 0 ? $" {{ {string.Join(", ", propertyBag)} }}" : ""; return($"new {GetComponentName(componentInfo)}(){properties}"); case TlcModule.DataKind.Unknown: return($"new {generatedClasses.GetApiName(fieldType, rootNameSpace)}()"); default: return(fieldValue.ToString()); } }
private static Float[] Train(IHost host, ColInfo[] infos, Arguments args, IDataView trainingData) { Contracts.AssertValue(host, "host"); host.AssertNonEmpty(infos); var avgDistances = new Float[infos.Length]; const int reservoirSize = 5000; bool[] activeColumns = new bool[trainingData.Schema.ColumnCount]; for (int i = 0; i < infos.Length; i++) { activeColumns[infos[i].Source] = true; } var reservoirSamplers = new ReservoirSamplerWithReplacement <VBuffer <Float> > [infos.Length]; using (var cursor = trainingData.GetRowCursor(col => activeColumns[col])) { var rng = args.Seed.HasValue ? RandomUtils.Create(args.Seed) : host.Rand; for (int i = 0; i < infos.Length; i++) { if (infos[i].TypeSrc.IsVector) { var get = cursor.GetGetter <VBuffer <Float> >(infos[i].Source); reservoirSamplers[i] = new ReservoirSamplerWithReplacement <VBuffer <Float> >(rng, reservoirSize, get); } else { var getOne = cursor.GetGetter <Float>(infos[i].Source); Float val = 0; ValueGetter <VBuffer <Float> > get = (ref VBuffer <Float> dst) => { getOne(ref val); dst = new VBuffer <float>(1, new[] { val }); }; reservoirSamplers[i] = new ReservoirSamplerWithReplacement <VBuffer <Float> >(rng, reservoirSize, get); } } while (cursor.MoveNext()) { for (int i = 0; i < infos.Length; i++) { reservoirSamplers[i].Sample(); } } for (int i = 0; i < infos.Length; i++) { reservoirSamplers[i].Lock(); } } for (int iinfo = 0; iinfo < infos.Length; iinfo++) { var instanceCount = reservoirSamplers[iinfo].NumSampled; // If the number of pairs is at most the maximum reservoir size / 2, we go over all the pairs, // so we get all the examples. Otherwise, get a sample with replacement. VBuffer <Float>[] res; int resLength; if (instanceCount < reservoirSize && instanceCount * (instanceCount - 1) <= reservoirSize) { res = reservoirSamplers[iinfo].GetCache(); resLength = reservoirSamplers[iinfo].Size; Contracts.Assert(resLength == instanceCount); } else { res = reservoirSamplers[iinfo].GetSample().ToArray(); resLength = res.Length; } // If the dataset contains only one valid Instance, then we can't learn anything anyway, so just return 1. if (instanceCount <= 1) { avgDistances[iinfo] = 1; } else { Float[] distances; var sub = args.Column[iinfo].MatrixGenerator; if (!sub.IsGood()) { sub = args.MatrixGenerator; } var info = ComponentCatalog.GetLoadableClassInfo(sub); bool gaussian = info != null && info.Type == typeof(GaussianFourierSampler); // If the number of pairs is at most the maximum reservoir size / 2, go over all the pairs. if (resLength < reservoirSize) { distances = new Float[instanceCount * (instanceCount - 1) / 2]; int count = 0; for (int i = 0; i < instanceCount; i++) { for (int j = i + 1; j < instanceCount; j++) { distances[count++] = gaussian ? VectorUtils.L2DistSquared(ref res[i], ref res[j]) : VectorUtils.L1Distance(ref res[i], ref res[j]); } } host.Assert(count == distances.Length); } else { distances = new Float[reservoirSize / 2]; for (int i = 0; i < reservoirSize - 1; i += 2) { // For Gaussian kernels, we scale by the L2 distance squared, since the kernel function is exp(-gamma ||x-y||^2). // For Laplacian kernels, we scale by the L1 distance, since the kernel function is exp(-gamma ||x-y||_1). distances[i / 2] = gaussian ? VectorUtils.L2DistSquared(ref res[i], ref res[i + 1]) : VectorUtils.L1Distance(ref res[i], ref res[i + 1]); } } // If by chance, in the random permutation all the pairs are the same instance we return 1. Float median = MathUtils.GetMedianInPlace(distances, distances.Length); avgDistances[iinfo] = median == 0 ? 1 : median; } } return(avgDistances); }
public static string GetInputType(ComponentCatalog catalog, Type inputType, GeneratedClasses generatedClasses, string rootNameSpace) { if (inputType.IsGenericType && inputType.GetGenericTypeDefinition() == typeof(Var <>)) { return($"Var<{GetCSharpTypeName(inputType.GetGenericTypeArgumentsEx()[0])}>"); } if (inputType.IsArray && Var <int> .CheckType(inputType.GetElementType())) { return($"ArrayVar<{GetCSharpTypeName(inputType.GetElementType())}>"); } if (inputType.IsGenericType && inputType.GetGenericTypeDefinition() == typeof(Dictionary <,>) && inputType.GetGenericTypeArgumentsEx()[0] == typeof(string)) { return($"DictionaryVar<{GetCSharpTypeName(inputType.GetGenericTypeArgumentsEx()[1])}>"); } if (Var <int> .CheckType(inputType)) { return($"Var<{GetCSharpTypeName(inputType)}>"); } var type = ExtractOptionalOrNullableType(inputType, out bool isNullable, out bool isOptional); var typeEnum = TlcModule.GetDataType(type); switch (typeEnum) { case TlcModule.DataKind.Float: case TlcModule.DataKind.Int: case TlcModule.DataKind.UInt: case TlcModule.DataKind.Char: case TlcModule.DataKind.String: case TlcModule.DataKind.Bool: case TlcModule.DataKind.DataView: case TlcModule.DataKind.TransformModel: case TlcModule.DataKind.PredictorModel: case TlcModule.DataKind.FileHandle: return(GetCSharpTypeName(inputType)); case TlcModule.DataKind.Array: return(GetInputType(catalog, inputType.GetElementType(), generatedClasses, rootNameSpace) + "[]"); case TlcModule.DataKind.Component: string kind; bool success = catalog.TryGetComponentKind(type, out kind); Contracts.Assert(success); return($"{kind}"); case TlcModule.DataKind.Enum: var enumName = generatedClasses.GetApiName(type, rootNameSpace); if (isNullable) { return($"{enumName}?"); } if (isOptional) { return($"Optional<{enumName}>"); } return($"{enumName}"); default: if (isNullable) { return(generatedClasses.GetApiName(type, rootNameSpace) + "?"); } if (isOptional) { return($"Optional<{generatedClasses.GetApiName(type, rootNameSpace)}>"); } return(generatedClasses.GetApiName(type, rootNameSpace)); } }
private void GenerateClasses(IndentedTextWriter writer, Type inputType, ComponentCatalog catalog, string currentNamespace) { foreach (var fieldInfo in inputType.GetFields()) { var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute; if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly) { continue; } var type = fieldInfo.FieldType; type = CSharpGeneratorUtils.ExtractOptionalOrNullableType(type); if (type.IsArray) { type = type.GetElementType(); } if (type == typeof(JArray) || type == typeof(JObject)) { continue; } if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>)) { continue; } if (type == typeof(CommonInputs.IEvaluatorInput)) { continue; } if (type == typeof(CommonOutputs.IEvaluatorOutput)) { continue; } var typeEnum = TlcModule.GetDataType(type); if (typeEnum == TlcModule.DataKind.State) { continue; } if (typeEnum != TlcModule.DataKind.Unknown) { continue; } if (_generatedClasses.IsGenerated(type.FullName)) { continue; } GenerateEnums(writer, type, currentNamespace); GenerateClasses(writer, type, catalog, currentNamespace); var apiName = _generatedClasses.GetApiName(type, currentNamespace); string classBase = ""; if (type.IsSubclassOf(typeof(OneToOneColumn))) { classBase = $" : OneToOneColumn<{apiName}>, IOneToOneColumn"; } else if (type.IsSubclassOf(typeof(ManyToOneColumn))) { classBase = $" : ManyToOneColumn<{apiName}>, IManyToOneColumn"; } writer.WriteLine($"public sealed partial class {apiName}{classBase}"); writer.WriteLine("{"); writer.Indent(); _generatedClasses.MarkAsGenerated(type.FullName); GenerateInputFields(writer, type, catalog, currentNamespace); writer.Outdent(); writer.WriteLine("}"); writer.WriteLineNoTabs(); } }
private static JObject BuildComponentManifest(IExceptionContext ectx, ComponentCatalog.ComponentInfo componentInfo, ComponentCatalog catalog) { Contracts.AssertValueOrNull(ectx); ectx.AssertValue(componentInfo); ectx.AssertValue(catalog); var result = new JObject(); result[FieldNames.Name] = componentInfo.Name; result[FieldNames.Desc] = componentInfo.Description; result[FieldNames.FriendlyName] = componentInfo.FriendlyName; if (Utils.Size(componentInfo.Aliases) > 0) { result[FieldNames.Aliases] = new JArray(componentInfo.Aliases); } result[FieldNames.Settings] = BuildInputManifest(ectx, componentInfo.ArgumentType, catalog); return(result); }
private void GenerateInput(IndentedTextWriter writer, ComponentCatalog.EntryPointInfo entryPointInfo, ComponentCatalog catalog) { var entryPointMetadata = CSharpGeneratorUtils.GetEntryPointMetadata(entryPointInfo); string classBase = ""; if (entryPointInfo.InputKinds != null) { classBase += $" : {string.Join(", ", entryPointInfo.InputKinds.Select(CSharpGeneratorUtils.GetCSharpTypeName))}"; if (entryPointInfo.InputKinds.Any(t => typeof(ITrainerInput).IsAssignableFrom(t) || typeof(ITransformInput).IsAssignableFrom(t))) { classBase += ", Microsoft.ML.Legacy.ILearningPipelineItem"; } } GenerateEnums(writer, entryPointInfo.InputType, _defaultNamespace + entryPointMetadata.Namespace); writer.WriteLineNoTabs(); GenerateClasses(writer, entryPointInfo.InputType, catalog, _defaultNamespace + entryPointMetadata.Namespace); CSharpGeneratorUtils.GenerateSummary(writer, entryPointInfo.Description, entryPointInfo.XmlInclude); if (entryPointInfo.ObsoleteAttribute != null) { writer.WriteLine($"[Obsolete(\"{entryPointInfo.ObsoleteAttribute.Message}\")]"); } writer.WriteLine($"public sealed partial class {entryPointMetadata.ClassName}{classBase}"); writer.WriteLine("{"); writer.Indent(); writer.WriteLineNoTabs(); if (entryPointInfo.InputKinds != null && entryPointInfo.InputKinds.Any(t => typeof(Legacy.ILearningPipelineLoader).IsAssignableFrom(t))) { CSharpGeneratorUtils.GenerateLoaderAddInputMethod(writer, entryPointMetadata.ClassName); } GenerateColumnAddMethods(writer, entryPointInfo.InputType, catalog, entryPointMetadata.ClassName, out Type transformType); writer.WriteLineNoTabs(); GenerateInputFields(writer, entryPointInfo.InputType, catalog, _defaultNamespace + entryPointMetadata.Namespace); writer.WriteLineNoTabs(); GenerateOutput(writer, entryPointInfo, out HashSet <string> outputVariableNames); GenerateApplyFunction(writer, entryPointMetadata.ClassName, transformType, outputVariableNames, entryPointInfo.InputKinds); writer.Outdent(); writer.WriteLine("}"); }
private static JObject BuildEntryPointManifest(IExceptionContext ectx, ComponentCatalog.EntryPointInfo entryPointInfo, ComponentCatalog catalog) { Contracts.CheckValueOrNull(ectx); ectx.CheckValue(entryPointInfo, nameof(entryPointInfo)); ectx.CheckValue(catalog, nameof(catalog)); var result = new JObject(); result[FieldNames.Name] = entryPointInfo.Name; result[FieldNames.Desc] = entryPointInfo.Description; result[FieldNames.FriendlyName] = entryPointInfo.FriendlyName; result[FieldNames.ShortName] = entryPointInfo.ShortName; // There supposed to be 2 parameters, env and input. result[FieldNames.Inputs] = BuildInputManifest(ectx, entryPointInfo.InputType, catalog); result[FieldNames.Outputs] = BuildOutputManifest(ectx, entryPointInfo.OutputType, catalog); if (entryPointInfo.InputKinds != null) { var jInputKinds = new JArray(); foreach (var kind in entryPointInfo.InputKinds) { jInputKinds.Add(kind.Name); } result[FieldNames.InputKind] = jInputKinds; } if (entryPointInfo.OutputKinds != null) { var jOutputKinds = new JArray(); foreach (var kind in entryPointInfo.OutputKinds) { jOutputKinds.Add(kind.Name); } result[FieldNames.OutputKind] = jOutputKinds; } return(result); }
private void GenerateComponent(IndentedTextWriter writer, ComponentCatalog.ComponentInfo component, ComponentCatalog catalog) { GenerateEnums(writer, component.ArgumentType, "Runtime"); writer.WriteLineNoTabs(); GenerateClasses(writer, component.ArgumentType, catalog, "Runtime"); writer.WriteLineNoTabs(); CSharpGeneratorUtils.GenerateSummary(writer, component.Description); writer.WriteLine($"public sealed class {CSharpGeneratorUtils.GetComponentName(component)} : {component.Kind}"); writer.WriteLine("{"); writer.Indent(); GenerateInputFields(writer, component.ArgumentType, catalog, "Runtime"); writer.WriteLine($"internal override string ComponentName => \"{component.Name}\";"); writer.Outdent(); writer.WriteLine("}"); writer.WriteLineNoTabs(); }
private static JArray BuildInputManifest(IExceptionContext ectx, Type inputType, ComponentCatalog catalog) { Contracts.AssertValueOrNull(ectx); ectx.AssertValue(inputType); ectx.AssertValue(catalog); // Instantiate a value of the input, to pull defaults out of. var defaults = Activator.CreateInstance(inputType); var inputs = new List <KeyValuePair <Double, JObject> >(); foreach (var fieldInfo in inputType.GetFields()) { var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute; if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly) { continue; } var jo = new JObject(); jo[FieldNames.Name] = inputAttr.Name ?? fieldInfo.Name; jo[FieldNames.Type] = BuildTypeToken(ectx, fieldInfo, fieldInfo.FieldType, catalog); jo[FieldNames.Desc] = inputAttr.HelpText; if (inputAttr.Aliases != null) { jo[FieldNames.Aliases] = new JArray(inputAttr.Aliases); } jo[FieldNames.Required] = inputAttr.IsRequired; jo[FieldNames.SortOrder] = inputAttr.SortOrder; jo[FieldNames.IsNullable] = fieldInfo.FieldType.IsGenericType && (fieldInfo.FieldType.GetGenericTypeDefinition() == typeof(Nullable <>)); var defaultValue = fieldInfo.GetValue(defaults); var dataType = TlcModule.GetDataType(fieldInfo.FieldType); if (!inputAttr.IsRequired || (dataType != TlcModule.DataKind.Unknown && defaultValue != null)) { jo[FieldNames.Default] = BuildValueToken(ectx, defaultValue, fieldInfo.FieldType, catalog); } if (fieldInfo.FieldType.IsGenericType && fieldInfo.FieldType.GetGenericTypeDefinition() == typeof(Optional <>)) { var val = fieldInfo.GetValue(defaults) as Optional; if (val == null && !inputAttr.IsRequired) { throw ectx.Except("Field '{0}' is an Optional<> type but is null by default, instead of set to a constructed implicit default.", fieldInfo.Name); } if (val != null && val.IsExplicit) { throw ectx.Except("Field '{0}' is an Optional<> type with a non-implicit default value.", fieldInfo.Name); } } var rangeAttr = fieldInfo.GetCustomAttributes(typeof(TlcModule.RangeAttribute), false).FirstOrDefault() as TlcModule.RangeAttribute; if (rangeAttr != null) { if (!TlcModule.IsNumericKind(TlcModule.GetDataType(fieldInfo.FieldType))) { throw ectx.Except("Field '{0}' has a range but is of a non-numeric type.", fieldInfo.Name); } if (!rangeAttr.Type.Equals(fieldInfo.FieldType)) { throw ectx.Except("Field '{0}' has a range attribute that uses a type which is not equal to the field's FieldType.", fieldInfo.Name); } var jRange = new JObject(); if (rangeAttr.Sup != null) { jRange[FieldNames.Range.Sup] = JToken.FromObject(rangeAttr.Sup); } if (rangeAttr.Inf != null) { jRange[FieldNames.Range.Inf] = JToken.FromObject(rangeAttr.Inf); } if (rangeAttr.Max != null) { jRange[FieldNames.Range.Max] = JToken.FromObject(rangeAttr.Max); } if (rangeAttr.Min != null) { jRange[FieldNames.Range.Min] = JToken.FromObject(rangeAttr.Min); } jo[FieldNames.Range.Type] = jRange; } // Handle deprecated/obsolete attributes, passing along the message to the manifest. if (fieldInfo.GetCustomAttributes(typeof(ObsoleteAttribute), false).FirstOrDefault() is ObsoleteAttribute obsAttr) { var jParam = new JObject { [FieldNames.Deprecated.Message] = JToken.FromObject(obsAttr.Message), }; jo[FieldNames.Deprecated.ToString()] = jParam; } if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableLongParamAttribute), false).FirstOrDefault() is TlcModule.SweepableLongParamAttribute slpAttr) { var jParam = new JObject { [FieldNames.SweepableLongParam.RangeType] = JToken.FromObject("Long"), [FieldNames.SweepableLongParam.Min] = JToken.FromObject(slpAttr.Min), [FieldNames.SweepableLongParam.Max] = JToken.FromObject(slpAttr.Max) }; if (slpAttr.StepSize != null) { jParam[FieldNames.SweepableLongParam.StepSize] = JToken.FromObject(slpAttr.StepSize); } if (slpAttr.NumSteps != null) { jParam[FieldNames.SweepableLongParam.NumSteps] = JToken.FromObject(slpAttr.NumSteps); } if (slpAttr.IsLogScale) { jParam[FieldNames.SweepableLongParam.IsLogScale] = JToken.FromObject(true); } jo[FieldNames.SweepableLongParam.ToString()] = jParam; } if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableFloatParamAttribute), false).FirstOrDefault() is TlcModule.SweepableFloatParamAttribute sfpAttr) { var jParam = new JObject { [FieldNames.SweepableFloatParam.RangeType] = JToken.FromObject("Float"), [FieldNames.SweepableFloatParam.Min] = JToken.FromObject(sfpAttr.Min), [FieldNames.SweepableFloatParam.Max] = JToken.FromObject(sfpAttr.Max) }; if (sfpAttr.StepSize != null) { jParam[FieldNames.SweepableFloatParam.StepSize] = JToken.FromObject(sfpAttr.StepSize); } if (sfpAttr.NumSteps != null) { jParam[FieldNames.SweepableFloatParam.NumSteps] = JToken.FromObject(sfpAttr.NumSteps); } if (sfpAttr.IsLogScale) { jParam[FieldNames.SweepableFloatParam.IsLogScale] = JToken.FromObject(true); } jo[FieldNames.SweepableFloatParam.ToString()] = jParam; } if (fieldInfo.GetCustomAttributes(typeof(TlcModule.SweepableDiscreteParamAttribute), false).FirstOrDefault() is TlcModule.SweepableDiscreteParamAttribute sdpAttr) { var jParam = new JObject { [FieldNames.SweepableDiscreteParam.RangeType] = JToken.FromObject("Discrete"), [FieldNames.SweepableDiscreteParam.Options] = JToken.FromObject(sdpAttr.Options) }; jo[FieldNames.SweepableDiscreteParam.ToString()] = jParam; } inputs.Add(new KeyValuePair <Double, JObject>(inputAttr.SortOrder, jo)); } return(new JArray(inputs.OrderBy(x => x.Key).Select(x => x.Value).ToArray())); }
private void ShowHelp(IndentingTextWriter writer, int?columns = null) { _env.AssertValue(_component); string name = _component.Trim(); string sig = _kind?.ToLowerInvariant(); // Note that we don't check IsHidden here. The current policy is when IsHidden is true, we don't // show the item in "list all" functionality, but will still show help when explicitly requested. var infos = ComponentCatalog.FindLoadableClasses(name) .OrderBy(x => ComponentCatalog.SignatureToString(x.SignatureTypes[0]).ToLowerInvariant()); var kinds = new StringBuilder(); var components = new List <Component>(); foreach (var info in infos) { _env.AssertValue(info.SignatureTypes); kinds.Clear(); bool foundSig = false; foreach (var signature in info.SignatureTypes) { _env.Assert(signature.BaseType == typeof(MulticastDelegate)); string kind; if (signature == typeof(SignatureDefault)) { kind = "Component"; if (sig == null || "default".StartsWithInvariantCulture(sig)) { foundSig = true; } } else { kind = ComponentCatalog.SignatureToString(signature); if (sig == null || kind.StartsWithInvariantCultureIgnoreCase(sig)) { foundSig = true; } } if (kinds.Length > 0) { kinds.Append(", "); } kinds.Append(kind); } if (foundSig) { string kindsStr = kinds.ToString(); var args = info.CreateArguments(); ShowUsage(writer, kindsStr, info.Summary, info.LoadNames[0], info.LoadNames, args, columns); components.Add(new Component(kindsStr, info, args)); } } if (components.Count == 0) { writer.WriteLine("Unknown component: '{0}'", name); } else { Serialize(components); } }
/// <summary> /// Builds a JSON representation of all entry points and components of the <paramref name="catalog"/>. /// </summary> /// <param name="ectx">The exception context to use</param> /// <param name="catalog">The module catalog</param> public static JObject BuildAllManifests(IExceptionContext ectx, ComponentCatalog catalog) { Contracts.CheckValueOrNull(ectx); ectx.CheckValue(catalog, nameof(catalog)); var jEntryPoints = new JArray(); var entryPointInfos = catalog.AllEntryPoints().ToArray(); foreach (var entryPointInfo in entryPointInfos.OrderBy(x => x.Name)) { jEntryPoints.Add(BuildEntryPointManifest(ectx, entryPointInfo, catalog)); } var jKinds = new JArray(); foreach (var kind in catalog.GetAllComponentKinds()) { var jKind = new JObject(); jKind[FieldNames.Kind] = kind; var jComponents = new JArray(); foreach (var component in catalog.GetAllComponents(kind)) { jComponents.Add(BuildComponentManifest(ectx, component, catalog)); } jKind[FieldNames.Components] = jComponents; jKinds.Add(jKind); } var jepKinds = new JArray(); var kinds = new List <Type>(); foreach (var entryPointInfo in entryPointInfos) { if (entryPointInfo.InputKinds != null) { kinds.AddRange(entryPointInfo.InputKinds); } if (entryPointInfo.OutputKinds != null) { kinds.AddRange(entryPointInfo.OutputKinds); } } foreach (var epKind in kinds.Distinct().OrderBy(k => k.Name)) { var jepKind = new JObject(); jepKind[FieldNames.Kind] = epKind.Name; var jepKindFields = new JArray(); var propertyInfos = epKind.GetProperties().AsEnumerable(); propertyInfos = epKind.GetInterfaces().Aggregate(propertyInfos, (current, face) => current.Union(face.GetProperties())); foreach (var fieldInfo in propertyInfos) { var jField = new JObject(); jField[FieldNames.Name] = fieldInfo.Name; var type = ExtractOptionalOrNullableType(fieldInfo.PropertyType); // Dive inside Var. if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>)) { type = type.GetGenericArguments()[0]; } var typeEnum = TlcModule.GetDataType(type); jField[FieldNames.Type] = typeEnum.ToString(); jepKindFields.Add(jField); } jepKind[FieldNames.Settings] = jepKindFields; jepKinds.Add(jepKind); } var jResult = new JObject(); jResult[FieldNames.TopEntryPoints] = jEntryPoints; jResult[FieldNames.TopComponents] = jKinds; jResult[FieldNames.TopEntryPointKinds] = jepKinds; return(jResult); }
/// <summary> /// The main method to invoke TLC, with some high level configuration options set. /// </summary> /// <param name="env">The environment used in this run of TLC, for the purpose of returning outputs.</param> /// <param name="args">The command line arguments.</param> /// <param name="alwaysPrintStacktrace">"Marked" exceptions are assumed to be sufficiently descriptive, so we /// do not print stack traces for them to the console, and instead print these only to a log file. /// However, throwing unmarked exceptions is considered a bug in TLC (even if due to bad user input), /// so we always write . If set to true though, this executable will also print stack traces from the /// marked exceptions as well.</param> /// <returns></returns> internal static int MainCore(ConsoleEnvironment env, string args, bool alwaysPrintStacktrace) { // REVIEW: How should extra dlls, tracking, etc be handled? Should the args objects for // all commands derive from a common base? var mainHost = env.Register("Main"); using (var telemetryPipe = mainHost.StartPipe <TelemetryMessage>("TelemetryPipe")) using (var ch = mainHost.Start("Main")) { int result; try { if (!CmdParser.TryGetFirstToken(args, out string kind, out string settings)) { telemetryPipe.Send(TelemetryMessage.CreateCommand("ArgumentParsingFailure", args)); Usage(); return(-1); } if (!ComponentCatalog.TryCreateInstance <ICommand, SignatureCommand>(mainHost, out ICommand cmd, kind, settings)) { // Telemetry: Log telemetryPipe.Send(TelemetryMessage.CreateCommand("UnknownCommand", settings)); ch.Error("Unknown command: '{0}'", kind); Usage(); return(-1); } // Telemetry: Log the command and settings. telemetryPipe.Send(TelemetryMessage.CreateCommand(kind.ToUpperInvariant(), settings)); cmd.Run(); result = 0; } catch (Exception ex) { var dumpFileDir = Path.Combine( Path.GetTempPath(), "TLC"); var dumpFilePath = Path.Combine(dumpFileDir, string.Format(CultureInfo.InvariantCulture, "Error_{0:yyyyMMdd_HHmmss}_{1}.log", DateTime.UtcNow, Guid.NewGuid())); bool isDumpSaved = false; try { Directory.CreateDirectory(dumpFileDir); // REVIEW: Should specify the encoding. using (var sw = new StreamWriter(new FileStream(dumpFilePath, FileMode.Create, FileAccess.Write))) { sw.WriteLine("--- Command line args ---"); sw.WriteLine(args); sw.WriteLine("--- Exception message ---"); PrintFullExceptionDetails(sw, ex); } isDumpSaved = true; } catch (Exception) { // Don't throw an exception if we failed to write to the dump file. } // Process exceptions that we understand. int count = 0; for (var e = ex; e != null; e = e.InnerException) { // Telemetry: Log the exception telemetryPipe.Send(TelemetryMessage.CreateException(e)); if (e.IsMarked()) { ch.Error(e.Sensitivity(), e.Message); PrintExceptionData(ch, e, false); count++; } } if (count == 0) { // Didn't recognize any of the exceptions. ch.Error(MessageSensitivity.None, "***** Unexpected failure. Please refer to https://aka.ms/MLNetIssue to file an issue with details *****"); if (isDumpSaved) { ch.Error(MessageSensitivity.None, "***** Error log has been saved to '{0}', please refer to https://aka.ms/MLNetIssue to file an issue with details *****", dumpFilePath); } } else if (isDumpSaved) { ch.Error(MessageSensitivity.None, "Error log has been saved to '{0}'. Please refer to https://aka.ms/MLNetIssue if you need assistance.", dumpFilePath); } if (count == 0 || alwaysPrintStacktrace) { ch.Error(MessageSensitivity.None, "===== Begin detailed dump ====="); PrintFullExceptionDetails(ch, ex); ch.Error(MessageSensitivity.None, "====== End detailed dump ====="); } // Return a negative result code so AEther recognizes this as a failure. result = count > 0 ? -1 : -2; } finally { } telemetryPipe.Done(); return(result); } }
private static JArray BuildOutputManifest(IExceptionContext ectx, Type outputType, ComponentCatalog catalog) { Contracts.AssertValueOrNull(ectx); ectx.AssertValue(outputType); ectx.AssertValue(catalog); var outputs = new List <KeyValuePair <Double, JObject> >(); if (outputType.IsGenericType && outputType.GetGenericTypeDefinition() == typeof(CommonOutputs.MacroOutput <>)) { outputType = outputType.GetGenericArguments()[0]; } foreach (var fieldInfo in outputType.GetFields()) { var outputAttr = fieldInfo.GetCustomAttributes(typeof(TlcModule.OutputAttribute), false) .FirstOrDefault() as TlcModule.OutputAttribute; if (outputAttr == null) { continue; } var jo = new JObject(); jo[FieldNames.Name] = outputAttr.Name ?? fieldInfo.Name; jo[FieldNames.Type] = BuildTypeToken(ectx, fieldInfo, fieldInfo.FieldType, catalog); jo[FieldNames.Desc] = outputAttr.Desc; outputs.Add(new KeyValuePair <Double, JObject>(outputAttr.SortOrder, jo)); } return(new JArray(outputs.OrderBy(x => x.Key).Select(x => x.Value).ToArray())); }
public void SingletonComponentRegisteredWithComponentInstance_IsCleanedUp() { // this component instance is created outside, but the ownership // is transferred to the ComponentCatalog via cleanup delegate var disposed = false; var mortalServer = new ReleasableComponent { Handler = () => disposed = true }; var cat = new ComponentCatalog(); cat.RegisterComponent<ISampleComponent, ReleasableComponent>(mortalServer, v => ((ReleasableComponent)v).Release()); Assert.IsFalse(disposed); var instance = cat.GetComponent<ISampleComponent>(); AssertEx.IsInstanceOf<ReleasableComponent>(instance); var reg = cat.GetRegistration(typeof(ISampleComponent)); cat.CleanUpComponentInstance(reg, instance); Assert.IsTrue(disposed); }
private static JToken BuildTypeToken(IExceptionContext ectx, FieldInfo fieldInfo, Type type, ComponentCatalog catalog) { Contracts.AssertValueOrNull(ectx); ectx.AssertValue(type); ectx.AssertValue(catalog); // REVIEW: Allows newly introduced types to not break the manifest bulding process. // Where possible, these types should be replaced by component kinds. if (type == typeof(CommonInputs.IEvaluatorInput) || type == typeof(CommonOutputs.IEvaluatorOutput)) { var jo = new JObject(); var typeString = $"{type}".Replace("Microsoft.ML.EntryPoints.", ""); jo[FieldNames.Kind] = "EntryPoint"; jo[FieldNames.ItemType] = typeString; return(jo); } type = ExtractOptionalOrNullableType(type); // Dive inside Var. if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Var <>)) { type = type.GetGenericArguments()[0]; } var typeEnum = TlcModule.GetDataType(type); switch (typeEnum) { case TlcModule.DataKind.Unknown: var jo = new JObject(); if (type == typeof(JArray)) { jo[FieldNames.Kind] = TlcModule.DataKind.Array.ToString(); jo[FieldNames.ItemType] = "Node"; return(jo); } if (type == typeof(JObject)) { return("Bindings"); } var fields = BuildInputManifest(ectx, type, catalog); if (fields.Count == 0) { throw ectx.Except("Unexpected parameter type: {0}", type); } jo[FieldNames.Kind] = "Struct"; jo[FieldNames.Fields] = fields; return(jo); case TlcModule.DataKind.Float: case TlcModule.DataKind.Int: case TlcModule.DataKind.UInt: case TlcModule.DataKind.Char: case TlcModule.DataKind.String: case TlcModule.DataKind.Bool: case TlcModule.DataKind.DataView: case TlcModule.DataKind.TransformModel: case TlcModule.DataKind.PredictorModel: case TlcModule.DataKind.FileHandle: return(typeEnum.ToString()); case TlcModule.DataKind.Enum: jo = new JObject(); jo[FieldNames.Kind] = typeEnum.ToString(); var values = Enum.GetNames(type).Where(n => type.GetField(n).GetCustomAttribute <HideEnumValueAttribute>() == null); jo[FieldNames.Values] = new JArray(values); return(jo); case TlcModule.DataKind.Array: jo = new JObject(); jo[FieldNames.Kind] = typeEnum.ToString(); jo[FieldNames.ItemType] = BuildTypeToken(ectx, fieldInfo, type.GetElementType(), catalog); return(jo); case TlcModule.DataKind.Dictionary: jo = new JObject(); jo[FieldNames.Kind] = typeEnum.ToString(); jo[FieldNames.ItemType] = BuildTypeToken(ectx, fieldInfo, type.GetGenericArguments()[1], catalog); return(jo); case TlcModule.DataKind.Component: string kind; if (!catalog.TryGetComponentKind(type, out kind)) { throw ectx.Except("Field '{0}' is a component of unknown kind", fieldInfo.Name); } jo = new JObject(); jo[FieldNames.Kind] = typeEnum.ToString(); jo[FieldNames.ComponentKind] = kind; return(jo); default: ectx.Assert(false); throw ectx.ExceptNotSupp(); } }
public void SingletonComponentRegisteredWithComponentType_IsCleanedUp() { var disposed = false; var cat = new ComponentCatalog(); cat.RegisterComponent<ISampleComponent, ReleasableComponent>( ActivationType.Singleton, v => ((ReleasableComponent)v).Release()); Assert.IsFalse(disposed); var instance = cat.GetComponent<ISampleComponent>(); AssertEx.IsInstanceOf<ReleasableComponent>(instance); instance.Handler = () => disposed = true; var reg = cat.GetRegistration(typeof(ISampleComponent)); cat.CleanUpComponentInstance(reg, instance); Assert.IsTrue(disposed); }
private static JToken BuildValueToken(IExceptionContext ectx, object value, Type valueType, ComponentCatalog catalog) { Contracts.AssertValueOrNull(ectx); ectx.AssertValueOrNull(value); ectx.AssertValue(valueType); ectx.AssertValue(catalog); if (value == null) { return(null); } // Dive inside Nullable. if (valueType.IsGenericType && valueType.GetGenericTypeDefinition() == typeof(Nullable <>)) { valueType = valueType.GetGenericArguments()[0]; } // Dive inside Optional. if (valueType.IsGenericType && valueType.GetGenericTypeDefinition() == typeof(Optional <>)) { valueType = valueType.GetGenericArguments()[0]; value = ((Optional)value).GetValue(); } var dataType = TlcModule.GetDataType(valueType); switch (dataType) { case TlcModule.DataKind.Bool: case TlcModule.DataKind.Int: case TlcModule.DataKind.UInt: case TlcModule.DataKind.Float: case TlcModule.DataKind.String: return(new JValue(value)); case TlcModule.DataKind.Char: return(new JValue(value.ToString())); case TlcModule.DataKind.Array: var valArray = value as Array; var ja = new JArray(); foreach (var item in valArray) { ja.Add(BuildValueToken(ectx, item, item.GetType(), catalog)); } return(ja); case TlcModule.DataKind.Enum: return(value.ToString()); case TlcModule.DataKind.Dictionary: // REVIEW: need to figure out how to represent these. throw ectx.ExceptNotSupp("Dictionary and component default values are not supported"); case TlcModule.DataKind.Component: var factory = value as IComponentFactory; ectx.AssertValue(factory); return(BuildComponentToken(ectx, factory, catalog)); default: throw ectx.ExceptNotSupp("Encountered a default value for unsupported type {0}", dataType); } }
/// <summary> /// Build a token for component default value. This will look up the component in the catalog, and if it finds an entry, it will /// build a JSON structure that would be parsed into the default value. /// /// This is an inherently fragile setup in case when the factory is not trivial, but it will work well for 'property bag' factories /// that we are currently using. /// </summary> private static JToken BuildComponentToken(IExceptionContext ectx, IComponentFactory value, ComponentCatalog catalog) { Contracts.AssertValueOrNull(ectx); ectx.AssertValue(value); ectx.AssertValue(catalog); var type = value.GetType(); ComponentCatalog.ComponentInfo componentInfo; if (!catalog.TryFindComponent(type, out componentInfo)) { // The default component is not in the catalog. This is, technically, allowed, but it means that there's no JSON representation // for the default value. We will emit the one the won't parse back. return(new JValue("(custom component)")); } ectx.Assert(componentInfo.ArgumentType == type); // Try to invoke default ctor for the factory to obtain defaults. object defaults; try { defaults = Activator.CreateInstance(type); } catch (MissingMemberException ex) { // There was no default constructor found. // This should never happen, since ComponentCatalog would error out if there is no default ctor. ectx.Assert(false); throw ectx.Except(ex, "Couldn't find default constructor"); } var jResult = new JObject(); var jSettings = new JObject(); jResult[FieldNames.Name] = componentInfo.Name; // Iterate over all fields of the factory object, and compare the values with the defaults. // If the value differs, insert it into the settings object. bool anyValue = false; foreach (var fieldInfo in type.GetFields()) { var attr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute; if (attr == null || attr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly) { continue; } ectx.Assert(!fieldInfo.IsStatic && !fieldInfo.IsInitOnly && !fieldInfo.IsLiteral); bool needValue = false; object actualValue = fieldInfo.GetValue(value); if (attr.IsRequired) { needValue = true; } else { object defaultValue = fieldInfo.GetValue(defaults); needValue = !Equals(actualValue, defaultValue); } if (!needValue) { continue; } jSettings[attr.Name ?? fieldInfo.Name] = BuildValueToken(ectx, actualValue, fieldInfo.FieldType, catalog); anyValue = true; } if (anyValue) { jResult[FieldNames.Settings] = jSettings; } return(jResult); }
static int MainCore(DelegateEnvironment env, string args, bool alwaysPrintStacktrace) { var mainHost = env.Register("Main"); using (var ch = mainHost.Start("Main")) { int result; try { if (!CmdParser.TryGetFirstToken(args, out string kind, out string settings)) { Usage(); return(-1); } if (!ComponentCatalog.TryCreateInstance <ICommand, SignatureCommand>(mainHost, out ICommand cmd, kind, settings)) { ch.Error("Unknown command: '{0}'", kind); Usage(); return(-1); } var helpC = cmd as HelpCommand; if (helpC == null) { env.SetPrintElapsed(true); cmd.Run(); } else { int width = 80; try { width = Console.BufferWidth; } catch (Exception) { } env.SetPrintElapsed(false); helpC.Run(width); } result = 0; } catch (Exception ex) { int count = 0; for (var e = ex; e != null; e = e.InnerException) { // Telemetry: Log the exception if (e.IsMarked()) { ch.Error(e.Sensitivity(), e.Message); PrintExceptionData(ch, e, false); count++; } } if (count == 0) { ch.Error(MessageSensitivity.None, "Unexpected failure."); } if (count == 0 || alwaysPrintStacktrace) { ch.Error(MessageSensitivity.None, "===== Begin detailed dump ====="); PrintFullExceptionDetails(ch, ex); if (env.VerboseLevel >= 3) { ch.Error("= LoadedAssemblies ="); var assemblies = AppDomain.CurrentDomain.GetAssemblies().Select(x => InfoAssembly(x)).OrderBy(c => c); foreach (var a in assemblies) { ch.Error(a); } } ch.Error(MessageSensitivity.None, "====== End detailed dump ====="); } // Return a negative result code so AEther recognizes this as a failure. result = count > 0 ? -1 : -2; } finally { } return(result); } }