public void Save(ModelSaveContext ctx) { _ectx.AssertValue(ctx); // *** Binary format *** // int: G - number of group columns // int[G]: ids of group column names // int: K: number of keep columns // int[K]: ids of keep column names _ectx.AssertNonEmpty(_groupColumns); ctx.Writer.Write(_groupColumns.Length); foreach (var name in _groupColumns) { _ectx.AssertNonEmpty(name); ctx.SaveString(name); } _ectx.AssertValue(_keepColumns); ctx.Writer.Write(_keepColumns.Length); foreach (var name in _keepColumns) { _ectx.AssertNonEmpty(name); ctx.SaveString(name); } }
private static void CheckAndBind(IExceptionContext ectx, ISchema inputSchema, string[] pivotColumns, out PivotColumnInfo[] infos) { Contracts.AssertValueOrNull(ectx); ectx.AssertValue(inputSchema); ectx.AssertNonEmpty(pivotColumns); infos = new PivotColumnInfo[pivotColumns.Length]; for (int i = 0; i < pivotColumns.Length; i++) { var name = pivotColumns[i]; // REVIEW: replace Check with CheckUser, once existing CheckUser is renamed to CheckUserArg or something. ectx.CheckUserArg(!string.IsNullOrEmpty(name), nameof(Arguments.Column), "Column name cannot be empty"); int col; if (!inputSchema.TryGetColumnIndex(name, out col)) { throw ectx.ExceptUserArg(nameof(Arguments.Column), "Pivot column '{0}' is not found", name); } var colType = inputSchema.GetColumnType(col); if (!colType.IsVector || !colType.ItemType.IsPrimitive) { throw ectx.ExceptUserArg(nameof(Arguments.Column), "Pivot column '{0}' has type '{1}', but must be a vector of primitive types", name, colType); } infos[i] = new PivotColumnInfo(name, col, colType.VectorSize, colType.ItemType.AsPrimitive); } }
public ColumnInfo(IExceptionContext ectx, string[] inputColumnNames, DataViewType[] inputTypes, string expression, string outputColumnName, int vectorInputColumn, LambdaNode node, int[] perm) { ectx.AssertNonEmpty(inputTypes); ectx.Assert(Utils.Size(inputTypes) == Utils.Size(inputColumnNames)); ectx.AssertNonWhiteSpace(expression); ectx.AssertNonWhiteSpace(outputColumnName); ectx.AssertValue(node); InputColumnNames = inputColumnNames; OutputColumnName = outputColumnName; OutputColumnItemType = node.ResultType as PrimitiveDataViewType; ectx.AssertValue(OutputColumnItemType); VectorInputColumn = vectorInputColumn; Perm = perm; Expression = expression; InputKinds = new InternalDataKind[inputTypes.Length]; for (int i = 0; i < inputTypes.Length; i++) { InputKinds[i] = inputTypes[i].GetRawKind(); } Del = LambdaCompiler.Compile(out var errors, node); if (Utils.Size(errors) > 0) { throw ectx.Except($"generating code failed: {errors[0].GetMessage()}"); } }
/// <summary> /// Returns true if the input type is something recognizable as being oen of the standard /// builtin types. This method will also throw if something is detected as being definitely /// wrong (e.g., the input type does not descend from <see cref="PipelineColumn"/> at all, /// or a <see cref="Key{T}"/> is declared with a <see cref="string"/> type parameter or /// something. /// </summary> private static bool IsStandard(IExceptionContext ectx, Type t) { Contracts.AssertValue(ectx); ectx.AssertValue(t); if (!typeof(PipelineColumn).IsAssignableFrom(t)) { throw ectx.ExceptParam(nameof(t), $"Type {t} was not even of {nameof(PipelineColumn)}"); } var gt = t.IsGenericType ? t.GetGenericTypeDefinition() : t; if (gt != typeof(Scalar <>) && gt != typeof(Key <>) && gt != typeof(Key <,>) && gt != typeof(VarKey <>) && gt != typeof(Vector <>) && gt != typeof(VarVector <>) && gt != typeof(NormVector <>)) { throw ectx.ExceptParam(nameof(t), $"Type {t} was not one of the standard subclasses of {nameof(PipelineColumn)}"); } ectx.Assert(t.IsGenericType); var ga = t.GetGenericArguments(); ectx.AssertNonEmpty(ga); if (gt == typeof(Key <>) || gt == typeof(Key <,>) || gt == typeof(VarKey <>)) { ectx.Assert((gt == typeof(Key <,>) && ga.Length == 2) || ga.Length == 1); var kt = ga[0]; if (kt != typeof(byte) && kt != typeof(ushort) && kt != typeof(uint) && kt != typeof(ulong)) { throw ectx.ExceptParam(nameof(t), $"Type parameter {kt.Name} is not a valid type for key"); } return(gt != typeof(Key <,>) || IsStandardCore(ga[1])); } ectx.Assert(ga.Length == 1); return(IsStandardCore(ga[0])); }
/// <summary> /// This is essentially the inverse function to <see cref="GetSerializedStaticDelegate"/>. If the function /// is not recoverable for any reason, this will return <c>null</c>, and the error parameter will be set. /// </summary> /// <param name="ectx">Exception context.</param> /// <param name="serialized">The serialized bytes, as returned by <see cref="GetSerializedStaticDelegate"/></param> /// <param name="inner">An exception the caller may raise as an inner exception if the return value is /// <c>null</c>, else, this itself will be <c>null</c></param> /// <returns>The recovered function wrapping the recovered method, or <c>null</c> if it could not /// be created, for some reason</returns> public static LambdaTransform.LoadDelegate DeserializeStaticDelegateOrNull(IExceptionContext ectx, byte[] serialized, out Exception inner) { Contracts.AssertValue(ectx); ectx.AssertNonEmpty(serialized); MethodInfo info = null; try { using (var ms = new MemoryStream(serialized, false)) { #if CORECLR var formatter = new BinaryFormatter(); object obj = formatter.Deserialize(ms); var hack = obj as CoreHackMethodInfo; var assembly = Assembly.Load(new AssemblyName(hack.AssemblyName)); Type t = assembly.GetType(hack.ClassName); info = t.GetTypeInfo().GetDeclaredMethod(hack.MethodName); #else var formatter = new BinaryFormatter(); object obj = formatter.Deserialize(ms); info = obj as MethodInfo; #endif } } catch (Exception e) { inner = ectx.ExceptDecode(e, "Failed to deserialize a .NET object"); return(null); } // Either it's not the right type, or obj itself may be null. Either way we have an error. switch (info) { case info == null: inner = ectx.ExceptDecode("Failed to deserialize the method"); return(null); break; case !info.IsStatic: inner = ectx.ExceptDecode("Deserialized method is not static"); return(null); break; } try { var del = info.CreateDelegate(typeof(LambdaTransform.LoadDelegate)); inner = null; return((LambdaTransform.LoadDelegate)del); } catch (Exception) { inner = ectx.ExceptDecode("Deserialized method has wrong signature"); return(null); } }
public EntryPointVariable(IExceptionContext ectx, string name, Type type) { Contracts.AssertValueOrNull(ectx); _ectx = ectx; _ectx.AssertNonEmpty(name); Name = name; ectx.Assert(IsValidType(type)); Type = type; }
/// <summary> /// Retreives the field index for a field with the given alias, or -1 if /// that field alias is not found. /// </summary> private int GetFieldIndex(string name) { _ectx.AssertNonEmpty(name); for (int i = 0; i < _attrs.Length; i++) { if (name == (_attrs[i].Input.Name ?? _fields[i].Name) || AnyMatch(name, _attrs[i].Input.Aliases)) { return(i); } } return(-1); }
private FieldInfo GetField(string name) { _ectx.AssertNonEmpty(name); for (int i = 0; i < _attrs.Length; i++) { if (name == (_attrs[i].Name ?? _fields[i].Name)) { return(_fields[i]); } } return(null); }
public UngroupBinding(IExceptionContext ectx, Schema inputSchema, UngroupMode mode, string[] pivotColumns) { Contracts.AssertValueOrNull(ectx); _ectx = ectx; _ectx.AssertValue(inputSchema); _ectx.AssertNonEmpty(pivotColumns); _inputSchema = inputSchema; // This also makes InputColumnCount valid. Mode = mode; Bind(_ectx, inputSchema, pivotColumns, out _infos); _pivotIndex = Utils.CreateArray(InputColumnCount, -1); for (int i = 0; i < _infos.Length; i++) { var info = _infos[i]; _ectx.Assert(_pivotIndex[info.Index] == -1); _pivotIndex[info.Index] = i; } var schemaBuilder = new SchemaBuilder(); // Iterate through input columns. Input columns which are not pivot columns will be copied to output schema with the same column index unchanged. // Input columns which are pivot columns would also be copied but with different data types and different metadata. for (int i = 0; i < InputColumnCount; ++i) { if (_pivotIndex[i] < 0) { // i-th input column is not a pivot column. Let's do a naive copy. schemaBuilder.AddColumn(inputSchema[i].Name, inputSchema[i].Type, inputSchema[i].Metadata); } else { // i-th input column is a pivot column. Let's calculate proper type and metadata for it. var metadataBuilder = new MetadataBuilder(); metadataBuilder.Add(inputSchema[i].Metadata, metadataName => ShouldPreserveMetadata(metadataName)); // To explain the output type of pivot columns, let's consider a row // Age UserID // 18 {"Amy", "Willy"} // where "Age" and "UserID" are column names and 18/{"Amy", "Willy"} is "Age"/"UserID" column in this example row. // If the only pivot column is "UserID", the ungroup may produce // Age UserID // 18 "Amy" // 18 "Willy" // One can see that "UserID" column (in output data) has a type identical to the element's type of the "UserID" column in input data. schemaBuilder.AddColumn(inputSchema[i].Name, inputSchema[i].Type.GetItemType(), metadataBuilder.GetMetadata()); } } OutputSchema = schemaBuilder.GetSchema(); }
public GroupBinding(IExceptionContext ectx, Schema inputSchema, string[] groupColumns, string[] keepColumns) { Contracts.AssertValue(ectx); _ectx = ectx; _ectx.AssertValue(inputSchema); _ectx.AssertNonEmpty(groupColumns); _ectx.AssertValue(keepColumns); _inputSchema = inputSchema; _groupColumns = groupColumns; GroupColumnIndexes = GetColumnIds(inputSchema, groupColumns, x => _ectx.ExceptUserArg(nameof(Arguments.GroupKeys), x)); _keepColumns = keepColumns; KeepColumnIndexes = GetColumnIds(inputSchema, keepColumns, x => _ectx.ExceptUserArg(nameof(Arguments.Columns), x)); // Compute output schema from the specified input schema. OutputSchema = BuildOutputSchema(inputSchema); }
internal ComponentInfo(IExceptionContext ectx, Type interfaceType, string kind, Type argumentType, TlcModule.ComponentAttribute attribute) { Contracts.AssertValueOrNull(ectx); ectx.AssertValue(interfaceType); ectx.AssertNonEmpty(kind); ectx.AssertValue(argumentType); ectx.AssertValue(attribute); Name = attribute.Name; Description = attribute.Desc; if (string.IsNullOrWhiteSpace(attribute.FriendlyName)) { FriendlyName = Name; } else { FriendlyName = attribute.FriendlyName; } Kind = kind; if (!IsValidName(Kind)) { throw ectx.Except("Invalid component kind: '{0}'", Kind); } Aliases = attribute.Aliases; if (!IsValidName(Name)) { throw ectx.Except("Component name '{0}' is not valid.", Name); } if (Aliases != null && Aliases.Any(x => !IsValidName(x))) { throw ectx.Except("Component '{0}' has an invalid alias '{1}'", Name, Aliases.First(x => !IsValidName(x))); } if (!typeof(IComponentFactory).IsAssignableFrom(argumentType)) { throw ectx.Except("Component '{0}' must inherit from IComponentFactory", argumentType); } ArgumentType = argumentType; InterfaceType = interfaceType; }
public GroupSchema(IExceptionContext ectx, ISchema inputSchema, string[] groupColumns, string[] keepColumns) { Contracts.AssertValue(ectx); _ectx = ectx; _ectx.AssertValue(inputSchema); _ectx.AssertNonEmpty(groupColumns); _ectx.AssertValue(keepColumns); _input = inputSchema; _groupColumns = groupColumns; GroupIds = GetColumnIds(inputSchema, groupColumns, x => _ectx.ExceptUserArg(nameof(Arguments.GroupKey), x)); _groupCount = GroupIds.Length; _keepColumns = keepColumns; KeepIds = GetColumnIds(inputSchema, keepColumns, x => _ectx.ExceptUserArg(nameof(Arguments.Column), x)); _columnTypes = BuildColumnTypes(_input, KeepIds); _columnNameMap = BuildColumnNameMap(); }
private static IComponentFactory GetComponentJson(IExceptionContext ectx, Type signatureType, string name, JObject settings, ComponentCatalog catalog) { Contracts.AssertValue(ectx); ectx.AssertValue(signatureType); ectx.AssertNonEmpty(name); ectx.AssertValueOrNull(settings); ectx.AssertValue(catalog); if (!catalog.TryGetComponentKind(signatureType, out string kind)) { throw ectx.Except($"Component type '{signatureType}' is not a valid signature type."); } if (!catalog.TryFindComponent(kind, name, out ComponentCatalog.ComponentInfo component)) { var available = catalog.GetAllComponents(kind).Select(x => $"'{x.Name}'"); throw ectx.Except($"Component '{name}' of kind '{kind}' is not found. Available components are: {string.Join(", ", available)}"); } var inputBuilder = new InputBuilder(ectx, component.ArgumentType, catalog); if (settings != null) { foreach (var pair in settings) { if (!inputBuilder.TrySetValueJson(pair.Key, pair.Value)) { throw ectx.Except($"Unexpected value for component '{name}', field '{pair.Key}': '{pair.Value}'"); } } } var missing = inputBuilder.GetMissingValues().ToArray(); if (missing.Length > 0) { throw ectx.Except($"The following required inputs were not provided for component '{name}': {string.Join(", ", missing)}"); } return(inputBuilder.GetInstance() as IComponentFactory); }
public SchemaImpl(IExceptionContext ectx, ISchema inputSchema, UngroupMode mode, string[] pivotColumns) { Contracts.AssertValueOrNull(ectx); _ectx = ectx; _ectx.AssertValue(inputSchema); _ectx.AssertNonEmpty(pivotColumns); _inputSchema = inputSchema; Mode = mode; CheckAndBind(_ectx, inputSchema, pivotColumns, out _infos); _pivotColMap = new Dictionary <string, int>(); _pivotIndex = Utils.CreateArray(_inputSchema.ColumnCount, -1); for (int i = 0; i < _infos.Length; i++) { var info = _infos[i]; _pivotColMap[info.Name] = info.Index; _ectx.Assert(_pivotIndex[info.Index] == -1); _pivotIndex[info.Index] = i; } }
/// <summary> /// Returns true if added new variable, false if variable already exists. /// </summary> public Boolean AddOutputVariable(string name, Type type) { _ectx.AssertNonEmpty(name); _ectx.AssertValue(type); EntryPointVariable v; if (!_vars.TryGetValue(name, out v)) { v = new EntryPointVariable(_ectx, name, type); _vars[name] = v; } else { if (v.Type != type) { throw _ectx.Except($"Variable '{v.Name}' is used as {v.Type} and as {type}"); } return(false); } v.MarkUsage(false); return(true); }