private EstimatorChain(IHostEnvironment env, IEstimator <ITransformer>[] estimators, TransformerScope[] scopes, bool[] needCacheAfter)
{
Contracts.AssertValueOrNull(env);
Contracts.AssertValueOrNull(estimators);
Contracts.AssertValueOrNull(scopes);
Contracts.AssertValueOrNull(needCacheAfter);
Contracts.Assert(Utils.Size(estimators) == Utils.Size(scopes));
Contracts.Assert(Utils.Size(estimators) == Utils.Size(needCacheAfter));
_host = env?.Register(nameof(EstimatorChain <TLastTransformer>));
_estimators = estimators ?? new IEstimator <ITransformer> [0];
_scopes = scopes ?? new TransformerScope[0];
LastEstimator = estimators.LastOrDefault() as IEstimator <TLastTransformer>;
_needCacheAfter = needCacheAfter ?? new bool[0];
Contracts.Assert((_host != null) == _needCacheAfter.Any(x => x));
Contracts.Assert((_estimators.Length > 0) == (LastEstimator != null));
}
// Marks telemetry in the rule. Needed in canvas, not needed in PowerFx.Core
// DelegationTracker.SetDelegationTrackerStatus(DelegationStatus.NotANumberArgType, callNode, binding, this, DelegationTelemetryInfo.CreateEmptyDelegationTelemetryInfo());
public static void SetDelegationTrackerStatus(DelegationStatus status, TexlNode node, TexlBinding binding, TexlFunction func, DelegationTelemetryInfo logInfo = null)
{
Contracts.AssertValue(node);
Contracts.AssertValue(binding);
Contracts.AssertValue(func);
Contracts.AssertValueOrNull(logInfo);
IExternalRule rule;
// The rule need not exist on ControlInfo yet. This happens when we are attempting
// to create a namemap in which case we try to bind the rule before adding it to the control.
if (!TryGetCurrentRule(binding, out rule))
{
return;
}
rule.SetDelegationTrackerStatus(node, status, logInfo ?? DelegationTelemetryInfo.CreateEmptyDelegationTelemetryInfo(), func);
}
private static JObject BuildComponentManifest(IExceptionContext ectx, ComponentCatalog.ComponentInfo componentInfo, ComponentCatalog catalog)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(componentInfo);
ectx.AssertValue(catalog);
var result = new JObject();
result[FieldNames.Name] = componentInfo.Name;
result[FieldNames.Desc] = componentInfo.Description;
result[FieldNames.FriendlyName] = componentInfo.FriendlyName;
if (Utils.Size(componentInfo.Aliases) > 0)
{
result[FieldNames.Aliases] = new JArray(componentInfo.Aliases);
}
result[FieldNames.Settings] = BuildInputManifest(ectx, componentInfo.ArgumentType, catalog);
return(result);
}
public CalibratedRowMapper(RoleMappedSchema schema, SchemaBindableBinaryPredictorWrapper parent)
{
Contracts.AssertValue(parent);
Contracts.Assert(parent._distMapper != null);
Contracts.AssertValue(schema);
Contracts.AssertValueOrNull(schema.Feature);
_parent = parent;
InputRoleMappedSchema = schema;
Schema = Schema.Create(new BinaryClassifierSchema());
if (schema.Feature != null)
{
var typeSrc = InputRoleMappedSchema.Feature.Type;
Contracts.Check(typeSrc.IsKnownSizeVector && typeSrc.ItemType == NumberType.Float,
"Invalid feature column type");
}
}
public Dense(int count, string[] names)
{
Contracts.Assert(count >= 0);
Contracts.AssertValueOrNull(names);
_count = count;
int size = Math.Min(Utils.Size(names), count);
_names = new string[size];
if (size > 0)
{
Array.Copy(names, _names, size);
}
// REVIEW: This seems wrong. The default feature column name is "Features" yet the role is named "Feature".
Schema = new RoleMappedSchema(new FeatureNameCollectionSchema(this),
roles: RoleMappedSchema.ColumnRole.Feature.Bind(RoleMappedSchema.ColumnRole.Feature.Value));
}
internal ComponentInfo(IExceptionContext ectx, Type interfaceType, string kind, Type argumentType, TlcModule.ComponentAttribute attribute)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(interfaceType);
ectx.AssertNonEmpty(kind);
ectx.AssertValue(argumentType);
ectx.AssertValue(attribute);
Name = attribute.Name;
Description = attribute.Desc;
if (string.IsNullOrWhiteSpace(attribute.FriendlyName))
{
FriendlyName = Name;
}
else
{
FriendlyName = attribute.FriendlyName;
}
Kind = kind;
if (!IsValidName(Kind))
{
throw ectx.Except("Invalid component kind: '{0}'", Kind);
}
Aliases = attribute.Aliases;
if (!IsValidName(Name))
{
throw ectx.Except("Component name '{0}' is not valid.", Name);
}
if (Aliases != null && Aliases.Any(x => !IsValidName(x)))
{
throw ectx.Except("Component '{0}' has an invalid alias '{1}'", Name, Aliases.First(x => !IsValidName(x)));
}
if (!typeof(IComponentFactory).IsAssignableFrom(argumentType))
{
throw ectx.Except("Component '{0}' must inherit from IComponentFactory", argumentType);
}
ArgumentType = argumentType;
InterfaceType = interfaceType;
}
private TexlNode Parse(ref List <TexlError> errors)
{
Contracts.AssertValueOrNull(errors);
_errors = errors;
TexlNode node;
Token firstToken = _curs.TokCur;
_before = new SourceList(ParseTrivia());
if (_curs.TidCur == TokKind.Eof)
{
if (firstToken.Kind == TokKind.Comment && firstToken.As <CommentToken>().IsOpenBlock)
{
// This provides an error message for when a block comment missing a closing '*/' is the only token in the formula bar
PostBlockCommentMissingClosingError();
errors = _errors;
}
;
node = new BlankNode(ref _idNext, _curs.TokCur);
}
else
{
node = ParseExpr(Precedence.None);
if (_curs.TidCur != TokKind.Eof)
{
PostError(_curs.TokCur, TexlStrings.ErrBadToken);
}
_after = _after == null ? new SourceList(ParseTrivia()) : new SourceList(new SpreadSource(_after.Sources), new SpreadSource(ParseTrivia()));
// This checks for and provides an error message for any block comments missing a closing '*/'
PostBlockCommentMissingClosingError();
errors = _errors;
}
// The top node (of the parse tree) should end up with the biggest id. We use this fact when binding.
Contracts.Assert(node.Id == _idNext - 1);
return(node);
}
protected override void SaveCore(ModelSaveContext ctx)
{
// *** Binary format ***
// (Base class)
// LinearModelStatistics: model statistics (optional, in a separate stream)
base.SaveCore(ctx);
Contracts.AssertValueOrNull(_stats);
if (_stats != null)
{
using (var statsCtx = new ModelSaveContext(ctx.Repository,
Path.Combine(ctx.Directory ?? "", ModelStatsSubModelFilename), ModelLoadContext.ModelStreamName))
{
_stats.Save(statsCtx);
statsCtx.Done();
}
}
ctx.SetVersionInfo(GetVersionInfo());
}
public override bool Equals(object other)
{
Contracts.AssertValueOrNull(other);
if (other == null)
{
return(false);
}
IntellisenseSuggestion otherSuggestion = other as IntellisenseSuggestion;
if (otherSuggestion == null)
{
return(false);
}
else
{
return(Equals(otherSuggestion));
}
}
private bool TryGetDsInfo(FirstNameNode firstName, TexlBinding binding, out IExternalDataSource dsInfo)
{
Contracts.AssertValueOrNull(firstName);
Contracts.AssertValue(binding);
dsInfo = null;
if (firstName == null || !binding.GetType(firstName).IsTable)
{
return(false);
}
var firstNameInfo = binding.GetInfo(firstName);
if (firstNameInfo == null || firstNameInfo.Kind != BindKind.Data)
{
return(false);
}
return(binding.EntityScope != null &&
binding.EntityScope.TryGetEntity(firstNameInfo.Name, out dsInfo));
}
/// <summary>
/// Create the bindings given the env, bindable, input schema, column roles, and column name suffix.
/// </summary>
private static Bindings Create(IHostEnvironment env, ISchemaBindableMapper bindable, DataViewSchema input,
IEnumerable <KeyValuePair <RoleMappedSchema.ColumnRole, string> > roles, string suffix, bool user = true)
{
Contracts.AssertValue(env);
Contracts.AssertValue(bindable);
Contracts.AssertValue(input);
Contracts.AssertValue(roles);
Contracts.AssertValueOrNull(suffix);
var mapper = bindable.Bind(env, new RoleMappedSchema(input, roles));
// We don't actually depend on this invariant, but if this assert fires it means the bindable
// did the wrong thing.
Contracts.Assert(mapper.InputRoleMappedSchema.Schema == input);
var rowMapper = mapper as ISchemaBoundRowMapper;
Contracts.Check(rowMapper != null, "Predictor expected to be a RowMapper!");
return(Create(input, rowMapper, suffix, user));
}
private static Float L2DistSquaredHalfSparse(Float[] valuesA, int lengthA, Float[] valuesB, int[] indicesB, int countB)
{
Contracts.AssertValueOrNull(valuesA);
Contracts.AssertValueOrNull(valuesB);
Contracts.AssertValueOrNull(indicesB);
Contracts.Assert(0 <= lengthA && lengthA <= Utils.Size(valuesA));
Contracts.Assert(0 <= countB && countB <= Utils.Size(indicesB));
Contracts.Assert(countB <= Utils.Size(valuesB));
var normA = CpuMathUtils.SumSq(valuesA.AsSpan(0, lengthA));
if (countB == 0)
{
return(normA);
}
var normB = CpuMathUtils.SumSq(valuesB.AsSpan(0, countB));
var dotP = CpuMathUtils.DotProductSparse(valuesA, valuesB, indicesB, countB);
var res = normA + normB - 2 * dotP;
return(res < 0 ? 0 : res);
}
public void AssertRep()
{
// Check that all fields have values.
Contracts.AssertNonEmpty(ColumnName);
Contracts.AssertValue(ColumnType);
Contracts.AssertValueOrNull(Generator);
// If Column is computed type, it must have a generator.
Contracts.Assert(IsComputed == (Generator != null));
// Column must have either a generator or a fieldInfo value.
Contracts.Assert((Generator == null) != (FieldInfo == null));
// Additional Checks if there is a generator.
if (Generator == null)
{
return;
}
Contracts.AssertValue(ReturnParameterInfo);
// Checks input parameters are (someClass, long, ref value) in that order.
var parameterInfos = Generator.GetMethodInfo().GetParameters().ToArray();
var parameterTypes = (from pInfo in parameterInfos select pInfo.ParameterType).ToArray();
Contracts.Assert(parameterTypes.Length == 3);
Contracts.Assert(parameterTypes[2].IsByRef);
Contracts.Assert(parameterTypes[1] == typeof(long));
Contracts.Assert(!(parameterTypes[0].GetTypeInfo().IsPrimitive || parameterTypes[0] == typeof(string)));
// Check that generator returns void.
Contracts.Assert(Generator.GetMethodInfo().ReturnType == typeof(void));
// Checks that the return type of the generator is compatible with ColumnType.
bool isVector;
DataKind datakind;
GetVectorAndKind(ReturnType, "return type", out isVector, out datakind);
Contracts.Assert(isVector == ColumnType.IsVector);
Contracts.Assert(datakind == ColumnType.ItemType.RawKind);
}
public CallNode(ref int idNext, Token primaryToken, SourceList sourceList, Identifier head, TexlNode headNode, ListNode args, Token tokParenClose)
: base(ref idNext, primaryToken, sourceList)
{
Contracts.AssertValue(head);
Contracts.AssertValueOrNull(headNode);
Contracts.AssertValue(args);
Contracts.AssertValueOrNull(tokParenClose);
Head = head;
HeadNode = headNode;
Args = args;
Args.Parent = this;
ParenClose = tokParenClose;
int headDepth = HeadNode == null ? 0 : HeadNode.Depth;
_depth = 1 + (args.Depth > headDepth ? args.Depth : headDepth);
if (headNode != null)
{
MinChildID = Math.Min(headNode.MinChildID, MinChildID);
}
if (args != null)
{
MinChildID = Math.Min(args.MinChildID, MinChildID);
}
#pragma warning disable 420
// A volatile field should not normally be passed using a ref or out parameter, since it will not be treated
// as volatile within the scope of the function. There are exceptions to this, such as when calling an interlocked API.
// Hence disabling the warning for this instance.
int invocationId = Interlocked.Increment(ref _uniqueInvocationIdNext);
#pragma warning restore 420
// We need to generate a globally unique name for this function invocation, so we use
// a new (hardcoded) guid as well as the unique counter to avoid colliding with any
// other data sources that may be imported by the user.
UniqueInvocationId = string.Format("Inv_7339A45FDB3141D49CB36063B712F5E0_{0}", invocationId);
}
private bool FirstIsSuffix<T>(IEnumerator<T> suffix, IEnumerator<T> seq, Func<T, T, bool> equalFunc = null)
{
Contracts.AssertValue(suffix);
Contracts.AssertValue(seq);
Contracts.AssertValueOrNull(equalFunc);
// It is possible to compose something that only stores part of suffix, and none of sequence,
// but this is relatively harder to code.
if (equalFunc == null)
equalFunc = EqualityComparer<T>.Default.Equals;
List<T> suffixList = new List<T>();
List<T> seqList = new List<T>();
while (suffix.MoveNext())
{
if (!seq.MoveNext())
{
Fail("Baseline sequence had {0} items, but the suffix seems to have more", suffixList.Count);
return false;
}
suffixList.Add(suffix.Current);
seqList.Add(seq.Current);
}
if (suffixList.Count == 0) // Empty suffix is trivially a suffix of anything.
return true;
Contracts.Assert(suffixList.Count == seqList.Count);
int idx = 0;
while (seq.MoveNext())
seqList[idx++ % seqList.Count] = seq.Current;
Log("Suffix of length {0} compared against sequence of length {1}", suffixList.Count, seqList.Count + idx);
for (int i = 0; i < suffixList.Count; ++i)
{
if (!equalFunc(suffixList[i], seqList[(idx + i) % seqList.Count]))
{
Fail("Baseline sequence mismatched {0} length suffix {1} item from the end",
suffixList.Count, suffixList.Count - i - 1);
return false;
}
}
return true;
}
public CodecFactory(IHostEnvironment env, MemoryStreamPool memPool = null)
{
Contracts.AssertValue(env, "env");
Contracts.AssertValueOrNull(memPool);
_host = env.Register("CodecFactory");
_memPool = memPool ?? new MemoryStreamPool();
_encoding = Encoding.UTF8;
_loadNameToCodecCreator = new Dictionary <string, GetCodecFromStreamDelegate>();
_simpleCodecTypeMap = new Dictionary <Type, IValueCodec>();
// Register the current codecs.
RegisterSimpleCodec(new UnsafeTypeCodec <sbyte>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <byte>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <short>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <ushort>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <int>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <uint>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <long>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <ulong>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <float>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <double>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <TimeSpan>(this));
RegisterSimpleCodec(new TextCodec(this));
RegisterSimpleCodec(new BoolCodec(this));
RegisterSimpleCodec(new DateTimeCodec(this));
RegisterSimpleCodec(new DateTimeOffsetCodec(this));
RegisterSimpleCodec(new UnsafeTypeCodec <RowId>(this));
// Register the old type system reading codec.
RegisterOtherCodec("DvBool", new OldBoolCodec(this).GetCodec);
RegisterOtherCodec("DvDateTimeZone", new DateTimeOffsetCodec(this).GetCodec);
RegisterOtherCodec("DvDateTime", new DateTimeCodec(this).GetCodec);
RegisterOtherCodec("DvTimeSpan", new UnsafeTypeCodec <TimeSpan>(this).GetCodec);
RegisterOtherCodec("VBuffer", GetVBufferCodec);
RegisterOtherCodec("Key2", GetKeyCodec);
RegisterOtherCodec("Key", GetKeyCodecOld);
}
/// <summary>
/// Append a field declaration to the provided <see cref="StringBuilder"/>.
/// </summary>
public static void AppendFieldDeclaration(CSharpCodeProvider codeProvider, StringBuilder target, int columnIndex,
string fieldName, ColumnType colType, bool appendInitializer, bool useVBuffer)
{
Contracts.AssertValueOrNull(codeProvider);
Contracts.AssertValue(target);
Contracts.Assert(columnIndex >= 0);
Contracts.AssertNonEmpty(fieldName);
Contracts.AssertValue(colType);
var attributes = new List <string>();
string generatedCsTypeName = GetBackingTypeName(colType, useVBuffer, attributes);
if (codeProvider != null && !codeProvider.IsValidIdentifier(fieldName))
{
attributes.Add(string.Format("[ColumnName({0})]", GetCSharpString(codeProvider, fieldName)));
fieldName = string.Format("Column{0}", columnIndex);
}
const string indent = " ";
if (attributes.Count > 0)
{
foreach (var attr in attributes)
{
target.Append(indent);
target.AppendLine(attr);
}
}
target.Append(indent);
target.AppendFormat("public {0} {1}", generatedCsTypeName, fieldName);
if (appendInitializer && colType is VectorType vecColType && vecColType.Size > 0 && !useVBuffer)
{
Contracts.Assert(generatedCsTypeName.EndsWith("[]"));
var csItemType = generatedCsTypeName.Substring(0, generatedCsTypeName.Length - 2);
target.AppendFormat(" = new {0}[{1}]", csItemType, vecColType.Size);
}
target.AppendLine(";");
}
public override bool CheckInvocation(TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType returnType, out Dictionary <TexlNode, DType> nodeToCoercedTypeMap)
{
Contracts.AssertValue(args);
Contracts.AssertAllValues(args);
Contracts.AssertValue(argTypes);
Contracts.Assert(args.Length == argTypes.Length);
Contracts.AssertValue(errors);
if (!base.CheckInvocation(args, argTypes, errors, out returnType, out nodeToCoercedTypeMap))
{
return(false);
}
// Option Set values need to be checked with their own function since they have a special return for "blank" values.
if (argTypes[0].Kind == DKind.OptionSetValue)
{
return(false);
}
if (argTypes[0] is IExternalControlType controlType)
{
// A control will never be null. It never worked as intended.
// We coerce the control to control.primaryOutProperty.
var primaryOutputProperty = controlType.ControlTemplate.VerifyValue().PrimaryOutputProperty;
Contracts.AssertValueOrNull(primaryOutputProperty);
if (primaryOutputProperty != null)
{
if (nodeToCoercedTypeMap == null)
{
nodeToCoercedTypeMap = new Dictionary <TexlNode, DType>();
}
nodeToCoercedTypeMap.Add(args[0], primaryOutputProperty.GetOpaqueType());
}
}
return(true);
}
public ParamNode(Token tok, string name, int index, DataViewType type)
: base(tok)
{
Contracts.AssertNonEmpty(name);
Contracts.Assert(index >= 0);
Contracts.AssertValueOrNull(type);
Name = name;
Index = index;
Type = type;
if (type == null)
{
ExprType = ExprTypeKind.Error;
}
else if (type is TextDataViewType)
{
ExprType = ExprTypeKind.TX;
}
else if (type is BooleanDataViewType)
{
ExprType = ExprTypeKind.BL;
}
else if (type == NumberDataViewType.Int32)
{
ExprType = ExprTypeKind.I4;
}
else if (type == NumberDataViewType.Int64)
{
ExprType = ExprTypeKind.I8;
}
else if (type == NumberDataViewType.Single)
{
ExprType = ExprTypeKind.R4;
}
else if (type == NumberDataViewType.Double)
{
ExprType = ExprTypeKind.R8;
}
}
internal EntryPointInfo(IExceptionContext ectx, MethodInfo method,
TlcModule.EntryPointAttribute attribute, ObsoleteAttribute obsoleteAttribute)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(method);
ectx.AssertValue(attribute);
Name = attribute.Name ?? string.Join(".", method.DeclaringType.Name, method.Name);
Description = attribute.Desc;
Method = method;
ShortName = attribute.ShortName;
FriendlyName = attribute.UserName;
XmlInclude = attribute.XmlInclude;
ObsoleteAttribute = obsoleteAttribute;
// There are supposed to be 2 parameters, env and input for non-macro nodes.
// Macro nodes have a 3rd parameter, the entry point node.
var parameters = method.GetParameters();
if (parameters.Length != 2 && parameters.Length != 3)
{
throw ectx.Except("Method '{0}' has {1} parameters, but must have 2 or 3", method.Name, parameters.Length);
}
if (parameters[0].ParameterType != typeof(IHostEnvironment))
{
throw ectx.Except("Method '{0}', 1st parameter is {1}, but must be IHostEnvironment", method.Name, parameters[0].ParameterType);
}
InputType = parameters[1].ParameterType;
var outputType = method.ReturnType;
if (!outputType.IsClass)
{
throw ectx.Except("Method '{0}' returns {1}, but must return a class", method.Name, outputType);
}
OutputType = outputType;
InputKinds = FindEntryPointKinds(InputType);
OutputKinds = FindEntryPointKinds(OutputType);
}
public bool IsValidFirstNameNode(FirstNameNode node, TexlBinding binding, IOpDelegationStrategy opDelStrategy)
{
Contracts.AssertValue(node);
Contracts.AssertValue(binding);
Contracts.AssertValueOrNull(opDelStrategy);
var isRowScoped = binding.IsRowScope(node);
var isValid = IsValidNode(node, binding);
if (isValid && !isRowScoped)
{
return(true);
}
// If invalid node then return immediately.
if (!isValid)
{
return(false);
}
return(IsDelegatableColumnNode(node, binding, opDelStrategy, _function.FunctionDelegationCapability));
}
private void PostPartial(string path, long total, ref long batch, int index, LineInfo[] infos)
{
Contracts.AssertValueOrNull(path);
Contracts.Assert(0 <= total);
Contracts.Assert(0 <= index && index < Utils.Size(infos));
// Queue the last partial batch.
if (index <= 0)
{
return;
}
Array.Resize(ref infos, index);
batch++;
while (!_queue.TryAdd(new LineBatch(path, total, batch, infos), TimeOut))
{
if (_abort)
{
return;
}
}
}
public SchemaImpl(IExceptionContext ectx, ISchema inputSchema, UngroupMode mode, string[] pivotColumns)
{
Contracts.AssertValueOrNull(ectx);
_ectx = ectx;
_ectx.AssertValue(inputSchema);
_ectx.AssertNonEmpty(pivotColumns);
_inputSchema = inputSchema;
Mode = mode;
CheckAndBind(_ectx, inputSchema, pivotColumns, out _infos);
_pivotColMap = new Dictionary <string, int>();
_pivotIndex = Utils.CreateArray(_inputSchema.ColumnCount, -1);
for (int i = 0; i < _infos.Length; i++)
{
var info = _infos[i];
_pivotColMap[info.Name] = info.Index;
_ectx.Assert(_pivotIndex[info.Index] == -1);
_pivotIndex[info.Index] = i;
}
}
// Assumes ownership of all of the array args.
public RecordNode(ref int idNext, Token primaryTokens, SourceList sourceList, Identifier[] ids, TexlNode[] exprs, Token[] commas, Token[] colons, Token curlyCloseToken, TexlNode sourceRestriction = null)
: base(ref idNext, primaryTokens, sourceList, exprs)
{
Contracts.AssertValue(ids);
Contracts.AssertValue(exprs);
Contracts.Assert(ids.Length == exprs.Length);
Contracts.AssertValueOrNull(commas);
Contracts.AssertValueOrNull(colons);
Contracts.AssertValueOrNull(curlyCloseToken);
Contracts.AssertValueOrNull(sourceRestriction);
Ids = ids;
Commas = commas;
Colons = colons;
CurlyClose = curlyCloseToken;
SourceRestriction = sourceRestriction;
if (sourceRestriction != null)
{
sourceRestriction.Parent = this;
MinChildID = Math.Min(sourceRestriction.MinChildID, MinChildID);
}
}
private static string[] GetOutputNames(ISchemaBoundMapper mapper, string suffix, string[] namesDerived)
{
Contracts.AssertValue(mapper);
Contracts.AssertValueOrNull(suffix);
Contracts.AssertValue(namesDerived);
var schema = mapper.OutputSchema;
int count = namesDerived.Length + schema.Count;
var res = new string[count];
int dst = 0;
for (int i = 0; i < namesDerived.Length; i++)
{
res[dst++] = namesDerived[i] + suffix;
}
for (int i = 0; i < schema.Count; i++)
{
res[dst++] = schema[i].Name + suffix;
}
Contracts.Assert(dst == count);
return(res);
}
public CodecFactory(IHostEnvironment env, MemoryStreamPool memPool = null)
{
Contracts.AssertValue(env, "env");
Contracts.AssertValueOrNull(memPool);
_host = env.Register("CodecFactory");
_memPool = memPool ?? new MemoryStreamPool();
_encoding = Encoding.UTF8;
_loadNameToCodecCreator = new Dictionary <string, GetCodecFromStreamDelegate>();
_simpleCodecTypeMap = new Dictionary <DataKind, IValueCodec>();
// Register the current codecs.
RegisterSimpleCodec(new UnsafeTypeCodec <DvInt1>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <byte>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <DvInt2>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <ushort>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <DvInt4>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <uint>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <DvInt8>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <ulong>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <Single>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <Double>(this));
RegisterSimpleCodec(new UnsafeTypeCodec <DvTimeSpan>(this));
RegisterSimpleCodec(new DvTextCodec(this));
RegisterSimpleCodec(new BoolCodec(this));
RegisterSimpleCodec(new DateTimeCodec(this));
RegisterSimpleCodec(new DateTimeZoneCodec(this));
RegisterSimpleCodec(new UnsafeTypeCodec <UInt128>(this));
// Register the old boolean reading codec.
var oldBool = new OldBoolCodec(this);
RegisterOtherCodec(oldBool.LoadName, oldBool.GetCodec);
RegisterOtherCodec("VBuffer", GetVBufferCodec);
RegisterOtherCodec("Key", GetKeyCodec);
}
public static SchemaImpl Create(ModelLoadContext ctx, IExceptionContext ectx, Schema inputSchema)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(ctx);
ectx.AssertValue(inputSchema);
// *** Binary format ***
// int: ungroup mode
// int: K - number of pivot columns
// int[K]: ids of pivot column names
int modeIndex = ctx.Reader.ReadInt32();
ectx.CheckDecode(Enum.IsDefined(typeof(UngroupMode), modeIndex));
UngroupMode mode = (UngroupMode)modeIndex;
int k = ctx.Reader.ReadInt32();
ectx.CheckDecode(k > 0);
var pivotColumns = new string[k];
for (int i = 0; i < k; i++)
pivotColumns[i] = ctx.LoadNonEmptyString();
return new SchemaImpl(ectx, inputSchema, mode, pivotColumns);
}
public int CompareTo(IntellisenseSuggestion other)
{
Contracts.AssertValueOrNull(other);
if (other == null)
{
return(-1);
}
bool thisIsExactMatch = IsExactMatch(Text, ExactMatch);
bool otherIsExactMatch = IsExactMatch(other.Text, other.ExactMatch);
if (thisIsExactMatch && !otherIsExactMatch)
{
return(-1);
}
else if (!thisIsExactMatch && otherIsExactMatch)
{
return(1);
}
return(SortPriority == other.SortPriority ? Text.CompareTo(other.Text) : (int)(other.SortPriority - SortPriority));
}
private static void CheckAndBind(IExceptionContext ectx, Schema inputSchema,
string[] pivotColumns, out PivotColumnInfo[] infos)
{
Contracts.AssertValueOrNull(ectx);
ectx.AssertValue(inputSchema);
ectx.AssertNonEmpty(pivotColumns);
infos = new PivotColumnInfo[pivotColumns.Length];
for (int i = 0; i < pivotColumns.Length; i++)
{
var name = pivotColumns[i];
// REVIEW: replace Check with CheckUser, once existing CheckUser is renamed to CheckUserArg or something.
ectx.CheckUserArg(!string.IsNullOrEmpty(name), nameof(Arguments.Column), "Column name cannot be empty");
int col;
if (!inputSchema.TryGetColumnIndex(name, out col))
throw ectx.ExceptUserArg(nameof(Arguments.Column), "Pivot column '{0}' is not found", name);
var colType = inputSchema.GetColumnType(col);
if (!colType.IsVector || !colType.ItemType.IsPrimitive)
throw ectx.ExceptUserArg(nameof(Arguments.Column),
"Pivot column '{0}' has type '{1}', but must be a vector of primitive types", name, colType);
infos[i] = new PivotColumnInfo(name, col, colType.VectorSize, (PrimitiveType)colType.ItemType);
}
}
internal MulticlassLogisticRegressionPredictor(IHostEnvironment env, ref VBuffer <float> weights, int numClasses, int numFeatures, string[] labelNames, LinearModelStatistics stats = null)
: base(env, RegistrationName)
{
Contracts.Assert(weights.Length == numClasses + numClasses * numFeatures);
_numClasses = numClasses;
_numFeatures = numFeatures;
// weights contains both bias and feature weights in a flat vector
// Biases are stored in the first _numClass elements
// followed by one weight vector for each class, in turn, all concatenated
// (i.e.: in "row major", if we encode each weight vector as a row of a matrix)
Contracts.Assert(weights.Length == _numClasses + _numClasses * _numFeatures);
_biases = new float[_numClasses];
for (int i = 0; i < _biases.Length; i++)
{
weights.GetItemOrDefault(i, ref _biases[i]);
}
_weights = new VBuffer <float> [_numClasses];
for (int i = 0; i < _weights.Length; i++)
{
weights.CopyTo(ref _weights[i], _numClasses + i * _numFeatures, _numFeatures);
}
if (_weights.All(v => v.IsDense))
{
_weightsDense = _weights;
}
InputType = new VectorType(NumberType.R4, _numFeatures);
OutputType = new VectorType(NumberType.R4, _numClasses);
Contracts.Assert(labelNames == null || labelNames.Length == numClasses);
_labelNames = labelNames;
Contracts.AssertValueOrNull(stats);
_stats = stats;
}
