public static RepositoryReader Open(Stream stream, IExceptionContext ectx = null, bool useFileSystem = true)
{
Contracts.CheckValueOrNull(ectx);
ectx.CheckValue(stream, nameof(stream));
return(new RepositoryReader(stream, ectx, useFileSystem));
}
internal static string CreateSplitColumn(IHostEnvironment env, ref IDataView data, string samplingKeyColumn, int?seed = null, bool fallbackInEnvSeed = false)
{
Contracts.CheckValue(env, nameof(env));
Contracts.CheckValueOrNull(samplingKeyColumn);
var splitColumnName = data.Schema.GetTempColumnName("SplitColumn");
int?seedToUse;
if (seed.HasValue)
{
seedToUse = seed.Value;
}
else if (fallbackInEnvSeed)
{
ISeededEnvironment seededEnv = (ISeededEnvironment)env;
seedToUse = seededEnv.Seed;
}
else
{
seedToUse = null;
}
// We need to handle two cases: if samplingKeyColumn is not provided, we generate a random number.
if (samplingKeyColumn == null)
{
data = new GenerateNumberTransform(env, data, splitColumnName, (uint?)seedToUse);
}
else
{
// If samplingKeyColumn was provided we will make a new column based on it, but using a temporary
// name, as it might be dropped elsewhere in the code
if (!data.Schema.TryGetColumnIndex(samplingKeyColumn, out int samplingColIndex))
{
throw env.ExceptSchemaMismatch(nameof(samplingKeyColumn), "SamplingKeyColumn", samplingKeyColumn);
}
var type = data.Schema[samplingColIndex].Type;
if (!RangeFilter.IsValidRangeFilterColumnType(env, type))
{
var hashInputColumnName = samplingKeyColumn;
// HashingEstimator currently handles all primitive types except for DateTime, DateTimeOffset and TimeSpan.
var itemType = type.GetItemType();
if (itemType is DateTimeDataViewType || itemType is DateTimeOffsetDataViewType || itemType is TimeSpanDataViewType)
{
data = new TypeConvertingTransformer(env, splitColumnName, DataKind.Int64, samplingKeyColumn).Transform(data);
hashInputColumnName = splitColumnName;
}
var columnOptions =
seedToUse.HasValue ?
new HashingEstimator.ColumnOptions(splitColumnName, hashInputColumnName, 30, (uint)seedToUse.Value, combine: true) :
new HashingEstimator.ColumnOptions(splitColumnName, hashInputColumnName, 30, combine: true);
data = new HashingEstimator(env, columnOptions).Fit(data).Transform(data);
}
else
{
if (type != NumberDataViewType.Single && type != NumberDataViewType.Double)
{
data = new ColumnCopyingEstimator(env, (splitColumnName, samplingKeyColumn)).Fit(data).Transform(data);
}
else
{
data = new NormalizingEstimator(env, new NormalizingEstimator.MinMaxColumnOptions(splitColumnName, samplingKeyColumn, ensureZeroUntouched: false)).Fit(data).Transform(data);
}
}
}
return(splitColumnName);
}
protected LoadableClassAttributeBase(string summary, Type instType, Type loaderType, Type argType, Type[] sigTypes, string userName, params string[] loadNames)
{
Contracts.CheckValueOrNull(summary);
Contracts.CheckValue(instType, nameof(instType));
Contracts.CheckValue(loaderType, nameof(loaderType));
Contracts.CheckNonEmpty(sigTypes, nameof(sigTypes));
if (Utils.Size(loadNames) == 0)
{
loadNames = new string[] { userName }
}
;
if (loadNames.Any(s => string.IsNullOrWhiteSpace(s)))
{
throw Contracts.ExceptEmpty(nameof(loadNames), "LoadableClass loadName parameter can't be empty");
}
var sigType = sigTypes[0];
Contracts.CheckValue(sigType, nameof(sigTypes));
Type[] types;
Contracts.CheckParam(sigType.BaseType == typeof(System.MulticastDelegate), nameof(sigTypes), "LoadableClass signature type must be a delegate type");
var meth = sigType.GetMethod("Invoke");
Contracts.CheckParam(meth != null, nameof(sigTypes), "LoadableClass signature type must be a delegate type");
Contracts.CheckParam(meth.ReturnType == typeof(void), nameof(sigTypes), "LoadableClass signature type must be a delegate type with void return");
var parms = meth.GetParameters();
int itypeBase = 0;
if (argType != null)
{
types = new Type[1 + parms.Length];
types[itypeBase++] = argType;
}
else if (parms.Length > 0)
{
types = new Type[parms.Length];
}
else
{
types = Type.EmptyTypes;
}
for (int itype = 0; itype < parms.Length; itype++)
{
var parm = parms[itype];
if ((parm.Attributes & (ParameterAttributes.Out | ParameterAttributes.Retval)) != 0)
{
throw Contracts.Except("Invalid signature parameter attributes");
}
types[itypeBase + itype] = parm.ParameterType;
}
for (int i = 1; i < sigTypes.Length; i++)
{
sigType = sigTypes[i];
Contracts.CheckValue(sigType, nameof(sigTypes));
Contracts.Check(sigType.BaseType == typeof(System.MulticastDelegate), "LoadableClass signature type must be a delegate type");
meth = sigType.GetMethod("Invoke");
Contracts.CheckParam(meth != null, nameof(sigTypes), "LoadableClass signature type must be a delegate type");
Contracts.CheckParam(meth.ReturnType == typeof(void), nameof(sigTypes), "LoadableClass signature type must be a delegate type with void return");
parms = meth.GetParameters();
Contracts.CheckParam(parms.Length + itypeBase == types.Length, nameof(sigTypes), "LoadableClass signatures must have the same number of parameters");
for (int itype = 0; itype < parms.Length; itype++)
{
var parm = parms[itype];
if ((parm.Attributes & (ParameterAttributes.Out | ParameterAttributes.Retval)) != 0)
{
throw Contracts.ExceptParam(nameof(sigTypes), "Invalid signature parameter attributes");
}
Contracts.CheckParam(types[itypeBase + itype] == parm.ParameterType, nameof(sigTypes),
"LoadableClass signatures must have the same set of parameters");
}
}
InstanceType = instType;
LoaderType = loaderType;
ArgType = argType;
SigTypes = sigTypes;
CtorTypes = types;
Summary = summary;
UserName = userName;
LoadNames = loadNames;
}
}
