/// <summary>
/// Load an optional object from the repository directory.
/// Returns false iff no stream was found for the object, iff result is set to null.
/// Throws if loading fails for any other reason.
/// </summary>
public static bool LoadModelOrNull <TRes, TSig>(IHostEnvironment env, out TRes result, RepositoryReader rep, string dir, params object[] extra)
where TRes : class
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(rep, nameof(rep));
var ent = rep.OpenEntryOrNull(dir, ModelStreamName);
if (ent != null)
{
using (ent)
{
// Provide the repository, entry, and directory name to the loadable class ctor.
env.Assert(ent.Stream.Position == 0);
LoadModel <TRes, TSig>(env, out result, rep, ent, dir, extra);
return(true);
}
}
if ((ent = rep.OpenEntryOrNull(dir, NameBinary)) != null)
{
using (ent)
{
env.Assert(ent.Stream.Position == 0);
LoadModel <TRes, TSig>(env, out result, ent.Stream, extra);
return(true);
}
}
result = null;
return(false);
}
/// <summary>
/// Loads data view (loader and transforms) from <paramref name="rep"/> if <paramref name="loadTransforms"/> is set to true,
/// otherwise loads loader only.
/// </summary>
public static IDataLoader LoadLoader(IHostEnvironment env, RepositoryReader rep, IMultiStreamSource files, bool loadTransforms)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(rep, nameof(rep));
env.CheckValue(files, nameof(files));
IDataLoader loader;
// If loadTransforms is false, load the loader only, not the transforms.
Repository.Entry ent = null;
string dir = "";
if (!loadTransforms)
{
ent = rep.OpenEntryOrNull(dir = Path.Combine(DirDataLoaderModel, "Loader"), ModelLoadContext.ModelStreamName);
}
if (ent == null) // either loadTransforms is true, or it's not a composite loader
{
ent = rep.OpenEntry(dir = DirDataLoaderModel, ModelLoadContext.ModelStreamName);
}
env.CheckDecode(ent != null, "Loader is not found.");
env.AssertNonEmpty(dir);
using (ent)
{
env.Assert(ent.Stream.Position == 0);
ModelLoadContext.LoadModel <IDataLoader, SignatureLoadDataLoader>(env, out loader, rep, ent, dir, files);
}
return(loader);
}
/// <summary>
/// Loads all transforms from the model stream, applies them sequentially to the provided data, and returns
/// the resulting data. If there are no transforms in the stream, or if there's no DataLoader stream at all
/// (this can happen if the model is produced by old TL), returns the source data.
/// If the DataLoader stream is invalid, throws.
/// </summary>
/// <param name="env">The host environment to use.</param>
/// <param name="data">The starting data view.</param>
/// <param name="rep">The repository reader.</param>
/// <returns>The resulting data view.</returns>
public static IDataView LoadTransforms(IHostEnvironment env, IDataView data, RepositoryReader rep)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(data, nameof(data));
env.CheckValue(rep, nameof(rep));
using (var ent = rep.OpenEntryOrNull(DirDataLoaderModel, ModelLoadContext.ModelStreamName))
{
if (ent == null)
return data;
var ctx = new ModelLoadContext(rep, ent, DirDataLoaderModel);
return CompositeDataLoader.LoadSelectedTransforms(ctx, data, env, x => true);
}
}
/// <summary>
/// Return role/column-name pairs loaded from a repository.
/// </summary>
public static IEnumerable <KeyValuePair <ColumnRole, string> > LoadRoleMappingsOrNull(IHostEnvironment env, RepositoryReader rep)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register("RoleMappingUtils");
var list = new List <KeyValuePair <string, string> >();
var entry = rep.OpenEntryOrNull(DirTrainingInfo, RoleMappingFile);
if (entry == null)
{
return(null);
}
entry.Dispose();
using (var ch = h.Start("Loading role mappings"))
{
// REVIEW: Should really validate the schema here, and consider
// ignoring this stream if it isn't as expected.
var loaderSub = new SubComponent <IDataLoader, SignatureDataLoader>("Text");
var loader = loaderSub.CreateInstance(env,
new RepositoryStreamWrapper(rep, DirTrainingInfo, RoleMappingFile));
using (var cursor = loader.GetRowCursor(c => true))
{
var roleGetter = cursor.GetGetter <DvText>(0);
var colGetter = cursor.GetGetter <DvText>(1);
var role = default(DvText);
var col = default(DvText);
while (cursor.MoveNext())
{
roleGetter(ref role);
colGetter(ref col);
string roleStr = role.ToString();
string colStr = col.ToString();
h.CheckDecode(!string.IsNullOrWhiteSpace(roleStr), "Role name must not be empty");
h.CheckDecode(!string.IsNullOrWhiteSpace(colStr), "Column name must not be empty");
list.Add(new KeyValuePair <string, string>(roleStr, colStr));
}
}
ch.Done();
}
return(TrainUtils.CheckAndGenerateCustomColumns(env, list.ToArray()));
}
/// <summary>
/// REVIEW: consider adding an overload that returns <see cref="VBuffer{DvText}"/>
/// Loads optionally feature names from the repository directory.
/// Returns false iff no stream was found for feature names, iff result is set to null.
/// </summary>
public static bool TryLoadFeatureNames(out FeatureNameCollection featureNames, RepositoryReader rep)
{
Contracts.CheckValue(rep, nameof(rep));
using (var ent = rep.OpenEntryOrNull(ModelFileUtils.DirTrainingInfo, "FeatureNames.bin"))
{
if (ent != null)
{
using (var ctx = new ModelLoadContext(rep, ent, ModelFileUtils.DirTrainingInfo))
{
featureNames = FeatureNameCollection.Create(ctx);
return(true);
}
}
}
featureNames = null;
return(false);
}