/// <summary>
/// Loads all transforms from the model stream, applies them sequentially to the provided data, and returns
/// the resulting data. If there are no transforms in the stream, or if there's no DataLoader stream at all
/// (this can happen if the model is produced by old TL), returns the source data.
/// If the DataLoader stream is invalid, throws.
/// </summary>
/// <param name="env">The host environment to use.</param>
/// <param name="data">The starting data view.</param>
/// <param name="rep">The repository reader.</param>
/// <returns>The resulting data view.</returns>
public static IDataView LoadTransforms(IHostEnvironment env, IDataView data, RepositoryReader rep)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(data, nameof(data));
env.CheckValue(rep, nameof(rep));
using (var ent = rep.OpenEntryOrNull(DirDataLoaderModel, ModelLoadContext.ModelStreamName))
{
if (ent == null)
return data;
var ctx = new ModelLoadContext(rep, ent, DirDataLoaderModel);
return CompositeDataLoader.LoadSelectedTransforms(ctx, data, env, x => true);
}
}
/// <summary>
/// Save a transformer model and the loader used to create its input data to the stream.
/// </summary>
/// <param name="model">The trained model to be saved. Note that this can be <see langword="null"/>, as a shorthand
/// for an empty transformer chain. Upon loading with <see cref="LoadWithDataLoader(Stream, out IDataLoader{IMultiStreamSource})"/>
/// the returned value will be an empty <see cref="TransformerChain{TLastTransformer}"/>.</param>
/// <param name="loader">The loader that was used to create data to train the model.</param>
/// <param name="stream">A writeable, seekable stream to save to.</param>
public void Save <TSource>(ITransformer model, IDataLoader <TSource> loader, Stream stream)
{
_env.CheckValue(loader, nameof(loader));
_env.CheckValueOrNull(model);
_env.CheckValue(stream, nameof(stream));
// For the sake of consistency of this API specifically, when called upon we save any transformer
// in a single element transformer chain.
var chainedModel = model == null ? null : new TransformerChain <ITransformer>(model);
var compositeLoader = new CompositeDataLoader <TSource, ITransformer>(loader, chainedModel);
using (var rep = RepositoryWriter.CreateNew(stream))
{
ModelSaveContext.SaveModel(rep, compositeLoader, null);
rep.Commit();
}
}
// Factory method for SignatureDataTransform.
private static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register("LoadTransform");
h.CheckValue(args, nameof(args));
h.CheckValue(input, nameof(input));
h.CheckUserArg(File.Exists(args.ModelFile), nameof(args.ModelFile), "File does not exist");
IDataView currentView;
// If there are no 'tag' parameters, we load everything, regardless of 'comp'.
bool complement = args.Complement || Utils.Size(args.Tag) == 0;
var allTags = new HashSet <string>();
for (int i = 0; i < Utils.Size(args.Tag); i++)
{
var curList = args.Tag[i];
if (string.IsNullOrWhiteSpace(curList))
{
continue;
}
foreach (var tag in curList.Split(','))
{
if (!string.IsNullOrWhiteSpace(tag))
{
allTags.Add(tag.ToLower());
}
}
}
Func <string, bool> predicate =
tag =>
{
bool found = allTags.Contains(tag.ToLower());
return(found == !complement);
};
using (var file = h.OpenInputFile(args.ModelFile))
using (var strm = file.OpenReadStream())
using (var rep = RepositoryReader.Open(strm, h))
using (var pipeLoaderEntry = rep.OpenEntry(ModelFileUtils.DirDataLoaderModel, ModelLoadContext.ModelStreamName))
using (var ctx = new ModelLoadContext(rep, pipeLoaderEntry, ModelFileUtils.DirDataLoaderModel))
{
currentView = CompositeDataLoader.LoadSelectedTransforms(ctx, input, h, predicate);
if (currentView == input)
{
// REVIEW: we are required to return an IDataTransform. Therefore, if we don't introduce a new transform
// on top of 'input', we must throw (since input may not be a data transform).
// We could of course introduce a 'no-op transform', or we could lift the requirement to always return an IDataTransform
// associated with SignatureDataTransform.
var criteria = string.Format(
complement
? "transforms that don't have tags from the list: '{0}'"
: "transforms that have tags from the list: '{0}'",
string.Join(",", allTags));
throw h.ExceptUserArg(nameof(args.Tag), "No transforms were found that match the search criteria ({0})", criteria);
}
}
h.Assert(currentView is IDataTransform);
return((IDataTransform)currentView);
}
