/// <summary> /// Loads all transforms from the model stream, applies them sequentially to the provided data, and returns /// the resulting data. If there are no transforms in the stream, or if there's no DataLoader stream at all /// (this can happen if the model is produced by old TL), returns the source data. /// If the DataLoader stream is invalid, throws. /// </summary> /// <param name="env">The host environment to use.</param> /// <param name="data">The starting data view.</param> /// <param name="rep">The repository reader.</param> /// <returns>The resulting data view.</returns> public static IDataView LoadTransforms(IHostEnvironment env, IDataView data, RepositoryReader rep) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(data, nameof(data)); env.CheckValue(rep, nameof(rep)); using (var ent = rep.OpenEntryOrNull(DirDataLoaderModel, ModelLoadContext.ModelStreamName)) { if (ent == null) return data; var ctx = new ModelLoadContext(rep, ent, DirDataLoaderModel); return CompositeDataLoader.LoadSelectedTransforms(ctx, data, env, x => true); } }
/// <summary> /// Save a transformer model and the loader used to create its input data to the stream. /// </summary> /// <param name="model">The trained model to be saved. Note that this can be <see langword="null"/>, as a shorthand /// for an empty transformer chain. Upon loading with <see cref="LoadWithDataLoader(Stream, out IDataLoader{IMultiStreamSource})"/> /// the returned value will be an empty <see cref="TransformerChain{TLastTransformer}"/>.</param> /// <param name="loader">The loader that was used to create data to train the model.</param> /// <param name="stream">A writeable, seekable stream to save to.</param> public void Save <TSource>(ITransformer model, IDataLoader <TSource> loader, Stream stream) { _env.CheckValue(loader, nameof(loader)); _env.CheckValueOrNull(model); _env.CheckValue(stream, nameof(stream)); // For the sake of consistency of this API specifically, when called upon we save any transformer // in a single element transformer chain. var chainedModel = model == null ? null : new TransformerChain <ITransformer>(model); var compositeLoader = new CompositeDataLoader <TSource, ITransformer>(loader, chainedModel); using (var rep = RepositoryWriter.CreateNew(stream)) { ModelSaveContext.SaveModel(rep, compositeLoader, null); rep.Commit(); } }
// Factory method for SignatureDataTransform. private static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input) { Contracts.CheckValue(env, nameof(env)); var h = env.Register("LoadTransform"); h.CheckValue(args, nameof(args)); h.CheckValue(input, nameof(input)); h.CheckUserArg(File.Exists(args.ModelFile), nameof(args.ModelFile), "File does not exist"); IDataView currentView; // If there are no 'tag' parameters, we load everything, regardless of 'comp'. bool complement = args.Complement || Utils.Size(args.Tag) == 0; var allTags = new HashSet <string>(); for (int i = 0; i < Utils.Size(args.Tag); i++) { var curList = args.Tag[i]; if (string.IsNullOrWhiteSpace(curList)) { continue; } foreach (var tag in curList.Split(',')) { if (!string.IsNullOrWhiteSpace(tag)) { allTags.Add(tag.ToLower()); } } } Func <string, bool> predicate = tag => { bool found = allTags.Contains(tag.ToLower()); return(found == !complement); }; using (var file = h.OpenInputFile(args.ModelFile)) using (var strm = file.OpenReadStream()) using (var rep = RepositoryReader.Open(strm, h)) using (var pipeLoaderEntry = rep.OpenEntry(ModelFileUtils.DirDataLoaderModel, ModelLoadContext.ModelStreamName)) using (var ctx = new ModelLoadContext(rep, pipeLoaderEntry, ModelFileUtils.DirDataLoaderModel)) { currentView = CompositeDataLoader.LoadSelectedTransforms(ctx, input, h, predicate); if (currentView == input) { // REVIEW: we are required to return an IDataTransform. Therefore, if we don't introduce a new transform // on top of 'input', we must throw (since input may not be a data transform). // We could of course introduce a 'no-op transform', or we could lift the requirement to always return an IDataTransform // associated with SignatureDataTransform. var criteria = string.Format( complement ? "transforms that don't have tags from the list: '{0}'" : "transforms that have tags from the list: '{0}'", string.Join(",", allTags)); throw h.ExceptUserArg(nameof(args.Tag), "No transforms were found that match the search criteria ({0})", criteria); } } h.Assert(currentView is IDataTransform); return((IDataTransform)currentView); }