protected virtual bool TryUnparseCore(StringBuilder sb, string extra) { Contracts.AssertNonEmpty(extra); if (string.IsNullOrWhiteSpace(Name) || Utils.Size(Source) == 0) { return(false); } if (CmdQuoter.NeedsQuoting(Name)) { return(false); } if (Source.Any(x => CmdQuoter.NeedsQuoting(x) || x.Contains(","))) { return(false); } sb.Append(Name).Append(':').Append(extra).Append(':'); string pre = ""; foreach (var src in Source) { sb.Append(pre).Append(src); pre = ","; } return(true); }
internal bool TryUnparse(StringBuilder sb) { Contracts.AssertValue(sb); if (ResultType == null && KeyCount == null) return TryUnparseCore(sb); if (!TrySanitize()) return false; if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source)) return false; int ich = sb.Length; sb.Append(Name); sb.Append(':'); if (ResultType != null) sb.Append(ResultType.Value.GetString()); if (KeyCount != null) { sb.Append('['); if (!KeyCount.TryUnparse(sb)) { sb.Length = ich; return false; } sb.Append(']'); } else if (!string.IsNullOrEmpty(Range)) sb.Append(Range); sb.Append(':'); sb.Append(Source); return true; }
public static PredictorAndArgs NnBinInline(string netDefinition) { var quoted = new StringBuilder(); CmdQuoter.QuoteValue(netDefinition, quoted); return(new PredictorAndArgs { Trainer = new SubComponent("BinaryNeuralNetwork", "net=" + quoted + " accel=sse"), Tag = "Inline", }); }
public static List <Sweep> GenerateCandidates(IHostEnvironment env, string dataFile, string schemaDefinitionFile) { var patterns = new List <Sweep>(); string loaderSettings; Type predictorType; TransformInference.InferenceResult inferenceResult; // Get the initial recipes for this data. RecipeInference.SuggestedRecipe[] recipes = RecipeInference.InferRecipesFromData(env, dataFile, schemaDefinitionFile, out predictorType, out loaderSettings, out inferenceResult); //get all the trainers for this task, and generate the initial set of candidates. // Exclude the hidden learners, and the metalinear learners. var trainers = env.ComponentCatalog.GetAllDerivedClasses(typeof(ITrainer), predictorType).Where(cls => !cls.IsHidden); if (!string.IsNullOrEmpty(loaderSettings)) { StringBuilder sb = new StringBuilder(); CmdQuoter.QuoteValue(loaderSettings, sb, true); loaderSettings = sb.ToString(); } string loader = $" loader=TextLoader{loaderSettings}"; // REVIEW: there are more learners than recipes atm. // Flip looping through recipes, then through learners if the cardinality changes. foreach (ComponentCatalog.LoadableClassInfo cl in trainers) { string learnerSettings; TrainerSweeper trainerSweeper = new TrainerSweeper(); trainerSweeper.Parameters.AddRange(RecipeInference.GetLearnerSettingsAndSweepParams(env, cl, out learnerSettings)); foreach (var recipe in recipes) { RecipeInference.SuggestedRecipe.SuggestedLearner learner = new RecipeInference.SuggestedRecipe.SuggestedLearner { LoadableClassInfo = cl, Settings = learnerSettings }; Pattern pattern = new Pattern(recipe.Transforms, learner, loader); Sweep sweep = new Sweep(pattern, trainerSweeper); patterns.Add(sweep); } } return(patterns); }
private protected virtual bool TryUnparseCore(StringBuilder sb, string extra) { Contracts.AssertValue(sb); Contracts.AssertNonEmpty(extra); if (!TrySanitize()) { return(false); } if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source)) { return(false); } sb.Append(Name).Append(':').Append(extra).Append(':').Append(Source); return(true); }
public override string ToString() { if (IsEmpty) { return("{}"); } if (_settings.Length == 0) { return(_kind); } string str = CmdParser.CombineSettings(_settings); StringBuilder sb = new StringBuilder(); CmdQuoter.QuoteValue(str, sb, true); return(_kind + sb.ToString()); }
/// <summary> /// The core unparsing functionality, for generating succinct command line forms "name" and "name:source". /// </summary> protected virtual bool TryUnparseCore(StringBuilder sb) { Contracts.AssertValue(sb); if (!TrySanitize()) { return(false); } if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source)) { return(false); } sb.Append(Name); if (Source != Name) { sb.Append(':').Append(Source); } return(true); }
public bool TryUnparse(StringBuilder sb) { Contracts.AssertValue(sb); if (ResultType == null && KeyCount == null) { return(TryUnparseCore(sb)); } if (!TrySanitize()) { return(false); } if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source)) { return(false); } int ich = sb.Length; sb.Append(Name); sb.Append(':'); if (ResultType != null) { sb.Append(ResultType.Value); } if (KeyCount != null) { sb.Append('['); if (!KeyCount.TryUnparse(sb)) { sb.Length = ich; return(false); } sb.Append(']'); } sb.Append(':'); sb.Append(Source); return(true); }
private string CreateLoaderArguments(ISchema schema, ValueWriter[] pipes, bool hasHeader, IChannel ch) { StringBuilder sb = new StringBuilder(); if (hasHeader) { sb.Append("header+ "); } sb.AppendFormat("sep={0}", SeparatorCharToString(_sepChar)); // This variable indicates the start index of each column. // If null, it means the index cannot be determined. int?index = 0; for (int i = 0; i < pipes.Length; i++) { int src = pipes[i].Source; string name = schema.GetColumnName(src); var type = schema.GetColumnType(src); var column = GetColumn(name, type, index); sb.Append(" col="); if (!column.TryUnparse(sb)) { var settings = CmdParser.GetSettings(ch, column, new TextLoader.Column()); CmdQuoter.QuoteValue(settings, sb, true); } if (type.IsVector && !type.IsKnownSizeVector && i != pipes.Length - 1) { ch.Warning("Column '{0}' is variable length, so it must be the last, or the file will be unreadable. Consider switching to binary format or use xf=Choose to make '{0}' the last column.", name); index = null; } index += type.ValueCount; } return(sb.ToString()); }
// Returns true if a normalizer was added. public static bool AddNormalizerIfNeeded(IHostEnvironment env, IChannel ch, ITrainer trainer, ref IDataView view, string featureColumn, NormalizeOption autoNorm) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(ch, nameof(ch)); ch.CheckValue(trainer, nameof(trainer)); ch.CheckValue(view, nameof(view)); ch.CheckValueOrNull(featureColumn); ch.CheckUserArg(Enum.IsDefined(typeof(NormalizeOption), autoNorm), nameof(TrainCommand.Arguments.NormalizeFeatures), "Normalize option is invalid. Specify one of 'norm=No', 'norm=Warn', 'norm=Auto', or 'norm=Yes'."); if (autoNorm == NormalizeOption.No) { ch.Info("Not adding a normalizer."); return(false); } if (string.IsNullOrEmpty(featureColumn)) { return(false); } int featCol; var schema = view.Schema; if (schema.TryGetColumnIndex(featureColumn, out featCol)) { if (autoNorm != NormalizeOption.Yes) { var nn = trainer as ITrainerEx; DvBool isNormalized = DvBool.False; if (nn == null || !nn.NeedNormalization || (schema.TryGetMetadata(BoolType.Instance, MetadataUtils.Kinds.IsNormalized, featCol, ref isNormalized) && isNormalized.IsTrue)) { ch.Info("Not adding a normalizer."); return(false); } if (autoNorm == NormalizeOption.Warn) { ch.Warning("A normalizer is needed for this trainer. Either add a normalizing transform or use the 'norm=Auto', 'norm=Yes' or 'norm=No' options."); return(false); } } ch.Info("Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off."); // Quote the feature column name string quotedFeatureColumnName = featureColumn; StringBuilder sb = new StringBuilder(); if (CmdQuoter.QuoteValue(quotedFeatureColumnName, sb)) { quotedFeatureColumnName = sb.ToString(); } var component = new SubComponent <IDataTransform, SignatureDataTransform>("MinMax", string.Format("col={{ name={0} source={0} }}", quotedFeatureColumnName)); var loader = view as IDataLoader; if (loader != null) { view = CompositeDataLoader.Create(env, loader, new KeyValuePair <string, SubComponent <IDataTransform, SignatureDataTransform> >(null, component)); } else { view = component.CreateInstance(env, view); } return(true); } return(false); }