protected virtual bool TryUnparseCore(StringBuilder sb, string extra)
{
Contracts.AssertNonEmpty(extra);
if (string.IsNullOrWhiteSpace(Name) || Utils.Size(Source) == 0)
{
return(false);
}
if (CmdQuoter.NeedsQuoting(Name))
{
return(false);
}
if (Source.Any(x => CmdQuoter.NeedsQuoting(x) || x.Contains(",")))
{
return(false);
}
sb.Append(Name).Append(':').Append(extra).Append(':');
string pre = "";
foreach (var src in Source)
{
sb.Append(pre).Append(src);
pre = ",";
}
return(true);
}
internal bool TryUnparse(StringBuilder sb)
{
Contracts.AssertValue(sb);
if (ResultType == null && KeyCount == null)
return TryUnparseCore(sb);
if (!TrySanitize())
return false;
if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source))
return false;
int ich = sb.Length;
sb.Append(Name);
sb.Append(':');
if (ResultType != null)
sb.Append(ResultType.Value.GetString());
if (KeyCount != null)
{
sb.Append('[');
if (!KeyCount.TryUnparse(sb))
{
sb.Length = ich;
return false;
}
sb.Append(']');
}
else if (!string.IsNullOrEmpty(Range))
sb.Append(Range);
sb.Append(':');
sb.Append(Source);
return true;
}
public static PredictorAndArgs NnBinInline(string netDefinition)
{
var quoted = new StringBuilder();
CmdQuoter.QuoteValue(netDefinition, quoted);
return(new PredictorAndArgs
{
Trainer = new SubComponent("BinaryNeuralNetwork", "net=" + quoted + " accel=sse"),
Tag = "Inline",
});
}
public static List <Sweep> GenerateCandidates(IHostEnvironment env, string dataFile, string schemaDefinitionFile)
{
var patterns = new List <Sweep>();
string loaderSettings;
Type predictorType;
TransformInference.InferenceResult inferenceResult;
// Get the initial recipes for this data.
RecipeInference.SuggestedRecipe[] recipes = RecipeInference.InferRecipesFromData(env, dataFile, schemaDefinitionFile, out predictorType, out loaderSettings, out inferenceResult);
//get all the trainers for this task, and generate the initial set of candidates.
// Exclude the hidden learners, and the metalinear learners.
var trainers = env.ComponentCatalog.GetAllDerivedClasses(typeof(ITrainer), predictorType).Where(cls => !cls.IsHidden);
if (!string.IsNullOrEmpty(loaderSettings))
{
StringBuilder sb = new StringBuilder();
CmdQuoter.QuoteValue(loaderSettings, sb, true);
loaderSettings = sb.ToString();
}
string loader = $" loader=TextLoader{loaderSettings}";
// REVIEW: there are more learners than recipes atm.
// Flip looping through recipes, then through learners if the cardinality changes.
foreach (ComponentCatalog.LoadableClassInfo cl in trainers)
{
string learnerSettings;
TrainerSweeper trainerSweeper = new TrainerSweeper();
trainerSweeper.Parameters.AddRange(RecipeInference.GetLearnerSettingsAndSweepParams(env, cl, out learnerSettings));
foreach (var recipe in recipes)
{
RecipeInference.SuggestedRecipe.SuggestedLearner learner = new RecipeInference.SuggestedRecipe.SuggestedLearner
{
LoadableClassInfo = cl,
Settings = learnerSettings
};
Pattern pattern = new Pattern(recipe.Transforms, learner, loader);
Sweep sweep = new Sweep(pattern, trainerSweeper);
patterns.Add(sweep);
}
}
return(patterns);
}
private protected virtual bool TryUnparseCore(StringBuilder sb, string extra)
{
Contracts.AssertValue(sb);
Contracts.AssertNonEmpty(extra);
if (!TrySanitize())
{
return(false);
}
if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source))
{
return(false);
}
sb.Append(Name).Append(':').Append(extra).Append(':').Append(Source);
return(true);
}
public override string ToString()
{
if (IsEmpty)
{
return("{}");
}
if (_settings.Length == 0)
{
return(_kind);
}
string str = CmdParser.CombineSettings(_settings);
StringBuilder sb = new StringBuilder();
CmdQuoter.QuoteValue(str, sb, true);
return(_kind + sb.ToString());
}
/// <summary>
/// The core unparsing functionality, for generating succinct command line forms "name" and "name:source".
/// </summary>
protected virtual bool TryUnparseCore(StringBuilder sb)
{
Contracts.AssertValue(sb);
if (!TrySanitize())
{
return(false);
}
if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source))
{
return(false);
}
sb.Append(Name);
if (Source != Name)
{
sb.Append(':').Append(Source);
}
return(true);
}
public bool TryUnparse(StringBuilder sb)
{
Contracts.AssertValue(sb);
if (ResultType == null && KeyCount == null)
{
return(TryUnparseCore(sb));
}
if (!TrySanitize())
{
return(false);
}
if (CmdQuoter.NeedsQuoting(Name) || CmdQuoter.NeedsQuoting(Source))
{
return(false);
}
int ich = sb.Length;
sb.Append(Name);
sb.Append(':');
if (ResultType != null)
{
sb.Append(ResultType.Value);
}
if (KeyCount != null)
{
sb.Append('[');
if (!KeyCount.TryUnparse(sb))
{
sb.Length = ich;
return(false);
}
sb.Append(']');
}
sb.Append(':');
sb.Append(Source);
return(true);
}
private string CreateLoaderArguments(ISchema schema, ValueWriter[] pipes, bool hasHeader, IChannel ch)
{
StringBuilder sb = new StringBuilder();
if (hasHeader)
{
sb.Append("header+ ");
}
sb.AppendFormat("sep={0}", SeparatorCharToString(_sepChar));
// This variable indicates the start index of each column.
// If null, it means the index cannot be determined.
int?index = 0;
for (int i = 0; i < pipes.Length; i++)
{
int src = pipes[i].Source;
string name = schema.GetColumnName(src);
var type = schema.GetColumnType(src);
var column = GetColumn(name, type, index);
sb.Append(" col=");
if (!column.TryUnparse(sb))
{
var settings = CmdParser.GetSettings(ch, column, new TextLoader.Column());
CmdQuoter.QuoteValue(settings, sb, true);
}
if (type.IsVector && !type.IsKnownSizeVector && i != pipes.Length - 1)
{
ch.Warning("Column '{0}' is variable length, so it must be the last, or the file will be unreadable. Consider switching to binary format or use xf=Choose to make '{0}' the last column.", name);
index = null;
}
index += type.ValueCount;
}
return(sb.ToString());
}
// Returns true if a normalizer was added.
public static bool AddNormalizerIfNeeded(IHostEnvironment env, IChannel ch, ITrainer trainer, ref IDataView view, string featureColumn, NormalizeOption autoNorm)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ch, nameof(ch));
ch.CheckValue(trainer, nameof(trainer));
ch.CheckValue(view, nameof(view));
ch.CheckValueOrNull(featureColumn);
ch.CheckUserArg(Enum.IsDefined(typeof(NormalizeOption), autoNorm), nameof(TrainCommand.Arguments.NormalizeFeatures),
"Normalize option is invalid. Specify one of 'norm=No', 'norm=Warn', 'norm=Auto', or 'norm=Yes'.");
if (autoNorm == NormalizeOption.No)
{
ch.Info("Not adding a normalizer.");
return(false);
}
if (string.IsNullOrEmpty(featureColumn))
{
return(false);
}
int featCol;
var schema = view.Schema;
if (schema.TryGetColumnIndex(featureColumn, out featCol))
{
if (autoNorm != NormalizeOption.Yes)
{
var nn = trainer as ITrainerEx;
DvBool isNormalized = DvBool.False;
if (nn == null || !nn.NeedNormalization ||
(schema.TryGetMetadata(BoolType.Instance, MetadataUtils.Kinds.IsNormalized, featCol, ref isNormalized) &&
isNormalized.IsTrue))
{
ch.Info("Not adding a normalizer.");
return(false);
}
if (autoNorm == NormalizeOption.Warn)
{
ch.Warning("A normalizer is needed for this trainer. Either add a normalizing transform or use the 'norm=Auto', 'norm=Yes' or 'norm=No' options.");
return(false);
}
}
ch.Info("Automatically adding a MinMax normalization transform, use 'norm=Warn' or 'norm=No' to turn this behavior off.");
// Quote the feature column name
string quotedFeatureColumnName = featureColumn;
StringBuilder sb = new StringBuilder();
if (CmdQuoter.QuoteValue(quotedFeatureColumnName, sb))
{
quotedFeatureColumnName = sb.ToString();
}
var component = new SubComponent <IDataTransform, SignatureDataTransform>("MinMax", string.Format("col={{ name={0} source={0} }}", quotedFeatureColumnName));
var loader = view as IDataLoader;
if (loader != null)
{
view = CompositeDataLoader.Create(env, loader,
new KeyValuePair <string, SubComponent <IDataTransform, SignatureDataTransform> >(null, component));
}
else
{
view = component.CreateInstance(env, view);
}
return(true);
}
return(false);
}
