public TrainingSession(IUserSettings settings, ITimer timer, ITrainer trainer, ITaunter taunter, IMetronome metronome, ISequencer sequencer, IContentCollection content,
IContentViewer viewer, IRandomNumberGenerator randomNumberGenerator)
{
_taunter = taunter;
Settings = settings;
Trainer = trainer;
Taunter = taunter;
Metronome = metronome;
Sequencer = sequencer;
Content = content;
Viewer = viewer;
Timer = timer;
RNG = randomNumberGenerator;
}
/// <summary>
/// Callback from the CV method to apply the transforms to the train data.
/// </summary>
private RoleMappedData CreateRoleMappedData(IHostEnvironment env, IChannel ch, IDataView data, ITrainer trainer)
{
foreach (var kvp in Args.Transform)
{
data = kvp.Value.CreateComponent(env, data);
}
var schema = data.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.LabelColumn), Args.LabelColumn, DefaultColumnNames.Label);
string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.FeatureColumn), Args.FeatureColumn, DefaultColumnNames.Features);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.WeightColumn), Args.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.NameColumn), Args.NameColumn, DefaultColumnNames.Name);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Args.GroupColumn), Args.GroupColumn, DefaultColumnNames.GroupId);
TrainUtils.AddNormalizerIfNeeded(env, ch, trainer, ref data, features, Args.NormalizeFeatures);
// Training pipe and examples.
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
return(new RoleMappedData(data, label, features, group, weight, name, customCols));
}
protected void DebugChecking2(RoleMappedData td, ITrainer trainer)
{
var scorer = PredictorHelper.CreateDefaultScorer(Host, td, trainer.CreatePredictor());
if (trainer.PredictionKind == PredictionKind.Ranking)
{
string schemas = SchemaHelper.ToString(scorer.Schema);
if (!schemas.Contains("Score"))
{
throw Host.Except("Issue with the schema: {0}", schemas);
}
}
using (var cursor = scorer.GetRowCursor(i => true))
{
int ilab, ipred, ifeat;
cursor.Schema.TryGetColumnIndex(td.Schema.Label.Name, out ilab);
if (trainer.PredictionKind == PredictionKind.Ranking)
{
cursor.Schema.TryGetColumnIndex("Score", out ipred);
cursor.Schema.TryGetColumnIndex(td.Schema.Feature.Name, out ifeat);
var getter = cursor.GetGetter <uint>(ilab);
var fgetter = cursor.GetGetter <VBuffer <float> >(ifeat);
var pgetter = cursor.GetGetter <float>(ipred);
if (pgetter == null)
{
throw Host.Except("Issue with the schema: {0}", SchemaHelper.ToString(cursor.Schema));
}
uint lab = 0;
var counts = new Dictionary <uint, int>();
var counts_pred = new Dictionary <float, int>();
float pre = 0;
VBuffer <float> features = default(VBuffer <float>);
int nbrows = 0;
int err = 0;
while (cursor.MoveNext())
{
getter(ref lab);
pgetter(ref pre);
fgetter(ref features);
counts[lab] = counts.ContainsKey(lab) ? counts[lab] + 1 : 0;
counts_pred[pre] = counts_pred.ContainsKey(pre) ? counts_pred[pre] + 1 : 0;
if (trainer.PredictionKind == PredictionKind.Ranking)
{
var elab = features.Values[features.Count - 1];
if (pre > 0 && lab < 0)
{
++err;
}
}
else if (!lab.Equals(pre))
{
++err;
}
++nbrows;
}
if (nbrows == 0)
{
throw Host.Except("No results.");
}
if (err * 2 > nbrows)
{
throw Host.Except("No training.");
}
}
else
{
cursor.Schema.TryGetColumnIndex("PredictedLabel", out ipred);
var getter = cursor.GetGetter <bool>(ilab);
var pgetter = cursor.GetGetter <bool>(ipred);
var counts = new Dictionary <bool, int>();
var counts_pred = new Dictionary <bool, int>();
bool lab = false;
bool pre = false;
int nbrows = 0;
int err = 0;
while (cursor.MoveNext())
{
getter(ref lab);
pgetter(ref pre);
counts[lab] = counts.ContainsKey(lab) ? counts[lab] + 1 : 0;
counts_pred[pre] = counts_pred.ContainsKey(pre) ? counts_pred[pre] + 1 : 0;
if (!lab.Equals(pre))
{
++err;
}
++nbrows;
}
if (nbrows == 0)
{
throw Host.Except("No results.");
}
if (err * 2 > nbrows)
{
throw Host.Except("No training.");
}
}
}
}
/// <summary>
/// Convenience train extension for the case where one has only a training set with no auxiliary information.
/// Equivalent to calling <see cref="ITrainer{TPredictor}.Train(TrainContext)"/>
/// on a <see cref="TrainContext"/> constructed with <paramref name="trainData"/>.
/// </summary>
/// <param name="trainer">The trainer</param>
/// <param name="trainData">The training data.</param>
/// <returns>The trained predictor</returns>
public static TPredictor Train <TPredictor>(this ITrainer <TPredictor> trainer, RoleMappedData trainData) where TPredictor : IPredictor
=> trainer.Train(new TrainContext(trainData));
public static IPredictor Train(IHostEnvironment env, IChannel ch, RoleMappedData data, ITrainer trainer, string name, RoleMappedData validData,
SubComponent <ICalibratorTrainer, SignatureCalibrator> calibrator, int maxCalibrationExamples, bool?cacheData, IPredictor inpPredictor = null)
{
ICalibratorTrainer caliTrainer = !calibrator.IsGood() ? null : calibrator.CreateInstance(env);
return(TrainCore(env, ch, data, trainer, name, validData, caliTrainer, maxCalibrationExamples, cacheData, inpPredictor));
}
public TrainCommand(ITrainer trainer, OpportunityRepository repository)
{
this.trainer = trainer;
this.repository = repository;
}
public static IPredictor Train(IHostEnvironment env, IChannel ch, RoleMappedData data, ITrainer trainer, RoleMappedData validData,
IComponentFactory <ICalibratorTrainer> calibrator, int maxCalibrationExamples, bool?cacheData, IPredictor inputPredictor = null)
{
return(TrainCore(env, ch, data, trainer, validData, calibrator, maxCalibrationExamples, cacheData, inputPredictor));
}
public BatchController()
{
BatchRepo = new Batch();
TrainerRepo = new Trainer();
}
public ILecture CreateLecture(string name, string date, ITrainer trainer)
{
return(new Lecture(name, date, trainer));
}
public MedianRatingRecommendationSystem(ITrainer <IMedianRatingModel> trainer, IRecommender <IMedianRatingModel> recommender)
{
Trainer = trainer;
Recommender = recommender;
}
private void RunCore(IChannel ch, string cmd)
{
Host.AssertValue(ch);
Host.AssertNonEmpty(cmd);
ch.Trace("Constructing trainer");
ITrainer trainer = Args.Trainer.CreateComponent(Host);
IPredictor inputPredictor = null;
if (Args.ContinueTrain && !TrainUtils.TryLoadPredictor(ch, Host, Args.InputModelFile, out inputPredictor))
{
ch.Warning("No input model file specified or model file did not contain a predictor. The model state cannot be initialized.");
}
ch.Trace("Constructing the training pipeline");
IDataView trainPipe = CreateLoader();
ISchema schema = trainPipe.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn),
Args.LabelColumn, DefaultColumnNames.Label);
string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.FeatureColumn),
Args.FeatureColumn, DefaultColumnNames.Features);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn),
Args.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn),
Args.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.NameColumn),
Args.NameColumn, DefaultColumnNames.Name);
TrainUtils.AddNormalizerIfNeeded(Host, ch, trainer, ref trainPipe, features, Args.NormalizeFeatures);
ch.Trace("Binding columns");
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
var data = new RoleMappedData(trainPipe, label, features, group, weight, name, customCols);
RoleMappedData validData = null;
if (!string.IsNullOrWhiteSpace(Args.ValidationFile))
{
if (!trainer.Info.SupportsValidation)
{
ch.Warning("Ignoring validationFile: Trainer does not accept validation dataset.");
}
else
{
ch.Trace("Constructing the validation pipeline");
IDataView validPipe = CreateRawLoader(dataFile: Args.ValidationFile);
validPipe = ApplyTransformUtils.ApplyAllTransformsToData(Host, trainPipe, validPipe);
validData = new RoleMappedData(validPipe, data.Schema.GetColumnRoleNames());
}
}
var predictor = TrainUtils.Train(Host, ch, data, trainer, validData,
Args.Calibrator, Args.MaxCalibrationExamples, Args.CacheData, inputPredictor);
IDataLoader testPipe;
using (var file = !string.IsNullOrEmpty(Args.OutputModelFile) ?
Host.CreateOutputFile(Args.OutputModelFile) : Host.CreateTempFile(".zip"))
{
TrainUtils.SaveModel(Host, ch, file, predictor, data, cmd);
ch.Trace("Constructing the testing pipeline");
using (var stream = file.OpenReadStream())
using (var rep = RepositoryReader.Open(stream, ch))
testPipe = LoadLoader(rep, Args.TestFile, true);
}
// Score.
ch.Trace("Scoring and evaluating");
ch.Assert(Args.Scorer == null || Args.Scorer is ICommandLineComponentFactory, "TrainTestCommand should only be used from the command line.");
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(Args.Scorer, predictor, testPipe, features, group, customCols, Host, data.Schema);
// Evaluate.
var evaluator = Args.Evaluator?.CreateComponent(Host) ??
EvaluateUtils.GetEvaluator(Host, scorePipe.Schema);
var dataEval = new RoleMappedData(scorePipe, label, features,
group, weight, name, customCols, opt: true);
var metrics = evaluator.Evaluate(dataEval);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out var overall))
{
throw ch.Except("No overall metrics found");
}
overall = evaluator.GetOverallResults(overall);
MetricWriter.PrintOverallMetrics(Host, ch, Args.SummaryFilename, overall, 1);
evaluator.PrintAdditionalMetrics(ch, metrics);
Dictionary <string, IDataView>[] metricValues = { metrics };
SendTelemetryMetric(metricValues);
if (!string.IsNullOrWhiteSpace(Args.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(dataEval);
var perInstData = new RoleMappedData(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, Args.OutputDataFile, idv);
}
}
public ExtendedTrainer(ITrainer trainer, string loadName)
{
_loadName = loadName;
_trainer = trainer;
}
public ILecture CreateLecture(string name, string date, ITrainer trainer)
{
// TODO: Implement this
return(new Lecture(name, DateTime.Parse(date), trainer));
}
public Lecture(string name, DateTime date, ITrainer trainer)
{
this.Name = name;
this.Date = date;
this.Trainer = trainer;
}
public BatchController(IBatch batchRepo, ITrainer trainerRepo)
{
BatchRepo = batchRepo;
TrainerRepo = trainerRepo;
}
public AverageRatingRecommendationSystem(ITrainer<IAverageRatingModel> trainer, IRecommender<IAverageRatingModel> recommender)
{
Trainer = trainer;
Recommender = recommender;
}
public SimpleKnnRecommendationSystem(ITrainer<ISimpleKnnModel> trainer, IRecommender<ISimpleKnnModel> recommender)
{
Trainer = trainer;
Recommender = recommender;
}
public ILecture CreateLecture(string name, string date, ITrainer trainer)
{
// TODO: Implement this
throw new NotImplementedException("Lecture class not attached to factory.");
}
public static IPredictor Train(IHostEnvironment env, IChannel ch, RoleMappedData data, ITrainer trainer,
IComponentFactory <ICalibratorTrainer> calibrator, int maxCalibrationExamples)
{
return(TrainCore(env, ch, data, trainer, null, calibrator, maxCalibrationExamples, false));
}
public FeedbackController()
{
FeedbackRepo = new Feedback();
InducteeRepo = new Inductee();
TrainerRepo = new Trainer();
}
private static IPredictor TrainCore(IHostEnvironment env, IChannel ch, RoleMappedData data, ITrainer trainer, RoleMappedData validData,
IComponentFactory <ICalibratorTrainer> calibrator, int maxCalibrationExamples, bool?cacheData, IPredictor inputPredictor = null)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ch, nameof(ch));
ch.CheckValue(data, nameof(data));
ch.CheckValue(trainer, nameof(trainer));
ch.CheckValueOrNull(validData);
ch.CheckValueOrNull(inputPredictor);
AddCacheIfWanted(env, ch, trainer, ref data, cacheData);
ch.Trace("Training");
if (validData != null)
{
AddCacheIfWanted(env, ch, trainer, ref validData, cacheData);
}
if (inputPredictor != null && !trainer.Info.SupportsIncrementalTraining)
{
ch.Warning("Ignoring " + nameof(TrainCommand.Arguments.InputModelFile) +
": Trainer does not support incremental training.");
inputPredictor = null;
}
ch.Assert(validData == null || trainer.Info.SupportsValidation);
var predictor = trainer.Train(new TrainContext(data, validData, inputPredictor));
var caliTrainer = calibrator?.CreateComponent(env);
return(CalibratorUtils.TrainCalibratorIfNeeded(env, ch, caliTrainer, maxCalibrationExamples, trainer, predictor, data));
}
public MainStory(ITrainer player) => _player = player;
public static IPredictor Train(IHostEnvironment env, IChannel ch, RoleMappedData data, ITrainer trainer, string name,
ICalibratorTrainerFactory calibrator, int maxCalibrationExamples)
{
var caliTrainer = calibrator?.CreateComponent(env);
return(TrainCore(env, ch, data, trainer, name, null, caliTrainer, maxCalibrationExamples, false));
}
private void RunCore(IChannel ch, string cmd)
{
Host.AssertValue(ch);
Host.AssertNonEmpty(cmd);
ch.Trace("Constructing trainer");
ITrainer trainer = ImplOptions.Trainer.CreateComponent(Host);
IPredictor inputPredictor = null;
if (ImplOptions.ContinueTrain && !TrainUtils.TryLoadPredictor(ch, Host, ImplOptions.InputModelFile, out inputPredictor))
{
ch.Warning("No input model file specified or model file did not contain a predictor. The model state cannot be initialized.");
}
ch.Trace("Constructing the training pipeline");
IDataView trainPipe = CreateLoader();
var schema = trainPipe.Schema;
string label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn),
ImplOptions.LabelColumn, DefaultColumnNames.Label);
string features = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.FeatureColumn),
ImplOptions.FeatureColumn, DefaultColumnNames.Features);
string group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn),
ImplOptions.GroupColumn, DefaultColumnNames.GroupId);
string weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn),
ImplOptions.WeightColumn, DefaultColumnNames.Weight);
string name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.NameColumn),
ImplOptions.NameColumn, DefaultColumnNames.Name);
TrainUtils.AddNormalizerIfNeeded(Host, ch, trainer, ref trainPipe, features, ImplOptions.NormalizeFeatures);
ch.Trace("Binding columns");
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, ImplOptions.CustomColumns);
var data = new RoleMappedData(trainPipe, label, features, group, weight, name, customCols);
RoleMappedData validData = null;
if (!string.IsNullOrWhiteSpace(ImplOptions.ValidationFile))
{
if (!trainer.Info.SupportsValidation)
{
ch.Warning("Ignoring validationFile: Trainer does not accept validation dataset.");
}
else
{
ch.Trace("Constructing the validation pipeline");
IDataView validPipe = CreateRawLoader(dataFile: ImplOptions.ValidationFile);
validPipe = ApplyTransformUtils.ApplyAllTransformsToData(Host, trainPipe, validPipe);
validData = new RoleMappedData(validPipe, data.Schema.GetColumnRoleNames());
}
}
// In addition to the training set, some trainers can accept two data sets, validation set and test set,
// in training phase. The major difference between validation set and test set is that training process may
// indirectly use validation set to improve the model but the learned model should totally independent of test set.
// Similar to validation set, the trainer can report the scores computed using test set.
RoleMappedData testDataUsedInTrainer = null;
if (!string.IsNullOrWhiteSpace(ImplOptions.TestFile))
{
// In contrast to the if-else block for validation above, we do not throw a warning if test file is provided
// because this is TrainTest command.
if (trainer.Info.SupportsTest)
{
ch.Trace("Constructing the test pipeline");
IDataView testPipeUsedInTrainer = CreateRawLoader(dataFile: ImplOptions.TestFile);
testPipeUsedInTrainer = ApplyTransformUtils.ApplyAllTransformsToData(Host, trainPipe, testPipeUsedInTrainer);
testDataUsedInTrainer = new RoleMappedData(testPipeUsedInTrainer, data.Schema.GetColumnRoleNames());
}
}
var predictor = TrainUtils.Train(Host, ch, data, trainer, validData,
ImplOptions.Calibrator, ImplOptions.MaxCalibrationExamples, ImplOptions.CacheData, inputPredictor, testDataUsedInTrainer);
ILegacyDataLoader testPipe;
bool hasOutfile = !string.IsNullOrEmpty(ImplOptions.OutputModelFile);
var tempFilePath = hasOutfile ? null : Path.GetTempFileName();
using (var file = new SimpleFileHandle(ch, hasOutfile ? ImplOptions.OutputModelFile : tempFilePath, true, !hasOutfile))
{
TrainUtils.SaveModel(Host, ch, file, predictor, data, cmd);
ch.Trace("Constructing the testing pipeline");
using (var stream = file.OpenReadStream())
using (var rep = RepositoryReader.Open(stream, ch))
testPipe = LoadLoader(rep, ImplOptions.TestFile, true);
}
// Score.
ch.Trace("Scoring and evaluating");
ch.Assert(ImplOptions.Scorer == null || ImplOptions.Scorer is ICommandLineComponentFactory, "TrainTestCommand should only be used from the command line.");
IDataScorerTransform scorePipe = ScoreUtils.GetScorer(ImplOptions.Scorer, predictor, testPipe, features, group, customCols, Host, data.Schema);
// Evaluate.
var evaluator = ImplOptions.Evaluator?.CreateComponent(Host) ??
EvaluateUtils.GetEvaluator(Host, scorePipe.Schema);
var dataEval = new RoleMappedData(scorePipe, label, features,
group, weight, name, customCols, opt: true);
var metrics = evaluator.Evaluate(dataEval);
MetricWriter.PrintWarnings(ch, metrics);
evaluator.PrintFoldResults(ch, metrics);
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out var overall))
{
throw ch.Except("No overall metrics found");
}
overall = evaluator.GetOverallResults(overall);
MetricWriter.PrintOverallMetrics(Host, ch, ImplOptions.SummaryFilename, overall, 1);
evaluator.PrintAdditionalMetrics(ch, metrics);
Dictionary <string, IDataView>[] metricValues = { metrics };
SendTelemetryMetric(metricValues);
if (!string.IsNullOrWhiteSpace(ImplOptions.OutputDataFile))
{
var perInst = evaluator.GetPerInstanceMetrics(dataEval);
var perInstData = new RoleMappedData(perInst, label, null, group, weight, name, customCols);
var idv = evaluator.GetPerInstanceDataViewToSave(perInstData);
MetricWriter.SavePerInstance(Host, ch, ImplOptions.OutputDataFile, idv);
}
}
private static IPredictor TrainCore(IHostEnvironment env, IChannel ch, RoleMappedData data, ITrainer trainer, string name, RoleMappedData validData,
ICalibratorTrainer calibrator, int maxCalibrationExamples, bool?cacheData, IPredictor inpPredictor = null)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ch, nameof(ch));
ch.CheckValue(data, nameof(data));
ch.CheckValue(trainer, nameof(trainer));
ch.CheckNonEmpty(name, nameof(name));
ch.CheckValueOrNull(validData);
ch.CheckValueOrNull(inpPredictor);
var trainerRmd = trainer as ITrainer <RoleMappedData>;
if (trainerRmd == null)
{
throw ch.ExceptUserArg(nameof(TrainCommand.Arguments.Trainer), "Trainer '{0}' does not accept known training data type", name);
}
Action <IChannel, ITrainer, Action <object>, object, object, object> trainCoreAction = TrainCore;
IPredictor predictor;
AddCacheIfWanted(env, ch, trainer, ref data, cacheData);
ch.Trace("Training");
if (validData != null)
{
AddCacheIfWanted(env, ch, trainer, ref validData, cacheData);
}
var genericExam = trainCoreAction.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(
typeof(RoleMappedData),
inpPredictor != null ? inpPredictor.GetType() : typeof(IPredictor));
Action <RoleMappedData> trainExam = trainerRmd.Train;
genericExam.Invoke(null, new object[] { ch, trainerRmd, trainExam, data, validData, inpPredictor });
ch.Trace("Constructing predictor");
predictor = trainerRmd.CreatePredictor();
return(CalibratorUtils.TrainCalibratorIfNeeded(env, ch, calibrator, maxCalibrationExamples, trainer, predictor, data));
}
public Lecture(string name, string date, ITrainer trainer)
{
this.Name = name;
this.Date = DateTime.Parse(date);
this.Trainer = trainer;
}
/// <summary> /// Convenience train extension for the case where one has only a training set with no auxiliary information. /// Equivalent to calling <see cref="ITrainer.Train(TrainContext)"/> /// on a <see cref="TrainContext"/> constructed with <paramref name="trainData"/>. /// </summary> /// <param name="trainer">The trainer</param> /// <param name="trainData">The training data.</param> /// <returns>The trained predictor</returns> public static IPredictor Train(this ITrainer trainer, RoleMappedData trainData) => trainer.Train(new TrainContext(trainData));
private static TrainerChartPanel <TChart, TSeries, TChartValues, TData> CreateChartPanel <TChart, TSeries, TChartValues, TData>(string title, ITrainer trainer, string hookedValue, ITimeStep timestep, bool averageMode = false) where TChart : Chart, new() where TSeries : Series, new() where TChartValues : IList <TData>, IChartValues, new()
{
var panel = new TrainerChartPanel <TChart, TSeries, TChartValues, TData>(title, trainer, hookedValue, timestep, averageMode);
panel.Fast();
panel.MaxPoints = 50;
return(panel);
}
public ILecture CreateLecture(string name, string date, ITrainer trainer)
{
var parsedDate = DateTime.Parse(date);
return(new Lecture(name, parsedDate, trainer));
}
private static void Main()
{
SigmaEnvironment.EnableLogging();
SigmaEnvironment sigma = SigmaEnvironment.Create("sigma_demo");
// create a new mnist trainer
string name = DemoMode.Name;
ITrainer trainer = DemoMode.CreateTrainer(sigma);
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.weights", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_weights_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.weights", (a, h) => h.StandardDeviation(a), "shared.network_weights_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.biases", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_biases_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*.biases", (a, h) => h.StandardDeviation(a), "shared.network_biases_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("optimiser.updates", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.optimiser_updates_average"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("optimiser.updates", (a, h) => h.StandardDeviation(a), "shared.optimiser_updates_stddev"));
trainer.AddLocalHook(new MetricProcessorHook <INDArray>("network.layers.*<external_output>._outputs.default.activations", (a, h) => h.Divide(h.Sum(a), a.Length), "shared.network_activations_mean"));
// create and attach a new UI framework
WPFMonitor gui = sigma.AddMonitor(new WPFMonitor(name, DemoMode.Language));
gui.ColourManager.Dark = DemoMode.Dark;
gui.ColourManager.PrimaryColor = DemoMode.PrimarySwatch;
StatusBarLegendInfo iris = new StatusBarLegendInfo(name, MaterialColour.Blue);
StatusBarLegendInfo general = new StatusBarLegendInfo("General", MaterialColour.Grey);
gui.AddLegend(iris);
gui.AddLegend(general);
// create a tab
gui.AddTabs("Overview", "Metrics", "Validation", "Maximisation", "Reproduction", "Update");
// access the window inside the ui thread
gui.WindowDispatcher(window =>
{
// enable initialisation
window.IsInitializing = true;
window.TabControl["Metrics"].GridSize = new GridSize(2, 4);
window.TabControl["Validation"].GridSize = new GridSize(2, 5);
window.TabControl["Maximisation"].GridSize = new GridSize(2, 5);
window.TabControl["Reproduction"].GridSize = new GridSize(2, 5);
window.TabControl["Update"].GridSize = new GridSize(1, 1);
window.TabControl["Overview"].GridSize.Rows -= 1;
window.TabControl["Overview"].GridSize.Columns -= 1;
// add a panel that controls the learning process
window.TabControl["Overview"].AddCumulativePanel(new ControlPanel("Control", trainer), legend: iris);
ITimeStep reportTimeStep = DemoMode.Slow ? TimeStep.Every(1, TimeScale.Iteration) : TimeStep.Every(10, TimeScale.Epoch);
var cost1 = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Cost / Epoch", trainer, "optimiser.cost_total", TimeStep.Every(1, TimeScale.Epoch)).Linearify();
var cost2 = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Cost / Epoch", trainer, "optimiser.cost_total", reportTimeStep);
var weightAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Weights / Epoch", trainer, "shared.network_weights_average", reportTimeStep, averageMode: true).Linearify();
var weightStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Weights / Epoch", trainer, "shared.network_weights_stddev", reportTimeStep, averageMode: true).Linearify();
var biasesAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Biases / Epoch", trainer, "shared.network_biases_average", reportTimeStep, averageMode: true).Linearify();
var biasesStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Biases / Epoch", trainer, "shared.network_biases_stddev", reportTimeStep, averageMode: true).Linearify();
var updateAverage = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Parameter Updates / Epoch", trainer, "shared.optimiser_updates_average", reportTimeStep, averageMode: true).Linearify();
var updateStddev = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Standard Deviation of Parameter Updates / Epoch", trainer, "shared.optimiser_updates_stddev", reportTimeStep, averageMode: true).Linearify();
var outputActivationsMean = CreateChartPanel <CartesianChart, GLineSeries, GearedValues <double>, double>("Mean of Output Activations", trainer, "shared.network_activations_mean", reportTimeStep, averageMode: true).Linearify();
AccuracyPanel accuracy1 = null, accuracy2 = null;
if (DemoMode != DemoType.Wdbc && DemoMode != DemoType.Parkinsons)
{
accuracy1 = new AccuracyPanel("Validation Accuracy", trainer, DemoMode.Slow ? TimeStep.Every(1, TimeScale.Epoch) : reportTimeStep, null, 1, 2);
accuracy1.Fast().Linearify();
accuracy2 = new AccuracyPanel("Validation Accuracy", trainer, DemoMode.Slow ? TimeStep.Every(1, TimeScale.Epoch) : reportTimeStep, null, 1, 2);
accuracy2.Fast().Linearify();
}
IRegistry regTest = new Registry();
regTest.Add("test", DateTime.Now);
var parameter = new ParameterPanel("Parameters", sigma, window);
parameter.Add("Time", typeof(DateTime), regTest, "test");
ValueSourceReporter valueHook = new ValueSourceReporter(TimeStep.Every(1, TimeScale.Epoch), "optimiser.cost_total");
trainer.AddGlobalHook(valueHook);
sigma.SynchronisationHandler.AddSynchronisationSource(valueHook);
var costBlock = (UserControlParameterVisualiser)parameter.Content.Add("Cost", typeof(double), trainer.Operator.Registry, "optimiser.cost_total");
costBlock.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Epoch));
var learningBlock = (UserControlParameterVisualiser)parameter.Content.Add("Learning rate", typeof(double), trainer.Operator.Registry, "optimiser.learning_rate");
learningBlock.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Epoch));
var paramCount = (UserControlParameterVisualiser)parameter.Content.Add("Parameter count", typeof(long), trainer.Operator.Registry, "network.parameter_count");
paramCount.AutoPollValues(trainer, TimeStep.Every(1, TimeScale.Start));
window.TabControl["Overview"].AddCumulativePanel(cost1, 1, 2, legend: iris);
window.TabControl["Overview"].AddCumulativePanel(parameter);
//window.TabControl["Overview"].AddCumulativePanel(accuracy1, 1, 2, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(cost2, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(weightAverage, legend: iris);
//window.TabControl["Metrics"].AddCumulativePanel(biasesAverage, legend: iris);
window.TabControl["Update"].AddCumulativePanel(updateAverage, legend: iris);
if (accuracy2 != null)
{
window.TabControl["Metrics"].AddCumulativePanel(accuracy2, legend: iris);
}
window.TabControl["Metrics"].AddCumulativePanel(weightStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(biasesStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(updateStddev, legend: iris);
window.TabControl["Metrics"].AddCumulativePanel(outputActivationsMean, legend: iris);
if (DemoMode == DemoType.Mnist)
{
NumberPanel outputpanel = new NumberPanel("Numbers", trainer);
DrawPanel drawPanel = new DrawPanel("Draw", trainer, 560, 560, 20, outputpanel);
window.TabControl["Validation"].AddCumulativePanel(drawPanel, 2, 3);
window.TabControl["Validation"].AddCumulativePanel(outputpanel, 2);
window.TabControl["Validation"].AddCumulativePanel(weightAverage);
window.TabControl["Validation"].AddCumulativePanel(biasesAverage);
for (int i = 0; i < 10; i++)
{
window.TabControl["Maximisation"].AddCumulativePanel(new MnistBitmapHookPanel($"Target Maximisation {i}", i, trainer, TimeStep.Every(1, TimeScale.Epoch)));
}
}
if (DemoMode == DemoType.TicTacToe)
{
window.TabControl["Overview"].AddCumulativePanel(new TicTacToePanel("Play TicTacToe!", trainer));
}
//for (int i = 0; i < 10; i++)
//{
// window.TabControl["Reproduction"].AddCumulativePanel(new MnistBitmapHookPanel($"Target Maximisation 7-{i}", 8, 28, 28, trainer, TimeStep.Every(1, TimeScale.Start)));
//}
});
if (DemoMode == DemoType.Mnist)
{
sigma.AddMonitor(new HttpMonitor("http://+:8080/sigma/"));
}
// the operators should not run instantly but when the user clicks play
sigma.StartOperatorsOnRun = false;
sigma.Prepare();
sigma.RunAsync();
gui.WindowDispatcher(window => window.IsInitializing = false);
}
public AccountController()
{
_Trainer = new Trainer();
}
private FoldResult RunFold(int fold)
{
var host = GetHost();
host.Assert(0 <= fold && fold <= _numFolds);
// REVIEW: Make channels buffered in multi-threaded environments.
using (var ch = host.Start($"Fold {fold}"))
{
ch.Trace("Constructing trainer");
ITrainer trainer = _trainer.CreateInstance(host);
// Train pipe.
var trainFilter = new RangeFilter.Arguments();
trainFilter.Column = _splitColumn;
trainFilter.Min = (Double)fold / _numFolds;
trainFilter.Max = (Double)(fold + 1) / _numFolds;
trainFilter.Complement = true;
IDataView trainPipe = new RangeFilter(host, trainFilter, _inputDataView);
trainPipe = new OpaqueDataView(trainPipe);
var trainData = _createExamples(host, ch, trainPipe, trainer);
// Test pipe.
var testFilter = new RangeFilter.Arguments();
testFilter.Column = trainFilter.Column;
testFilter.Min = trainFilter.Min;
testFilter.Max = trainFilter.Max;
ch.Assert(!testFilter.Complement);
IDataView testPipe = new RangeFilter(host, testFilter, _inputDataView);
testPipe = new OpaqueDataView(testPipe);
var testData = _applyTransformsToTestData(host, ch, testPipe, trainData, trainPipe);
// Validation pipe and examples.
RoleMappedData validData = null;
if (_getValidationDataView != null)
{
ch.Assert(_applyTransformsToValidationData != null);
if (!trainer.Info.SupportsValidation)
{
ch.Warning("Trainer does not accept validation dataset.");
}
else
{
ch.Trace("Constructing the validation pipeline");
IDataView validLoader = _getValidationDataView();
var validPipe = ApplyTransformUtils.ApplyAllTransformsToData(host, _inputDataView, validLoader);
validPipe = new OpaqueDataView(validPipe);
validData = _applyTransformsToValidationData(host, ch, validPipe, trainData, trainPipe);
}
}
// Train.
var predictor = TrainUtils.Train(host, ch, trainData, trainer, validData,
_calibrator, _maxCalibrationExamples, _cacheData, _inputPredictor);
// Score.
ch.Trace("Scoring and evaluating");
ch.Assert(_scorer == null || _scorer is ICommandLineComponentFactory, "CrossValidationCommand should only be used from the command line.");
var bindable = ScoreUtils.GetSchemaBindableMapper(host, predictor, scorerFactorySettings: _scorer as ICommandLineComponentFactory);
ch.AssertValue(bindable);
var mapper = bindable.Bind(host, testData.Schema);
var scorerComp = _scorer ?? ScoreUtils.GetScorerComponent(mapper);
IDataScorerTransform scorePipe = scorerComp.CreateComponent(host, testData.Data, mapper, trainData.Schema);
// Save per-fold model.
string modelFileName = ConstructPerFoldName(_outputModelFile, fold);
if (modelFileName != null && _loader != null)
{
using (var file = host.CreateOutputFile(modelFileName))
{
var rmd = new RoleMappedData(
CompositeDataLoader.ApplyTransform(host, _loader, null, null,
(e, newSource) => ApplyTransformUtils.ApplyAllTransformsToData(e, trainData.Data, newSource)),
trainData.Schema.GetColumnRoleNames());
TrainUtils.SaveModel(host, ch, file, predictor, rmd, _cmd);
}
}
// Evaluate.
var evalComp = _evaluator;
if (!evalComp.IsGood())
{
evalComp = EvaluateUtils.GetEvaluatorType(ch, scorePipe.Schema);
}
var eval = evalComp.CreateInstance(host);
// Note that this doesn't require the provided columns to exist (because of the "opt" parameter).
// We don't normally expect the scorer to drop columns, but if it does, we should not require
// all the columns in the test pipeline to still be present.
var dataEval = new RoleMappedData(scorePipe, testData.Schema.GetColumnRoleNames(), opt: true);
var dict = eval.Evaluate(dataEval);
RoleMappedData perInstance = null;
if (_savePerInstance)
{
var perInst = eval.GetPerInstanceMetrics(dataEval);
perInstance = new RoleMappedData(perInst, dataEval.Schema.GetColumnRoleNames(), opt: true);
}
ch.Done();
return(new FoldResult(dict, dataEval.Schema.Schema, perInstance, trainData.Schema));
}
}
public SimpleKnnRecommendationSystem(ITrainer<ISimpleKnnModel> trainer)
: this(trainer, new SimpleKnnRecommender())
{
}
private void RunCore(IChannel ch, string cmd)
{
Host.AssertValue(ch);
Host.AssertNonEmpty(cmd);
ch.Trace("Constructing trainer");
ITrainer trainer = _trainer.CreateComponent(Host);
IPredictor inputPredictor = null;
if (Args.ContinueTrain && !TrainUtils.TryLoadPredictor(ch, Host, Args.InputModelFile, out inputPredictor))
{
ch.Warning("No input model file specified or model file did not contain a predictor. The model state cannot be initialized.");
}
ch.Trace("Constructing data pipeline");
IDataView view = CreateLoader();
ISchema schema = view.Schema;
var label = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.LabelColumn), _labelColumn, DefaultColumnNames.Label);
var feature = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.FeatureColumn), _featureColumn, DefaultColumnNames.Features);
var group = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.GroupColumn), _groupColumn, DefaultColumnNames.GroupId);
var weight = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.WeightColumn), _weightColumn, DefaultColumnNames.Weight);
var name = TrainUtils.MatchNameOrDefaultOrNull(ch, schema, nameof(Arguments.NameColumn), _nameColumn, DefaultColumnNames.Name);
TrainUtils.AddNormalizerIfNeeded(Host, ch, trainer, ref view, feature, Args.NormalizeFeatures);
ch.Trace("Binding columns");
var customCols = TrainUtils.CheckAndGenerateCustomColumns(ch, Args.CustomColumn);
var data = new RoleMappedData(view, label, feature, group, weight, name, customCols);
// REVIEW: Unify the code that creates validation examples in Train, TrainTest and CV commands.
RoleMappedData validData = null;
if (!string.IsNullOrWhiteSpace(Args.ValidationFile))
{
if (!trainer.Info.SupportsValidation)
{
ch.Warning("Ignoring validationFile: Trainer does not accept validation dataset.");
}
else
{
ch.Trace("Constructing the validation pipeline");
IDataView validPipe = CreateRawLoader(dataFile: Args.ValidationFile);
validPipe = ApplyTransformUtils.ApplyAllTransformsToData(Host, view, validPipe);
validData = new RoleMappedData(validPipe, data.Schema.GetColumnRoleNames());
}
}
// In addition to the training set, some trainers can accept two extra data sets, validation set and test set,
// in training phase. The major difference between validation set and test set is that training process may
// indirectly use validation set to improve the model but the learned model should totally independent of test set.
// Similar to validation set, the trainer can report the scores computed using test set.
RoleMappedData testDataUsedInTrainer = null;
if (!string.IsNullOrWhiteSpace(Args.TestFile))
{
// In contrast to the if-else block for validation above, we do not throw a warning if test file is provided
// because this is TrainTest command.
if (trainer.Info.SupportsTest)
{
ch.Trace("Constructing the test pipeline");
IDataView testPipeUsedInTrainer = CreateRawLoader(dataFile: Args.TestFile);
testPipeUsedInTrainer = ApplyTransformUtils.ApplyAllTransformsToData(Host, view, testPipeUsedInTrainer);
testDataUsedInTrainer = new RoleMappedData(testPipeUsedInTrainer, data.Schema.GetColumnRoleNames());
}
}
var predictor = TrainUtils.Train(Host, ch, data, trainer, validData,
Args.Calibrator, Args.MaxCalibrationExamples, Args.CacheData, inputPredictor, testDataUsedInTrainer);
using (var file = Host.CreateOutputFile(Args.OutputModelFile))
TrainUtils.SaveModel(Host, ch, file, predictor, data, cmd);
}
public MedianRatingRecommendationSystem(ITrainer<IMedianRatingModel> trainer, IRecommender<IMedianRatingModel> recommender)
{
Trainer = trainer;
Recommender = recommender;
}
IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input, out IDataView sourceCtx)
{
sourceCtx = input;
Contracts.CheckValue(env, "env");
env.CheckValue(args, "args");
env.CheckValue(input, "input");
env.CheckValue(args.tag, "tag is empty");
env.CheckValue(args.trainer, "trainer",
"Trainer cannot be null. If your model is already trained, please use ScoreTransform instead.");
var views = TagHelper.EnumerateTaggedView(true, input).Where(c => c.Item1 == args.tag);
if (views.Any())
{
throw env.Except("Tag '{0}' is already used.", args.tag);
}
var host = env.Register("TagTrainOrScoreTransform");
using (var ch = host.Start("Train"))
{
ch.Trace("Constructing trainer");
var trainerSett = ScikitSubComponent <ITrainer, SignatureTrainer> .AsSubComponent(args.trainer);
ITrainer trainer = trainerSett.CreateInstance(host);
var customCols = TrainUtils.CheckAndGenerateCustomColumns(env, args.CustomColumns);
string feat;
string group;
var data = CreateDataFromArgs(_host, ch, new OpaqueDataView(input), args, out feat, out group);
ICalibratorTrainer calibrator = args.calibrator == null
? null
: ScikitSubComponent <ICalibratorTrainer, SignatureCalibrator> .AsSubComponent(args.calibrator).CreateInstance(host);
var nameTrainer = args.trainer.ToString().Replace("{", "").Replace("}", "").Replace(" ", "").Replace("=", "").Replace("+", "Y").Replace("-", "N");
var extTrainer = new ExtendedTrainer(trainer, nameTrainer);
_predictor = extTrainer.Train(host, ch, data, null, calibrator, args.maxCalibrationExamples);
if (!string.IsNullOrEmpty(args.outputModel))
{
ch.Info("Saving model into '{0}'", args.outputModel);
using (var fs = File.Create(args.outputModel))
TrainUtils.SaveModel(env, ch, fs, _predictor, data);
ch.Info("Done.");
}
if (_cali != null)
{
throw ch.ExceptNotImpl("Calibrator is not implemented yet.");
}
ch.Trace("Scoring");
if (_args.scorer != null)
{
var mapper = new SchemaBindablePredictorWrapper(_predictor);
var roles = new RoleMappedSchema(input.Schema, null, feat, group: group);
var bound = (mapper as ISchemaBindableMapper).Bind(_host, roles);
var scorPars = ScikitSubComponent <IDataScorerTransform, SignatureDataScorer> .AsSubComponent(_args.scorer);
_scorer = scorPars.CreateInstance(_host, input, bound, roles);
}
else
{
_scorer = PredictorHelper.CreateDefaultScorer(_host, input, feat, group, _predictor);
}
ch.Info("Tagging with tag '{0}'.", args.tag);
var ar = new TagViewTransform.Arguments {
tag = args.tag
};
var res = new TagViewTransform(env, ar, _scorer, _predictor);
return(res);
}
}