/// <summary>
/// Run the predictor with given args and check if it adds up
/// </summary>
protected void Run(RunContext ctx)
{
Contracts.Assert(IsActive);
List <string> args = new List <string>();
if (ctx.Command != Cmd.Test)
{
AddIfNotEmpty(args, ctx.Predictor.Trainer, "tr");
}
string dataName = ctx.Command == Cmd.Test ? ctx.Dataset.testFilename : ctx.Dataset.trainFilename;
AddIfNotEmpty(args, GetDataPath(dataName), "data");
AddIfNotEmpty(args, 1, "seed");
//AddIfNotEmpty(args, false, "threads");
Log("Running '{0}' on '{1}'", ctx.Predictor.Trainer.Kind, ctx.Dataset.name);
string dir = ctx.BaselineDir;
if (ctx.Command == Cmd.TrainTest)
{
AddIfNotEmpty(args, GetDataPath(ctx.Dataset.testFilename), "test");
}
if (ctx.Command == Cmd.TrainTest || ctx.Command == Cmd.Train)
{
AddIfNotEmpty(args, GetDataPath(ctx.Dataset.validFilename), "valid");
}
// Add in the loader args, and keep a location so we can backtrack and remove it later.
int loaderArgIndex = -1;
string loaderArgs = GetLoaderTransformSettings(ctx.Dataset);
if (!string.IsNullOrWhiteSpace(loaderArgs))
{
loaderArgIndex = args.Count;
args.Add(loaderArgs);
}
// Add in the dataset transforms. These need to come before the predictor imposed transforms.
if (ctx.Dataset.mamlExtraSettings != null)
{
args.AddRange(ctx.Dataset.mamlExtraSettings);
}
// Model file output, used only for train/traintest.
var modelPath = ctx.Command == Cmd.Train || ctx.Command == Cmd.TrainTest ? ctx.ModelPath() : null;
AddIfNotEmpty(args, modelPath, "out");
string basePrefix = ctx.BaselineNamePrefix;
// Predictions output, for all types of commands except train.
OutputPath predOutPath = ctx.Command == Cmd.Train ? null : ctx.InitPath(".txt");
AddIfNotEmpty(args, predOutPath, "dout");
if (ctx.Predictor.MamlArgs != null)
{
args.AddRange(ctx.Predictor.MamlArgs);
}
// If CV, do not run the CV in multiple threads.
if (ctx.Command == Cmd.CV)
{
args.Add("threads-");
}
if (ctx.ExtraArgs != null)
{
foreach (string arg in ctx.ExtraArgs)
{
args.Add(arg);
}
}
AddIfNotEmpty(args, ctx.Predictor.Scorer, "scorer");
if (ctx.Command != Cmd.Test)
{
AddIfNotEmpty(args, ctx.Predictor.Tester, "eval");
}
else
{
AddIfNotEmpty(args, ctx.ModelOverride.Path, "in");
}
string runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));
Log(" Running as: {0} {1}", ctx.Command, runcmd);
int res;
if (basePrefix == null)
{
// Not capturing into a specific log.
Log("*** Start raw predictor output");
res = MainForTest(Env, LogWriter, string.Join(" ", ctx.Command, runcmd), ctx.BaselineProgress);
Log("*** End raw predictor output, return={0}", res);
return;
}
var consOutPath = ctx.StdoutPath();
TestCore(ctx, ctx.Command.ToString(), runcmd);
bool matched = consOutPath.CheckEqualityNormalized();
if (modelPath != null && (ctx.Summary || ctx.SaveAsIni))
{
// Save the predictor summary and compare it to baseline.
string str = string.Format("SavePredictorAs in={{{0}}}", modelPath.Path);
List <string> files = new List <string>();
if (ctx.Summary)
{
var summaryName = basePrefix + "-summary.txt";
files.Add(summaryName);
var summaryPath = DeleteOutputPath(dir, summaryName);
str += string.Format(" sum={{{0}}}", summaryPath);
Log(" Saving summary with: {0}", str);
}
if (ctx.SaveAsIni)
{
var iniName = basePrefix + ".ini";
files.Add(iniName);
var iniPath = DeleteOutputPath(dir, iniName);
str += string.Format(" ini={{{0}}}", iniPath);
Log(" Saving ini file: {0}", str);
}
MainForTest(Env, LogWriter, str);
files.ForEach(file => CheckEqualityNormalized(dir, file));
}
if (ctx.Command == Cmd.Train || ctx.Command == Cmd.Test || ctx.ExpectedToFail)
{
return;
}
// ResultProcessor output
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // -rp.txt files are not getting generated for Non-Windows Os
{
string rpName = basePrefix + "-rp.txt";
string rpOutPath = DeleteOutputPath(dir, rpName);
string[] rpArgs = null;
if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+")))
{
rpArgs = new string[] { "opf+" }
}
;
// Run result processor on the console output.
RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs);
CheckEqualityNormalized(dir, rpName);
}
// Check the prediction output against its baseline.
Contracts.Assert(predOutPath != null);
predOutPath.CheckEquality();
if (ctx.Command == Cmd.TrainTest)
{
// Adjust the args so that we no longer have the loader and transform
// arguments in there.
if (loaderArgIndex >= 0)
{
args.RemoveAt(loaderArgIndex);
}
bool foundOut = false;
List <int> toRemove = new List <int>();
HashSet <string> removeArgs = new HashSet <string>();
removeArgs.Add("tr=");
removeArgs.Add("data=");
removeArgs.Add("valid=");
removeArgs.Add("norm=");
removeArgs.Add("cali=");
removeArgs.Add("numcali=");
removeArgs.Add("xf=");
removeArgs.Add("cache-");
removeArgs.Add("sf=");
for (int i = 0; i < args.Count; ++i)
{
if (string.IsNullOrWhiteSpace(args[i]))
{
continue;
}
if (removeArgs.Any(x => args[i].StartsWith(x)))
{
toRemove.Add(i);
}
if (args[i].StartsWith("out="))
{
args[i] = string.Format("in={0}", args[i].Substring(4));
}
if (args[i].StartsWith("test="))
{
args[i] = string.Format("data={0}", args[i].Substring(5));
}
foundOut = true;
}
Contracts.Assert(foundOut);
toRemove.Reverse();
foreach (int i in toRemove)
{
args.RemoveAt(i);
}
runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));
// Redirect output to the individual log and run the test.
var ctx2 = ctx.TestCtx();
OutputPath consOutPath2 = ctx2.StdoutPath();
TestCore(ctx2, "Test", runcmd);
if (CheckTestOutputMatchesTrainTest(consOutPath.Path, consOutPath2.Path, 1))
{
File.Delete(consOutPath2.Path);
}
else if (matched)
{
// The TrainTest output matched the baseline, but the SaveLoadTest output did not, so
// append some stuff to the .txt output so comparing output to baselines in BeyondCompare
// will show the issue.
using (var writer = OpenWriter(consOutPath.Path, true))
{
writer.WriteLine("*** Unit Test Failure! ***");
writer.WriteLine("Loaded predictor test results differ! Compare baseline with {0}", consOutPath2.Path);
writer.WriteLine("*** Unit Test Failure! ***");
}
}
// REVIEW: There is nothing analogous to the old predictor output comparison here.
// The MAML command does not "export" the result of its training programmatically, that would
// allow us to compare it to the loaded model. To verify that the result of the trained model
// is the same as its programmatic
}
}
