/// <summary>
/// Run INI test for a pair of predictor and dataset.
/// </summary>
/// <param name="debugInformation"></param>
/// <param name="predictor"></param>
/// <param name="dataset"></param>
/// <param name="evaluationOutputDirPrefix"></param>
/// <param name="extraSettings"></param>
/// <param name="extraTag"></param>
public void RunIniFileEvaluationTest(
List <IniModelTestInformation> successTestInformation,
List <IniModelTestInformation> failureTestInformation,
PredictorAndArgs predictor,
TestDataset dataset,
string evaluationOutputDirPrefix,
string[] extraSettings = null,
string extraTag = ""
)
{
string outName = ExpectedFilename("Train", predictor, dataset, extraTag);
string[] extraTrainingSettings = JoinOptions(GetInstancesSettings(dataset), extraSettings);
string trainDataset = dataset.testFilename;
InternalLearnRunParameters runParameters = TrainForIniModel(
predictor,
trainDataset,
outName,
extraTrainingSettings,
ModelType.ModelKind.Ini);
CheckEqualityNormalized(runParameters.BaselineDir, runParameters.ModelFilename);
string modelFilePath = GetOutputPath(runParameters.BaselineDir, runParameters.ModelFilename);
string trainDatasetPath = GetDataPath(trainDataset);
string evaluationOutputDir = GetOutputDir(evaluationOutputDirPrefix + @"\Dirs\" + outName);
Assert.IsNull(EnsureEmptyDirectory(evaluationOutputDir));
string cmd = string.Format(EvaluationCommandLineFormat, modelFilePath, evaluationOutputDir, trainDatasetPath);
string dir = Path.GetFullPath(EvaluationExecutorDir);
Log("Working directory for evaluation: {0}", dir);
Log("Evaluation command line: {0}", cmd);
ProcessDebugInformation processDebugInformation = RunCommandLine(cmd, dir);
if (processDebugInformation.ExitCode == 0)
{
KeyValuePair <Exception, List <string> > baselineCheckDebugInformation =
DirectoryBaselineCheck(evaluationOutputDir);
IniModelTestInformation iniModelTestInformation =
new IniModelTestInformation(modelFilePath, trainDatasetPath, evaluationOutputDir, cmd, runParameters, processDebugInformation, baselineCheckDebugInformation);
if (baselineCheckDebugInformation.Key == null)
{
successTestInformation.Add(iniModelTestInformation);
}
else
{
failureTestInformation.Add(iniModelTestInformation);
}
}
else
{
IniModelTestInformation iniModelTestInformation =
new IniModelTestInformation(modelFilePath, trainDatasetPath, evaluationOutputDir, cmd, runParameters, processDebugInformation, new KeyValuePair <Exception, List <string> >(null, null));
failureTestInformation.Add(iniModelTestInformation);
}
}
public IniModelTestInformation(
string modelFilePath,
string trainDatasetPath,
string evaluationOutputDir,
string evaluationCommandLine,
InternalLearnRunParameters runParameters,
ProcessDebugInformation processDebugInformation,
KeyValuePair <Exception, List <string> > baselineDebugInformation
)
{
this.ModelFilePath = modelFilePath;
this.TrainDatasetPath = trainDatasetPath;
this.EvaluationOutputDir = evaluationOutputDir;
this.EvaluationCommandLine = evaluationCommandLine;
this.RunParameters = runParameters;
this.ProcessDebugInformation = processDebugInformation;
this.BaselineDebugInformation = baselineDebugInformation;
}
/// <summary>
/// Run a train unit test
/// </summary>
public InternalLearnRunParameters TrainForIniModel(
PredictorAndArgs predictor,
string trainDataset,
string outName,
string[] extra = null,
ModelType.ModelKind modelKind = ModelType.ModelKind.Model)
{
InternalLearnRunParameters runParams = new InternalLearnRunParameters
{
Command = "Train",
Trainer = predictor.Trainer,
Dataset = trainDataset,
BaselineDir = IniSubDirectory,
BaselineFilename = outName,
ModelFilename = outName + ModelType.GetModelType(modelKind).ModelExtension,
ModelType = ModelType.GetModelType(modelKind),
extraArgs = extra,
NoTest = true,
};
RunPredictor(runParams);
return(runParams);
}
