/// <summary>
/// Run a command-line script.
/// </summary>
/// <param name="commandLine"></param>
public ProcessDebugInformation RunCommandLine(string commandLine, string dir)
{
// Create the process.
var proc = new System.Diagnostics.Process();
proc.StartInfo.UseShellExecute = false;
proc.StartInfo.FileName = "cmd.exe";
proc.StartInfo.CreateNoWindow = true;
proc.StartInfo.Arguments = "/C" + " " + commandLine;
proc.StartInfo.RedirectStandardOutput = true;
proc.StartInfo.RedirectStandardError = true;
proc.StartInfo.WorkingDirectory = dir;
// Run the process.
proc.Start();
string standardOutput = proc.StandardOutput.ReadToEnd();
string standardError = proc.StandardError.ReadToEnd();
Log("---- execution standard output: " + standardOutput);
Log("---- execution standard error: " + standardError);
// Wait for the process to finish.
proc.WaitForExit();
ProcessDebugInformation info = new ProcessDebugInformation(proc.ExitCode, standardError, standardOutput);
proc.Close();
return(info);
}
/// <summary>
/// Run INI test for a pair of predictor and dataset.
/// </summary>
/// <param name="debugInformation"></param>
/// <param name="predictor"></param>
/// <param name="dataset"></param>
/// <param name="evaluationOutputDirPrefix"></param>
/// <param name="extraSettings"></param>
/// <param name="extraTag"></param>
public void RunIniFileEvaluationTest(
List <IniModelTestInformation> successTestInformation,
List <IniModelTestInformation> failureTestInformation,
PredictorAndArgs predictor,
TestDataset dataset,
string evaluationOutputDirPrefix,
string[] extraSettings = null,
string extraTag = ""
)
{
string outName = ExpectedFilename("Train", predictor, dataset, extraTag);
string[] extraTrainingSettings = JoinOptions(GetInstancesSettings(dataset), extraSettings);
string trainDataset = dataset.testFilename;
InternalLearnRunParameters runParameters = TrainForIniModel(
predictor,
trainDataset,
outName,
extraTrainingSettings,
ModelType.ModelKind.Ini);
CheckEqualityNormalized(runParameters.BaselineDir, runParameters.ModelFilename);
string modelFilePath = GetOutputPath(runParameters.BaselineDir, runParameters.ModelFilename);
string trainDatasetPath = GetDataPath(trainDataset);
string evaluationOutputDir = GetOutputDir(evaluationOutputDirPrefix + @"\Dirs\" + outName);
Assert.IsNull(EnsureEmptyDirectory(evaluationOutputDir));
string cmd = string.Format(EvaluationCommandLineFormat, modelFilePath, evaluationOutputDir, trainDatasetPath);
string dir = Path.GetFullPath(EvaluationExecutorDir);
Log("Working directory for evaluation: {0}", dir);
Log("Evaluation command line: {0}", cmd);
ProcessDebugInformation processDebugInformation = RunCommandLine(cmd, dir);
if (processDebugInformation.ExitCode == 0)
{
KeyValuePair <Exception, List <string> > baselineCheckDebugInformation =
DirectoryBaselineCheck(evaluationOutputDir);
IniModelTestInformation iniModelTestInformation =
new IniModelTestInformation(modelFilePath, trainDatasetPath, evaluationOutputDir, cmd, runParameters, processDebugInformation, baselineCheckDebugInformation);
if (baselineCheckDebugInformation.Key == null)
{
successTestInformation.Add(iniModelTestInformation);
}
else
{
failureTestInformation.Add(iniModelTestInformation);
}
}
else
{
IniModelTestInformation iniModelTestInformation =
new IniModelTestInformation(modelFilePath, trainDatasetPath, evaluationOutputDir, cmd, runParameters, processDebugInformation, new KeyValuePair <Exception, List <string> >(null, null));
failureTestInformation.Add(iniModelTestInformation);
}
}
public IniModelTestInformation(
string modelFilePath,
string trainDatasetPath,
string evaluationOutputDir,
string evaluationCommandLine,
InternalLearnRunParameters runParameters,
ProcessDebugInformation processDebugInformation,
KeyValuePair <Exception, List <string> > baselineDebugInformation
)
{
this.ModelFilePath = modelFilePath;
this.TrainDatasetPath = trainDatasetPath;
this.EvaluationOutputDir = evaluationOutputDir;
this.EvaluationCommandLine = evaluationCommandLine;
this.RunParameters = runParameters;
this.ProcessDebugInformation = processDebugInformation;
this.BaselineDebugInformation = baselineDebugInformation;
}
