private void ExecuteDefine(object param)
{
var dialog = param as TraceLab.UI.WPF.Views.DefineBenchmarkDialog;
if (dialog != null)
{
if (PublishContest == true)
{
//continue to next screen - authenticate
CurrentState = DefiningBenchmarkDialogState.Authentication;
}
else
{
//otherwise simply define benchmark, save it on disk and close dialog
m_definingBenchmark.Define();
dialog.Close();
}
}
}
public void DefiningBenchmarkTest()
{
Assert.Fail("Test temporarily broken. Ignored till contest feature is going to be revisited.");
string baseExperimentFilename = "DefiningBenchmarkTestExperiment.teml";
string testingSolutionFilename = "DefiningBenchmarkTestingSolution.teml";
//create temporary directory for defining benchmark
string benchmarkTemporaryDirectory = System.IO.Path.Combine(AppContext.BaseTestDirectory, "DefiningBenchmarkTest");
System.IO.Directory.CreateDirectory(benchmarkTemporaryDirectory);
string newBenchmarkFilePath = System.IO.Path.Combine(benchmarkTemporaryDirectory, "newDefinedBenchmark.tbml");
//copy the test data into temporary benchmark directory
string testData = System.IO.Path.Combine(AppContext.BaseTestDirectory, "DefiningBenchmarkTestData.xml");
System.IO.File.Copy(testData, System.IO.Path.Combine(benchmarkTemporaryDirectory, "DefiningBenchmarkTestData.xml"));
// load the experiment from which the benchmark is going to be defined from
string baseExperimentFilePath = System.IO.Path.Combine(AppContext.BaseTestDirectory, baseExperimentFilename);
Experiment baseExperimentForDefiningBenchmark = ExperimentManager.Load(baseExperimentFilePath, AppContext.Components);
var benchmarkDefiner = new DefiningBenchmark(baseExperimentForDefiningBenchmark, AppContext.Components, AppContext.WorkspaceInstance, AppContext.PackageManager, AppContext.WorkspaceInstance.TypeDirectories, null);
Assert.AreEqual(1, benchmarkDefiner.TemplatizableComponents.Count);
Assert.AreEqual("Preprocessor", benchmarkDefiner.TemplatizableComponents[0].Data.Metadata.Label);
//select preprocessor template as Component Template for benchmarking
benchmarkDefiner.SelectedTemplateNode = benchmarkDefiner.TemplatizableComponents[0];
//select new benchmark path
benchmarkDefiner.BenchmarkInfo.FilePath = newBenchmarkFilePath;
//set some values for benchmark info
string benchmarkName = "Testing defining new benchmark";
string author = "Re test author";
string contributors = "Re test contributors";
string description = "Re test description";
string shortDescription = "Re test short description";
DateTime deadline = DateTime.Now;
string fakeExperimentResultsUnitname = "fakeunitname";
string webpageLink = "test://test.webpage.link";
benchmarkDefiner.BenchmarkInfo.Name = benchmarkName;
benchmarkDefiner.BenchmarkInfo.Author = author;
benchmarkDefiner.BenchmarkInfo.Contributors = contributors;
benchmarkDefiner.BenchmarkInfo.Description = description;
benchmarkDefiner.BenchmarkInfo.ShortDescription = shortDescription;
benchmarkDefiner.BenchmarkInfo.Deadline = deadline;
benchmarkDefiner.BenchmarkInfo.ExperimentResultsUnitname = fakeExperimentResultsUnitname;
benchmarkDefiner.BenchmarkInfo.WebPageLink = new Uri(webpageLink);
//assure file does not exists prior defining
Assert.IsFalse(System.IO.File.Exists(benchmarkDefiner.BenchmarkInfo.FilePath));
//set some mock experiment results as baseline
TraceLabSDK.Types.Contests.TLExperimentResults fakeBaseline = CreateDummyExperimentResults("FAKE-BASELINE");
benchmarkDefiner.SelectedExperimentResults = fakeBaseline;
//call define benchmark
benchmarkDefiner.Define();
//check if new benchmark has been created
Assert.IsTrue(System.IO.File.Exists(benchmarkDefiner.BenchmarkInfo.FilePath));
//load newly defined benchmark
List <Benchmark> benchmarks = BenchmarkLoader.LoadBenchmarksInfo(benchmarkTemporaryDirectory);
Benchmark testBenchmark = benchmarks[0]; //there should be only 1, since the directory has been just created
//check if new test benchmark has previously defined properties
Assert.AreEqual(benchmarkName, testBenchmark.BenchmarkInfo.Name);
Assert.AreEqual(author, testBenchmark.BenchmarkInfo.Author);
Assert.AreEqual(contributors, testBenchmark.BenchmarkInfo.Contributors);
Assert.AreEqual(description, testBenchmark.BenchmarkInfo.Description);
Assert.AreEqual(shortDescription, testBenchmark.BenchmarkInfo.ShortDescription);
Assert.AreEqual(deadline.ToString(), testBenchmark.BenchmarkInfo.Deadline.ToString());
Assert.AreEqual(fakeExperimentResultsUnitname, testBenchmark.BenchmarkInfo.ExperimentResultsUnitname);
//check if baseline results has been saved properly, by loading it from xml
TraceLabSDK.Types.Contests.TLExperimentResults baseline = BenchmarkLoader.ReadBaseline(benchmarkDefiner.BenchmarkInfo.FilePath);
Assert.AreEqual(fakeBaseline.TechniqueName, baseline.TechniqueName);
Assert.AreEqual(fakeBaseline.Score, baseline.Score);
Assert.AreEqual(fakeBaseline.AcrossAllDatasetsResults, baseline.AcrossAllDatasetsResults);
Assert.IsTrue(fakeBaseline.DatasetsResults.SequenceEqual(baseline.DatasetsResults));
// load the experiment to be run against new defined benchmark
string experimentFilename = System.IO.Path.Combine(AppContext.BaseTestDirectory, testingSolutionFilename);
Experiment testingSolutionExperiment = ExperimentManager.Load(experimentFilename, AppContext.Components);
//finally prepare benchmark experiment
testBenchmark.PrepareBenchmarkExperiment(testingSolutionExperiment, AppContext.Components);
//run benchmark
MockProgress progress = new MockProgress();
using (var dispatcher = CreateExperiment(testBenchmark.BenchmarkExperiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
Assert.AreEqual(5, progress.NumSteps);
Assert.IsFalse(progress.HasError);
}
}