public void TestExecuteExperiment_Decision() { MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("graph_decision.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); }
public void TestExecuteExperimentErrorCannotInitializeComponent() { MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("basic_graph_with_non_existing_component.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsTrue(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerErrorMessage, progress.CurrentStatus); }
public void TestExecuteExperimentDecisionConnectsToTwoNodesWithSameLabel() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("graph_decision_bug_56.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsTrue(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerErrorMessage, progress.CurrentStatus); }
public void TestTerminateExperiment() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment currentExperiment = LoadExperiment("infinite_loop_terminate_experiment_test.teml"); RunExperimentAndTerminate(currentExperiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsTrue(progress.HasError); Assert.AreEqual(Messages.ExperimentExecutionTerminated, progress.CurrentStatus); }
public void TestExecuteExperimentWithCompositeComponent() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("experiment_with_composite_component.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); // the composite component defined that // 'test1' is outputted as test_x // 'test2' was not visible in top component, so it should stay with name including composite component node id // 'test3' was not visible in top component, so it should stay with name including composite component node id // 'test4' is outputted as test_y int test_x = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_x"); Assert.AreEqual(10, test_x); int test_y = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_y"); Assert.AreEqual(6, test_y); string compositeComponentNodeId = "46b30cf2-28e8-40ef-8fae-797255c09dc3"; object test_1 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test1"); Assert.IsNull(test_1); //assure that test_1 is not in workspace, because it should have been renamed as test_x object test_2 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test2"); Assert.IsNull(test_2); object test_3 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test3"); Assert.IsNull(test_3); object test_4 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test4"); Assert.IsNull(test_4); //assure that test_1 is not in workspace, because it should have been renamed as test_y }
private void RunExperimentAndTerminate(Experiment experiment, MockProgress progress, Workspace workspace, ComponentsLibrary library) { using (var waiter = new System.Threading.ManualResetEventSlim()) { experiment.RunExperiment(progress, workspace, library); experiment.ExperimentStarted += (sender, args) => { experiment.StopRunningExperiment(); }; experiment.ExperimentCompleted += (sender, args) => { waiter.Set(); }; waiter.Wait(); } }
public void TestExecuteExperimentDecisionGraphTwoNodesComingToDecision() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("graph_loop_two_nodes_coming_to_decision_bug_72.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); int test_a = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_a"); Assert.AreEqual(9, test_a); }
public void TestExecuteExperimentWithLoopNode() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("LoopExperimentTest.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); int integer = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".integer"); Assert.AreEqual(5, integer); }
public void TestExecuteExperimentBasicGraph2() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("basic_graph_2.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); //the test value after experiment is expected to be equal 6 int test = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".testoutput"); Assert.AreEqual(test, 6); }
public void TestExecuteExperimentComplexLoopWithManyOutGoingPaths() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("graph_complex_loop_with_many_outgoing_paths.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); //the test value after experiment is expected to be equal 7 int test = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test"); //all values are prefixed with experiment id Assert.AreEqual(test, 7); }
public void TestExecuteExperimentLogs() { var target = new MockLogTarget(); var logConfig = new NLog.Config.LoggingConfiguration(); logConfig.AddTarget("MockLogger", target); NLog.LogManager.Configuration = logConfig; MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("basic_graph.teml"); using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(experiment, AppContext.WorkspaceInstance, AppContext.Components)) { dispatcher.ExecuteExperiment(progress); } // only two nodes log, because the other two are start/end Assert.AreEqual(2, target.NumEvents); }
public void TestRunTwoExperimentsSimultanouslyAndTerminateOneExperiment() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); var done = new System.Threading.CountdownEvent(2); //start experiment one (takes more than 2 seconds to finish successfully) Experiment experimentOne = LoadExperiment("long-experiment.teml"); MockProgress progressOne = new MockProgress(); experimentOne.RunExperiment(progressOne, AppContext.WorkspaceInstance, AppContext.Components); experimentOne.ExperimentCompleted += (sender, args) => { done.Signal(); }; //start experiment two and terminate it MockProgress progressTwo = new MockProgress(); Experiment experimentTwo = LoadExperiment("infinite_loop_terminate_experiment_test.teml"); experimentTwo.RunExperiment(progressTwo, AppContext.WorkspaceInstance, AppContext.Components); experimentTwo.ExperimentStarted += (sender, args) => { experimentTwo.StopRunningExperiment(); }; experimentTwo.ExperimentCompleted += (sender, args) => { done.Signal(); }; //wait for both experiment to end done.Wait(5000); //the experiment one should have ended successfully (it SHOULD NOT have been terminated) Assert.IsFalse(progressOne.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progressOne.CurrentStatus); //while exepriment two should be terminated Assert.IsTrue(progressTwo.HasError); Assert.AreEqual(Messages.ExperimentExecutionTerminated, progressTwo.CurrentStatus); }
public void TestExecuteExperimentDecisionGraphBug71() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("graph_decision_bug_71.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); //all counters should be equal 3 int counter1 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter1"); int counter2 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter2"); int counter3 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter3"); Assert.AreEqual(3, counter1); Assert.AreEqual(3, counter2); Assert.AreEqual(3, counter3); }
public void TestTemplateGraphDisposal_ValidNodes() { IEditableExperiment experiment = ((IEditableExperiment)ExperimentManager.New()); experiment.ExperimentInfo.FilePath = "C:\\somefakelocation\\mockExperiment.teml"; //construct some simple m_experiment ExperimentNode node1 = experiment.AddComponentFromDefinition(m_emptyComponentMetaDefinition, 5, 5); ExperimentNode node2 = experiment.AddComponentFromDefinition(m_emptyComponentMetaDefinition, 5, 5); experiment.AddConnection(experiment.StartNode, node1); experiment.AddConnection(node1, node2); experiment.AddConnection(node2, experiment.EndNode); //initiate mockNodesFactory MockNodesFactory mockNodesFactory = new MockNodesFactory(); RunnableExperimentBase template = GraphAdapter.Adapt(experiment, mockNodesFactory, AppContext.Components, AppContext.WorkspaceInstance.TypeDirectories); foreach (MockNode node in mockNodesFactory.CreatedNodes) { Assert.IsFalse(node.disposed); } //execute disptacher, because in case of correct graph dispatcher disposes nodes MockProgress progress = new MockProgress(); using (var dispatcher = ExperimentRunnerFactory.CreateExperimentRunner(template)) { dispatcher.ExecuteExperiment(progress); Assert.IsFalse(progress.HasError); } foreach (MockNode node in mockNodesFactory.CreatedNodes) { Assert.IsTrue(node.disposed); } }
public void TestTerminateExperimentWithCompositeComponents() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment currentExperiment = LoadExperiment("infinite_loop_terminate_experiment_test.teml"); RunExperimentAndTerminate(currentExperiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsTrue(progress.HasError); Assert.AreEqual(Messages.ExperimentExecutionTerminated, progress.CurrentStatus); RunExperimentAndTerminate(currentExperiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsTrue(progress.HasError); Assert.AreEqual(Messages.ExperimentExecutionTerminated, progress.CurrentStatus); }
public void DefiningBenchmarkTest() { Assert.Fail("Test temporarily broken. Ignored till contest feature is going to be revisited."); string baseExperimentFilename = "DefiningBenchmarkTestExperiment.teml"; string testingSolutionFilename = "DefiningBenchmarkTestingSolution.teml"; //create temporary directory for defining benchmark string benchmarkTemporaryDirectory = System.IO.Path.Combine(AppContext.BaseTestDirectory, "DefiningBenchmarkTest"); System.IO.Directory.CreateDirectory(benchmarkTemporaryDirectory); string newBenchmarkFilePath = System.IO.Path.Combine(benchmarkTemporaryDirectory, "newDefinedBenchmark.tbml"); //copy the test data into temporary benchmark directory string testData = System.IO.Path.Combine(AppContext.BaseTestDirectory, "DefiningBenchmarkTestData.xml"); System.IO.File.Copy(testData, System.IO.Path.Combine(benchmarkTemporaryDirectory, "DefiningBenchmarkTestData.xml")); // load the experiment from which the benchmark is going to be defined from string baseExperimentFilePath = System.IO.Path.Combine(AppContext.BaseTestDirectory, baseExperimentFilename); Experiment baseExperimentForDefiningBenchmark = ExperimentManager.Load(baseExperimentFilePath, AppContext.Components); var benchmarkDefiner = new DefiningBenchmark(baseExperimentForDefiningBenchmark, AppContext.Components, AppContext.WorkspaceInstance, AppContext.PackageManager, AppContext.WorkspaceInstance.TypeDirectories, null); Assert.AreEqual(1, benchmarkDefiner.TemplatizableComponents.Count); Assert.AreEqual("Preprocessor", benchmarkDefiner.TemplatizableComponents[0].Data.Metadata.Label); //select preprocessor template as Component Template for benchmarking benchmarkDefiner.SelectedTemplateNode = benchmarkDefiner.TemplatizableComponents[0]; //select new benchmark path benchmarkDefiner.BenchmarkInfo.FilePath = newBenchmarkFilePath; //set some values for benchmark info string benchmarkName = "Testing defining new benchmark"; string author = "Re test author"; string contributors = "Re test contributors"; string description = "Re test description"; string shortDescription = "Re test short description"; DateTime deadline = DateTime.Now; string fakeExperimentResultsUnitname = "fakeunitname"; string webpageLink = "test://test.webpage.link"; benchmarkDefiner.BenchmarkInfo.Name = benchmarkName; benchmarkDefiner.BenchmarkInfo.Author = author; benchmarkDefiner.BenchmarkInfo.Contributors = contributors; benchmarkDefiner.BenchmarkInfo.Description = description; benchmarkDefiner.BenchmarkInfo.ShortDescription = shortDescription; benchmarkDefiner.BenchmarkInfo.Deadline = deadline; benchmarkDefiner.BenchmarkInfo.ExperimentResultsUnitname = fakeExperimentResultsUnitname; benchmarkDefiner.BenchmarkInfo.WebPageLink = new Uri(webpageLink); //assure file does not exists prior defining Assert.IsFalse(System.IO.File.Exists(benchmarkDefiner.BenchmarkInfo.FilePath)); //set some mock experiment results as baseline TraceLabSDK.Types.Contests.TLExperimentResults fakeBaseline = CreateDummyExperimentResults("FAKE-BASELINE"); benchmarkDefiner.SelectedExperimentResults = fakeBaseline; //call define benchmark benchmarkDefiner.Define(); //check if new benchmark has been created Assert.IsTrue(System.IO.File.Exists(benchmarkDefiner.BenchmarkInfo.FilePath)); //load newly defined benchmark List<Benchmark> benchmarks = BenchmarkLoader.LoadBenchmarksInfo(benchmarkTemporaryDirectory); Benchmark testBenchmark = benchmarks[0]; //there should be only 1, since the directory has been just created //check if new test benchmark has previously defined properties Assert.AreEqual(benchmarkName, testBenchmark.BenchmarkInfo.Name); Assert.AreEqual(author, testBenchmark.BenchmarkInfo.Author); Assert.AreEqual(contributors, testBenchmark.BenchmarkInfo.Contributors); Assert.AreEqual(description, testBenchmark.BenchmarkInfo.Description); Assert.AreEqual(shortDescription, testBenchmark.BenchmarkInfo.ShortDescription); Assert.AreEqual(deadline.ToString(), testBenchmark.BenchmarkInfo.Deadline.ToString()); Assert.AreEqual(fakeExperimentResultsUnitname, testBenchmark.BenchmarkInfo.ExperimentResultsUnitname); //check if baseline results has been saved properly, by loading it from xml TraceLabSDK.Types.Contests.TLExperimentResults baseline = BenchmarkLoader.ReadBaseline(benchmarkDefiner.BenchmarkInfo.FilePath); Assert.AreEqual(fakeBaseline.TechniqueName, baseline.TechniqueName); Assert.AreEqual(fakeBaseline.Score, baseline.Score); Assert.AreEqual(fakeBaseline.AcrossAllDatasetsResults, baseline.AcrossAllDatasetsResults); Assert.IsTrue(fakeBaseline.DatasetsResults.SequenceEqual(baseline.DatasetsResults)); // load the experiment to be run against new defined benchmark string experimentFilename = System.IO.Path.Combine(AppContext.BaseTestDirectory, testingSolutionFilename); Experiment testingSolutionExperiment = ExperimentManager.Load(experimentFilename, AppContext.Components); //finally prepare benchmark experiment testBenchmark.PrepareBenchmarkExperiment(testingSolutionExperiment, AppContext.Components); //run benchmark MockProgress progress = new MockProgress(); using (var dispatcher = CreateExperiment(testBenchmark.BenchmarkExperiment, AppContext.WorkspaceInstance, AppContext.Components)) { dispatcher.ExecuteExperiment(progress); Assert.AreEqual(5, progress.NumSteps); Assert.IsFalse(progress.HasError); } }
public void TestExecuteExperimentComplexLoopWithManyOutGoingPaths() { //assure workspace is empty AppContext.WorkspaceInstance.Clear(); MockProgress progress = new MockProgress(); Experiment experiment = LoadExperiment("graph_complex_loop_with_many_outgoing_paths.teml"); RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components); Assert.IsFalse(progress.HasError); Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus); //the test value after experiment is expected to be equal 7 int test = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id+".test"); //all values are prefixed with experiment id Assert.AreEqual(test, 7); }
public void ScopeWithCompositeComponent_ScopeNestedWorkspaceWrapperTearDownTest() { MockProgress progress = new MockProgress(); string testExperimentFilepath = System.IO.Path.Combine(AppContext.BaseTestDirectory, "Decision Scope with composite component.teml"); Experiment experiment = ExperimentManager.Load(testExperimentFilepath, AppContext.Components); using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(experiment, AppContext.WorkspaceInstance, AppContext.Components)) { dispatcher.ExecuteExperiment(progress); } Assert.AreEqual(3, AppContext.WorkspaceInstance.Units.Count); HashSet<string> expectedUnitNames = new HashSet<string>(); expectedUnitNames.Add("targetArtifacts"); expectedUnitNames.Add("sourceArtifacts"); expectedUnitNames.Add("similarityMatrix"); foreach (WorkspaceUnit unit in AppContext.WorkspaceInstance.Units) { bool isCorrect = false; if (expectedUnitNames.Contains(unit.FriendlyUnitName)) isCorrect = true; Assert.IsTrue(isCorrect); } }
public void NestedWorkspaceWrapperTest_Bug180() { MockProgress progress = new MockProgress(); Experiment experiment = ExperimentManager.Load("compositecomponentbug180.teml", AppContext.Components); using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(experiment, AppContext.WorkspaceInstance, AppContext.Components)) { dispatcher.ExecuteExperiment(progress); } Assert.AreEqual(3, AppContext.WorkspaceInstance.Units.Count); foreach (WorkspaceUnit unit in AppContext.WorkspaceInstance.Units) { bool isCorrect = false; if(unit.FriendlyUnitName.Equals("a") && (int)unit.Data == 10) isCorrect = true; else if(unit.FriendlyUnitName.Equals("b") && (int)unit.Data == 5) isCorrect = true; else if(unit.FriendlyUnitName.Equals("y") && (int)unit.Data == 6) isCorrect = true; Assert.IsTrue(isCorrect); } }
public void PrepareAndRunBenchmarkExperiment() { Assert.Fail("Test temporarily broken. Ignored till contest feature is going to be revisited."); List<Benchmark> benchmarks = BenchmarkLoader.LoadBenchmarksInfo(BenchmarkDirectory); Benchmark testBenchmark = benchmarks[0]; // load the experiment to be run against benchmark string experimentFilename = System.IO.Path.Combine(AppContext.BaseTestDirectory, "experiment_to_be_benchmarked.gml"); Experiment experimentToBeBenchmarked = ExperimentManager.Load(experimentFilename, AppContext.Components); //prepare matching io testBenchmark.PrepareMatchingIOByType(experimentToBeBenchmarked); Assert.AreEqual(2, testBenchmark.BenchmarkInputSetting.Count); Assert.AreEqual(1, testBenchmark.BenchmarkOutputsSetting.Count); //match benchmarkSourceArtifact with original source artifacts foreach (BenchmarkItemSetting<IOItem> pair in testBenchmark.BenchmarkInputSetting) { IOItem item = pair.Item; ItemSettingCollection candidates = pair.CandidateSettings; if (item.MappedTo.Equals("benchmarkSourceArtifacts")) { //we found the item we want to remap pair.SelectedSetting = candidates["originalSourceArtifacts"]; } } //finally prepare benchmark experiment testBenchmark.PrepareBenchmarkExperiment(experimentToBeBenchmarked, AppContext.Components); //assert that only two inputs are included in the export settings and one output int includedInputs = 0; foreach (KeyValuePair<string, ItemSetting> pair in testBenchmark.Setup.InputSettings) { if (pair.Value.Include == true) includedInputs++; } Assert.AreEqual(2, includedInputs); int includedOutputs = 0; foreach (KeyValuePair<string, ItemSetting> pair in testBenchmark.Setup.OutputSettings) { if (pair.Value.Include == true) includedOutputs++; } Assert.AreEqual(1, includedOutputs); Assert.IsNotNull(testBenchmark.BenchmarkExperiment); //for debug output file // string path = System.IO.Path.Combine(AppContext.BaseTestDirectory, "benchmarkTest1.gml"); // AppContext.ExperimentManager.Save(testBenchmark.BenchmarkExperiment, path); MockProgress progress = new MockProgress(); using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(testBenchmark.BenchmarkExperiment, AppContext.WorkspaceInstance, AppContext.Components)) { dispatcher.ExecuteExperiment(progress); Assert.AreEqual(7, progress.NumSteps); Assert.IsFalse(progress.HasError); } }