public void TestExecuteExperiment_Decision()
{
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_decision.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
}
public void TestExecuteExperimentErrorCannotInitializeComponent()
{
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("basic_graph_with_non_existing_component.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsTrue(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerErrorMessage, progress.CurrentStatus);
}
public void TestExecuteExperimentDecisionConnectsToTwoNodesWithSameLabel()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_decision_bug_56.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsTrue(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerErrorMessage, progress.CurrentStatus);
}
public void TestTerminateExperiment()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment currentExperiment = LoadExperiment("infinite_loop_terminate_experiment_test.teml");
RunExperimentAndTerminate(currentExperiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsTrue(progress.HasError);
Assert.AreEqual(Messages.ExperimentExecutionTerminated, progress.CurrentStatus);
}
public void TestExecuteExperimentWithCompositeComponent()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("experiment_with_composite_component.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
// the composite component defined that
// 'test1' is outputted as test_x
// 'test2' was not visible in top component, so it should stay with name including composite component node id
// 'test3' was not visible in top component, so it should stay with name including composite component node id
// 'test4' is outputted as test_y
int test_x = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_x");
Assert.AreEqual(10, test_x);
int test_y = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_y");
Assert.AreEqual(6, test_y);
string compositeComponentNodeId = "46b30cf2-28e8-40ef-8fae-797255c09dc3";
object test_1 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test1");
Assert.IsNull(test_1); //assure that test_1 is not in workspace, because it should have been renamed as test_x
object test_2 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test2");
Assert.IsNull(test_2);
object test_3 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test3");
Assert.IsNull(test_3);
object test_4 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test4");
Assert.IsNull(test_4); //assure that test_1 is not in workspace, because it should have been renamed as test_y
}
private void RunExperimentAndTerminate(Experiment experiment, MockProgress progress, Workspace workspace, ComponentsLibrary library)
{
using (var waiter = new System.Threading.ManualResetEventSlim())
{
experiment.RunExperiment(progress, workspace, library);
experiment.ExperimentStarted += (sender, args) =>
{
experiment.StopRunningExperiment();
};
experiment.ExperimentCompleted += (sender, args) =>
{
waiter.Set();
};
waiter.Wait();
}
}
public void TestExecuteExperimentDecisionGraphTwoNodesComingToDecision()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_loop_two_nodes_coming_to_decision_bug_72.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
int test_a = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_a");
Assert.AreEqual(9, test_a);
}
public void TestExecuteExperimentWithLoopNode()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("LoopExperimentTest.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
int integer = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".integer");
Assert.AreEqual(5, integer);
}
public void TestExecuteExperimentBasicGraph2()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("basic_graph_2.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
//the test value after experiment is expected to be equal 6
int test = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".testoutput");
Assert.AreEqual(test, 6);
}
public void TestExecuteExperimentComplexLoopWithManyOutGoingPaths()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_complex_loop_with_many_outgoing_paths.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
//the test value after experiment is expected to be equal 7
int test = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test"); //all values are prefixed with experiment id
Assert.AreEqual(test, 7);
}
public void TestExecuteExperimentLogs()
{
var target = new MockLogTarget();
var logConfig = new NLog.Config.LoggingConfiguration();
logConfig.AddTarget("MockLogger", target);
NLog.LogManager.Configuration = logConfig;
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("basic_graph.teml");
using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(experiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
}
// only two nodes log, because the other two are start/end
Assert.AreEqual(2, target.NumEvents);
}
public void TestRunTwoExperimentsSimultanouslyAndTerminateOneExperiment()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
var done = new System.Threading.CountdownEvent(2);
//start experiment one (takes more than 2 seconds to finish successfully)
Experiment experimentOne = LoadExperiment("long-experiment.teml");
MockProgress progressOne = new MockProgress();
experimentOne.RunExperiment(progressOne, AppContext.WorkspaceInstance, AppContext.Components);
experimentOne.ExperimentCompleted += (sender, args) =>
{
done.Signal();
};
//start experiment two and terminate it
MockProgress progressTwo = new MockProgress();
Experiment experimentTwo = LoadExperiment("infinite_loop_terminate_experiment_test.teml");
experimentTwo.RunExperiment(progressTwo, AppContext.WorkspaceInstance, AppContext.Components);
experimentTwo.ExperimentStarted += (sender, args) =>
{
experimentTwo.StopRunningExperiment();
};
experimentTwo.ExperimentCompleted += (sender, args) =>
{
done.Signal();
};
//wait for both experiment to end
done.Wait(5000);
//the experiment one should have ended successfully (it SHOULD NOT have been terminated)
Assert.IsFalse(progressOne.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progressOne.CurrentStatus);
//while exepriment two should be terminated
Assert.IsTrue(progressTwo.HasError);
Assert.AreEqual(Messages.ExperimentExecutionTerminated, progressTwo.CurrentStatus);
}
public void TestExecuteExperimentDecisionGraphBug71()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_decision_bug_71.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
//all counters should be equal 3
int counter1 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter1");
int counter2 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter2");
int counter3 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter3");
Assert.AreEqual(3, counter1);
Assert.AreEqual(3, counter2);
Assert.AreEqual(3, counter3);
}
public void TestTemplateGraphDisposal_ValidNodes()
{
IEditableExperiment experiment = ((IEditableExperiment)ExperimentManager.New());
experiment.ExperimentInfo.FilePath = "C:\\somefakelocation\\mockExperiment.teml";
//construct some simple m_experiment
ExperimentNode node1 = experiment.AddComponentFromDefinition(m_emptyComponentMetaDefinition, 5, 5);
ExperimentNode node2 = experiment.AddComponentFromDefinition(m_emptyComponentMetaDefinition, 5, 5);
experiment.AddConnection(experiment.StartNode, node1);
experiment.AddConnection(node1, node2);
experiment.AddConnection(node2, experiment.EndNode);
//initiate mockNodesFactory
MockNodesFactory mockNodesFactory = new MockNodesFactory();
RunnableExperimentBase template = GraphAdapter.Adapt(experiment, mockNodesFactory, AppContext.Components, AppContext.WorkspaceInstance.TypeDirectories);
foreach (MockNode node in mockNodesFactory.CreatedNodes)
{
Assert.IsFalse(node.disposed);
}
//execute disptacher, because in case of correct graph dispatcher disposes nodes
MockProgress progress = new MockProgress();
using (var dispatcher = ExperimentRunnerFactory.CreateExperimentRunner(template))
{
dispatcher.ExecuteExperiment(progress);
Assert.IsFalse(progress.HasError);
}
foreach (MockNode node in mockNodesFactory.CreatedNodes)
{
Assert.IsTrue(node.disposed);
}
}
private void RunExperimentAndTerminate(Experiment experiment, MockProgress progress, Workspace workspace, ComponentsLibrary library)
{
using (var waiter = new System.Threading.ManualResetEventSlim())
{
experiment.RunExperiment(progress, workspace, library);
experiment.ExperimentStarted += (sender, args) =>
{
experiment.StopRunningExperiment();
};
experiment.ExperimentCompleted += (sender, args) =>
{
waiter.Set();
};
waiter.Wait();
}
}
public void TestTerminateExperimentWithCompositeComponents()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment currentExperiment = LoadExperiment("infinite_loop_terminate_experiment_test.teml");
RunExperimentAndTerminate(currentExperiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsTrue(progress.HasError);
Assert.AreEqual(Messages.ExperimentExecutionTerminated, progress.CurrentStatus);
RunExperimentAndTerminate(currentExperiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsTrue(progress.HasError);
Assert.AreEqual(Messages.ExperimentExecutionTerminated, progress.CurrentStatus);
}
public void TestRunTwoExperimentsSimultanouslyAndTerminateOneExperiment()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
var done = new System.Threading.CountdownEvent(2);
//start experiment one (takes more than 2 seconds to finish successfully)
Experiment experimentOne = LoadExperiment("long-experiment.teml");
MockProgress progressOne = new MockProgress();
experimentOne.RunExperiment(progressOne, AppContext.WorkspaceInstance, AppContext.Components);
experimentOne.ExperimentCompleted += (sender, args) =>
{
done.Signal();
};
//start experiment two and terminate it
MockProgress progressTwo = new MockProgress();
Experiment experimentTwo = LoadExperiment("infinite_loop_terminate_experiment_test.teml");
experimentTwo.RunExperiment(progressTwo, AppContext.WorkspaceInstance, AppContext.Components);
experimentTwo.ExperimentStarted += (sender, args) =>
{
experimentTwo.StopRunningExperiment();
};
experimentTwo.ExperimentCompleted += (sender, args) =>
{
done.Signal();
};
//wait for both experiment to end
done.Wait(5000);
//the experiment one should have ended successfully (it SHOULD NOT have been terminated)
Assert.IsFalse(progressOne.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progressOne.CurrentStatus);
//while exepriment two should be terminated
Assert.IsTrue(progressTwo.HasError);
Assert.AreEqual(Messages.ExperimentExecutionTerminated, progressTwo.CurrentStatus);
}
public void TestExecuteExperimentWithCompositeComponent()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("experiment_with_composite_component.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
// the composite component defined that
// 'test1' is outputted as test_x
// 'test2' was not visible in top component, so it should stay with name including composite component node id
// 'test3' was not visible in top component, so it should stay with name including composite component node id
// 'test4' is outputted as test_y
int test_x = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_x");
Assert.AreEqual(10, test_x);
int test_y = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_y");
Assert.AreEqual(6, test_y);
string compositeComponentNodeId = "46b30cf2-28e8-40ef-8fae-797255c09dc3";
object test_1 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test1");
Assert.IsNull(test_1); //assure that test_1 is not in workspace, because it should have been renamed as test_x
object test_2 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test2");
Assert.IsNull(test_2);
object test_3 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test3");
Assert.IsNull(test_3);
object test_4 = AppContext.WorkspaceInstance.Load(compositeComponentNodeId + ".test4");
Assert.IsNull(test_4); //assure that test_1 is not in workspace, because it should have been renamed as test_y
}
public void TestExecuteExperimentWithLoopNode()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("LoopExperimentTest.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
int integer = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".integer");
Assert.AreEqual(5, integer);
}
public void DefiningBenchmarkTest()
{
Assert.Fail("Test temporarily broken. Ignored till contest feature is going to be revisited.");
string baseExperimentFilename = "DefiningBenchmarkTestExperiment.teml";
string testingSolutionFilename = "DefiningBenchmarkTestingSolution.teml";
//create temporary directory for defining benchmark
string benchmarkTemporaryDirectory = System.IO.Path.Combine(AppContext.BaseTestDirectory, "DefiningBenchmarkTest");
System.IO.Directory.CreateDirectory(benchmarkTemporaryDirectory);
string newBenchmarkFilePath = System.IO.Path.Combine(benchmarkTemporaryDirectory, "newDefinedBenchmark.tbml");
//copy the test data into temporary benchmark directory
string testData = System.IO.Path.Combine(AppContext.BaseTestDirectory, "DefiningBenchmarkTestData.xml");
System.IO.File.Copy(testData, System.IO.Path.Combine(benchmarkTemporaryDirectory, "DefiningBenchmarkTestData.xml"));
// load the experiment from which the benchmark is going to be defined from
string baseExperimentFilePath = System.IO.Path.Combine(AppContext.BaseTestDirectory, baseExperimentFilename);
Experiment baseExperimentForDefiningBenchmark = ExperimentManager.Load(baseExperimentFilePath, AppContext.Components);
var benchmarkDefiner = new DefiningBenchmark(baseExperimentForDefiningBenchmark, AppContext.Components, AppContext.WorkspaceInstance, AppContext.PackageManager, AppContext.WorkspaceInstance.TypeDirectories, null);
Assert.AreEqual(1, benchmarkDefiner.TemplatizableComponents.Count);
Assert.AreEqual("Preprocessor", benchmarkDefiner.TemplatizableComponents[0].Data.Metadata.Label);
//select preprocessor template as Component Template for benchmarking
benchmarkDefiner.SelectedTemplateNode = benchmarkDefiner.TemplatizableComponents[0];
//select new benchmark path
benchmarkDefiner.BenchmarkInfo.FilePath = newBenchmarkFilePath;
//set some values for benchmark info
string benchmarkName = "Testing defining new benchmark";
string author = "Re test author";
string contributors = "Re test contributors";
string description = "Re test description";
string shortDescription = "Re test short description";
DateTime deadline = DateTime.Now;
string fakeExperimentResultsUnitname = "fakeunitname";
string webpageLink = "test://test.webpage.link";
benchmarkDefiner.BenchmarkInfo.Name = benchmarkName;
benchmarkDefiner.BenchmarkInfo.Author = author;
benchmarkDefiner.BenchmarkInfo.Contributors = contributors;
benchmarkDefiner.BenchmarkInfo.Description = description;
benchmarkDefiner.BenchmarkInfo.ShortDescription = shortDescription;
benchmarkDefiner.BenchmarkInfo.Deadline = deadline;
benchmarkDefiner.BenchmarkInfo.ExperimentResultsUnitname = fakeExperimentResultsUnitname;
benchmarkDefiner.BenchmarkInfo.WebPageLink = new Uri(webpageLink);
//assure file does not exists prior defining
Assert.IsFalse(System.IO.File.Exists(benchmarkDefiner.BenchmarkInfo.FilePath));
//set some mock experiment results as baseline
TraceLabSDK.Types.Contests.TLExperimentResults fakeBaseline = CreateDummyExperimentResults("FAKE-BASELINE");
benchmarkDefiner.SelectedExperimentResults = fakeBaseline;
//call define benchmark
benchmarkDefiner.Define();
//check if new benchmark has been created
Assert.IsTrue(System.IO.File.Exists(benchmarkDefiner.BenchmarkInfo.FilePath));
//load newly defined benchmark
List<Benchmark> benchmarks = BenchmarkLoader.LoadBenchmarksInfo(benchmarkTemporaryDirectory);
Benchmark testBenchmark = benchmarks[0]; //there should be only 1, since the directory has been just created
//check if new test benchmark has previously defined properties
Assert.AreEqual(benchmarkName, testBenchmark.BenchmarkInfo.Name);
Assert.AreEqual(author, testBenchmark.BenchmarkInfo.Author);
Assert.AreEqual(contributors, testBenchmark.BenchmarkInfo.Contributors);
Assert.AreEqual(description, testBenchmark.BenchmarkInfo.Description);
Assert.AreEqual(shortDescription, testBenchmark.BenchmarkInfo.ShortDescription);
Assert.AreEqual(deadline.ToString(), testBenchmark.BenchmarkInfo.Deadline.ToString());
Assert.AreEqual(fakeExperimentResultsUnitname, testBenchmark.BenchmarkInfo.ExperimentResultsUnitname);
//check if baseline results has been saved properly, by loading it from xml
TraceLabSDK.Types.Contests.TLExperimentResults baseline = BenchmarkLoader.ReadBaseline(benchmarkDefiner.BenchmarkInfo.FilePath);
Assert.AreEqual(fakeBaseline.TechniqueName, baseline.TechniqueName);
Assert.AreEqual(fakeBaseline.Score, baseline.Score);
Assert.AreEqual(fakeBaseline.AcrossAllDatasetsResults, baseline.AcrossAllDatasetsResults);
Assert.IsTrue(fakeBaseline.DatasetsResults.SequenceEqual(baseline.DatasetsResults));
// load the experiment to be run against new defined benchmark
string experimentFilename = System.IO.Path.Combine(AppContext.BaseTestDirectory, testingSolutionFilename);
Experiment testingSolutionExperiment = ExperimentManager.Load(experimentFilename, AppContext.Components);
//finally prepare benchmark experiment
testBenchmark.PrepareBenchmarkExperiment(testingSolutionExperiment, AppContext.Components);
//run benchmark
MockProgress progress = new MockProgress();
using (var dispatcher = CreateExperiment(testBenchmark.BenchmarkExperiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
Assert.AreEqual(5, progress.NumSteps);
Assert.IsFalse(progress.HasError);
}
}
public void TestExecuteExperimentDecisionConnectsToTwoNodesWithSameLabel()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_decision_bug_56.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsTrue(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerErrorMessage, progress.CurrentStatus);
}
public void TestExecuteExperimentBasicGraph2()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("basic_graph_2.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
//the test value after experiment is expected to be equal 6
int test = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".testoutput");
Assert.AreEqual(test, 6);
}
public void TestExecuteExperimentDecisionGraphBug71()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_decision_bug_71.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
//all counters should be equal 3
int counter1 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter1");
int counter2 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter2");
int counter3 = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".counter3");
Assert.AreEqual(3, counter1);
Assert.AreEqual(3, counter2);
Assert.AreEqual(3, counter3);
}
public void TestExecuteExperiment_Decision()
{
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_decision.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
}
public void TestExecuteExperimentComplexLoopWithManyOutGoingPaths()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_complex_loop_with_many_outgoing_paths.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
//the test value after experiment is expected to be equal 7
int test = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id+".test"); //all values are prefixed with experiment id
Assert.AreEqual(test, 7);
}
public void TestExecuteExperimentErrorCannotInitializeComponent()
{
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("basic_graph_with_non_existing_component.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsTrue(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerErrorMessage, progress.CurrentStatus);
}
public void TestExecuteExperimentLogs()
{
var target = new MockLogTarget();
var logConfig = new NLog.Config.LoggingConfiguration();
logConfig.AddTarget("MockLogger", target);
NLog.LogManager.Configuration = logConfig;
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("basic_graph.teml");
using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(experiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
}
// only two nodes log, because the other two are start/end
Assert.AreEqual(2, target.NumEvents);
}
public void ScopeWithCompositeComponent_ScopeNestedWorkspaceWrapperTearDownTest()
{
MockProgress progress = new MockProgress();
string testExperimentFilepath = System.IO.Path.Combine(AppContext.BaseTestDirectory, "Decision Scope with composite component.teml");
Experiment experiment = ExperimentManager.Load(testExperimentFilepath, AppContext.Components);
using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(experiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
}
Assert.AreEqual(3, AppContext.WorkspaceInstance.Units.Count);
HashSet<string> expectedUnitNames = new HashSet<string>();
expectedUnitNames.Add("targetArtifacts");
expectedUnitNames.Add("sourceArtifacts");
expectedUnitNames.Add("similarityMatrix");
foreach (WorkspaceUnit unit in AppContext.WorkspaceInstance.Units)
{
bool isCorrect = false;
if (expectedUnitNames.Contains(unit.FriendlyUnitName)) isCorrect = true;
Assert.IsTrue(isCorrect);
}
}
public void TestTemplateGraphDisposal_ValidNodes()
{
IEditableExperiment experiment = ((IEditableExperiment)ExperimentManager.New());
experiment.ExperimentInfo.FilePath = "C:\\somefakelocation\\mockExperiment.teml";
//construct some simple m_experiment
ExperimentNode node1 = experiment.AddComponentFromDefinition(m_emptyComponentMetaDefinition, 5, 5);
ExperimentNode node2 = experiment.AddComponentFromDefinition(m_emptyComponentMetaDefinition, 5, 5);
experiment.AddConnection(experiment.StartNode, node1);
experiment.AddConnection(node1, node2);
experiment.AddConnection(node2, experiment.EndNode);
//initiate mockNodesFactory
MockNodesFactory mockNodesFactory = new MockNodesFactory();
RunnableExperimentBase template = GraphAdapter.Adapt(experiment, mockNodesFactory, AppContext.Components, AppContext.WorkspaceInstance.TypeDirectories);
foreach (MockNode node in mockNodesFactory.CreatedNodes)
{
Assert.IsFalse(node.disposed);
}
//execute disptacher, because in case of correct graph dispatcher disposes nodes
MockProgress progress = new MockProgress();
using (var dispatcher = ExperimentRunnerFactory.CreateExperimentRunner(template))
{
dispatcher.ExecuteExperiment(progress);
Assert.IsFalse(progress.HasError);
}
foreach (MockNode node in mockNodesFactory.CreatedNodes)
{
Assert.IsTrue(node.disposed);
}
}
public void TestExecuteExperimentDecisionGraphTwoNodesComingToDecision()
{
//assure workspace is empty
AppContext.WorkspaceInstance.Clear();
MockProgress progress = new MockProgress();
Experiment experiment = LoadExperiment("graph_loop_two_nodes_coming_to_decision_bug_72.teml");
RunExperimentAsync(experiment, progress, AppContext.WorkspaceInstance, AppContext.Components);
Assert.IsFalse(progress.HasError);
Assert.AreEqual(Messages.ExperimentRunnerSuccessMessage, progress.CurrentStatus);
int test_a = (int)AppContext.WorkspaceInstance.Load(experiment.ExperimentInfo.Id + ".test_a");
Assert.AreEqual(9, test_a);
}
public void NestedWorkspaceWrapperTest_Bug180()
{
MockProgress progress = new MockProgress();
Experiment experiment = ExperimentManager.Load("compositecomponentbug180.teml", AppContext.Components);
using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(experiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
}
Assert.AreEqual(3, AppContext.WorkspaceInstance.Units.Count);
foreach (WorkspaceUnit unit in AppContext.WorkspaceInstance.Units)
{
bool isCorrect = false;
if(unit.FriendlyUnitName.Equals("a") && (int)unit.Data == 10) isCorrect = true;
else if(unit.FriendlyUnitName.Equals("b") && (int)unit.Data == 5) isCorrect = true;
else if(unit.FriendlyUnitName.Equals("y") && (int)unit.Data == 6) isCorrect = true;
Assert.IsTrue(isCorrect);
}
}
public void PrepareAndRunBenchmarkExperiment()
{
Assert.Fail("Test temporarily broken. Ignored till contest feature is going to be revisited.");
List<Benchmark> benchmarks = BenchmarkLoader.LoadBenchmarksInfo(BenchmarkDirectory);
Benchmark testBenchmark = benchmarks[0];
// load the experiment to be run against benchmark
string experimentFilename = System.IO.Path.Combine(AppContext.BaseTestDirectory, "experiment_to_be_benchmarked.gml");
Experiment experimentToBeBenchmarked = ExperimentManager.Load(experimentFilename, AppContext.Components);
//prepare matching io
testBenchmark.PrepareMatchingIOByType(experimentToBeBenchmarked);
Assert.AreEqual(2, testBenchmark.BenchmarkInputSetting.Count);
Assert.AreEqual(1, testBenchmark.BenchmarkOutputsSetting.Count);
//match benchmarkSourceArtifact with original source artifacts
foreach (BenchmarkItemSetting<IOItem> pair in testBenchmark.BenchmarkInputSetting)
{
IOItem item = pair.Item;
ItemSettingCollection candidates = pair.CandidateSettings;
if (item.MappedTo.Equals("benchmarkSourceArtifacts"))
{
//we found the item we want to remap
pair.SelectedSetting = candidates["originalSourceArtifacts"];
}
}
//finally prepare benchmark experiment
testBenchmark.PrepareBenchmarkExperiment(experimentToBeBenchmarked, AppContext.Components);
//assert that only two inputs are included in the export settings and one output
int includedInputs = 0;
foreach (KeyValuePair<string, ItemSetting> pair in testBenchmark.Setup.InputSettings)
{
if (pair.Value.Include == true) includedInputs++;
}
Assert.AreEqual(2, includedInputs);
int includedOutputs = 0;
foreach (KeyValuePair<string, ItemSetting> pair in testBenchmark.Setup.OutputSettings)
{
if (pair.Value.Include == true) includedOutputs++;
}
Assert.AreEqual(1, includedOutputs);
Assert.IsNotNull(testBenchmark.BenchmarkExperiment);
//for debug output file
// string path = System.IO.Path.Combine(AppContext.BaseTestDirectory, "benchmarkTest1.gml");
// AppContext.ExperimentManager.Save(testBenchmark.BenchmarkExperiment, path);
MockProgress progress = new MockProgress();
using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(testBenchmark.BenchmarkExperiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
Assert.AreEqual(7, progress.NumSteps);
Assert.IsFalse(progress.HasError);
}
}
