/// <summary>
/// Initializes a new instance of the <see cref="DefiningBenchmark"/> class.
/// </summary>
/// <param name="baseExperiment">The base experiment represents experiment based on which the benchmark is being defined.</param>
public DefiningBenchmark(Experiment baseExperiment, ComponentsLibrary library,
Workspace workspace, PackageSystem.PackageManager manager,
IEnumerable<string> workspaceTypeDirectories, string webserviceAddress)
{
if (baseExperiment == null)
throw new ArgumentNullException("baseExperiment");
if (library == null)
throw new ArgumentNullException("library");
if (workspace == null)
throw new ArgumentNullException("workspace");
if (workspaceTypeDirectories == null)
throw new ArgumentNullException("workspaceTypeDirectories");
// these are needed to create experiment serializing in Define method
m_packageManager = manager;
m_library = library;
WebserviceAddress = webserviceAddress;
if (webserviceAddress != null)
{
m_webService = new WebserviceAccessor(webserviceAddress, true);
}
m_baseExperiment = baseExperiment;
m_workspace = workspace;
//wrap the workspace into experiment workspace wrapper, so that we can load units only from experiment namespace
m_experimentWorkspaceWrapper = new ExperimentWorkspaceWrapper(workspace, m_baseExperiment.ExperimentInfo.Id);
//prefil benchmark info
PrefillBenchmarkInfo();
//get nodes that can be selected as template
GetTemplatizableComponents();
//get the ExperimentResults variables that can be selected as publishable results
//from the ExperimentResults all metric names, their descriptions and dataset names are going to be extracted
//when contest is going to be published
GetPublishableExperimentResults();
}
public void NestedWorkspaceWrapperSetupAndTearDownTest()
{
string experimentNamespace = "experiment_namespace";
ExperimentWorkspaceWrapper experimentWorkspaceWrapper = new ExperimentWorkspaceWrapper(AppContext.WorkspaceInstance, experimentNamespace);
// let store some objects in the workspace using experiment wrapper
// note that experiment wrapper does not have any restriction and can store anything in workspace - it has direct access to Workspace
string unitname1 = "testunitname1";
string unitname2 = "testunitname2";
TestObject object1 = new TestObject();
object1.Value = "some value";
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(unitname1));
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname1));
experimentWorkspaceWrapper.Store(unitname1, object1);
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(unitname1));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname1));
int testInt = 5;
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(unitname2));
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname2));
experimentWorkspaceWrapper.Store(unitname2, testInt);
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(unitname2));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname2));
//prepare io spec for first nested workspace
IOSpec mockIoSpec1 = new IOSpec();
string nest1unitname1 = "nest1unitname1"; //input nest1unitname1 from unitname1
string nest1unitname2 = "nest1unitname2"; //input nest1unitname2 from unitname2
mockIoSpec1.Input.Add(nest1unitname1, new IOItem(new IOItemDefinition(nest1unitname1, typeof(TestObject).FullName, "mockdescription", TraceLabSDK.IOSpecType.Input), unitname1));
mockIoSpec1.Input.Add(nest1unitname2, new IOItem(new IOItemDefinition(nest1unitname2, typeof(int).FullName, "mockdescription", TraceLabSDK.IOSpecType.Input), unitname2));
string nestedWorkspace1Namespace = "nested_workspace1";
NestedWorkspaceWrapper nestedWorkspace1 = new NestedWorkspaceWrapper(mockIoSpec1, experimentWorkspaceWrapper, nestedWorkspace1Namespace);
//prepare io spec for 2nd nested workspace
IOSpec mockIoSpec2 = new IOSpec();
string nest2unitname1 = "nest2unitname1"; //input nest2unitname1 from nest1unitname1
string nest2unitname2 = "nest2unitname2"; //input nest2unitname2 from nest1unitname2
//output matters only in teardown
string unitname3 = "unitname3";
string unitname3_OutputAS = "nest2unitname3";
string localScopeUnitname = "localScope";
mockIoSpec2.Input.Add(nest2unitname1, new IOItem(new IOItemDefinition(nest2unitname1, typeof(TestObject).FullName, "mockdescription", TraceLabSDK.IOSpecType.Input), nest1unitname1));
mockIoSpec2.Input.Add(nest2unitname2, new IOItem(new IOItemDefinition(nest2unitname2, typeof(int).FullName, "mockdescription", TraceLabSDK.IOSpecType.Input), nest1unitname2));
mockIoSpec2.Output.Add(unitname3, new IOItem(new IOItemDefinition(unitname3, typeof(int).FullName, "mockdescription", TraceLabSDK.IOSpecType.Output), unitname3_OutputAS));
string nestedWorkspace2Namespace = "nested_workspace2";
NestedWorkspaceWrapper nestedWorkspace2 = new NestedWorkspaceWrapper(mockIoSpec2, nestedWorkspace1, nestedWorkspace2Namespace);
IOSpec mockIoSpec3 = new IOSpec();
mockIoSpec3.Output.Add(unitname3, new IOItem(new IOItemDefinition(unitname3, typeof(int).FullName, "mockdescription", TraceLabSDK.IOSpecType.Output), unitname3));
mockIoSpec3.Output.Add(localScopeUnitname, new IOItem(new IOItemDefinition(localScopeUnitname, typeof(int).FullName, "mockdescription", TraceLabSDK.IOSpecType.Output), localScopeUnitname));
WorkspaceWrapper workspaceWrapper = new WorkspaceWrapper(mockIoSpec3, nestedWorkspace2);
// call in order
// 1. setup nested workspace 1
// 2. setup nested workspace 2
// 3. store a object with unitname3 using workspace wrapper, that is output to the higher level scope
// 4. store another object, that is NOT output to higher level scope and should be discarded in teardown
// 4. tear down workspace 2
// 5. tear down workspace 1
nestedWorkspace1.Setup();
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname1)); //it actually should stay in experiment scope
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname2));
Assert.IsTrue(experimentWorkspaceWrapper.Exists(unitname1));
Assert.IsTrue(experimentWorkspaceWrapper.Exists(unitname2));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname1));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname2));
Assert.IsTrue(nestedWorkspace1.Exists(nest1unitname1));
Assert.IsTrue(nestedWorkspace1.Exists(nest1unitname2));
nestedWorkspace2.Setup();
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname1));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname2));
Assert.IsTrue(nestedWorkspace1.Exists(nest1unitname1));
Assert.IsTrue(nestedWorkspace1.Exists(nest1unitname2));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + nest2unitname1));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + nest2unitname2));
Assert.IsTrue(nestedWorkspace2.Exists(nest2unitname1));
Assert.IsTrue(nestedWorkspace2.Exists(nest2unitname2));
//store unit 3
int unit3 = 10;
workspaceWrapper.Store(unitname3, unit3);
//it should store it with namespace
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + unitname3));
//store localscope variable
int localScopeUnit = 25;
workspaceWrapper.Store(localScopeUnitname, localScopeUnit);
//it should store it with namespace
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + localScopeUnitname));
nestedWorkspace2.TearDown();
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + nest2unitname1));
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + nest2unitname2));
Assert.IsFalse(nestedWorkspace2.Exists(nest2unitname1));
Assert.IsFalse(nestedWorkspace2.Exists(nest2unitname2));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname1));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname2));
Assert.IsTrue(nestedWorkspace1.Exists(nest1unitname1));
Assert.IsTrue(nestedWorkspace1.Exists(nest1unitname2));
//unitname 3 should be tear down and renamed to unitnameOutpusAs with proper namespace
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + unitname3));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + unitname3_OutputAS));
//local scope should also be cleared, although it was not output to higher level scope
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nestedWorkspace2Namespace + DOT + localScopeUnitname));
nestedWorkspace1.TearDown();
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname1));
Assert.IsFalse(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + nestedWorkspace1Namespace + DOT + nest1unitname2));
Assert.IsFalse(nestedWorkspace1.Exists(nest1unitname1));
Assert.IsFalse(nestedWorkspace1.Exists(nest1unitname2));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname1));
Assert.IsTrue(AppContext.WorkspaceInstance.Exists(experimentNamespace + DOT + unitname2));
Assert.IsTrue(experimentWorkspaceWrapper.Exists(unitname1));
Assert.IsTrue(experimentWorkspaceWrapper.Exists(unitname2));
}
public void ExecutePublishResults(string ticket, string username, Callback<ContestResultsPublishedResponse> publishingResultsCallback)
{
//get the unitname to be loaded from workspace
string experimentResultsUnitname = SelectedBenchmark.BenchmarkInfo.ExperimentResultsUnitname;
//create an experiment workspace wrapper with experiment info
var benchmarkWorkspaceWrapper = new ExperimentWorkspaceWrapper(m_workspace, SelectedBenchmark.BenchmarkInfo.Id);
var experimentResults = benchmarkWorkspaceWrapper.Load(experimentResultsUnitname) as TraceLabSDK.Types.Contests.TLExperimentResults;
if (experimentResults != null)
{
ContestResults results = BenchmarkResultsHelper.PrepareBaselineContestRestults(SelectedBenchmark.BenchmarkInfo.Id,
experimentResults,
TechniqueName, TechniqueDescription);
WebService.PublishContestResults(ticket, username, results, publishingResultsCallback);
}
else
{
throw new InvalidOperationException("The experiment results outputed in the contest are empty! Empty results cannot be publish! ");
}
}
