public BenchmarkItemSetting(T item, ItemSettingCollection candidates)
{
if (item == null)
{
throw new ArgumentNullException("item");
}
if (candidates == null)
{
throw new ArgumentNullException("candidates");
}
Item = item;
CandidateSettings = candidates;
}
/// <summary>
/// Processes the IO.
/// </summary>
/// <param name="componentTemplateIODictionary">The io dictionary of either Inputs of Outputs of the component template.</param>
/// <param name="settings">input or output settings for the composite component that is being defined</param>
/// <param name="benchmarkMappingSettingsCollection">The benchmark setting collection .</param>
private static void ProcessIO(IDictionary <string, IOItem> componentTemplateIODictionary, SortedDictionary <string, ItemSetting> settings, BenchmarkSettingCollection <IOItem> benchmarkMappingSettingsCollection)
{
MultiDictionary <string, ItemSetting> lookupByType = new MultiDictionary <string, ItemSetting>();
//first prepare lookup by type
foreach (KeyValuePair <string, ItemSetting> pair in settings)
{
ItemSetting item = pair.Value;
//as default don't include it
item.Include = false;
lookupByType.Add(item.Type, item);
}
foreach (string itemKey in componentTemplateIODictionary.Keys)
{
IOItem item = componentTemplateIODictionary[itemKey];
//check if there are any item settings with matching type
IEnumerable <ItemSetting> matchingItemSettings;
if (lookupByType.TryGetValue(item.IOItemDefinition.Type, out matchingItemSettings))
{
ItemSettingCollection list = new ItemSettingCollection(matchingItemSettings);
//add all to the candidate matching items - these are items that user can choose from
var setting = new BenchmarkItemSetting <IOItem>(item, list);
benchmarkMappingSettingsCollection.Add(setting);
setting.SelectedSetting = setting.CandidateSettings[0];
}
else
{
//add empty list - no matching items - benchmarking cannot be executed
var setting = new BenchmarkItemSetting <IOItem>(item, new ItemSettingCollection());
benchmarkMappingSettingsCollection.Add(setting);
}
}
}
/// <summary>
/// Processes the IO.
/// </summary>
/// <param name="componentTemplateIODictionary">The io dictionary of either Inputs of Outputs of the component template.</param>
/// <param name="settings">input or output settings for the composite component that is being defined</param>
/// <param name="benchmarkMappingSettingsCollection">The benchmark setting collection .</param>
private static void ProcessIO(IDictionary<string, IOItem> componentTemplateIODictionary, SortedDictionary<string, ItemSetting> settings, BenchmarkSettingCollection<IOItem> benchmarkMappingSettingsCollection)
{
MultiDictionary<string, ItemSetting> lookupByType = new MultiDictionary<string, ItemSetting>();
//first prepare lookup by type
foreach (KeyValuePair<string, ItemSetting> pair in settings)
{
ItemSetting item = pair.Value;
//as default don't include it
item.Include = false;
lookupByType.Add(item.Type, item);
}
foreach (string itemKey in componentTemplateIODictionary.Keys)
{
IOItem item = componentTemplateIODictionary[itemKey];
//check if there are any item settings with matching type
IEnumerable<ItemSetting> matchingItemSettings;
if (lookupByType.TryGetValue(item.IOItemDefinition.Type, out matchingItemSettings))
{
ItemSettingCollection list = new ItemSettingCollection(matchingItemSettings);
//add all to the candidate matching items - these are items that user can choose from
var setting = new BenchmarkItemSetting<IOItem>(item, list);
benchmarkMappingSettingsCollection.Add(setting);
setting.SelectedSetting = setting.CandidateSettings[0];
}
else
{
//add empty list - no matching items - benchmarking cannot be executed
var setting = new BenchmarkItemSetting<IOItem>(item, new ItemSettingCollection());
benchmarkMappingSettingsCollection.Add(setting);
}
}
}
public void PrepareAndRunBenchmarkExperiment()
{
Assert.Fail("Test temporarily broken. Ignored till contest feature is going to be revisited.");
List <Benchmark> benchmarks = BenchmarkLoader.LoadBenchmarksInfo(BenchmarkDirectory);
Benchmark testBenchmark = benchmarks[0];
// load the experiment to be run against benchmark
string experimentFilename = System.IO.Path.Combine(AppContext.BaseTestDirectory, "experiment_to_be_benchmarked.gml");
Experiment experimentToBeBenchmarked = ExperimentManager.Load(experimentFilename, AppContext.Components);
//prepare matching io
testBenchmark.PrepareMatchingIOByType(experimentToBeBenchmarked);
Assert.AreEqual(2, testBenchmark.BenchmarkInputSetting.Count);
Assert.AreEqual(1, testBenchmark.BenchmarkOutputsSetting.Count);
//match benchmarkSourceArtifact with original source artifacts
foreach (BenchmarkItemSetting <IOItem> pair in testBenchmark.BenchmarkInputSetting)
{
IOItem item = pair.Item;
ItemSettingCollection candidates = pair.CandidateSettings;
if (item.MappedTo.Equals("benchmarkSourceArtifacts"))
{
//we found the item we want to remap
pair.SelectedSetting = candidates["originalSourceArtifacts"];
}
}
//finally prepare benchmark experiment
testBenchmark.PrepareBenchmarkExperiment(experimentToBeBenchmarked, AppContext.Components);
//assert that only two inputs are included in the export settings and one output
int includedInputs = 0;
foreach (KeyValuePair <string, ItemSetting> pair in testBenchmark.Setup.InputSettings)
{
if (pair.Value.Include == true)
{
includedInputs++;
}
}
Assert.AreEqual(2, includedInputs);
int includedOutputs = 0;
foreach (KeyValuePair <string, ItemSetting> pair in testBenchmark.Setup.OutputSettings)
{
if (pair.Value.Include == true)
{
includedOutputs++;
}
}
Assert.AreEqual(1, includedOutputs);
Assert.IsNotNull(testBenchmark.BenchmarkExperiment);
//for debug output file
// string path = System.IO.Path.Combine(AppContext.BaseTestDirectory, "benchmarkTest1.gml");
// AppContext.ExperimentManager.Save(testBenchmark.BenchmarkExperiment, path);
MockProgress progress = new MockProgress();
using (var dispatcher = ExperimentRunnerHelper.CreateExperimentRunner(testBenchmark.BenchmarkExperiment, AppContext.WorkspaceInstance, AppContext.Components))
{
dispatcher.ExecuteExperiment(progress);
Assert.AreEqual(7, progress.NumSteps);
Assert.IsFalse(progress.HasError);
}
}
