/// <summary>
/// Loads test module information from the specified assemblies.
/// </summary>
/// <param name="assemblies">The assemblies to consider.</param>
/// <param name="explorationSeed">The exploration seed to use when loading test items with a <see cref="TestMatrixAttribute"/>.</param>
/// <param name="explorationKind">The exploraiton kind to use when loading test items with a <see cref="TestMatrixAttribute"/>.</param>
/// <returns>
/// A <see cref="TestModuleData"/> instance representing the
/// module information found in the specified assemblies.
/// </returns>
public TestModuleData Load(IEnumerable<Assembly> assemblies, int explorationSeed, TestMatrixExplorationKind? explorationKind)
{
ExceptionUtilities.CheckArgumentNotNull(assemblies, "assemblies");
List<Type> testModuleTypes = new List<Type>();
foreach (Assembly assembly in assemblies)
{
List<Type> types = assembly.GetTypes().Where(t => typeof(TestModule).IsAssignableFrom(t)).ToList();
List<Type> filteredTypes = types.Where(t => t.IsDefined(typeof(TestModuleAttribute), false)).ToList();
testModuleTypes.AddRange(filteredTypes);
}
Type testModuleType = testModuleTypes.FirstOrDefault();
if (testModuleType == null)
{
throw new TaupoArgumentException("The specified assembly does not contain any test modules.");
}
var testModule = (TestModule)Activator.CreateInstance(testModuleType);
testModule.ExplorationSeed = explorationSeed;
testModule.ExplorationKind = explorationKind;
return new TestModuleData(testModule);
}
private IEnumerable<TestRow> Explore(TestConfigurationMatrix matrix, TestMatrixExplorationKind explorationKind, TestRunStrategy testRunStrategy)
{
List<KeyValuePair<Dimension, IExplorationStrategy>> testShellDimensionsWithStrategies = new List<KeyValuePair<Dimension, IExplorationStrategy>>();
List<IConstraint> testShellContraints = new List<IConstraint>();
Dictionary<string, TestDimension> testDimensionsToExpand = new Dictionary<string, TestDimension>();
foreach (var testDimension in matrix.Dimensions)
{
testDimensionsToExpand.Add(testDimension.Name, testDimension);
}
// Now override any dimensions for the run Strategy
if (testRunStrategy != null)
{
foreach (var testDimension in testRunStrategy.OverrideDimensions)
{
testDimensionsToExpand[testDimension.Name] = testDimension;
}
}
// Add test matrix combinations
this.GetFlattenDimensionsWithConstraints(testDimensionsToExpand.Values, testShellDimensionsWithStrategies, testShellContraints);
// Create test dimension for the platforms
var platformDimension = new Dimension<string>("Platform");
var platformExplorationStrategy = new ExhaustiveIEnumerableStrategy<string>(matrix.TestAssemblies.Select(ta => ta.PlatformType.ToString()));
testShellDimensionsWithStrategies.Add(new KeyValuePair<Dimension, IExplorationStrategy>(platformDimension, platformExplorationStrategy));
CombinatorialStrategy testConfigurationExplorationStrategy = null;
Matrix testShellMatrix = new Matrix(matrix.Name, testShellDimensionsWithStrategies.Select(pair => pair.Key).ToArray());
if (explorationKind == TestMatrixExplorationKind.Exhaustive)
{
testConfigurationExplorationStrategy = new ExhaustiveCombinatorialStrategy(testShellMatrix, testShellContraints);
}
else
{
testConfigurationExplorationStrategy = new PairwiseStrategy(testShellMatrix, this.RandomNumberGenerator.Next, testShellContraints);
}
foreach (var ds in testShellDimensionsWithStrategies)
{
testConfigurationExplorationStrategy.SetDimensionStrategy(ds.Key, ds.Value);
}
List<TestRow> testRows = new List<TestRow>();
foreach (var vector in testConfigurationExplorationStrategy.Explore())
{
Dictionary<string, string> rowValues = new Dictionary<string, string>();
foreach (var testShellDimensionWithStrategy in testShellDimensionsWithStrategies)
{
rowValues.Add(testShellDimensionWithStrategy.Key.Name, (string)vector.GetValue(testShellDimensionWithStrategy.Key));
}
testRows.Add(new TestRow(rowValues));
}
return testRows;
}
private IEnumerable <TAttribute> GenerateTestAttributesFromMatrix <TAttribute>(TestMatrixAttribute matrix, IEnumerable <TestMatrixDimensionAttribute> dimensions, IEnumerable <KeyValuePair <MethodInfo, object> > constraintsInfo)
where TAttribute : TestItemWithParametersAttribute, new()
{
var attributes = new List <TAttribute>();
int currentIdOffset = 0;
TestMatrixExplorationKind explorationKind = this.ExplorationKind.HasValue ? this.ExplorationKind.Value : matrix.ExplorationKind;
foreach (var row in this.GenerateMatrixFromDimensions(dimensions, constraintsInfo, explorationKind))
{
string suffix = " - " + string.Join(" ", row.Select(pair => pair.Key + "=" + pair.Value).ToArray());
int id = matrix.Id;
if (id > 0)
{
id += currentIdOffset++;
}
string name = null;
if (matrix.Name != null)
{
name = matrix.Name + suffix;
}
string description = null;
if (matrix.Description != null)
{
description = matrix.Description + suffix;
}
attributes.Add(new TAttribute()
{
Id = id,
Name = name,
Description = description,
Owner = matrix.Owner,
Parameters = row.Select(pair => pair.Value).ToArray(),
Priority = matrix.Priority,
SkipReason = matrix.SkipReason,
SkipUntil = matrix.SkipUntil,
Timeout = matrix.Timeout,
Version = matrix.Version,
});
}
return(attributes);
}
private IEnumerable <TestRow> Explore(TestConfigurationMatrix matrix, TestMatrixExplorationKind explorationKind, TestRunStrategy testRunStrategy)
{
List <KeyValuePair <Dimension, IExplorationStrategy> > testShellDimensionsWithStrategies = new List <KeyValuePair <Dimension, IExplorationStrategy> >();
List <IConstraint> testShellContraints = new List <IConstraint>();
Dictionary <string, TestDimension> testDimensionsToExpand = new Dictionary <string, TestDimension>();
foreach (var testDimension in matrix.Dimensions)
{
testDimensionsToExpand.Add(testDimension.Name, testDimension);
}
// Now override any dimensions for the run Strategy
if (testRunStrategy != null)
{
foreach (var testDimension in testRunStrategy.OverrideDimensions)
{
testDimensionsToExpand[testDimension.Name] = testDimension;
}
}
// Add test matrix combinations
this.GetFlattenDimensionsWithConstraints(testDimensionsToExpand.Values, testShellDimensionsWithStrategies, testShellContraints);
// Create test dimension for the platforms
var platformDimension = new Dimension <string>("Platform");
var platformExplorationStrategy = new ExhaustiveIEnumerableStrategy <string>(matrix.TestAssemblies.Select(ta => ta.PlatformType.ToString()));
testShellDimensionsWithStrategies.Add(new KeyValuePair <Dimension, IExplorationStrategy>(platformDimension, platformExplorationStrategy));
CombinatorialStrategy testConfigurationExplorationStrategy = null;
Matrix testShellMatrix = new Matrix(matrix.Name, testShellDimensionsWithStrategies.Select(pair => pair.Key).ToArray());
if (explorationKind == TestMatrixExplorationKind.Exhaustive)
{
testConfigurationExplorationStrategy = new ExhaustiveCombinatorialStrategy(testShellMatrix, testShellContraints);
}
else
{
testConfigurationExplorationStrategy = new PairwiseStrategy(testShellMatrix, this.RandomNumberGenerator.Next, testShellContraints);
}
foreach (var ds in testShellDimensionsWithStrategies)
{
testConfigurationExplorationStrategy.SetDimensionStrategy(ds.Key, ds.Value);
}
List <TestRow> testRows = new List <TestRow>();
foreach (var vector in testConfigurationExplorationStrategy.Explore())
{
Dictionary <string, string> rowValues = new Dictionary <string, string>();
foreach (var testShellDimensionWithStrategy in testShellDimensionsWithStrategies)
{
rowValues.Add(testShellDimensionWithStrategy.Key.Name, (string)vector.GetValue(testShellDimensionWithStrategy.Key));
}
testRows.Add(new TestRow(rowValues));
}
return(testRows);
}
/// <summary>
/// Explores the Test Configuration matrix
/// </summary>
/// <param name="configurationMatrix">Matrix to explore</param>
/// <param name="explorationKind">Exploration kind, pairwise or exhaustive</param>
/// <returns>Rows of the explored matrix</returns>
public IEnumerable <TestRow> Explore(TestConfigurationMatrix configurationMatrix, TestMatrixExplorationKind explorationKind)
{
ExceptionUtilities.CheckArgumentNotNull(configurationMatrix, "configurationMatrix");
return(this.Explore(configurationMatrix, explorationKind, null));
}
private IEnumerable <IEnumerable <KeyValuePair <string, object> > > GenerateMatrixFromDimensions(IEnumerable <TestMatrixDimensionAttribute> dimensions, IEnumerable <KeyValuePair <MethodInfo, object> > constraintsInfo, TestMatrixExplorationKind explorationKind)
{
int order = -1;
switch (explorationKind)
{
case TestMatrixExplorationKind.Pairwise:
order = 2;
break;
case TestMatrixExplorationKind.Exhaustive:
order = dimensions.Count();
break;
default:
throw new TaupoInvalidOperationException("Unhandled test matrix exploration kind: " + explorationKind);
}
return(this.PairwiseGenerateMatrixFromDimensions(dimensions, constraintsInfo, order));
}
/// <summary>
/// Initializes a new instance of the TestRunStrategy class.
/// </summary>
/// <param name="priority">Priority of the Strategy</param>
/// <param name="explorationKind">Exploration kind</param>
public TestRunStrategy(int priority, TestMatrixExplorationKind explorationKind)
{
this.Priority = priority;
this.ExplorationKind = explorationKind;
this.OverrideDimensions = new List <TestDimension>();
}
/// <summary>
/// Initializes a new instance of the TestRunStrategy class.
/// </summary>
/// <param name="priority">Priority of the Strategy</param>
/// <param name="explorationKind">Exploration kind</param>
public TestRunStrategy(int priority, TestMatrixExplorationKind explorationKind)
{
this.Priority = priority;
this.ExplorationKind = explorationKind;
this.OverrideDimensions = new List<TestDimension>();
}
/// <summary>
/// Explores the Test Configuration matrix
/// </summary>
/// <param name="configurationMatrix">Matrix to explore</param>
/// <param name="explorationKind">Exploration kind, pairwise or exhaustive</param>
/// <returns>Rows of the explored matrix</returns>
public IEnumerable<TestRow> Explore(TestConfigurationMatrix configurationMatrix, TestMatrixExplorationKind explorationKind)
{
ExceptionUtilities.CheckArgumentNotNull(configurationMatrix, "configurationMatrix");
return this.Explore(configurationMatrix, explorationKind, null);
}
