/// <summary> /// Loads test module information from the specified assemblies. /// </summary> /// <param name="assemblies">The assemblies to consider.</param> /// <param name="explorationSeed">The exploration seed to use when loading test items with a <see cref="TestMatrixAttribute"/>.</param> /// <param name="explorationKind">The exploraiton kind to use when loading test items with a <see cref="TestMatrixAttribute"/>.</param> /// <returns> /// A <see cref="TestModuleData"/> instance representing the /// module information found in the specified assemblies. /// </returns> public TestModuleData Load(IEnumerable<Assembly> assemblies, int explorationSeed, TestMatrixExplorationKind? explorationKind) { ExceptionUtilities.CheckArgumentNotNull(assemblies, "assemblies"); List<Type> testModuleTypes = new List<Type>(); foreach (Assembly assembly in assemblies) { List<Type> types = assembly.GetTypes().Where(t => typeof(TestModule).IsAssignableFrom(t)).ToList(); List<Type> filteredTypes = types.Where(t => t.IsDefined(typeof(TestModuleAttribute), false)).ToList(); testModuleTypes.AddRange(filteredTypes); } Type testModuleType = testModuleTypes.FirstOrDefault(); if (testModuleType == null) { throw new TaupoArgumentException("The specified assembly does not contain any test modules."); } var testModule = (TestModule)Activator.CreateInstance(testModuleType); testModule.ExplorationSeed = explorationSeed; testModule.ExplorationKind = explorationKind; return new TestModuleData(testModule); }
private IEnumerable<TestRow> Explore(TestConfigurationMatrix matrix, TestMatrixExplorationKind explorationKind, TestRunStrategy testRunStrategy) { List<KeyValuePair<Dimension, IExplorationStrategy>> testShellDimensionsWithStrategies = new List<KeyValuePair<Dimension, IExplorationStrategy>>(); List<IConstraint> testShellContraints = new List<IConstraint>(); Dictionary<string, TestDimension> testDimensionsToExpand = new Dictionary<string, TestDimension>(); foreach (var testDimension in matrix.Dimensions) { testDimensionsToExpand.Add(testDimension.Name, testDimension); } // Now override any dimensions for the run Strategy if (testRunStrategy != null) { foreach (var testDimension in testRunStrategy.OverrideDimensions) { testDimensionsToExpand[testDimension.Name] = testDimension; } } // Add test matrix combinations this.GetFlattenDimensionsWithConstraints(testDimensionsToExpand.Values, testShellDimensionsWithStrategies, testShellContraints); // Create test dimension for the platforms var platformDimension = new Dimension<string>("Platform"); var platformExplorationStrategy = new ExhaustiveIEnumerableStrategy<string>(matrix.TestAssemblies.Select(ta => ta.PlatformType.ToString())); testShellDimensionsWithStrategies.Add(new KeyValuePair<Dimension, IExplorationStrategy>(platformDimension, platformExplorationStrategy)); CombinatorialStrategy testConfigurationExplorationStrategy = null; Matrix testShellMatrix = new Matrix(matrix.Name, testShellDimensionsWithStrategies.Select(pair => pair.Key).ToArray()); if (explorationKind == TestMatrixExplorationKind.Exhaustive) { testConfigurationExplorationStrategy = new ExhaustiveCombinatorialStrategy(testShellMatrix, testShellContraints); } else { testConfigurationExplorationStrategy = new PairwiseStrategy(testShellMatrix, this.RandomNumberGenerator.Next, testShellContraints); } foreach (var ds in testShellDimensionsWithStrategies) { testConfigurationExplorationStrategy.SetDimensionStrategy(ds.Key, ds.Value); } List<TestRow> testRows = new List<TestRow>(); foreach (var vector in testConfigurationExplorationStrategy.Explore()) { Dictionary<string, string> rowValues = new Dictionary<string, string>(); foreach (var testShellDimensionWithStrategy in testShellDimensionsWithStrategies) { rowValues.Add(testShellDimensionWithStrategy.Key.Name, (string)vector.GetValue(testShellDimensionWithStrategy.Key)); } testRows.Add(new TestRow(rowValues)); } return testRows; }
private IEnumerable <TAttribute> GenerateTestAttributesFromMatrix <TAttribute>(TestMatrixAttribute matrix, IEnumerable <TestMatrixDimensionAttribute> dimensions, IEnumerable <KeyValuePair <MethodInfo, object> > constraintsInfo) where TAttribute : TestItemWithParametersAttribute, new() { var attributes = new List <TAttribute>(); int currentIdOffset = 0; TestMatrixExplorationKind explorationKind = this.ExplorationKind.HasValue ? this.ExplorationKind.Value : matrix.ExplorationKind; foreach (var row in this.GenerateMatrixFromDimensions(dimensions, constraintsInfo, explorationKind)) { string suffix = " - " + string.Join(" ", row.Select(pair => pair.Key + "=" + pair.Value).ToArray()); int id = matrix.Id; if (id > 0) { id += currentIdOffset++; } string name = null; if (matrix.Name != null) { name = matrix.Name + suffix; } string description = null; if (matrix.Description != null) { description = matrix.Description + suffix; } attributes.Add(new TAttribute() { Id = id, Name = name, Description = description, Owner = matrix.Owner, Parameters = row.Select(pair => pair.Value).ToArray(), Priority = matrix.Priority, SkipReason = matrix.SkipReason, SkipUntil = matrix.SkipUntil, Timeout = matrix.Timeout, Version = matrix.Version, }); } return(attributes); }
private IEnumerable <TestRow> Explore(TestConfigurationMatrix matrix, TestMatrixExplorationKind explorationKind, TestRunStrategy testRunStrategy) { List <KeyValuePair <Dimension, IExplorationStrategy> > testShellDimensionsWithStrategies = new List <KeyValuePair <Dimension, IExplorationStrategy> >(); List <IConstraint> testShellContraints = new List <IConstraint>(); Dictionary <string, TestDimension> testDimensionsToExpand = new Dictionary <string, TestDimension>(); foreach (var testDimension in matrix.Dimensions) { testDimensionsToExpand.Add(testDimension.Name, testDimension); } // Now override any dimensions for the run Strategy if (testRunStrategy != null) { foreach (var testDimension in testRunStrategy.OverrideDimensions) { testDimensionsToExpand[testDimension.Name] = testDimension; } } // Add test matrix combinations this.GetFlattenDimensionsWithConstraints(testDimensionsToExpand.Values, testShellDimensionsWithStrategies, testShellContraints); // Create test dimension for the platforms var platformDimension = new Dimension <string>("Platform"); var platformExplorationStrategy = new ExhaustiveIEnumerableStrategy <string>(matrix.TestAssemblies.Select(ta => ta.PlatformType.ToString())); testShellDimensionsWithStrategies.Add(new KeyValuePair <Dimension, IExplorationStrategy>(platformDimension, platformExplorationStrategy)); CombinatorialStrategy testConfigurationExplorationStrategy = null; Matrix testShellMatrix = new Matrix(matrix.Name, testShellDimensionsWithStrategies.Select(pair => pair.Key).ToArray()); if (explorationKind == TestMatrixExplorationKind.Exhaustive) { testConfigurationExplorationStrategy = new ExhaustiveCombinatorialStrategy(testShellMatrix, testShellContraints); } else { testConfigurationExplorationStrategy = new PairwiseStrategy(testShellMatrix, this.RandomNumberGenerator.Next, testShellContraints); } foreach (var ds in testShellDimensionsWithStrategies) { testConfigurationExplorationStrategy.SetDimensionStrategy(ds.Key, ds.Value); } List <TestRow> testRows = new List <TestRow>(); foreach (var vector in testConfigurationExplorationStrategy.Explore()) { Dictionary <string, string> rowValues = new Dictionary <string, string>(); foreach (var testShellDimensionWithStrategy in testShellDimensionsWithStrategies) { rowValues.Add(testShellDimensionWithStrategy.Key.Name, (string)vector.GetValue(testShellDimensionWithStrategy.Key)); } testRows.Add(new TestRow(rowValues)); } return(testRows); }
/// <summary> /// Explores the Test Configuration matrix /// </summary> /// <param name="configurationMatrix">Matrix to explore</param> /// <param name="explorationKind">Exploration kind, pairwise or exhaustive</param> /// <returns>Rows of the explored matrix</returns> public IEnumerable <TestRow> Explore(TestConfigurationMatrix configurationMatrix, TestMatrixExplorationKind explorationKind) { ExceptionUtilities.CheckArgumentNotNull(configurationMatrix, "configurationMatrix"); return(this.Explore(configurationMatrix, explorationKind, null)); }
private IEnumerable <IEnumerable <KeyValuePair <string, object> > > GenerateMatrixFromDimensions(IEnumerable <TestMatrixDimensionAttribute> dimensions, IEnumerable <KeyValuePair <MethodInfo, object> > constraintsInfo, TestMatrixExplorationKind explorationKind) { int order = -1; switch (explorationKind) { case TestMatrixExplorationKind.Pairwise: order = 2; break; case TestMatrixExplorationKind.Exhaustive: order = dimensions.Count(); break; default: throw new TaupoInvalidOperationException("Unhandled test matrix exploration kind: " + explorationKind); } return(this.PairwiseGenerateMatrixFromDimensions(dimensions, constraintsInfo, order)); }
/// <summary> /// Initializes a new instance of the TestRunStrategy class. /// </summary> /// <param name="priority">Priority of the Strategy</param> /// <param name="explorationKind">Exploration kind</param> public TestRunStrategy(int priority, TestMatrixExplorationKind explorationKind) { this.Priority = priority; this.ExplorationKind = explorationKind; this.OverrideDimensions = new List <TestDimension>(); }
/// <summary> /// Initializes a new instance of the TestRunStrategy class. /// </summary> /// <param name="priority">Priority of the Strategy</param> /// <param name="explorationKind">Exploration kind</param> public TestRunStrategy(int priority, TestMatrixExplorationKind explorationKind) { this.Priority = priority; this.ExplorationKind = explorationKind; this.OverrideDimensions = new List<TestDimension>(); }
/// <summary> /// Explores the Test Configuration matrix /// </summary> /// <param name="configurationMatrix">Matrix to explore</param> /// <param name="explorationKind">Exploration kind, pairwise or exhaustive</param> /// <returns>Rows of the explored matrix</returns> public IEnumerable<TestRow> Explore(TestConfigurationMatrix configurationMatrix, TestMatrixExplorationKind explorationKind) { ExceptionUtilities.CheckArgumentNotNull(configurationMatrix, "configurationMatrix"); return this.Explore(configurationMatrix, explorationKind, null); }