private IEnumerable <IEnumerable <KeyValuePair <string, object> > > PairwiseGenerateMatrixFromDimensions(IEnumerable <TestMatrixDimensionAttribute> dimensionAttributes, IEnumerable <KeyValuePair <MethodInfo, object> > constraintsInfo, int order) { var random = new Random(this.ExplorationSeed); var dimensionsWithStrategies = dimensionAttributes.OrderBy(a => a.Position).Select(a => this.GetDimensionWithStrategy(a)).ToArray(); var dimensions = dimensionsWithStrategies.Select(ds => ds.Key).ToArray(); var matrix = new Matrix(dimensions); var constraints = constraintsInfo.Select(ci => (IConstraint) new TestMatrixMethodBasedConstraint(ci.Key, ci.Value, dimensions)); var pairwise = new PairwiseStrategy(matrix, random.Next, constraints); pairwise.Order = order; foreach (var ds in dimensionsWithStrategies) { // Note: strategy can be null for closed domains like bool or enum types if (ds.Value != null) { pairwise.SetDimensionStrategy(ds.Key, ds.Value); } } var vectors = pairwise.Explore(); var result = vectors.Select(v => (IEnumerable <KeyValuePair <string, object> >)dimensions.Select(d => new KeyValuePair <string, object>(d.Name, v.GetValue(d))).ToList()).ToList(); return(result); }
public static void VerifyAllPairs <A, B, C>( Func <A, B, C, object> function, IEnumerable <A> aItems, IEnumerable <B> bItems, IEnumerable <C> cItems) { var testCases = new PairwiseStrategy().GetTestCases(new System.Collections.IEnumerable[] { aItems, bItems, cItems }); StringBuilder output = new StringBuilder(); foreach (NUnit.Framework.Interfaces.ITestCaseData testCase in testCases) { var result = function( (A)testCase.Arguments[0], (B)testCase.Arguments[1], (C)testCase.Arguments[2]); output.AppendLine( "[" + string.Join(",", testCase.Arguments) + "] => " + result); } Approvals.Verify(output); }
public void Test(int[] dimensions, int bestSoFar, int targetCases) { int features = dimensions.Length; string[][] sources = new string[features][]; for (int i = 0; i < features; i++) { string featureName = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".Substring(i, 1); int n = dimensions[i]; sources[i] = new string[n]; for (int j = 0; j < n; j++) { sources[i][j] = featureName + j.ToString(); } } ICombiningStrategy strategy = new PairwiseStrategy(); PairCounter pairs = new PairCounter(); int cases = 0; foreach (NUnit.Framework.Internal.TestCaseParameters parms in strategy.GetTestCases(sources)) { for (int i = 1; i < features; i++) { for (int j = 0; j < i; j++) { string a = parms.Arguments[i] as string; string b = parms.Arguments[j] as string; pairs[a + b] = null; } } ++cases; } int expectedPairs = 0; for (int i = 1; i < features; i++) { for (int j = 0; j < i; j++) { expectedPairs += dimensions[i] * dimensions[j]; } } Assert.That(pairs.Count, Is.EqualTo(expectedPairs), "Number of pairs is incorrect"); Assert.That(cases, Is.AtMost(bestSoFar), "Regression: Number of test cases exceeded target previously reached"); #if DEBUG //Assert.That(cases, Is.AtMost(targetCases), "Number of test cases exceeded target"); #endif }
private IEnumerable <TestRow> Explore(TestConfigurationMatrix matrix, TestMatrixExplorationKind explorationKind, TestRunStrategy testRunStrategy) { List <KeyValuePair <Dimension, IExplorationStrategy> > testShellDimensionsWithStrategies = new List <KeyValuePair <Dimension, IExplorationStrategy> >(); List <IConstraint> testShellContraints = new List <IConstraint>(); Dictionary <string, TestDimension> testDimensionsToExpand = new Dictionary <string, TestDimension>(); foreach (var testDimension in matrix.Dimensions) { testDimensionsToExpand.Add(testDimension.Name, testDimension); } // Now override any dimensions for the run Strategy if (testRunStrategy != null) { foreach (var testDimension in testRunStrategy.OverrideDimensions) { testDimensionsToExpand[testDimension.Name] = testDimension; } } // Add test matrix combinations this.GetFlattenDimensionsWithConstraints(testDimensionsToExpand.Values, testShellDimensionsWithStrategies, testShellContraints); // Create test dimension for the platforms var platformDimension = new Dimension <string>("Platform"); var platformExplorationStrategy = new ExhaustiveIEnumerableStrategy <string>(matrix.TestAssemblies.Select(ta => ta.PlatformType.ToString())); testShellDimensionsWithStrategies.Add(new KeyValuePair <Dimension, IExplorationStrategy>(platformDimension, platformExplorationStrategy)); CombinatorialStrategy testConfigurationExplorationStrategy = null; Matrix testShellMatrix = new Matrix(matrix.Name, testShellDimensionsWithStrategies.Select(pair => pair.Key).ToArray()); if (explorationKind == TestMatrixExplorationKind.Exhaustive) { testConfigurationExplorationStrategy = new ExhaustiveCombinatorialStrategy(testShellMatrix, testShellContraints); } else { testConfigurationExplorationStrategy = new PairwiseStrategy(testShellMatrix, this.RandomNumberGenerator.Next, testShellContraints); } foreach (var ds in testShellDimensionsWithStrategies) { testConfigurationExplorationStrategy.SetDimensionStrategy(ds.Key, ds.Value); } List <TestRow> testRows = new List <TestRow>(); foreach (var vector in testConfigurationExplorationStrategy.Explore()) { Dictionary <string, string> rowValues = new Dictionary <string, string>(); foreach (var testShellDimensionWithStrategy in testShellDimensionsWithStrategies) { rowValues.Add(testShellDimensionWithStrategy.Key.Name, (string)vector.GetValue(testShellDimensionWithStrategy.Key)); } testRows.Add(new TestRow(rowValues)); } return(testRows); }