public override List <TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method)
        {
            // build data
            var columnAttrs = method.GetCustomAttributes(typeof(MatrixColumnAttribute), false) as MatrixColumnAttribute[];

            Debug.Assert(columnAttrs.Length > 0);

            object[][] inputMatrix = new object[columnAttrs.Length][];

            Dictionary <string, int> columnName2PositionMap = new Dictionary <string, int>();

            foreach (var inputColumn in columnAttrs)
            {
                #region Asserts
                Debug.Assert(inputMatrix[inputColumn.ParamOrderId] == null);
                Debug.Assert(!columnName2PositionMap.ContainsKey(inputColumn.Name));
                Debug.Assert(!columnName2PositionMap.ContainsValue(inputColumn.ParamOrderId));
                #endregion
                inputMatrix[inputColumn.ParamOrderId]    = inputColumn.Values;
                columnName2PositionMap[inputColumn.Name] = inputColumn.ParamOrderId;
            }



            // Build Skip filters
            List <object[]> skipConstraints = CreateSkipConstraints(method, columnName2PositionMap);

            // Build Priority filters + default filter
            Dictionary <int, List <object[]> > priFilters = CreatePriorityConstraints(method, Priority, columnName2PositionMap);

            // Create Pairwise combinations based on the pri filters
            Dictionary <int, List <object[]> > combinations = Pairwise(inputMatrix, skipConstraints, priFilters);

            List <TestVariation> variations = new List <TestVariation>();
            int variationCounter            = 0;

            //string orderedColumnNames = string.Concat(columnName2PositionMap.OrderBy(p => p.Value).Select((p, i) => i == 0 ? p.Key : string.Concat(",", p.Key)).ToArray());

            foreach (int priority in combinations.Keys)
            {
                foreach (object[] line in combinations[priority])
                {
                    string        name = GetVariationName(columnName2PositionMap, line);
                    TestVariation tv   = new TestVariation(name, (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method));
                    tv.Priority = priority;

                    //// the last element in params will contain the parameter names (we need it to be able perform filters on the silverlight side);
                    //object[] linePlusNames = new object[line.Length + 1];
                    //Array.Copy(line, linePlusNames, line.Length);
                    //linePlusNames[line.Length] = orderedColumnNames ;
                    //tv.Params = linePlusNames;

                    tv.Params = line;
                    tv.Id     = ++variationCounter;
                    variations.Add(tv);
                }
            }

            return(variations);
        }
示例#2
0
 public void AddChild(TestFunc myFunction, int priority, string description, params object[] paramList)
 {
     TestVariation myVariation = new TestVariation(myFunction);
     VariationAttribute myAttribute = new VariationAttribute(description, paramList);
     myAttribute.Priority = priority;
     myVariation.Attribute = myAttribute;
     this.AddChild(myVariation);
 }
        public override List<TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method) {

            // build data 
            var columnAttrs = method.GetCustomAttributes(typeof(MatrixColumnAttribute), false) as MatrixColumnAttribute[];
            Debug.Assert(columnAttrs.Length > 0);

            object[][] inputMatrix = new object[columnAttrs.Length][];

            Dictionary<string, int> columnName2PositionMap = new Dictionary<string, int>();
            foreach (var inputColumn in columnAttrs) {
                #region Asserts
                Debug.Assert(inputMatrix[inputColumn.ParamOrderId] == null);
                Debug.Assert(!columnName2PositionMap.ContainsKey(inputColumn.Name));
                Debug.Assert(!columnName2PositionMap.ContainsValue(inputColumn.ParamOrderId));
                #endregion
                inputMatrix[inputColumn.ParamOrderId] = inputColumn.Values;
                columnName2PositionMap[inputColumn.Name] = inputColumn.ParamOrderId;
            }



            // Build Skip filters
            List<object[]> skipConstraints = CreateSkipConstraints(method, columnName2PositionMap);

            // Build Priority filters + default filter
            Dictionary<int, List<object[]>> priFilters = CreatePriorityConstraints(method, Priority, columnName2PositionMap);

            // Create Pairwise combinations based on the pri filters
            Dictionary<int, List<object[]>> combinations = Pairwise(inputMatrix, skipConstraints, priFilters);

            List<TestVariation> variations = new List<TestVariation>();
            int variationCounter = 0;

            //string orderedColumnNames = string.Concat(columnName2PositionMap.OrderBy(p => p.Value).Select((p, i) => i == 0 ? p.Key : string.Concat(",", p.Key)).ToArray());

            foreach (int priority in combinations.Keys) {
                foreach (object[] line in combinations[priority]) {
                    string name = GetVariationName(columnName2PositionMap, line);
                    TestVariation tv = new TestVariation(name, (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method));
                    tv.Priority = priority;

                    //// the last element in params will contain the parameter names (we need it to be able perform filters on the silverlight side);
                    //object[] linePlusNames = new object[line.Length + 1];
                    //Array.Copy(line, linePlusNames, line.Length);
                    //linePlusNames[line.Length] = orderedColumnNames ;
                    //tv.Params = linePlusNames;

                    tv.Params = line;
                    tv.Id = ++variationCounter;
                    variations.Add(tv);
                }
            }

            return variations;
        }
示例#4
0
        //Helpers
        protected override void                     DetermineChildren()
        {
            //Delegate (add any nested testcases)
            base.DetermineChildren();

            //Dynamically figure out what functions are variations...
            foreach (MethodInfo method in this.GetType().GetMethods())
            {
                //Loop through the Attributes for method
                foreach (VariationAttribute attr in method.GetCustomAttributes(typeof(VariationAttribute), false /*inhert*/))
                {
                    //Every method that has a [Variation attribute] = a variation
                    //Add this variation to our array
                    TestFunc func = null;
                    try
                    {
                        func = (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), this, method);
                    }
                    catch (Exception e)
                    {
                        e = new TestFailedException("Variation: '" + method + "' doesn't match expected signature of 'void func()', unable to add that variation.", null, null, e);
                        HandleException(e);
                        continue;
                    }

                    TestVariation var = this.CreateVariation(func);
                    attr.Name = func.Method.Name;                       //Name is always the function name (invoke)
                    if (attr.Desc == null)
                    {
                        attr.Desc = attr.Name;
                    }
                    var.Attribute = attr;
                    AddChild(var);
                }
            }

            //Sort
            //Default sort is based upon IComparable of each item
            Children.Sort();
        }
示例#5
0
        public TestVariation GenerateVariation(TestCase testcase, MethodInfo method, VariationAttribute testVarAttrib)
        {
            //do not generate variations for tests which have KeepInContent set to false and the version is not V2
            if (!this.KeepInContent && !Versioning.Server.SupportsV2Features)
            {
                return null;
            }

            TestVariation ffTestVariation;
            MethodInfo generateEpmMappingsMethod = testcase.GetType().GetMethod("GenerateEpmMappings");
            FilterResourcesLambda = GenerateLambda();

            TestFunc testMethodAction = delegate()
            {
                bool thisTestFailed = true;

                SkipInvalidRuns();

                try
                {
                    generateEpmMappingsMethod.Invoke(testcase, new object[]{
                    KeepInContent,//bool pKeepInContent,
                    RemoveUnEligibleTypes,//bool pRemoveUnEligibleTypes,
                    GenerateClientTypes,//bool pGenerateClientTypes,
                    MapToAtomElements,//bool pMapPropertiesToAtomElements,
                    MapToNonAtomElements,//bool pMapPropertiesToNonAtomElements,
                    GenerateServerEPMMappings,//bool GenerateServerEPMMappings,
                    KeepSameNamespace,//bool KeepSameNamespace,
                    IncludeComplexTypes,//bool  IncludeComplexTypes
                    PreDefinedPaths  ,//string[] PreDefinedPaths
                    FilterResourcesLambda //Func<ResourceType, bool> pFilterResourcesLambda
                });
                    method.Invoke(testcase, null);


                    thisTestFailed = AstoriaTestLog.FailureFound;
                }
                catch (Microsoft.OData.Client.DataServiceClientException dsException)
                {
                    if (dsException.InnerException != null)
                        AstoriaTestLog.WriteLine(dsException.InnerException.Message);
                }
                catch (TestFailedException testException)
                {

                    AstoriaTestLog.WriteLine("Test failed  with :{0}", testException.Message);
                    AstoriaTestLog.WriteLine("Repro Details\r\nClient Code");
                }
                finally
                {
                    //Disabling this as OOM errors prevent file copying
                    thisTestFailed = false;
                    #region //Clean out all the facets after the test , so that the next test is clear
                    object workspacePointer = null;
                    PropertyInfo testWorkspacesProperty = testcase.Parent.GetType().GetProperty("Workspaces");
                    workspacePointer = testcase.Parent;
                    if (testWorkspacesProperty == null)
                    {
                        testWorkspacesProperty = testcase.GetType().GetProperty("Workspaces");
                        workspacePointer = testcase;
                    }
                    Workspaces testWorkspaces = (Workspaces)testWorkspacesProperty.GetValue(workspacePointer, null);
                    foreach (Workspace workspace in testWorkspaces)
                    {
                        if (thisTestFailed)
                        {
                            string testFailureReproPath = String.Format("FF_Failure_{0}", System.Guid.NewGuid().ToString());
                            testFailureReproPath = System.IO.Path.Combine(Environment.CurrentDirectory, testFailureReproPath);
                            IOUtil.CopyFolder(workspace.DataService.DestinationFolder, testFailureReproPath, true);
                            AstoriaTestLog.WriteLine("Test failed , Repro available at : {0} ", testFailureReproPath);
                        }
                 
                        if (!(workspace is EdmWorkspace))
                        {
                            workspace.GenerateClientTypes = true;
                            workspace.GenerateServerMappings = false;
                            workspace.ApplyFriendlyFeeds();
                        }
                    }
                    #endregion

                }
            };
            ffTestVariation = new TestVariation(testMethodAction);
            if (testVarAttrib != null)
            {
                ffTestVariation.Params = testVarAttrib.Params;
                ffTestVariation.Param = testVarAttrib.Param;
                ffTestVariation.Desc = String.Format(
                "{3} , {0},{1},{2},{4}{5}{6}{7}",
                GenerateClientTypes ? "Client Mapped" : "",
                GenerateServerEPMMappings ? "Server Mapped" : "",
                MapToAtomElements ? "Mapped to Atom" : "Mapped to Non-Atom Elements",
                testVarAttrib.Desc, InheritanceFilter.ToString(),
                KeepSameNamespace ? "Same Namespace" : "Different NameSpaces",
                PreDefinedPaths != null ? ",PredefinedPaths" : "", KeepInContent.ToString());
                if (AstoriaTestProperties.MaxPriority == 0)
                {
                    ffTestVariation.Id = 1;
                }
                else
                {
                    ffTestVariation.Id = idCounter++;
                }
                ffTestVariation.Priority = this.Priority;
            }
            return ffTestVariation;

        }
示例#6
0
        public TestVariation GenerateVariation(TestCase testcase, MethodInfo method)
        {
            TestVariation rcTestVariation;
            TestFunc testMethodAction = delegate()
            {
                method.Invoke(testcase, null);
            };
            rcTestVariation = new TestVariation(testMethodAction);

            rcTestVariation.Params = this.Params;
            rcTestVariation.Param = this.Param;
            rcTestVariation.Desc = this.Desc;
            rcTestVariation.Id = this.Id;
            rcTestVariation.Priority = this.Priority;

            return rcTestVariation;
        }
示例#7
0
 public TestVariation CreateNewVariation(string descPrefix, string uri)
 {
     TestVariation variation = new TestVariation(CompareReaders, descPrefix);
     variation.Attribute = new VariationAttribute(descPrefix + " :: " + uri, new object[] { uri });
     return variation;
 }
示例#8
0
        protected override void DetermineChildren()
        {
            // Use previous settings if  client partial trust level is not medium
            // We only perform client partial-trust test here
            if (AstoriaTestProperties.ServiceTrustLevel != TrustLevel.Medium || (!this.Name.Contains("Client")))
            {
                base.DetermineChildren();
            }

            else
            {
                AppDomain myDomain = this.GetCustomizedAppDomain();
                ArrayList types = this.GetNestedTypes(this.GetType());
                foreach (Type type in types)
                {
                    //Were only interested in objects that inherit from TestCase.
                    //NOTE: We need to filter other objects, since some internal VB classes throw trying 
                    //to get Attributes
                    if (!type.IsSubclassOf(typeof(TestCase)))
                        continue;

                    //Every class that has a TestCase attribute, IS a TestCase
                    foreach (TestCaseAttribute attr in type.GetCustomAttributes(typeof(TestCaseAttribute), false/*inhert*/))
                    {
                        //Create this class (call the constructor with no arguments)
                        // Create testcase instance in another app domain with desired permission set.
                        TestCase testcase = (TestCase)myDomain.CreateInstanceAndUnwrap(type.Assembly.FullName, type.FullName);

                        if (attr.Name == null)
                            attr.Name = type.Name;
                        testcase.Attribute = attr;
                        this.AddChild(testcase);
                    }
                }

                //Sort
                //Default sort is based upon IComparable of each item
                Children.Sort();
            }


            if (AstoriaTestProperties.RuntimeEnvironment != TestRuntimeEnvironment.CheckinSuites)
            {
                // ensure that we don't get 0/0 = 0% results due to empty test cases
                foreach (TestItem child in this.Children.Cast<TestItem>().Where(c => !Descendents(c).OfType<TestVariation>().Any()))
                {
                    child.Children.Clear();

                    TestFunc func = new TestFunc(delegate() { throw new TestSkippedException("Placeholder"); });
                    TestVariation variation = new TestVariation("Placeholder for module with no tests", func);
                    variation.Attribute = new VariationAttribute("Placeholder for module with no tests")
                    {
                        Id = 1,
                        Implemented = true,
                        Priority = AstoriaTestProperties.MinPriority
                    };

                    variation.Parent = child;
                    child.AddChild(variation);
                }
            }
        }
示例#9
0
        public virtual List <TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method)
        {
            int             id   = StartingId;
            List <object[]> rows = new List <object[]>();
            List <MatrixColumnAttribute> matrixColumns = new List <MatrixColumnAttribute>();

            object [] attributes = method.GetCustomAttributes(false);
            List <MatrixSkipConstraintAttribute> skipAttributes     = new List <MatrixSkipConstraintAttribute>();
            List <MatrixPriorityFilterAttribute> priorityAttributes = new List <MatrixPriorityFilterAttribute>();

            foreach (MatrixColumnAttribute matrixColumn in attributes.OfType <MatrixColumnAttribute>())
            {
                matrixColumns.Insert(0, matrixColumn);
            }
            foreach (MatrixSkipConstraintAttribute skipConstraint in attributes.OfType <MatrixSkipConstraintAttribute>())
            {
                skipAttributes.Insert(0, skipConstraint);
            }
            foreach (MatrixPriorityFilterAttribute filter in attributes.OfType <MatrixPriorityFilterAttribute>())
            {
                // TODO: why aren't we just doing add? or .ToList()?
                priorityAttributes.Insert(0, filter);
            }

            //Verify that no ParamOrderId's match
            IEnumerable <IGrouping <int, MatrixColumnAttribute> > grouping = matrixColumns.GroupBy <MatrixColumnAttribute, int>(mc => mc.ParamOrderId);

            foreach (IGrouping <int, MatrixColumnAttribute> group in grouping)
            {
                if (group.Key != -1)
                {
                    if (group.Count() > 1)
                    {
                        throw new ArgumentException("For Matrix variation:" + this.PrefixName + " test method:" + method.Name + " Multiple MatrixColumnAttributes have the same ParamOrderId, this is invalid");
                    }
                }
            }
            //Sort it by ParamOrderId
            matrixColumns.Sort(CompareMatrixColumnParamId);
            foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
            {
                rows = MultiplyMatrixRowsByColumn(matrixColumn, rows);
            }
            List <TestVariation> variations = new List <TestVariation>();

            foreach (object[] matrixRow in rows)
            {
                //Every method that has a [Variation attribute] = a variation
                //Add this variation to our array
                TestFunc func = null;
                try
                {
                    func = (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method);
                }
                catch (Exception e)
                {
                    e = new TestFailedException("Variation: '" + method + "' doesn't match expected signature of 'void func()', unable to add that variation.", null, null, e);
                    TestLog.HandleException(e);
                    continue;
                }
                int    i           = 0;
                string description = null;
                //Calling from the list to reverse the attribute order so its in the order
                //as its in the code
                foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
                {
                    if (description == null)
                    {
                        description = this.PrefixName + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
                    }
                    else
                    {
                        description = description + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
                    }
                    i++;
                }
                if (!ShouldSkipVariation(testcase, matrixColumns, skipAttributes, matrixRow))
                {
                    TestVariation var = new TestVariation(description, func);

                    var.Params   = matrixRow;
                    var.Id       = id;
                    var.Desc     = description;
                    var.Priority = VariationPriority(testcase, matrixColumns, priorityAttributes, matrixRow);
                    variations.Add(var);
                    id++;
                }
            }
            return(variations);
        }
示例#10
0
        public override TestResult Execute()
        {
            base.Execute();
            TestModule module = GetModule();

            if (module == null)
            {
                throw new Exception("No parent module");
            }
            TestItems children = Children;

            if (children != null && children.Count > 0)
            {
                // this is not a leaf node, just invoke all the children's execute
                foreach (object child in children)
                {
                    if (child is TestCase)
                    {
                        TestCase t = child as TestCase;
                        t.Init();
                        t.Execute();
                        continue;
                    }
                    TestVariation var = child as TestVariation;
                    if (var != null)
                    {
                        const string indent = "\t";
                        try
                        {
                            CurVariation = var;
                            TestResult ret = var.Execute();
                            if (TestResult.Passed == ret)
                            {
                                module.PassCount++;
                            }
                            else if (TestResult.Failed == ret)
                            {
                                System.Console.WriteLine(indent + var.Desc);
                                System.Console.WriteLine(indent + " FAILED");
                                module.FailCount++;
                            }
                            else
                            {
                                System.Console.WriteLine(indent + var.Desc);
                                System.Console.WriteLine(indent + " SKIPPED");
                                module.SkipCount++;
                            }
                        }
                        catch (TestSkippedException tse)
                        {
                            if (!string.IsNullOrWhiteSpace(var.Desc))
                            {
                                System.Console.WriteLine(indent + var.Desc);
                            }
                            if (!string.IsNullOrWhiteSpace(tse.Message))
                            {
                                System.Console.WriteLine(indent + " SKIPPED" + ", Msg:" + tse.Message);
                            }
                            module.SkipCount++;
                        }
                        catch (Exception e)
                        {
                            System.Console.WriteLine(indent + var.Desc);
                            System.Console.WriteLine(e);
                            System.Console.WriteLine(indent + " FAILED");
                            module.FailCount++;
                        }
                    }
                }
            }
            return(TestResult.Passed);
        }
示例#11
0
        public virtual List<TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method)
        {
            int id= StartingId;
            List<object[]> rows = new List<object[]>();
            List<MatrixColumnAttribute> matrixColumns = new List<MatrixColumnAttribute>();
            object []attributes = method.GetCustomAttributes(false);
            List<MatrixSkipConstraintAttribute> skipAttributes = new List<MatrixSkipConstraintAttribute>();
            List<MatrixPriorityFilterAttribute> priorityAttributes = new List<MatrixPriorityFilterAttribute>();

            foreach (MatrixColumnAttribute matrixColumn in attributes.OfType < MatrixColumnAttribute>())
            {
                matrixColumns.Insert(0,matrixColumn);
            }
            foreach (MatrixSkipConstraintAttribute skipConstraint in attributes.OfType<MatrixSkipConstraintAttribute>())
            {
                skipAttributes.Insert(0, skipConstraint);
            }
            foreach (MatrixPriorityFilterAttribute filter in attributes.OfType<MatrixPriorityFilterAttribute>())
            {
                // TODO: why aren't we just doing add? or .ToList()?
                priorityAttributes.Insert(0, filter);
            }

            //Verify that no ParamOrderId's match
            IEnumerable<IGrouping<int, MatrixColumnAttribute>> grouping = matrixColumns.GroupBy<MatrixColumnAttribute, int>(mc => mc.ParamOrderId);
            foreach (IGrouping<int, MatrixColumnAttribute> group in grouping)
            {
                if (group.Key != -1)
                {
                    if (group.Count() > 1)
                        throw new ArgumentException("For Matrix variation:" + this.PrefixName + " test method:" + method.Name + " Multiple MatrixColumnAttributes have the same ParamOrderId, this is invalid");
                }
            }
            //Sort it by ParamOrderId
            matrixColumns.Sort(CompareMatrixColumnParamId);
            foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
            {
                rows = MultiplyMatrixRowsByColumn(matrixColumn, rows);
            }
            List<TestVariation> variations = new List<TestVariation>();
            foreach(object[] matrixRow in rows)
            {
                //Every method that has a [Variation attribute] = a variation
		        //Add this variation to our array
                TestFunc func = null;
		        try
                {
                    func = (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method);
                }
                catch(Exception e)
                {
                    e = new TestFailedException("Variation: '" + method + "' doesn't match expected signature of 'void func()', unable to add that variation.", null, null, e);
                    TestLog.HandleException(e);
                    continue;
                }
                int i = 0;
                string description = null;
                //Calling from the list to reverse the attribute order so its in the order
                //as its in the code
                foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
                {
                    if (description == null)
                        description = this.PrefixName + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
                    else
                        description = description + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
                    i++;
                }
                if (!ShouldSkipVariation(testcase, matrixColumns, skipAttributes, matrixRow))
                {
                    TestVariation var = new TestVariation(description, func);

                    var.Params = matrixRow;
                    var.Id = id;
                    var.Desc = description;
                    var.Priority = VariationPriority(testcase, matrixColumns, priorityAttributes, matrixRow);
                    variations.Add(var);
                    id++;
                }
            }
            return variations;
        }