public override List <TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method)
{
// build data
var columnAttrs = method.GetCustomAttributes(typeof(MatrixColumnAttribute), false) as MatrixColumnAttribute[];
Debug.Assert(columnAttrs.Length > 0);
object[][] inputMatrix = new object[columnAttrs.Length][];
Dictionary <string, int> columnName2PositionMap = new Dictionary <string, int>();
foreach (var inputColumn in columnAttrs)
{
#region Asserts
Debug.Assert(inputMatrix[inputColumn.ParamOrderId] == null);
Debug.Assert(!columnName2PositionMap.ContainsKey(inputColumn.Name));
Debug.Assert(!columnName2PositionMap.ContainsValue(inputColumn.ParamOrderId));
#endregion
inputMatrix[inputColumn.ParamOrderId] = inputColumn.Values;
columnName2PositionMap[inputColumn.Name] = inputColumn.ParamOrderId;
}
// Build Skip filters
List <object[]> skipConstraints = CreateSkipConstraints(method, columnName2PositionMap);
// Build Priority filters + default filter
Dictionary <int, List <object[]> > priFilters = CreatePriorityConstraints(method, Priority, columnName2PositionMap);
// Create Pairwise combinations based on the pri filters
Dictionary <int, List <object[]> > combinations = Pairwise(inputMatrix, skipConstraints, priFilters);
List <TestVariation> variations = new List <TestVariation>();
int variationCounter = 0;
//string orderedColumnNames = string.Concat(columnName2PositionMap.OrderBy(p => p.Value).Select((p, i) => i == 0 ? p.Key : string.Concat(",", p.Key)).ToArray());
foreach (int priority in combinations.Keys)
{
foreach (object[] line in combinations[priority])
{
string name = GetVariationName(columnName2PositionMap, line);
TestVariation tv = new TestVariation(name, (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method));
tv.Priority = priority;
//// the last element in params will contain the parameter names (we need it to be able perform filters on the silverlight side);
//object[] linePlusNames = new object[line.Length + 1];
//Array.Copy(line, linePlusNames, line.Length);
//linePlusNames[line.Length] = orderedColumnNames ;
//tv.Params = linePlusNames;
tv.Params = line;
tv.Id = ++variationCounter;
variations.Add(tv);
}
}
return(variations);
}
public void AddChild(TestFunc myFunction, int priority, string description, params object[] paramList)
{
TestVariation myVariation = new TestVariation(myFunction);
VariationAttribute myAttribute = new VariationAttribute(description, paramList);
myAttribute.Priority = priority;
myVariation.Attribute = myAttribute;
this.AddChild(myVariation);
}
public override List<TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method) {
// build data
var columnAttrs = method.GetCustomAttributes(typeof(MatrixColumnAttribute), false) as MatrixColumnAttribute[];
Debug.Assert(columnAttrs.Length > 0);
object[][] inputMatrix = new object[columnAttrs.Length][];
Dictionary<string, int> columnName2PositionMap = new Dictionary<string, int>();
foreach (var inputColumn in columnAttrs) {
#region Asserts
Debug.Assert(inputMatrix[inputColumn.ParamOrderId] == null);
Debug.Assert(!columnName2PositionMap.ContainsKey(inputColumn.Name));
Debug.Assert(!columnName2PositionMap.ContainsValue(inputColumn.ParamOrderId));
#endregion
inputMatrix[inputColumn.ParamOrderId] = inputColumn.Values;
columnName2PositionMap[inputColumn.Name] = inputColumn.ParamOrderId;
}
// Build Skip filters
List<object[]> skipConstraints = CreateSkipConstraints(method, columnName2PositionMap);
// Build Priority filters + default filter
Dictionary<int, List<object[]>> priFilters = CreatePriorityConstraints(method, Priority, columnName2PositionMap);
// Create Pairwise combinations based on the pri filters
Dictionary<int, List<object[]>> combinations = Pairwise(inputMatrix, skipConstraints, priFilters);
List<TestVariation> variations = new List<TestVariation>();
int variationCounter = 0;
//string orderedColumnNames = string.Concat(columnName2PositionMap.OrderBy(p => p.Value).Select((p, i) => i == 0 ? p.Key : string.Concat(",", p.Key)).ToArray());
foreach (int priority in combinations.Keys) {
foreach (object[] line in combinations[priority]) {
string name = GetVariationName(columnName2PositionMap, line);
TestVariation tv = new TestVariation(name, (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method));
tv.Priority = priority;
//// the last element in params will contain the parameter names (we need it to be able perform filters on the silverlight side);
//object[] linePlusNames = new object[line.Length + 1];
//Array.Copy(line, linePlusNames, line.Length);
//linePlusNames[line.Length] = orderedColumnNames ;
//tv.Params = linePlusNames;
tv.Params = line;
tv.Id = ++variationCounter;
variations.Add(tv);
}
}
return variations;
}
//Helpers
protected override void DetermineChildren()
{
//Delegate (add any nested testcases)
base.DetermineChildren();
//Dynamically figure out what functions are variations...
foreach (MethodInfo method in this.GetType().GetMethods())
{
//Loop through the Attributes for method
foreach (VariationAttribute attr in method.GetCustomAttributes(typeof(VariationAttribute), false /*inhert*/))
{
//Every method that has a [Variation attribute] = a variation
//Add this variation to our array
TestFunc func = null;
try
{
func = (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), this, method);
}
catch (Exception e)
{
e = new TestFailedException("Variation: '" + method + "' doesn't match expected signature of 'void func()', unable to add that variation.", null, null, e);
HandleException(e);
continue;
}
TestVariation var = this.CreateVariation(func);
attr.Name = func.Method.Name; //Name is always the function name (invoke)
if (attr.Desc == null)
{
attr.Desc = attr.Name;
}
var.Attribute = attr;
AddChild(var);
}
}
//Sort
//Default sort is based upon IComparable of each item
Children.Sort();
}
public TestVariation GenerateVariation(TestCase testcase, MethodInfo method, VariationAttribute testVarAttrib)
{
//do not generate variations for tests which have KeepInContent set to false and the version is not V2
if (!this.KeepInContent && !Versioning.Server.SupportsV2Features)
{
return null;
}
TestVariation ffTestVariation;
MethodInfo generateEpmMappingsMethod = testcase.GetType().GetMethod("GenerateEpmMappings");
FilterResourcesLambda = GenerateLambda();
TestFunc testMethodAction = delegate()
{
bool thisTestFailed = true;
SkipInvalidRuns();
try
{
generateEpmMappingsMethod.Invoke(testcase, new object[]{
KeepInContent,//bool pKeepInContent,
RemoveUnEligibleTypes,//bool pRemoveUnEligibleTypes,
GenerateClientTypes,//bool pGenerateClientTypes,
MapToAtomElements,//bool pMapPropertiesToAtomElements,
MapToNonAtomElements,//bool pMapPropertiesToNonAtomElements,
GenerateServerEPMMappings,//bool GenerateServerEPMMappings,
KeepSameNamespace,//bool KeepSameNamespace,
IncludeComplexTypes,//bool IncludeComplexTypes
PreDefinedPaths ,//string[] PreDefinedPaths
FilterResourcesLambda //Func<ResourceType, bool> pFilterResourcesLambda
});
method.Invoke(testcase, null);
thisTestFailed = AstoriaTestLog.FailureFound;
}
catch (Microsoft.OData.Client.DataServiceClientException dsException)
{
if (dsException.InnerException != null)
AstoriaTestLog.WriteLine(dsException.InnerException.Message);
}
catch (TestFailedException testException)
{
AstoriaTestLog.WriteLine("Test failed with :{0}", testException.Message);
AstoriaTestLog.WriteLine("Repro Details\r\nClient Code");
}
finally
{
//Disabling this as OOM errors prevent file copying
thisTestFailed = false;
#region //Clean out all the facets after the test , so that the next test is clear
object workspacePointer = null;
PropertyInfo testWorkspacesProperty = testcase.Parent.GetType().GetProperty("Workspaces");
workspacePointer = testcase.Parent;
if (testWorkspacesProperty == null)
{
testWorkspacesProperty = testcase.GetType().GetProperty("Workspaces");
workspacePointer = testcase;
}
Workspaces testWorkspaces = (Workspaces)testWorkspacesProperty.GetValue(workspacePointer, null);
foreach (Workspace workspace in testWorkspaces)
{
if (thisTestFailed)
{
string testFailureReproPath = String.Format("FF_Failure_{0}", System.Guid.NewGuid().ToString());
testFailureReproPath = System.IO.Path.Combine(Environment.CurrentDirectory, testFailureReproPath);
IOUtil.CopyFolder(workspace.DataService.DestinationFolder, testFailureReproPath, true);
AstoriaTestLog.WriteLine("Test failed , Repro available at : {0} ", testFailureReproPath);
}
if (!(workspace is EdmWorkspace))
{
workspace.GenerateClientTypes = true;
workspace.GenerateServerMappings = false;
workspace.ApplyFriendlyFeeds();
}
}
#endregion
}
};
ffTestVariation = new TestVariation(testMethodAction);
if (testVarAttrib != null)
{
ffTestVariation.Params = testVarAttrib.Params;
ffTestVariation.Param = testVarAttrib.Param;
ffTestVariation.Desc = String.Format(
"{3} , {0},{1},{2},{4}{5}{6}{7}",
GenerateClientTypes ? "Client Mapped" : "",
GenerateServerEPMMappings ? "Server Mapped" : "",
MapToAtomElements ? "Mapped to Atom" : "Mapped to Non-Atom Elements",
testVarAttrib.Desc, InheritanceFilter.ToString(),
KeepSameNamespace ? "Same Namespace" : "Different NameSpaces",
PreDefinedPaths != null ? ",PredefinedPaths" : "", KeepInContent.ToString());
if (AstoriaTestProperties.MaxPriority == 0)
{
ffTestVariation.Id = 1;
}
else
{
ffTestVariation.Id = idCounter++;
}
ffTestVariation.Priority = this.Priority;
}
return ffTestVariation;
}
public TestVariation GenerateVariation(TestCase testcase, MethodInfo method)
{
TestVariation rcTestVariation;
TestFunc testMethodAction = delegate()
{
method.Invoke(testcase, null);
};
rcTestVariation = new TestVariation(testMethodAction);
rcTestVariation.Params = this.Params;
rcTestVariation.Param = this.Param;
rcTestVariation.Desc = this.Desc;
rcTestVariation.Id = this.Id;
rcTestVariation.Priority = this.Priority;
return rcTestVariation;
}
public TestVariation CreateNewVariation(string descPrefix, string uri)
{
TestVariation variation = new TestVariation(CompareReaders, descPrefix);
variation.Attribute = new VariationAttribute(descPrefix + " :: " + uri, new object[] { uri });
return variation;
}
protected override void DetermineChildren()
{
// Use previous settings if client partial trust level is not medium
// We only perform client partial-trust test here
if (AstoriaTestProperties.ServiceTrustLevel != TrustLevel.Medium || (!this.Name.Contains("Client")))
{
base.DetermineChildren();
}
else
{
AppDomain myDomain = this.GetCustomizedAppDomain();
ArrayList types = this.GetNestedTypes(this.GetType());
foreach (Type type in types)
{
//Were only interested in objects that inherit from TestCase.
//NOTE: We need to filter other objects, since some internal VB classes throw trying
//to get Attributes
if (!type.IsSubclassOf(typeof(TestCase)))
continue;
//Every class that has a TestCase attribute, IS a TestCase
foreach (TestCaseAttribute attr in type.GetCustomAttributes(typeof(TestCaseAttribute), false/*inhert*/))
{
//Create this class (call the constructor with no arguments)
// Create testcase instance in another app domain with desired permission set.
TestCase testcase = (TestCase)myDomain.CreateInstanceAndUnwrap(type.Assembly.FullName, type.FullName);
if (attr.Name == null)
attr.Name = type.Name;
testcase.Attribute = attr;
this.AddChild(testcase);
}
}
//Sort
//Default sort is based upon IComparable of each item
Children.Sort();
}
if (AstoriaTestProperties.RuntimeEnvironment != TestRuntimeEnvironment.CheckinSuites)
{
// ensure that we don't get 0/0 = 0% results due to empty test cases
foreach (TestItem child in this.Children.Cast<TestItem>().Where(c => !Descendents(c).OfType<TestVariation>().Any()))
{
child.Children.Clear();
TestFunc func = new TestFunc(delegate() { throw new TestSkippedException("Placeholder"); });
TestVariation variation = new TestVariation("Placeholder for module with no tests", func);
variation.Attribute = new VariationAttribute("Placeholder for module with no tests")
{
Id = 1,
Implemented = true,
Priority = AstoriaTestProperties.MinPriority
};
variation.Parent = child;
child.AddChild(variation);
}
}
}
public virtual List <TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method)
{
int id = StartingId;
List <object[]> rows = new List <object[]>();
List <MatrixColumnAttribute> matrixColumns = new List <MatrixColumnAttribute>();
object [] attributes = method.GetCustomAttributes(false);
List <MatrixSkipConstraintAttribute> skipAttributes = new List <MatrixSkipConstraintAttribute>();
List <MatrixPriorityFilterAttribute> priorityAttributes = new List <MatrixPriorityFilterAttribute>();
foreach (MatrixColumnAttribute matrixColumn in attributes.OfType <MatrixColumnAttribute>())
{
matrixColumns.Insert(0, matrixColumn);
}
foreach (MatrixSkipConstraintAttribute skipConstraint in attributes.OfType <MatrixSkipConstraintAttribute>())
{
skipAttributes.Insert(0, skipConstraint);
}
foreach (MatrixPriorityFilterAttribute filter in attributes.OfType <MatrixPriorityFilterAttribute>())
{
// TODO: why aren't we just doing add? or .ToList()?
priorityAttributes.Insert(0, filter);
}
//Verify that no ParamOrderId's match
IEnumerable <IGrouping <int, MatrixColumnAttribute> > grouping = matrixColumns.GroupBy <MatrixColumnAttribute, int>(mc => mc.ParamOrderId);
foreach (IGrouping <int, MatrixColumnAttribute> group in grouping)
{
if (group.Key != -1)
{
if (group.Count() > 1)
{
throw new ArgumentException("For Matrix variation:" + this.PrefixName + " test method:" + method.Name + " Multiple MatrixColumnAttributes have the same ParamOrderId, this is invalid");
}
}
}
//Sort it by ParamOrderId
matrixColumns.Sort(CompareMatrixColumnParamId);
foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
{
rows = MultiplyMatrixRowsByColumn(matrixColumn, rows);
}
List <TestVariation> variations = new List <TestVariation>();
foreach (object[] matrixRow in rows)
{
//Every method that has a [Variation attribute] = a variation
//Add this variation to our array
TestFunc func = null;
try
{
func = (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method);
}
catch (Exception e)
{
e = new TestFailedException("Variation: '" + method + "' doesn't match expected signature of 'void func()', unable to add that variation.", null, null, e);
TestLog.HandleException(e);
continue;
}
int i = 0;
string description = null;
//Calling from the list to reverse the attribute order so its in the order
//as its in the code
foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
{
if (description == null)
{
description = this.PrefixName + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
}
else
{
description = description + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
}
i++;
}
if (!ShouldSkipVariation(testcase, matrixColumns, skipAttributes, matrixRow))
{
TestVariation var = new TestVariation(description, func);
var.Params = matrixRow;
var.Id = id;
var.Desc = description;
var.Priority = VariationPriority(testcase, matrixColumns, priorityAttributes, matrixRow);
variations.Add(var);
id++;
}
}
return(variations);
}
public override TestResult Execute()
{
base.Execute();
TestModule module = GetModule();
if (module == null)
{
throw new Exception("No parent module");
}
TestItems children = Children;
if (children != null && children.Count > 0)
{
// this is not a leaf node, just invoke all the children's execute
foreach (object child in children)
{
if (child is TestCase)
{
TestCase t = child as TestCase;
t.Init();
t.Execute();
continue;
}
TestVariation var = child as TestVariation;
if (var != null)
{
const string indent = "\t";
try
{
CurVariation = var;
TestResult ret = var.Execute();
if (TestResult.Passed == ret)
{
module.PassCount++;
}
else if (TestResult.Failed == ret)
{
System.Console.WriteLine(indent + var.Desc);
System.Console.WriteLine(indent + " FAILED");
module.FailCount++;
}
else
{
System.Console.WriteLine(indent + var.Desc);
System.Console.WriteLine(indent + " SKIPPED");
module.SkipCount++;
}
}
catch (TestSkippedException tse)
{
if (!string.IsNullOrWhiteSpace(var.Desc))
{
System.Console.WriteLine(indent + var.Desc);
}
if (!string.IsNullOrWhiteSpace(tse.Message))
{
System.Console.WriteLine(indent + " SKIPPED" + ", Msg:" + tse.Message);
}
module.SkipCount++;
}
catch (Exception e)
{
System.Console.WriteLine(indent + var.Desc);
System.Console.WriteLine(e);
System.Console.WriteLine(indent + " FAILED");
module.FailCount++;
}
}
}
}
return(TestResult.Passed);
}
public virtual List<TestVariation> EnumerateVariations(TestCase testcase, MethodInfo method)
{
int id= StartingId;
List<object[]> rows = new List<object[]>();
List<MatrixColumnAttribute> matrixColumns = new List<MatrixColumnAttribute>();
object []attributes = method.GetCustomAttributes(false);
List<MatrixSkipConstraintAttribute> skipAttributes = new List<MatrixSkipConstraintAttribute>();
List<MatrixPriorityFilterAttribute> priorityAttributes = new List<MatrixPriorityFilterAttribute>();
foreach (MatrixColumnAttribute matrixColumn in attributes.OfType < MatrixColumnAttribute>())
{
matrixColumns.Insert(0,matrixColumn);
}
foreach (MatrixSkipConstraintAttribute skipConstraint in attributes.OfType<MatrixSkipConstraintAttribute>())
{
skipAttributes.Insert(0, skipConstraint);
}
foreach (MatrixPriorityFilterAttribute filter in attributes.OfType<MatrixPriorityFilterAttribute>())
{
// TODO: why aren't we just doing add? or .ToList()?
priorityAttributes.Insert(0, filter);
}
//Verify that no ParamOrderId's match
IEnumerable<IGrouping<int, MatrixColumnAttribute>> grouping = matrixColumns.GroupBy<MatrixColumnAttribute, int>(mc => mc.ParamOrderId);
foreach (IGrouping<int, MatrixColumnAttribute> group in grouping)
{
if (group.Key != -1)
{
if (group.Count() > 1)
throw new ArgumentException("For Matrix variation:" + this.PrefixName + " test method:" + method.Name + " Multiple MatrixColumnAttributes have the same ParamOrderId, this is invalid");
}
}
//Sort it by ParamOrderId
matrixColumns.Sort(CompareMatrixColumnParamId);
foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
{
rows = MultiplyMatrixRowsByColumn(matrixColumn, rows);
}
List<TestVariation> variations = new List<TestVariation>();
foreach(object[] matrixRow in rows)
{
//Every method that has a [Variation attribute] = a variation
//Add this variation to our array
TestFunc func = null;
try
{
func = (TestFunc)Delegate.CreateDelegate(typeof(TestFunc), testcase, method);
}
catch(Exception e)
{
e = new TestFailedException("Variation: '" + method + "' doesn't match expected signature of 'void func()', unable to add that variation.", null, null, e);
TestLog.HandleException(e);
continue;
}
int i = 0;
string description = null;
//Calling from the list to reverse the attribute order so its in the order
//as its in the code
foreach (MatrixColumnAttribute matrixColumn in matrixColumns)
{
if (description == null)
description = this.PrefixName + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
else
description = description + " " + matrixColumn.Name + ":" + matrixRow[i].ToString();
i++;
}
if (!ShouldSkipVariation(testcase, matrixColumns, skipAttributes, matrixRow))
{
TestVariation var = new TestVariation(description, func);
var.Params = matrixRow;
var.Id = id;
var.Desc = description;
var.Priority = VariationPriority(testcase, matrixColumns, priorityAttributes, matrixRow);
variations.Add(var);
id++;
}
}
return variations;
}
