/// <summary>
/// Performs test case discovery from a DiscoveryTarget.
/// </summary>
/// <param name="discoveryTarget">Contains adaptor, manifest path, and default TestInfo data.</param>
/// <param name="discoveryAdaptorType">Type DiscoveryTarget.Adaptor maps to.</param>
/// <param name="discoveryRootDirectoryPath">Directory DiscoveryTarget.Path is relative to.</param>
/// <param name="defaultTestInfo">Default to info to use as prototype.</param>
/// <param name="filteringSettings">Filtering settings.</param>
/// <returns>Collection of TestInfos.</returns>
private static IEnumerable <TestInfo> Discover(DiscoveryTarget discoveryTarget, Type discoveryAdaptorType, DirectoryInfo discoveryRootDirectoryPath, TestInfo defaultTestInfo, FilteringSettings filteringSettings)
{
// Start with the default test info passed into the adaptor, and clone it.
// then merge in local values from the discovery target's default test info.
TestInfo compositedTestInfo = defaultTestInfo.Clone();
compositedTestInfo.Merge(discoveryTarget.DefaultTestInfo);
List <TestInfo> discoveredTests = new List <TestInfo>();
DiscoveryAdaptor discoveryAdaptor = (DiscoveryAdaptor)Activator.CreateInstance(discoveryAdaptorType);
IEnumerable <FileInfo> discoveryTargetFileNames = GetFiles(discoveryRootDirectoryPath, discoveryTarget.Path, compositedTestInfo, filteringSettings);
foreach (FileInfo discoveryTargetFileName in discoveryTargetFileNames)
{
IEnumerable <TestInfo> discoveryTargetTests;
try
{
discoveryTargetTests = discoveryAdaptor.Discover(discoveryTargetFileName, compositedTestInfo);
}
catch (Exception e)
{
throw new InvalidOperationException("Exception when running " + discoveryAdaptor.GetType().Name + " against test manifest " + discoveryTargetFileName.FullName + ": " + e.Message);
}
discoveredTests.AddRange(discoveryTargetTests);
}
return(discoveredTests);
}
/// <summary>
/// Parse an XTC file for metadata describing testcase(s).
/// </summary>
/// <param name="testManifestPath">Xtc file.</param>
/// <param name="defaultTestInfo">The default values for a TestInfo.</param>
/// <returns>List of TestInfo objects representing the testcase(s).</returns>
private static List <TestInfo> ParseXtc(FileInfo testManifestPath, TestInfo defaultTestInfo)
{
List <TestInfo> tests = new List <TestInfo>();
//
XmlTextReader xmlTextReader = new SkipDataReader(testManifestPath.FullName);
TestInfo xtcTestInfo = defaultTestInfo.Clone();
// Add the name of the xtc file as a driver parameter.
if (xtcTestInfo.DriverParameters == null)
{
xtcTestInfo.DriverParameters = new ContentPropertyBag();
}
xtcTestInfo.DriverParameters["XtcFileName"] = testManifestPath.Name;
int xtcTestIndex = 1;
// Process the text reader until we hit EOF. We are interested
// in Elements named DefaultTestInfo or Test, in which cases we
// deserialize into a TestInfo via ObjectSerializer. If the
// current text reader node was an element but of a different
// name, or not an element at all, we just Read() past it.
while (!xmlTextReader.EOF)
{
if (xmlTextReader.NodeType == XmlNodeType.Element)
{
if (xmlTextReader.Name == "DEFAULTTESTINFO")
{
TestInfo deserializedDefaultTestInfo = (TestInfo)ObjectSerializer.Deserialize(xmlTextReader, typeof(TestInfo), null);
xtcTestInfo.Merge(deserializedDefaultTestInfo);
}
else if (xmlTextReader.Name == "TEST")
{
TestInfo deserializedTest = (TestInfo)ObjectSerializer.Deserialize(xmlTextReader, typeof(TestInfo), null);
TestInfo test = xtcTestInfo.Clone();
test.Merge(deserializedTest);
test.DriverParameters["XtcTestIndex"] = xtcTestIndex.ToString(CultureInfo.InvariantCulture);
xtcTestIndex++;
tests.Add(test);
}
else
{
xmlTextReader.Read();
}
}
else
{
xmlTextReader.Read();
}
}
return(tests);
}
/// <summary>
/// Create TestInfo from a TestAttribute, Type and default TestInfo.
/// New test info will be added to tests List.
/// </summary>
protected virtual ICollection <TestInfo> BuildTestInfo(TestAttribute testAttribute, Type ownerType, TestInfo defaultTestInfo)
{
// NOTE:
// The old implementation restricted this method to only building a
// TestInfo from the exact AttributeType, not any class inheriting
// from TestAttribute. This logic was already being enforced by
// SearchClass/SearchMethods, so this restriction here has been
// lifted. It allows subclasses to delegate population of inherited
// properties to the base class and focus on just new properties.
List <TestInfo> newTests = new List <TestInfo>();
TestInfo testInfo = defaultTestInfo.Clone();
// General Test metadata
ApplyGeneralMetadata(testAttribute, ownerType, testInfo);
ApplyVersions(testAttribute, testInfo);
ApplyBugs(testAttribute, testInfo);
ApplyKeywords(testAttribute, testInfo);
ApplyConfigurations(testAttribute, testInfo);
ApplyDeployments(testAttribute, testInfo);
ApplySupportFiles(testAttribute, testInfo);
// Driver data. This is part of an implicit contract with sti.exe.
if (!ApplyDriverSettings(testAttribute, ownerType, testInfo))
{
return(newTests);
}
newTests.Add(testInfo);
return(newTests);
}
/// <summary>
/// Create TestInfo from a TestAttribute, Type, default TestInfo, and XTC file.
/// </summary>
protected override ICollection <TestInfo> BuildTestInfo(TestAttribute testAttribute, Type ownerType, TestInfo defaultTestInfo)
{
ModelAttribute modelAttribute = (ModelAttribute)testAttribute;
List <TestInfo> newTests = new List <TestInfo>();
if (String.IsNullOrEmpty(modelAttribute.XtcFileName))
{
// No xtc file name, return empty list.
Trace.TraceWarning("Xtc file name is null or empty. Aborting discovery on class " + ownerType.FullName + ".");
return(newTests);
}
if (modelAttribute.ModelStart > modelAttribute.ModelEnd)
{
// Invalid end index, return empty list.
Trace.TraceWarning("The model end index cannot be greater than the start index. Aborting discovery on class " + ownerType.FullName + ".");
return(newTests);
}
// Build test info as we would for normal test attribute.
// This should only return one test case.
IEnumerable <TestInfo> baseTests = base.BuildTestInfo(modelAttribute, ownerType, defaultTestInfo);
if (baseTests.Count() > 1)
{
// Too many tests, return empty list.
Trace.TraceWarning("Parsing single ModelAttribute produced multiple test infos before reading XTC, aborting discovery on class " + ownerType.FullName + ".");
return(newTests);
}
if (baseTests.Count() == 0)
{
// Too few tests, return empty list.
Trace.TraceWarning("Failure parsing ModelAttribute on class " + ownerType.FullName + " before reading XTC. Aborting discovery.");
return(newTests);
}
TestInfo baseTest = base.BuildTestInfo(modelAttribute, ownerType, defaultTestInfo).First();
baseTest.DriverParameters["ModelClass"] = baseTest.DriverParameters["Class"];
baseTest.DriverParameters["ModelAssembly"] = baseTest.DriverParameters["Assembly"];
baseTest.DriverParameters["XtcFileName"] = modelAttribute.XtcFileName;
TestSupportFile tsf = new TestSupportFile();
tsf.Source = modelAttribute.XtcFileName;
baseTest.SupportFiles.Add(tsf);
int modelStart, modelEnd;
try
{
GetStartEndTestCaseFromXtc(modelAttribute.XtcFileName, out modelStart, out modelEnd);
}
catch (ArgumentException e)
{
// Xtc file does not exist, return empty list.
Trace.TraceWarning(e.Message + " Discovery aborted on class " + ownerType.FullName + ".");
return(newTests);
}
// Attribute range overrides that found in the xtc file.
if (modelAttribute.ModelStart >= 0)
{
modelStart = modelAttribute.ModelStart;
}
if (modelAttribute.ModelEnd >= 0)
{
modelEnd = modelAttribute.ModelEnd;
}
if (modelAttribute.ExpandModelCases)
{
// Create new test info for each test in the xtc and pass TIndex to driver.
for (int testIndex = modelStart; testIndex <= modelEnd; testIndex++)
{
baseTest.DriverParameters["TIndex"] = testIndex.ToString(CultureInfo.InvariantCulture);
newTests.Add(baseTest.Clone());
}
}
else
{
// Create a single test info for all the tests in the xtc and pass range to driver.
baseTest.DriverParameters["ModelStart"] = modelStart.ToString(CultureInfo.InvariantCulture);
baseTest.DriverParameters["ModelEnd"] = modelEnd.ToString(CultureInfo.InvariantCulture);
newTests.Add(baseTest);
}
return(newTests);
}
/// <summary>
/// Discover DRTs from drt manifest.
/// </summary>
/// <param name="testManifestPath">Should be path to rundrtlist.txt</param>
/// <param name="defaultTestInfo"></param>
/// <returns>TestInfos for drts.</returns>
//List<TestInfo> Discover(string testBinRootPath, string targetFilename, TestInfo defaultTestInfo)
public override IEnumerable <TestInfo> Discover(FileInfo testManifestPath, TestInfo defaultTestInfo)
{
string targetFilename = testManifestPath.FullName;
if (!File.Exists(targetFilename))
{
throw new FileNotFoundException(Path.GetFullPath(targetFilename));
}
// Deserialize Drt manifest file into a list of Drt objects.
XmlTextReader reader = new XmlTextReader(targetFilename);
List <Drt> drtDefinitions = (List <Drt>)ObjectSerializer.Deserialize(reader, typeof(List <Drt>), null);
// Convert each Drt object into a TestInfo.
List <TestInfo> drts = new List <TestInfo>();
foreach (Drt drtDef in drtDefinitions)
{
// Initialize TestInfo.
TestInfo drtTestInfo = defaultTestInfo.Clone();
ContentPropertyBag driverArgs = drtTestInfo.DriverParameters;
// Store Executable, Owner, and Args in the property bag for DrtDriver to consume.
driverArgs["exe"] = drtDef.Executable;
if (!String.IsNullOrEmpty(drtDef.Args))
{
driverArgs["args"] = drtDef.Args;
}
driverArgs["owner"] = drtDef.Owner;
string exeFileNameWithoutExtension = Path.GetFileNameWithoutExtension(driverArgs["exe"]);
drtTestInfo.Name = exeFileNameWithoutExtension + "(" + drtDef.Args + ")";
drtTestInfo.DriverParameters = driverArgs;
drtTestInfo.Area = drtDef.Team;
if (drtDef.Timeout > 0)
{
drtTestInfo.Timeout = TimeSpan.FromSeconds(drtDef.Timeout);
}
SelectConfiguration(drtTestInfo, drtDef);
// Convert drt support files to a source/destination pair for TestInfo.
foreach (string file in drtDef.SupportFiles)
{
//The path may be to a directory or a file. If we think the path is to a directory
//then the destination and the source are the same
//otherwise the destination is the containing directory of the source file
//We assume people are not specifying files that have no extention
//and that * is used only in filenames
TestSupportFile supportFile = new TestSupportFile();
supportFile.Source = Path.Combine("DRT", file);
supportFile.Destination = string.IsNullOrEmpty(Path.GetExtension(supportFile.Source)) && !supportFile.Source.Contains("*") ? file : Path.GetDirectoryName(file);
drtTestInfo.SupportFiles.Add(supportFile);
}
// Add all needed build output files
List <string> buildOutputFilePaths =
new List <string>()
{
Path.Combine("DRT", driverArgs["exe"]),
};
// In .NET Core we need to add all outputs that are prefixed with the exe name
buildOutputFilePaths.AddRange(Directory.GetFiles("DRT", exeFileNameWithoutExtension + ".*"));
foreach (var path in buildOutputFilePaths)
{
if (!drtTestInfo.SupportFiles.Select(s => s.Source).Contains(path))
{
drtTestInfo.SupportFiles.Add(new TestSupportFile()
{
Source = path
});
}
}
// Append relative path to all deployments.
foreach (string deployment in drtDef.Deployments)
{
drtTestInfo.Deployments.Add(Path.Combine("DRT", deployment));
}
drts.Add(drtTestInfo);
}
return(drts);
}
/// <summary />
public override IEnumerable <TestInfo> Discover(FileInfo testManifestPath, TestInfo defaultTestInfo)
{
if (!(testManifestPath.Exists))
{
throw new FileNotFoundException(testManifestPath.FullName);
}
string targetFilename = testManifestPath.FullName;
Assembly testAssembly = Assembly.LoadFrom(targetFilename);
List <TestInfo> discoveredTests = new List <TestInfo>();
Type[] allTypes = testAssembly.GetTypes();
foreach (Type type in allTypes)
{
//if (type.IsSubclassOf(typeof(TestSuite)) && !type.IsAbstract)
if (HackyIsSubclassOf(type, "TestSuite") && !type.IsAbstract)
{
// Trace.TraceInformation("Discovering Suite '" + type.Name + "'.");
//TestSuite suite = (TestSuite)Activator.CreateInstance(type);
object suite = Activator.CreateInstance(type);
//IList<TestCase> tests = suite.TestCases;
IEnumerable tests = (IEnumerable)HackyGetProperty(suite, "TestCases");
//foreach (TestCase test in tests)
foreach (object test in tests)
{
//IList<TestVariation> variations = test.Variations;
IEnumerable variations = (IEnumerable)HackyGetProperty(test, "Variations");
//foreach (TestVariation variation in variations)
foreach (object variation in variations)
{
TestInfo testInfo = defaultTestInfo.Clone();
//testInfo.Disabled = test.IsDisabled;
testInfo.Disabled = (bool)HackyGetProperty(test, "IsDisabled");
//testInfo.Name = string.Format("{0}.{1}{2}", type.Name, test.Id,
// (variation.Parameters == null || variation.Parameters.Length == 0) ?
// string.Empty : string.Format(".{0}", string.Join(".", variation.Parameters)));
testInfo.Name = string.Format("{0}.{1}{2}", type.Name, HackyGetProperty(test, "Id"),
(HackyGetProperty(variation, "Parameters") == null || ((string[])HackyGetProperty(variation, "Parameters")).Length == 0) ?
string.Empty : string.Format(".{0}", string.Join(".", ((string[])HackyGetProperty(variation, "Parameters")))));
//testInfo.Priority = test.Priority;
testInfo.Priority = (int)HackyGetProperty(test, "Priority");
testInfo.SubArea = type.Name; //
//testInfo.Keywords.Add(test.Keywords);
if (testInfo.Keywords == null)
{
testInfo.Keywords = new Collection <string>();
}
testInfo.Keywords.Add((string)HackyGetProperty(test, "Keywords"));
// Group all tests defined in the same TestSuite in a single ExecutionGroup.
//
//testInfo.DriverParameters = AnnotationsTestSettings.ToDriverParameters(testAssembly, variation);
testInfo.DriverParameters = ToDriverParameters(testAssembly, variation);
//if (!(testInfo.Disabled.HasValue && testInfo.Disabled.Value) && test.Bugs.Length <= 0)
// discoveredTests.Add(testInfo);
if (!(testInfo.Disabled.HasValue && testInfo.Disabled.Value) && ((int[])HackyGetProperty(test, "Bugs")).Length <= 0)
{
discoveredTests.Add(testInfo);
}
}
}
}
}
return(discoveredTests);
}
public override IEnumerable <TestInfo> Discover(FileInfo testManifestPath, TestInfo defaultTestInfo)
{
if (testManifestPath == null)
{
throw new ArgumentNullException("testManifestPath");
}
if (!testManifestPath.Exists)
{
throw new FileNotFoundException("testManifestPath", testManifestPath.FullName);
}
if (defaultTestInfo == null)
{
throw new ArgumentNullException("defaultTestInfo");
}
string assemblyPath = testManifestPath.FullName;
if (Path.GetExtension(assemblyPath).Equals(".exe"))
{
assemblyPath = Path.ChangeExtension(assemblyPath, ".dll");
}
Assembly assembly = Assembly.LoadFrom(assemblyPath);
// Parse TestDefaults attributes on the assembly, these override the default TestInfo passed to the adapter.
TestDefaultsAttribute[] assemblyAttributes = (TestDefaultsAttribute[])assembly.GetCustomAttributes(typeof(TestDefaultsAttribute), false);
if (assemblyAttributes.Length > 0)
{
ApplyTestDefaultsAttribute(ref defaultTestInfo, assemblyAttributes[0]);
}
Type[] types;
try
{
types = assembly.GetTypes();
}
catch (ReflectionTypeLoadException e)
{
throw new ApplicationException("Reflection on Assembly failed due to:" + e.LoaderExceptions.ToCommaSeparatedList());
}
foreach (Type currentType in types)
{
// Get TestDefaults and Test attributes on the current type (not ancestors).
TestDefaultsAttribute[] testDefaultsAttributes = (TestDefaultsAttribute[])currentType.GetCustomAttributes(typeof(TestDefaultsAttribute), false);
TestAttribute[] classTestAttributes = (TestAttribute[])currentType.GetCustomAttributes(this.AttributeType, false);
// Can't have TestDefaults and Test attribute on same class.
if (testDefaultsAttributes.Length > 0 && classTestAttributes.Length > 0)
{
Trace.TraceWarning("Test and TestDefaults attribute are not allowed on the same class definition.");
continue;
}
// Save a default test info specific to this type.
TestInfo typeDefaultTestInfo = defaultTestInfo.Clone();
// Apply TestDefaults found on this type AND any of its ancestors.
ApplyAncestorTestDefaults(currentType, ref typeDefaultTestInfo);
if (testDefaultsAttributes.Length > 0)
{
SearchMethods(currentType, typeDefaultTestInfo);
}
else // classTestAttributes.Length > 0
{
// Handle per-type test case attributes.
SearchClass(classTestAttributes, currentType, typeDefaultTestInfo);
}
}
return(Tests);
}