private bool FindAllTestMethods(TestResult testResult, TestAssembly testAssembly, Type[] types)
{
foreach (var type in types)
{
object[] attrs = type.GetCustomAttributes(false);
if (type.IsNested)
// Ignore nested types
continue;
if (attrs.Length == 0 || FindAttribute(attrs, typeof(TestClassAttribute)) == null)
// No attributes or TestClass attribute, no tests
continue;
OrderAttribute priorityAttr;
WriteMessage(TestingResources.TestClassFound(type.ToString()));
TestClass testClass = new TestClass(type);
testResult.TestAssembly.TestClasses_Internal.Add(testClass);
// By default tests are run in the order that they are declared in the class.
// The OrderAttribute can be used to order running of methods AND classes.
priorityAttr = (OrderAttribute)FindAttribute(attrs, typeof(OrderAttribute));
if (priorityAttr != null)
testClass.Order = priorityAttr.Order;
List<DeploymentItem> deploymentItems = new List<DeploymentItem>();
foreach (var attr in attrs)
{
if (attr is DeploymentItemAttribute)
{
DeploymentItem deploymentItem = CreateDeploymentItem((DeploymentItemAttribute)attr);
if (deploymentItem != null)
deploymentItems.Add(deploymentItem);
}
}
if (deploymentItems != null)
testClass.DeploymentItems = deploymentItems.ToArray();
// Now get all the public methods in the class and search for ones marked with test attributes
MethodInfo[] methods = type.GetMethods(
BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static | BindingFlags.DeclaredOnly);
List<Type> expectedExceptions = new List<Type>();
foreach (var method in methods)
{
attrs = method.GetCustomAttributes(false);
TestMethod testMethod = null;
// We determine what type of test method we have by searching for the various
// explicit (TestMethod) and implicit (ClassInitialize, etc..) methods.
if (FindAttribute(attrs, typeof(TestMethodAttribute)) != null)
{
if (!CheckInstanceMethodVoidNoParams(method))
continue;
ExplicitTestMethod explicitTestMethod = new ExplicitTestMethod(method);
testMethod = explicitTestMethod;
priorityAttr = (OrderAttribute)FindAttribute(attrs, typeof(OrderAttribute));
if (priorityAttr != null)
explicitTestMethod.Order = priorityAttr.Order;
testClass.TestMethods_Internal.Add(explicitTestMethod);
}
else if (FindAttribute(attrs, typeof(AssemblyInitializeAttribute)) != null)
{
if (testAssembly.AssemblyInitializeMethod != null)
{
WriteError(TestingResources.SecondAssemblyInitializeFound(testClass.Name));
return false;
}
if (!CheckStaticMethodVoidNoParams(method))
continue;
testMethod = testAssembly.AssemblyInitializeMethod = new TestMethod(method);
}
else if (FindAttribute(attrs, typeof(AssemblyCleanupAttribute)) != null)
{
if (testAssembly.AssemblyCleanupMethod != null)
{
WriteError(TestingResources.SecondAssemblyCleanupFound(testClass.Name));
return false;
}
if (!CheckStaticMethodVoidNoParams(method))
continue;
testMethod = testAssembly.AssemblyCleanupMethod = new TestMethod(method);
}
else if (FindAttribute(attrs, typeof(ClassInitializeAttribute)) != null)
{
if (testClass.ClassInitializeMethod != null)
{
WriteError(TestingResources.SecondClassInitializeFound(testClass.Name));
return false;
}
if (!CheckStaticMethodVoidTestContextParam(method))
continue;
testMethod = testClass.ClassInitializeMethod = new TestMethod(method);
}
else if (FindAttribute(attrs, typeof(ClassCleanupAttribute)) != null)
{
if (testClass.ClassCleanupMethod != null)
{
WriteError(TestingResources.SecondClassCleanupFound(testClass.Name));
return false;
}
if (!CheckStaticMethodVoidNoParams(method))
continue;
testMethod = testClass.ClassCleanupMethod = new TestMethod(method);
}
else if (FindAttribute(attrs, typeof(TestInitializeAttribute)) != null)
{
if (testClass.TestInitializeMethod != null)
{
WriteError(TestingResources.SecondTestInitializeFound(testClass.Name));
return false;
}
if (!CheckInstanceMethodVoidNoParams(method))
continue;
testMethod = testClass.TestInitializeMethod = new TestMethod(method);
}
else if (FindAttribute(attrs, typeof(TestCleanupAttribute)) != null)
{
if (testClass.TestCleanupMethod != null)
{
WriteError(TestingResources.SecondTestCleanupFound(testClass.Name));
return false;
}
if (!CheckInstanceMethodVoidNoParams(method))
continue;
testMethod = testClass.TestCleanupMethod = new TestMethod(method);
}
if (testMethod != null)
{
// Check if we have any expected exception or deployment item attributes
expectedExceptions.Clear();
deploymentItems.Clear();
foreach (var attr in attrs)
{
ExpectedExceptionAttribute expectedExceptionAttr = attr as ExpectedExceptionAttribute;
DeploymentItemAttribute deploymentItemAttr = attr as DeploymentItemAttribute;
if (expectedExceptionAttr != null)
{
Type exceptionType = expectedExceptionAttr.ExceptionType;
if (!exceptionType.IsSubclassOf(typeof(Exception)))
WriteWarning(TestingResources.ExpectedExceptionMustBeException(testMethod.Name));
else
expectedExceptions.Add(exceptionType);
}
else if (deploymentItemAttr != null)
{
DeploymentItem deploymentItem = CreateDeploymentItem(deploymentItemAttr);
if (deploymentItem != null)
deploymentItems.Add(deploymentItem);
}
}
testMethod.ExpectedExceptions = expectedExceptions.ToArray();
testMethod.DeploymentItems = deploymentItems.ToArray();
}
}
// Go through each explicit test method and set the initialize and cleanup calls if they exist in the class
foreach (var testMethod in testClass.TestMethods_Internal)
{
if (testClass.TestInitializeMethod != null)
{
testMethod.TestInitializeMethod = new TestMethod(testClass.TestInitializeMethod);
}
if (testClass.TestCleanupMethod != null)
{
testMethod.TestCleanupMethod = new TestMethod(testClass.TestCleanupMethod);
}
}
if (testClass.TestCount == 0)
WriteWarning(TestingResources.TestClassHasNoTestMethods(type.Name));
}
// Sort classes in ascending order
testAssembly.TestClasses_Internal.Sort((testClass1, testClass2) => testClass1.Order - testClass2.Order);
// Sort tests in ascending order
foreach (var testClass in testAssembly.TestClasses_Internal)
{
testClass.TestMethods_Internal.Sort((testMethod1, testMethod2) => testMethod1.Order - testMethod2.Order);
}
return true;
}
public void Execute()
{
if (ShowHelp)
{
WriteMessage(Parser.LogoBanner);
WriteMessage(Parser.Usage);
return;
}
// Get the test results going by measuring the start time
TestResult testResult = new TestResult();
DateTime now = DateTime.Now;
testResult.StartDate = now.Date;
testResult.StartTime = now.TimeOfDay;
WriteMessage(
TestingResources.RuntimeVersion(
Marshal.SizeOf(typeof(IntPtr)) == 4 ? TestingResources.WordSize32 : TestingResources.WordSize64,
RuntimeEnvironment.GetSystemVersion().ToString(),
RuntimeEnvironment.GetRuntimeDirectory()));
if (TestFile == null)
{
WriteError(TestingResources.TestAssemblyNotSupplied);
return;
}
TestFile = TestFile.MakeFullPath();
if (!File.Exists(TestFile))
{
WriteError(TestingResources.TestAssemblyCannotBeFound(TestFile));
return;
}
if (DeploymentDir == null)
{
// If no deployment directory specified, then it is wherever the test assembly is
DeploymentDir = new ParsedPath(TestFile.VolumeAndDirectory, PathType.Directory);
WriteMessage(TestingResources.DeploymentDirIsTestAssemblyDir);
}
if (DeploymentItems == null)
{
// Avoid problems with nulls
DeploymentItems = new List<DeploymentItem>();
}
if (AssemblySearchDirs == null)
{
// Avoid nulls
AssemblySearchDirs = new List<ParsedPath>();
}
// Set test context results
ToastTool.TestContext.TestResult = testResult;
// Set properties for compatability with MSTest
ToastTool.TestContext.TestDeploymentDir = DeploymentDir;
// Save the current working directory as we will change it later
ToastTool.TestContext.InternalOriginalCurrentDirectory = new ParsedPath(Environment.CurrentDirectory, PathType.Directory);
// Initialize properties from the environment and from the command line
TestContext.Properties.AddFromEnvironment();
TestContext.Properties.AddFromPropertyString(Property);
TestContext.Properties.AddWellKnown(DeploymentDir);
// Expand all paths in any command line deployment items relative to the working directory
foreach (var deploymentItem in DeploymentItems)
{
deploymentItem.Path = deploymentItem.Path.MakeFullPath();
deploymentItem.OutputDirectory = DeploymentDir;
}
// Copy the test assembly to the deployment directory
if (!CopyDeploymentItem(new DeploymentItem(TestFile.MakeFullPath(), DeploymentDir)))
return;
// If any deployments fail it's an error because we assume we will not be
// able to load the test assembly, so change the outputter to upgrade any warnings we print.
WarningsAsErrors = true;
if (!CopyDeploymentItems(this.DeploymentItems.ToArray()))
return;
WarningsAsErrors = false;
TestAssembly testAssembly = null;
AppDomain.CurrentDomain.AssemblyResolve += OnAssemblyResolve;
AppDomain.CurrentDomain.AssemblyLoad += OnAssemblyLoad;
try
{
// We use Assembly.Load so that the test assembly and subsequently loaded
// assemblies end up in the correct load context. If the assembly cannot be
// found it will raise a AssemblyResolve event where we will search for the
// assembly.
testAssembly = testResult.TestAssembly = new TestAssembly(Assembly.Load(TestFile.FileAndExtension));
}
catch (Exception e)
{
if (ExceptionUtility.IsCriticalException(e))
throw;
WriteError(TestingResources.UnableToLoadTestAssembly(TestFile, e.ToString()));
// Not being able to load the test assembly pretty much shuts us down
return;
}
Type[] types;
// We won't get dependency errors until we actually try to reflect on all the types in the assembly
try
{
types = testAssembly.Assembly.GetTypes();
}
catch (ReflectionTypeLoadException e)
{
string message = TestingResources.UnableToReflectAssemblyTypes(testAssembly.Name);
// There is one entry in the exceptions array for each null in the types array,
// and they correspond positionally.
foreach (Exception ex in e.LoaderExceptions)
message += Environment.NewLine + " " + ex.Message;
WriteError(message);
// Not being able to reflect on classes in the test assembly is a critical error
return;
}
//
// Go through all the types in the test assembly and find all the test methods.
// An explicit test method is marked with a TestMethod attribute. An implicit test method
// are all ClassInitialize, TestCleanup, etc.. methods. They can cause the test run to fail
// just like explicit test methods.
//
try
{
if (!FindAllTestMethods(testResult, testAssembly, types))
return;
}
catch (Exception e)
{
if (ExceptionUtility.IsCriticalException(e))
throw;
// If we have any problems loading the test assembly that's a critical error
// that indicates a deployment problem and we must stop.
WriteError(TestingResources.ProblemProcessingTestAssembly(e.ToString()));
return;
}
//
// Start running the tests. Go through all assemblies, classes & methods do deployments, run tests.
//
// Change the current directory to the deployment directory
Environment.CurrentDirectory = DeploymentDir;
ProcessCycleStopwatch assemblyCycleStopwatch = null;
if (ProcessCycleStopwatch.IsSupported)
assemblyCycleStopwatch = ProcessCycleStopwatch.StartNew();
Stopwatch assemblyStopwatch = Stopwatch.StartNew();
RunTestMethod(null, testAssembly.AssemblyInitializeMethod);
// Now go through and run all the tests that we found
foreach (var testClass in testAssembly.TestClasses_Internal)
{
// If we have been given a class name then only run tests from that class
if (!String.IsNullOrEmpty(TestClassName) &&
!testClass.Name.EndsWith(TestClassName, StringComparison.CurrentCultureIgnoreCase))
{
testClass.SetAllTestOutcomesToSkipped();
WriteMessage(TestingResources.SkippingTestClass(testClass.Name));
continue;
}
ProcessCycleStopwatch classCycleStopwatch = null;
if (ProcessCycleStopwatch.IsSupported)
classCycleStopwatch = ProcessCycleStopwatch.StartNew();
Stopwatch classStopwatch = Stopwatch.StartNew();
WriteMessage(TestingResources.TestClassRunning(testClass.Name));
CopyDeploymentItems(testClass.DeploymentItems);
RunTestMethod(null, testClass.ClassInitializeMethod);
object testObj = null;
try
{
testObj = Activator.CreateInstance(testClass.Type);
}
catch (Exception e)
{
if (ExceptionUtility.IsCriticalException(e))
return;
if (e is TargetInvocationException)
{
WriteError(TestingResources.UnableToCreateInstanceOfTestClass(testClass.Name, e.InnerException.Message));
}
else
{
WriteError(TestingResources.UnableToCreateInstanceOfTestClass(testClass.Name, e.Message));
}
// Not being able to create the test class is a critical error and we need to exit
return;
}
if (testObj != null)
{
foreach (var testMethod in testClass.TestMethods)
{
// If we have been given a test method name then only run test methods that end with that name
if (!String.IsNullOrEmpty(TestMethodName) &&
!testMethod.Name.EndsWith(TestMethodName, StringComparison.CurrentCultureIgnoreCase))
{
testMethod.SetTestOutcomeToSkipped();
WriteMessage(TestingResources.SkippingTestMethod(testMethod.Name));
continue;
}
if (testMethod.TestInitializeMethod != null)
CopyDeploymentItems(testMethod.TestInitializeMethod.DeploymentItems);
RunTestMethod(testObj, testMethod.TestInitializeMethod);
CopyDeploymentItems(testMethod.DeploymentItems);
RunTestMethod(testObj, testMethod);
if (testMethod.TestCleanupMethod != null)
CopyDeploymentItems(testMethod.TestCleanupMethod.DeploymentItems);
RunTestMethod(testObj, testMethod.TestCleanupMethod);
}
}
RunTestMethod(null, testClass.ClassCleanupMethod);
classStopwatch.Stop();
testClass.ExecutionTime = new TimeSpan(classStopwatch.ElapsedTicks);
if (classCycleStopwatch != null)
{
classCycleStopwatch.Stop();
testClass.ExecutionCycles = unchecked((long)classCycleStopwatch.ElapsedCycles);
}
}
RunTestMethod(null, testAssembly.AssemblyCleanupMethod);
assemblyStopwatch.Stop();
testAssembly.ExecutionTime = new TimeSpan(assemblyStopwatch.ElapsedTicks);
if (assemblyCycleStopwatch != null)
{
assemblyCycleStopwatch.Stop();
testAssembly.ExecutionCycles = unchecked((long)assemblyCycleStopwatch.ElapsedCycles);
}
// Grab the end time
now = DateTime.Now;
testResult.EndDate = now.Date;
testResult.EndTime = now.TimeOfDay;
if (testResult.FailedTests > 0)
WriteWarning("{0} test{1} FAILED", testResult.FailedTests, testResult.FailedTests > 1 ? "s" : "");
else
WriteMessage("All tests PASSED");
if (OutputFile != null)
{
try
{
if (OutputFile.ToString() == "-")
{
testResult.WriteResults(Console.Error);
}
else
{
using (StreamWriter wr = new StreamWriter(OutputFile.ToString(), false, Encoding.Unicode))
{
testResult.WriteResults(wr);
}
WriteMessage("Test results written to '{0}'", OutputFile.ToString());
}
}
catch (Exception e)
{
if (ExceptionUtility.IsCriticalException(e))
throw;
WriteError(TestingResources.UnableToWriteOutputFile(OutputFile, e.Message));
OutputFile = null;
}
}
// Test may have failed, but the program HAS executed at this point
return;
}