public override TestResult[] Execute(ITestMethod testMethod)
{
var diagnosticAttributes = testMethod.GetAttributes <AssertionDiagnosticAttribute>(false);
var codeFixAttributes = testMethod.GetAttributes <AssertionCodeFixAttribute>(false);
var results = new List <TestResult>();
foreach (var diagnosticAttribute in diagnosticAttributes.Where(attribute => !attribute.Ignore))
{
foreach (var assertion in GetTestCases(diagnosticAttribute))
{
var result = testMethod.Invoke(new[] { assertion });
result.DisplayName = assertion;
results.Add(result);
}
}
foreach (var codeFixAttribute in codeFixAttributes.Where(attribute => !attribute.Ignore))
{
foreach (var(oldAssertion, newAssertion) in GetTestCases(codeFixAttribute))
{
var result = testMethod.Invoke(new[] { oldAssertion, newAssertion });
result.DisplayName = $"{Environment.NewLine}old: \"{oldAssertion}\" {Environment.NewLine}new: \"{newAssertion}\"";
results.Add(result);
}
}
return(results.ToArray());
}
/// <summary>
/// Executes the test method on the UI Thread.
/// </summary>
/// <param name="testMethod">
/// The test method.
/// </param>
/// <returns>
/// An array of <see cref="TestResult"/> instances.
/// </returns>
/// Throws <exception cref="NotSupportedException"> when run on an async test method.
/// </exception>
public override TestResult[] Execute(ITestMethod testMethod)
{
var attrib = testMethod.GetAttributes <AsyncStateMachineAttribute>(false);
if (attrib.Length > 0)
{
throw new NotSupportedException(FrameworkMessages.AsyncUITestMethodNotSupported);
}
TestResult result = null;
var dispatcher = DispatcherQueue ?? global::Microsoft.UI.Xaml.Window.Current?.DispatcherQueue;
if (dispatcher == null)
{
throw new InvalidOperationException(FrameworkMessages.AsyncUITestMethodWithNoDispatcherQueue);
}
if (dispatcher.HasThreadAccess)
{
try
{
result = testMethod.Invoke(Array.Empty <object>());
}
catch (Exception e)
{
return(new TestResult[] { new TestResult {
TestFailureException = e
} });
}
}
else
{
var taskCompletionSource = new global::System.Threading.Tasks.TaskCompletionSource <object>();
if (!dispatcher.TryEnqueue(Microsoft.UI.Dispatching.DispatcherQueuePriority.Normal, () =>
{
try
{
result = testMethod.Invoke(Array.Empty <object>());
taskCompletionSource.SetResult(null);
}
catch (Exception e)
{
result = new TestResult {
TestFailureException = e
};
taskCompletionSource.SetException(e);
}
}))
{
taskCompletionSource.SetResult(null);
}
taskCompletionSource.Task.GetAwaiter().GetResult();
}
return(new TestResult[] { result });
}
/// <inheritdoc />
public override TestResult[] Execute(ITestMethod testMethod)
{
var result = new List <TestResult>();
result.Add(testMethod.Invoke(new object[] { false }));
result.Add(testMethod.Invoke(new object[] { true }));
return(result.ToArray());
}
public override TestResult[] Execute(ITestMethod testMethod)
{
MyTest.Run = 1;
var result1 = testMethod.Invoke(null);
MyTest.Run = 2;
var result2 = testMethod.Invoke(null);
return(new TestResult[] { result1, result2 });
}
private TestResult ExecuteWithServer(ITestMethod testMethod)
{
var arguments = ExtendArguments(testMethod.Arguments);
var filesToCreate = testMethod.GetAttributes <CreateTestSpecificFileAttribute>(false);
TestEnvironmentImpl.AddBeforeAfterTest(async() => {
var interpreterConfiguration = GetInterpreterConfiguration(arguments);
var rootUri = TestSpecificRootUri ? TestData.GetTestSpecificRootUri() : null;
foreach (var file in filesToCreate)
{
await TestData.CreateTestSpecificFileAsync(file.RelativeFilePath, file.Content);
}
var server = await new Server().InitializeAsync(interpreterConfiguration, rootUri);
if (DefaultTypeshedPath)
{
var limits = server.Analyzer.Limits;
limits.UseTypeStubPackages = true;
server.Analyzer.Limits = limits;
server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() });
}
arguments[0] = server;
return(server);
});
return(testMethod.Invoke(arguments));
}
private TestResult[] Invoke(ITestMethod testMethod)
{
if (_testMethodAttribute != null)
{
return(_testMethodAttribute.Execute(testMethod));
}
return(new[] { testMethod.Invoke(null) });
}
public static TestResult[] Execute(ITestMethod testMethod)
{
// Get our test context
TestContext testContext = testMethod.GetTestContext();
// Get a possible Description attribute
var desc = testMethod.MethodInfo.GetCustomAttribute <DescriptionAttribute>(inherit: true)?.Description;
if (string.IsNullOrEmpty(desc))
{
desc = testMethod.MethodInfo.GetCustomAttribute <System.ComponentModel.DescriptionAttribute>(inherit: true)?.Description;
}
string customDisplayName = null;
// Test has method arguments, include them in the description
if (testMethod.Arguments?.Length > 0)
{
// Test has description, use it as a string formatter for the arguments
if (!string.IsNullOrEmpty(desc))
{
try
{
customDisplayName = testMethod.TestMethodName + " - " + string.Format(CultureInfo.InvariantCulture, desc, testMethod.Arguments);
}
catch (Exception ex)
{
// String format with description failed
customDisplayName = testMethod.TestMethodName + " - " + ex.Message;
}
}
else
{
// No description formatter, only append arguments to test name
customDisplayName = testMethod.TestMethodName + " - " + string.Join(", ", testMethod.Arguments);
}
}
// Test only has description, use in display name
else if (desc != null)
{
customDisplayName = testMethod.TestMethodName + " - " + desc;
}
if (!string.IsNullOrEmpty(customDisplayName))
{
testContext.GetInternalTestState().CustomDisplayName = customDisplayName;
}
var executeResult = testMethod.Invoke(testMethod.Arguments);
if (!string.IsNullOrEmpty(customDisplayName))
{
executeResult.DisplayName = customDisplayName;
}
return(new[] { executeResult });
}
/// <summary>
/// Execute tests on method
/// </summary>
/// <param name="testMethod"></param>
/// <returns></returns>
public override TestResult[] Execute(ITestMethod testMethod)
{
var parameterValues = AttributeHelper.GetData(testMethod.MethodInfo, _parameters);
var result = testMethod.Invoke(parameterValues);
result.DisplayName = $"{testMethod.TestMethodName} ({string.Join(",", parameterValues)})";
return(new[] { result });
}
public override TestResult[] Execute(ITestMethod testMethod)
{
TestResult result = null;
CoreApplication.MainView.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
result = testMethod.Invoke(new object[0]);
}).AsTask().GetAwaiter().GetResult();
return(new TestResult[] { result });
}
public override TestResult[] Execute(ITestMethod testMethod)
{
TestResult result = null;
CoreApplication.MainView.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
result = testMethod.Invoke(new object[0]);
}).AsTask().GetAwaiter().GetResult();
return new TestResult[] { result };
}
/// <summary>
/// Executes a test method.
/// </summary>
/// <param name="testMethod">The test method to execute.</param>
/// <returns>An array of TestResult objects that represent the outcome(s) of the test.</returns>
/// <remarks>Extensions can override this method to customize running a TestMethod.</remarks>
public virtual TestResult[] Execute(ITestMethod testMethod)
{
ITestDataSource[] dataSources = testMethod.GetAttributes <Attribute>(true)?.Where(a => a is ITestDataSource).OfType <ITestDataSource>().ToArray();
if (dataSources == null || dataSources.Length == 0)
{
return(new TestResult[] { testMethod.Invoke(null) });
}
return(DataTestMethodAttribute.RunDataDrivenTest(testMethod, dataSources));
}
/// <summary>
/// Executes a test method.
/// </summary>
/// <param name="testMethod">The test method to execute.</param>
/// <returns>An array of TestResult objects that represent the outcome(s) of the test.</returns>
/// <remarks>Extensions can override this method to customize running a TestMethod.</remarks>
public virtual TestResult[] Execute(ITestMethod testMethod)
{
DataRowAttribute[] dataRows = testMethod.GetAttributes <DataRowAttribute>(false);
if (dataRows == null || dataRows.Length == 0)
{
return(new TestResult[] { testMethod.Invoke(null) });
}
return(DataTestMethodAttribute.RunDataDrivenTest(testMethod, dataRows));
}
public override TestResult[] Execute(ITestMethod testMethod)
{
if (Thread.CurrentThread.GetApartmentState() == ApartmentState.STA)
{
return new TestResult[] { testMethod.Invoke(null) }
}
;
TestResult[] result = null;
Thread thread = new Thread(() =>
{
result = new TestResult[] { testMethod.Invoke(null) };
});
thread.SetApartmentState(ApartmentState.STA);
thread.Start();
thread.Join();
return(result);
}
}
public TestResult Invoke(object[] arguments)
{
try
{
TestContextProvider.Initialize(_target.MethodInfo, arguments);
return(_target.Invoke(arguments));
}
finally
{
TestContextProvider.Clear();
}
}
private TestResult ExecuteWithServer(ITestMethod testMethod)
{
var arguments = ExtendArguments(testMethod.Arguments);
TestEnvironmentImpl.AddBeforeAfterTest(async() => {
var interpreterConfiguration = GetInterpreterConfiguration(arguments);
var server = await new Server().InitializeAsync(interpreterConfiguration);
arguments[0] = server;
return(server);
});
return(testMethod.Invoke(arguments));
}
public override TestResult[] Execute(ITestMethod testMethod)
{
TestResult testResult = testMethod.Invoke(null);
var testContextMessages = JsonConvert.DeserializeObject <List <dynamic> >(testResult.TestContextMessages);
var count = 1;
while (count++ < _count)
{
testResult.InnerResultsCount = count;
var iteration = testMethod.Invoke(null);
var iterationContextMessages = JsonConvert.DeserializeObject <List <dynamic> >(iteration.TestContextMessages);
testContextMessages.AddRange(iterationContextMessages);
}
testResult.TestContextMessages = JsonConvert.SerializeObject(testContextMessages, _formatting);
return(new TestResult[] { testResult });
}
/// <summary>
/// Invokes the underlying method on the instance and marks the
/// test work item as complete.
/// </summary>
/// <returns>False, noting the completion of the work item.</returns>
public override bool Invoke()
{
if (_testMethod != null)
{
_testMethod.Invoke(_instance);
}
else if (_method != null)
{
_method.Invoke(_instance, None);
}
WorkItemComplete();
return(base.Invoke());
}
public override TestResult[] Execute(ITestMethod testMethod)
{
if (testMethod.GetAttributes <AsyncStateMachineAttribute>(false).Length != 0)
{
throw new NotSupportedException("async TestMethod with UITestMethodAttribute are not supported. Either remove async, wrap the code with RunAsync() or use TestMethodAttribute.");
}
TestResult result = null;
CoreApplication.MainView.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => {
result = testMethod.Invoke(new object[0]);
}).AsTask().GetAwaiter().GetResult();
return(new TestResult[] { result });
}
public override TestResult[] Execute(ITestMethod testMethod)
{
var results = new List <TestResult>();
foreach (var testInput in _items.Value)
{
var result = testMethod.Invoke(new object[] { testInput });
var overriddenDisplayName = GetDisplayNameForTestItem(testInput);
if (!string.IsNullOrEmpty(overriddenDisplayName))
{
result.DisplayName = overriddenDisplayName;
}
results.Add(result);
}
return(results.ToArray());
}
/// <inheritdoc />
public override TestResult[] Execute(ITestMethod testMethod)
{
var dataSourceAttributes = testMethod.MethodInfo.GetCustomAttributes().OfType <ITestDataSource>().Where(dataSourceAttribute => !(dataSourceAttribute is AutoDataAttribute));
if (dataSourceAttributes.Any())
{
return(base.Execute(testMethod));
}
// If there are no other ITestDataSource attributes on this test method, then use AutoFixture to generate the arguments for a single test run.
var testMethodInfo = testMethod.ToTestMethodInfo();
var fixture = TestMethodFixtureSource.GetFixture(testMethodInfo);
var arguments = TestMethodArgumentResolver.GetTestMethodArguments(fixture, testMethodInfo);
return(new[] { testMethod.Invoke(arguments.ToArray()) });
}
private TestResult [] Invoke(ITestMethod testMethod)
{
if (TestMethodAttribute != null)
{
return(TestMethodAttribute.Execute(testMethod));
}
IEnumerable <IParameterInfo>
infos = testMethod.ParameterTypes.Select(x => new ParameterInfo(x));
var arguments = Generator.Create(infos);
return(new []
{
testMethod.Invoke(arguments)
});
}
/// <summary>
/// Run data driven test method.
/// </summary>
/// <param name="testMethod"> Test method to execute. </param>
/// <param name="testDataSources">Test data sources. </param>
/// <returns> Results of execution. </returns>
internal static TestResult[] RunDataDrivenTest(ITestMethod testMethod, ITestDataSource[] testDataSources)
{
List <TestResult> results = new List <TestResult>();
foreach (var testDataSource in testDataSources)
{
foreach (var data in testDataSource.GetData(testMethod.MethodInfo))
{
TestResult result = testMethod.Invoke(data);
result.DisplayName = testDataSource.GetDisplayName(testMethod.MethodInfo, data);
results.Add(result);
}
}
return(results.ToArray());
}
/// <summary>
/// Executes the test method on the UI Thread.
/// </summary>
/// <param name="testMethod">
/// The test method.
/// </param>
/// <returns>
/// An array of <see cref="TestResult"/> instances.
/// </returns>
/// Throws <exception cref="NotSupportedException"> when run on an async test method.
/// </exception>
public override TestResult[] Execute(ITestMethod testMethod)
{
var attrib = testMethod.GetAttributes <AsyncStateMachineAttribute>(false);
if (attrib.Length > 0)
{
throw new NotSupportedException(FrameworkMessages.AsyncUITestMethodNotSupported);
}
TestResult result = null;
Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(
Windows.UI.Core.CoreDispatcherPriority.Normal,
() =>
{
result = testMethod.Invoke(new object[] { });
}).AsTask().GetAwaiter().GetResult();
return(new TestResult[] { result });
}
/// <summary>
/// Run data driven test method.
/// </summary>
/// <param name="testMethod"> Test method to execute. </param>
/// <param name="dataRows"> Data Row. </param>
/// <returns> Results of execution. </returns>
internal static TestResult[] RunDataDrivenTest(ITestMethod testMethod, DataRowAttribute[] dataRows)
{
List <TestResult> results = new List <TestResult>();
foreach (var dataRow in dataRows)
{
TestResult result = testMethod.Invoke(dataRow.Data);
if (!string.IsNullOrEmpty(dataRow.DisplayName))
{
result.DisplayName = dataRow.DisplayName;
}
else
{
result.DisplayName = string.Format(CultureInfo.CurrentCulture, "{0} ({1})", testMethod.TestMethodName, string.Join(",", dataRow.Data));
}
results.Add(result);
}
return(results.ToArray());
}
/// <summary>
/// Invokes the underlying method on the instance and marks the
/// test work item as complete.
/// </summary>
/// <returns>False, noting the completion of the work item.</returns>
public override bool Invoke()
{
if (_testMethod != null)
{
System.Console.WriteLine("{0}.{1}", _testMethod.Method.DeclaringType.FullName, _testMethod.Method.Name);
_testMethod.Invoke(_instance);
#if HEAPVIZ
Console.WriteLine("yo");
GC.Collect();
GC.WaitForPendingFinalizers();
string name = string.Format("{0}.{1}.heap", _testMethod.Method.DeclaringType.FullName, _testMethod.Method.Name).Replace('.', '_');
Deployment.Current.GraphManagedHeap(name);
#endif
}
else if (_method != null)
{
_method.Invoke(_instance, None);
}
WorkItemComplete();
return(base.Invoke());
}
/// <summary>
/// Executes a combinatorial test method.
/// </summary>
/// <param name="testMethod">The test method to execute.</param>
/// <returns>An array of TestResult objects that represent the outcomes of the test.</returns>
public override TestResult[] Execute(ITestMethod testMethod)
{
try
{
ValidateAttributes(testMethod);
object[][] args = GetArgumentsInOrder(testMethod);
ValidateArguments(testMethod, args);
// get the count of tests so we know how to format the # in the test name.
int totalTestsDigits = ComputeTotalTestCountDigits(args);
var indexes = new int[args.Length];
bool done;
var results = new List <TestResult>();
do
{
// run the test with the current values.
object[] argsToPass = Enumerable.Range(0, args.Length)
.Select(i => args[i][indexes[i]])
.ToArray();
TestResult result = testMethod.Invoke(argsToPass);
// In case the results are alphabetized (I'm looking at you
// VS test explorer), prepend the test # padded with 0s.
string number = (results.Count + 1).ToString().PadLeft(totalTestsDigits, '0');
result.DisplayName = $"#{number}: {GetTestDisplayName(testMethod, argsToPass)}";
results.Add(result);
// update the current indexes.
int updateIdx = indexes.Length - 1;
for (; updateIdx >= 0; updateIdx--)
{
indexes[updateIdx]++;
if (indexes[updateIdx] < args[updateIdx].Length)
{
// still values left for this parameter.
break;
}
// carry to the next slot
indexes[updateIdx] = 0;
}
// If we carried off the end of the array, we're done
done = (updateIdx < 0);
} while (!done);
return(results.ToArray());
}
catch (Exception ex)
{
// something went wrong - fail the test with the exception info.
// note: I believe the MSTest executor will catch any exceptions that escape
// tests, and return a failed TestResult (if it wasn't an ExpectedException).
return(new[]
{
new TestResult
{
Outcome = UnitTestOutcome.Error,
TestFailureException = ex
}
});
}
}
private TestResult[] Invoke(ITestMethod testMethod)
{
return(new[] { testMethod.Invoke(null) });
}
public override TestResult[] Execute(ITestMethod testMethod)
{
// Get our test context
TestContext testContext = testMethod.GetTestContext();
// Create a new internal test state.
InternalTestState internalTestState = testContext.ResetInternalTestState();
// Convert test argument types
object[] args = null;
if (testMethod.Arguments != null)
{
args = new object[testMethod.Arguments.Length];
for (var i = 0; i < args.Length; i++)
{
args[i] = TypeConversion.ConvertValue(testMethod.ParameterTypes[i].ParameterType, testMethod.Arguments[i]);
}
}
// Execute our test method on our main node.
TestResult mainResult = testMethod.Invoke(args);
// Set a more accurate time elapse duration (end of init to start of cleanup)
mainResult.Duration = internalTestState.EndTime - internalTestState.StartTime;
// Obtain the display name for our test.
string testDisplayName = mainResult.DisplayName ?? testMethod.TestMethodName;
// Reset the internal test state for this test context
internalTestState = testContext.ResetInternalTestState();
// If we have a parallel node to run tests against..
if (Global.ExternalNodeTestServices != null)
{
// If we are only running against the external node
if (Global.OnlyUseExternalNode.GetValueOrDefault())
{
// Only using the external node means the above method invocation already ran on
// the external node, so we quit with the singular result.
return(new[] { mainResult });
}
// Begin using the external node.
internalTestState.InExternalNodeContext = true;
// Execute our test method on our parallel node
TestResult parallelResult = testMethod.Invoke(args);
// Set a more accurate time elapse duration (end of init to start of cleanup)
parallelResult.Duration = internalTestState.EndTime - internalTestState.StartTime;
// Stop using the external node.
internalTestState.InExternalNodeContext = false;
// Set our test names
mainResult.DisplayName = $"{testDisplayName} (built-in)";
parallelResult.DisplayName = $"{testDisplayName} (external)";
return(new[] { mainResult, parallelResult });
}
else
{
var parallelResult = new TestResult
{
Outcome = UnitTestOutcome.Error,
TestFailureException = new Exception("Failed to run parallel test because parallel test node services failed to initialize properly."),
};
// Set our test names
mainResult.DisplayName = $"{testDisplayName} (built-in)";
parallelResult.DisplayName = $"{testDisplayName} (external)";
return(new[] { mainResult, parallelResult });
}
}
private TestResult RunTestMethod(object testClassInstance, ITestMethod testMethod)
{
TestResult testResult = null;
string className = testClassInstance.GetType().Name;
string methodName = testMethod.Name;
try {
testMethod.Invoke(testClassInstance);
testResult = new TestResult(className, methodName, true, "The test ran successfully.");
} catch (Exception ex) {
testResult = new TestResult(className, methodName, false, "The test failed: ", ex);
}
return testResult;
}
/// <summary>
/// Executes a test method.
/// </summary>
/// <param name="testMethod">The test method to execute.</param>
/// <returns>An array of TestResult objects that represent the outcome(s) of the test.</returns>
/// <remarks>Extensions can override this method to customize running a TestMethod.</remarks>
public virtual TestResult[] Execute(ITestMethod testMethod)
{
return(new TestResult[] { testMethod.Invoke(null) });
}
public override TestResult[] Execute(ITestMethod testMethod)
{
TestResult[] errorResults = ValidateElevated(testMethod);
if (errorResults != null)
{
return(errorResults);
}
Exception signatureException = GetMethodSignatureException(testMethod);
if (signatureException != null)
{
return(testMethod.CreateExceptionResult(signatureException));
}
var runParameters = TestRunParameters.Read();
string logFolder = runParameters.LogFolder;
bool shouldLog = !string.IsNullOrEmpty(logFolder);
if (shouldLog)
{
try
{
logFolder = CreateLogFolder(logFolder, testMethod.TestMethodName);
}
catch (Exception e)
{
return(testMethod.CreateExceptionResult(e));
}
}
int iterations = runParameters.Iterations;
var results = new TestResult[iterations];
for (int iteration = 1; iteration <= iterations; iteration++)
{
string sessionName = $"{testMethod.TestMethodName}-{iteration}";
using (var session = new TraceEventSession(sessionName))
{
EnableKernelProviders(session, shouldLog);
TraceEventDispatcher source;
ZippedETLWriter writer = null;
if (shouldLog)
{
string etlPath = Path.Combine(logFolder, $"Iteration{iteration}.etl");
source = new ETWReloggerTraceEventSource(sessionName, TraceEventSourceType.Session, etlPath);
writer = new ZippedETLWriter(etlPath);
}
else
{
source = session.Source;
}
EnableProviders(session);
PerformanceTestContext context = CreateContext(source);
Task <TestResult> testTask = Task.Run(() => testMethod.Invoke(new object[] { context }));
// This is a blocking call that in the case of ETWReloggerTraceEventSource, must be run on the same
// thread as ETWReloggerTraceEventSource was created on. It will become unblocked when the
// PerformanceTestContext calls StopProcessing on the source.
source.Process();
TestResult result = testTask.Result;
string displayName = testMethod.TestMethodName;
if (iterations > 1)
{
displayName += $" [{iteration}/{iterations}]";
}
result.DisplayName = displayName;
session.Flush();
OnIterationEnded(context);
context.LogScenarios();
context.LogMemoryDelta();
context.LogMessage($"{displayName} completed. {session.EventsLost} events lost.");
context.WriteLogsToResult(result, writer);
results[iteration - 1] = result;
}
}
return(results);
}
