public override TestResult[] Execute(ITestMethod testMethod) { var diagnosticAttributes = testMethod.GetAttributes <AssertionDiagnosticAttribute>(false); var codeFixAttributes = testMethod.GetAttributes <AssertionCodeFixAttribute>(false); var results = new List <TestResult>(); foreach (var diagnosticAttribute in diagnosticAttributes.Where(attribute => !attribute.Ignore)) { foreach (var assertion in GetTestCases(diagnosticAttribute)) { var result = testMethod.Invoke(new[] { assertion }); result.DisplayName = assertion; results.Add(result); } } foreach (var codeFixAttribute in codeFixAttributes.Where(attribute => !attribute.Ignore)) { foreach (var(oldAssertion, newAssertion) in GetTestCases(codeFixAttribute)) { var result = testMethod.Invoke(new[] { oldAssertion, newAssertion }); result.DisplayName = $"{Environment.NewLine}old: \"{oldAssertion}\" {Environment.NewLine}new: \"{newAssertion}\""; results.Add(result); } } return(results.ToArray()); }
/// <summary> /// Executes the test method on the UI Thread. /// </summary> /// <param name="testMethod"> /// The test method. /// </param> /// <returns> /// An array of <see cref="TestResult"/> instances. /// </returns> /// Throws <exception cref="NotSupportedException"> when run on an async test method. /// </exception> public override TestResult[] Execute(ITestMethod testMethod) { var attrib = testMethod.GetAttributes <AsyncStateMachineAttribute>(false); if (attrib.Length > 0) { throw new NotSupportedException(FrameworkMessages.AsyncUITestMethodNotSupported); } TestResult result = null; var dispatcher = DispatcherQueue ?? global::Microsoft.UI.Xaml.Window.Current?.DispatcherQueue; if (dispatcher == null) { throw new InvalidOperationException(FrameworkMessages.AsyncUITestMethodWithNoDispatcherQueue); } if (dispatcher.HasThreadAccess) { try { result = testMethod.Invoke(Array.Empty <object>()); } catch (Exception e) { return(new TestResult[] { new TestResult { TestFailureException = e } }); } } else { var taskCompletionSource = new global::System.Threading.Tasks.TaskCompletionSource <object>(); if (!dispatcher.TryEnqueue(Microsoft.UI.Dispatching.DispatcherQueuePriority.Normal, () => { try { result = testMethod.Invoke(Array.Empty <object>()); taskCompletionSource.SetResult(null); } catch (Exception e) { result = new TestResult { TestFailureException = e }; taskCompletionSource.SetException(e); } })) { taskCompletionSource.SetResult(null); } taskCompletionSource.Task.GetAwaiter().GetResult(); } return(new TestResult[] { result }); }
/// <inheritdoc /> public override TestResult[] Execute(ITestMethod testMethod) { var result = new List <TestResult>(); result.Add(testMethod.Invoke(new object[] { false })); result.Add(testMethod.Invoke(new object[] { true })); return(result.ToArray()); }
public override TestResult[] Execute(ITestMethod testMethod) { MyTest.Run = 1; var result1 = testMethod.Invoke(null); MyTest.Run = 2; var result2 = testMethod.Invoke(null); return(new TestResult[] { result1, result2 }); }
private TestResult ExecuteWithServer(ITestMethod testMethod) { var arguments = ExtendArguments(testMethod.Arguments); var filesToCreate = testMethod.GetAttributes <CreateTestSpecificFileAttribute>(false); TestEnvironmentImpl.AddBeforeAfterTest(async() => { var interpreterConfiguration = GetInterpreterConfiguration(arguments); var rootUri = TestSpecificRootUri ? TestData.GetTestSpecificRootUri() : null; foreach (var file in filesToCreate) { await TestData.CreateTestSpecificFileAsync(file.RelativeFilePath, file.Content); } var server = await new Server().InitializeAsync(interpreterConfiguration, rootUri); if (DefaultTypeshedPath) { var limits = server.Analyzer.Limits; limits.UseTypeStubPackages = true; server.Analyzer.Limits = limits; server.Analyzer.SetTypeStubPaths(new[] { TestData.GetDefaultTypeshedPath() }); } arguments[0] = server; return(server); }); return(testMethod.Invoke(arguments)); }
private TestResult[] Invoke(ITestMethod testMethod) { if (_testMethodAttribute != null) { return(_testMethodAttribute.Execute(testMethod)); } return(new[] { testMethod.Invoke(null) }); }
public static TestResult[] Execute(ITestMethod testMethod) { // Get our test context TestContext testContext = testMethod.GetTestContext(); // Get a possible Description attribute var desc = testMethod.MethodInfo.GetCustomAttribute <DescriptionAttribute>(inherit: true)?.Description; if (string.IsNullOrEmpty(desc)) { desc = testMethod.MethodInfo.GetCustomAttribute <System.ComponentModel.DescriptionAttribute>(inherit: true)?.Description; } string customDisplayName = null; // Test has method arguments, include them in the description if (testMethod.Arguments?.Length > 0) { // Test has description, use it as a string formatter for the arguments if (!string.IsNullOrEmpty(desc)) { try { customDisplayName = testMethod.TestMethodName + " - " + string.Format(CultureInfo.InvariantCulture, desc, testMethod.Arguments); } catch (Exception ex) { // String format with description failed customDisplayName = testMethod.TestMethodName + " - " + ex.Message; } } else { // No description formatter, only append arguments to test name customDisplayName = testMethod.TestMethodName + " - " + string.Join(", ", testMethod.Arguments); } } // Test only has description, use in display name else if (desc != null) { customDisplayName = testMethod.TestMethodName + " - " + desc; } if (!string.IsNullOrEmpty(customDisplayName)) { testContext.GetInternalTestState().CustomDisplayName = customDisplayName; } var executeResult = testMethod.Invoke(testMethod.Arguments); if (!string.IsNullOrEmpty(customDisplayName)) { executeResult.DisplayName = customDisplayName; } return(new[] { executeResult }); }
/// <summary> /// Execute tests on method /// </summary> /// <param name="testMethod"></param> /// <returns></returns> public override TestResult[] Execute(ITestMethod testMethod) { var parameterValues = AttributeHelper.GetData(testMethod.MethodInfo, _parameters); var result = testMethod.Invoke(parameterValues); result.DisplayName = $"{testMethod.TestMethodName} ({string.Join(",", parameterValues)})"; return(new[] { result }); }
public override TestResult[] Execute(ITestMethod testMethod) { TestResult result = null; CoreApplication.MainView.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { result = testMethod.Invoke(new object[0]); }).AsTask().GetAwaiter().GetResult(); return(new TestResult[] { result }); }
public override TestResult[] Execute(ITestMethod testMethod) { TestResult result = null; CoreApplication.MainView.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { result = testMethod.Invoke(new object[0]); }).AsTask().GetAwaiter().GetResult(); return new TestResult[] { result }; }
/// <summary> /// Executes a test method. /// </summary> /// <param name="testMethod">The test method to execute.</param> /// <returns>An array of TestResult objects that represent the outcome(s) of the test.</returns> /// <remarks>Extensions can override this method to customize running a TestMethod.</remarks> public virtual TestResult[] Execute(ITestMethod testMethod) { ITestDataSource[] dataSources = testMethod.GetAttributes <Attribute>(true)?.Where(a => a is ITestDataSource).OfType <ITestDataSource>().ToArray(); if (dataSources == null || dataSources.Length == 0) { return(new TestResult[] { testMethod.Invoke(null) }); } return(DataTestMethodAttribute.RunDataDrivenTest(testMethod, dataSources)); }
/// <summary> /// Executes a test method. /// </summary> /// <param name="testMethod">The test method to execute.</param> /// <returns>An array of TestResult objects that represent the outcome(s) of the test.</returns> /// <remarks>Extensions can override this method to customize running a TestMethod.</remarks> public virtual TestResult[] Execute(ITestMethod testMethod) { DataRowAttribute[] dataRows = testMethod.GetAttributes <DataRowAttribute>(false); if (dataRows == null || dataRows.Length == 0) { return(new TestResult[] { testMethod.Invoke(null) }); } return(DataTestMethodAttribute.RunDataDrivenTest(testMethod, dataRows)); }
public override TestResult[] Execute(ITestMethod testMethod) { if (Thread.CurrentThread.GetApartmentState() == ApartmentState.STA) { return new TestResult[] { testMethod.Invoke(null) } } ; TestResult[] result = null; Thread thread = new Thread(() => { result = new TestResult[] { testMethod.Invoke(null) }; }); thread.SetApartmentState(ApartmentState.STA); thread.Start(); thread.Join(); return(result); } }
public TestResult Invoke(object[] arguments) { try { TestContextProvider.Initialize(_target.MethodInfo, arguments); return(_target.Invoke(arguments)); } finally { TestContextProvider.Clear(); } }
private TestResult ExecuteWithServer(ITestMethod testMethod) { var arguments = ExtendArguments(testMethod.Arguments); TestEnvironmentImpl.AddBeforeAfterTest(async() => { var interpreterConfiguration = GetInterpreterConfiguration(arguments); var server = await new Server().InitializeAsync(interpreterConfiguration); arguments[0] = server; return(server); }); return(testMethod.Invoke(arguments)); }
public override TestResult[] Execute(ITestMethod testMethod) { TestResult testResult = testMethod.Invoke(null); var testContextMessages = JsonConvert.DeserializeObject <List <dynamic> >(testResult.TestContextMessages); var count = 1; while (count++ < _count) { testResult.InnerResultsCount = count; var iteration = testMethod.Invoke(null); var iterationContextMessages = JsonConvert.DeserializeObject <List <dynamic> >(iteration.TestContextMessages); testContextMessages.AddRange(iterationContextMessages); } testResult.TestContextMessages = JsonConvert.SerializeObject(testContextMessages, _formatting); return(new TestResult[] { testResult }); }
/// <summary> /// Invokes the underlying method on the instance and marks the /// test work item as complete. /// </summary> /// <returns>False, noting the completion of the work item.</returns> public override bool Invoke() { if (_testMethod != null) { _testMethod.Invoke(_instance); } else if (_method != null) { _method.Invoke(_instance, None); } WorkItemComplete(); return(base.Invoke()); }
public override TestResult[] Execute(ITestMethod testMethod) { if (testMethod.GetAttributes <AsyncStateMachineAttribute>(false).Length != 0) { throw new NotSupportedException("async TestMethod with UITestMethodAttribute are not supported. Either remove async, wrap the code with RunAsync() or use TestMethodAttribute."); } TestResult result = null; CoreApplication.MainView.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { result = testMethod.Invoke(new object[0]); }).AsTask().GetAwaiter().GetResult(); return(new TestResult[] { result }); }
public override TestResult[] Execute(ITestMethod testMethod) { var results = new List <TestResult>(); foreach (var testInput in _items.Value) { var result = testMethod.Invoke(new object[] { testInput }); var overriddenDisplayName = GetDisplayNameForTestItem(testInput); if (!string.IsNullOrEmpty(overriddenDisplayName)) { result.DisplayName = overriddenDisplayName; } results.Add(result); } return(results.ToArray()); }
/// <inheritdoc /> public override TestResult[] Execute(ITestMethod testMethod) { var dataSourceAttributes = testMethod.MethodInfo.GetCustomAttributes().OfType <ITestDataSource>().Where(dataSourceAttribute => !(dataSourceAttribute is AutoDataAttribute)); if (dataSourceAttributes.Any()) { return(base.Execute(testMethod)); } // If there are no other ITestDataSource attributes on this test method, then use AutoFixture to generate the arguments for a single test run. var testMethodInfo = testMethod.ToTestMethodInfo(); var fixture = TestMethodFixtureSource.GetFixture(testMethodInfo); var arguments = TestMethodArgumentResolver.GetTestMethodArguments(fixture, testMethodInfo); return(new[] { testMethod.Invoke(arguments.ToArray()) }); }
private TestResult [] Invoke(ITestMethod testMethod) { if (TestMethodAttribute != null) { return(TestMethodAttribute.Execute(testMethod)); } IEnumerable <IParameterInfo> infos = testMethod.ParameterTypes.Select(x => new ParameterInfo(x)); var arguments = Generator.Create(infos); return(new [] { testMethod.Invoke(arguments) }); }
/// <summary> /// Run data driven test method. /// </summary> /// <param name="testMethod"> Test method to execute. </param> /// <param name="testDataSources">Test data sources. </param> /// <returns> Results of execution. </returns> internal static TestResult[] RunDataDrivenTest(ITestMethod testMethod, ITestDataSource[] testDataSources) { List <TestResult> results = new List <TestResult>(); foreach (var testDataSource in testDataSources) { foreach (var data in testDataSource.GetData(testMethod.MethodInfo)) { TestResult result = testMethod.Invoke(data); result.DisplayName = testDataSource.GetDisplayName(testMethod.MethodInfo, data); results.Add(result); } } return(results.ToArray()); }
/// <summary> /// Executes the test method on the UI Thread. /// </summary> /// <param name="testMethod"> /// The test method. /// </param> /// <returns> /// An array of <see cref="TestResult"/> instances. /// </returns> /// Throws <exception cref="NotSupportedException"> when run on an async test method. /// </exception> public override TestResult[] Execute(ITestMethod testMethod) { var attrib = testMethod.GetAttributes <AsyncStateMachineAttribute>(false); if (attrib.Length > 0) { throw new NotSupportedException(FrameworkMessages.AsyncUITestMethodNotSupported); } TestResult result = null; Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync( Windows.UI.Core.CoreDispatcherPriority.Normal, () => { result = testMethod.Invoke(new object[] { }); }).AsTask().GetAwaiter().GetResult(); return(new TestResult[] { result }); }
/// <summary> /// Run data driven test method. /// </summary> /// <param name="testMethod"> Test method to execute. </param> /// <param name="dataRows"> Data Row. </param> /// <returns> Results of execution. </returns> internal static TestResult[] RunDataDrivenTest(ITestMethod testMethod, DataRowAttribute[] dataRows) { List <TestResult> results = new List <TestResult>(); foreach (var dataRow in dataRows) { TestResult result = testMethod.Invoke(dataRow.Data); if (!string.IsNullOrEmpty(dataRow.DisplayName)) { result.DisplayName = dataRow.DisplayName; } else { result.DisplayName = string.Format(CultureInfo.CurrentCulture, "{0} ({1})", testMethod.TestMethodName, string.Join(",", dataRow.Data)); } results.Add(result); } return(results.ToArray()); }
/// <summary> /// Invokes the underlying method on the instance and marks the /// test work item as complete. /// </summary> /// <returns>False, noting the completion of the work item.</returns> public override bool Invoke() { if (_testMethod != null) { System.Console.WriteLine("{0}.{1}", _testMethod.Method.DeclaringType.FullName, _testMethod.Method.Name); _testMethod.Invoke(_instance); #if HEAPVIZ Console.WriteLine("yo"); GC.Collect(); GC.WaitForPendingFinalizers(); string name = string.Format("{0}.{1}.heap", _testMethod.Method.DeclaringType.FullName, _testMethod.Method.Name).Replace('.', '_'); Deployment.Current.GraphManagedHeap(name); #endif } else if (_method != null) { _method.Invoke(_instance, None); } WorkItemComplete(); return(base.Invoke()); }
/// <summary> /// Executes a combinatorial test method. /// </summary> /// <param name="testMethod">The test method to execute.</param> /// <returns>An array of TestResult objects that represent the outcomes of the test.</returns> public override TestResult[] Execute(ITestMethod testMethod) { try { ValidateAttributes(testMethod); object[][] args = GetArgumentsInOrder(testMethod); ValidateArguments(testMethod, args); // get the count of tests so we know how to format the # in the test name. int totalTestsDigits = ComputeTotalTestCountDigits(args); var indexes = new int[args.Length]; bool done; var results = new List <TestResult>(); do { // run the test with the current values. object[] argsToPass = Enumerable.Range(0, args.Length) .Select(i => args[i][indexes[i]]) .ToArray(); TestResult result = testMethod.Invoke(argsToPass); // In case the results are alphabetized (I'm looking at you // VS test explorer), prepend the test # padded with 0s. string number = (results.Count + 1).ToString().PadLeft(totalTestsDigits, '0'); result.DisplayName = $"#{number}: {GetTestDisplayName(testMethod, argsToPass)}"; results.Add(result); // update the current indexes. int updateIdx = indexes.Length - 1; for (; updateIdx >= 0; updateIdx--) { indexes[updateIdx]++; if (indexes[updateIdx] < args[updateIdx].Length) { // still values left for this parameter. break; } // carry to the next slot indexes[updateIdx] = 0; } // If we carried off the end of the array, we're done done = (updateIdx < 0); } while (!done); return(results.ToArray()); } catch (Exception ex) { // something went wrong - fail the test with the exception info. // note: I believe the MSTest executor will catch any exceptions that escape // tests, and return a failed TestResult (if it wasn't an ExpectedException). return(new[] { new TestResult { Outcome = UnitTestOutcome.Error, TestFailureException = ex } }); } }
private TestResult[] Invoke(ITestMethod testMethod) { return(new[] { testMethod.Invoke(null) }); }
public override TestResult[] Execute(ITestMethod testMethod) { // Get our test context TestContext testContext = testMethod.GetTestContext(); // Create a new internal test state. InternalTestState internalTestState = testContext.ResetInternalTestState(); // Convert test argument types object[] args = null; if (testMethod.Arguments != null) { args = new object[testMethod.Arguments.Length]; for (var i = 0; i < args.Length; i++) { args[i] = TypeConversion.ConvertValue(testMethod.ParameterTypes[i].ParameterType, testMethod.Arguments[i]); } } // Execute our test method on our main node. TestResult mainResult = testMethod.Invoke(args); // Set a more accurate time elapse duration (end of init to start of cleanup) mainResult.Duration = internalTestState.EndTime - internalTestState.StartTime; // Obtain the display name for our test. string testDisplayName = mainResult.DisplayName ?? testMethod.TestMethodName; // Reset the internal test state for this test context internalTestState = testContext.ResetInternalTestState(); // If we have a parallel node to run tests against.. if (Global.ExternalNodeTestServices != null) { // If we are only running against the external node if (Global.OnlyUseExternalNode.GetValueOrDefault()) { // Only using the external node means the above method invocation already ran on // the external node, so we quit with the singular result. return(new[] { mainResult }); } // Begin using the external node. internalTestState.InExternalNodeContext = true; // Execute our test method on our parallel node TestResult parallelResult = testMethod.Invoke(args); // Set a more accurate time elapse duration (end of init to start of cleanup) parallelResult.Duration = internalTestState.EndTime - internalTestState.StartTime; // Stop using the external node. internalTestState.InExternalNodeContext = false; // Set our test names mainResult.DisplayName = $"{testDisplayName} (built-in)"; parallelResult.DisplayName = $"{testDisplayName} (external)"; return(new[] { mainResult, parallelResult }); } else { var parallelResult = new TestResult { Outcome = UnitTestOutcome.Error, TestFailureException = new Exception("Failed to run parallel test because parallel test node services failed to initialize properly."), }; // Set our test names mainResult.DisplayName = $"{testDisplayName} (built-in)"; parallelResult.DisplayName = $"{testDisplayName} (external)"; return(new[] { mainResult, parallelResult }); } }
private TestResult RunTestMethod(object testClassInstance, ITestMethod testMethod) { TestResult testResult = null; string className = testClassInstance.GetType().Name; string methodName = testMethod.Name; try { testMethod.Invoke(testClassInstance); testResult = new TestResult(className, methodName, true, "The test ran successfully."); } catch (Exception ex) { testResult = new TestResult(className, methodName, false, "The test failed: ", ex); } return testResult; }
/// <summary> /// Executes a test method. /// </summary> /// <param name="testMethod">The test method to execute.</param> /// <returns>An array of TestResult objects that represent the outcome(s) of the test.</returns> /// <remarks>Extensions can override this method to customize running a TestMethod.</remarks> public virtual TestResult[] Execute(ITestMethod testMethod) { return(new TestResult[] { testMethod.Invoke(null) }); }
public override TestResult[] Execute(ITestMethod testMethod) { TestResult[] errorResults = ValidateElevated(testMethod); if (errorResults != null) { return(errorResults); } Exception signatureException = GetMethodSignatureException(testMethod); if (signatureException != null) { return(testMethod.CreateExceptionResult(signatureException)); } var runParameters = TestRunParameters.Read(); string logFolder = runParameters.LogFolder; bool shouldLog = !string.IsNullOrEmpty(logFolder); if (shouldLog) { try { logFolder = CreateLogFolder(logFolder, testMethod.TestMethodName); } catch (Exception e) { return(testMethod.CreateExceptionResult(e)); } } int iterations = runParameters.Iterations; var results = new TestResult[iterations]; for (int iteration = 1; iteration <= iterations; iteration++) { string sessionName = $"{testMethod.TestMethodName}-{iteration}"; using (var session = new TraceEventSession(sessionName)) { EnableKernelProviders(session, shouldLog); TraceEventDispatcher source; ZippedETLWriter writer = null; if (shouldLog) { string etlPath = Path.Combine(logFolder, $"Iteration{iteration}.etl"); source = new ETWReloggerTraceEventSource(sessionName, TraceEventSourceType.Session, etlPath); writer = new ZippedETLWriter(etlPath); } else { source = session.Source; } EnableProviders(session); PerformanceTestContext context = CreateContext(source); Task <TestResult> testTask = Task.Run(() => testMethod.Invoke(new object[] { context })); // This is a blocking call that in the case of ETWReloggerTraceEventSource, must be run on the same // thread as ETWReloggerTraceEventSource was created on. It will become unblocked when the // PerformanceTestContext calls StopProcessing on the source. source.Process(); TestResult result = testTask.Result; string displayName = testMethod.TestMethodName; if (iterations > 1) { displayName += $" [{iteration}/{iterations}]"; } result.DisplayName = displayName; session.Flush(); OnIterationEnded(context); context.LogScenarios(); context.LogMemoryDelta(); context.LogMessage($"{displayName} completed. {session.EventsLost} events lost."); context.WriteLogsToResult(result, writer); results[iteration - 1] = result; } } return(results); }