public void ExecuteAsync_Sets_ObjectContent_Value_As_Content() { var result = new TestResult(); Assert.AreSame( result, (ExecuteAsync(result).Result.Content as ObjectContent<TestResult>).Value); }
private static void FailedTest(string key, TestResult testResult, IssueModel issue) { JiraInfoProvider.SaveStackTrace(key, GetStackTrace(testResult.TestFailureException)); issue.Summary = testResult.TestFailureException.Message; issue.Status = Status.Failed; IssueManager.AddIssue(issue); }
public static void SendTestResult(string key, TestResult testResult) { var issueModel = new IssueModel() { Key = key, Time = testResult.Duration.Milliseconds, IsTestComplete = true }; switch (testResult.Outcome) { case UnitTestOutcome.Failed: FailedTest(key, testResult, issueModel); break; case UnitTestOutcome.Passed: PassedTest(key, testResult, issueModel); break; default: SkippedTest(key, testResult, issueModel); break; } }
internal UnitTestResult[] RunTestMethod() { Debug.Assert(this.test != null, "Test should not be null."); Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null."); List <UTF.TestResult> results = new List <UTF.TestResult>(); if (this.testMethodInfo.TestMethodOptions.Executor != null) { UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false); if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) { Stopwatch watch = new Stopwatch(); watch.Start(); try { IEnumerable <object> dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext); if (dataRows == null) { watch.Stop(); var inconclusiveResult = new UTF.TestResult(); inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; inconclusiveResult.Duration = watch.Elapsed; results.Add(inconclusiveResult); } else { try { int rowIndex = 0; foreach (object dataRow in dataRows) { watch.Reset(); watch.Start(); this.testContext.SetDataRow(dataRow); UTF.TestResult[] testResults; try { testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { testResults = new[] { new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } }; } watch.Stop(); foreach (var testResult in testResults) { testResult.DatarowIndex = rowIndex; testResult.Duration = watch.Elapsed; } rowIndex++; results.AddRange(testResults); } } finally { this.testContext.SetDataConnection(null); this.testContext.SetDataRow(null); } } } catch (Exception ex) { watch.Stop(); var failedResult = new UTF.TestResult(); failedResult.Outcome = UTF.UnitTestOutcome.Error; failedResult.TestFailureException = ex; failedResult.Duration = watch.Elapsed; results.Add(failedResult); } } else { UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes <Attribute>(true)?.Where(a => a is UTF.ITestDataSource).OfType <UTF.ITestDataSource>().ToArray(); if (testDataSources != null && testDataSources.Length > 0) { foreach (var testDataSource in testDataSources) { foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) { this.testMethodInfo.SetArguments(data); UTF.TestResult[] testResults; try { testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { testResults = new[] { new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } }; } foreach (var testResult in testResults) { testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); } results.AddRange(testResults); this.testMethodInfo.SetArguments(null); } } } else { try { results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)); } catch (Exception ex) { results.Add(new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }); } } } } else { PlatformServiceProvider.Instance.AdapterTraceLogger.LogError( "Not able to get executor for method {0}.{1}", this.testMethodInfo.TestClassName, this.testMethodInfo.TestMethodName); } if (results != null && results.Count > 0) { // aggregate for data driven tests UTF.UnitTestOutcome aggregateOutcome = UTF.UnitTestOutcome.Passed; foreach (var result in results) { if (result.Outcome != UTF.UnitTestOutcome.Passed) { if (aggregateOutcome != UTF.UnitTestOutcome.Failed) { if (result.Outcome == UTF.UnitTestOutcome.Failed || aggregateOutcome != UTF.UnitTestOutcome.Timeout) { aggregateOutcome = result.Outcome; } } } } this.testContext.SetOutcome(aggregateOutcome); } else { this.testContext.SetOutcome(UTF.UnitTestOutcome.Unknown); results.Add(new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Unknown, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) }); } return(results.ToArray().ToUnitTestResults()); }
private void ExtractMethodTest(string input, Func<Span> extract, TestResult expected, string scopeName = null, string targetName = "g", Version version = null, params string[] parameters) { var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version ?? new Version(2, 7)); var serviceProvider = PythonToolsTestUtilities.CreateMockServiceProvider(); using (var analyzer = new VsProjectAnalyzer(serviceProvider, fact)) { var buffer = new MockTextBuffer(input, "Python", "C:\\fob.py"); var view = new MockTextView(buffer); buffer.AddProperty(typeof(VsProjectAnalyzer), analyzer); analyzer.MonitorTextBufferAsync(buffer).Wait(); var extractInput = new ExtractMethodTestInput(true, scopeName, targetName, parameters ?? new string[0]); view.Selection.Select( new SnapshotSpan(view.TextBuffer.CurrentSnapshot, extract()), false ); new MethodExtractor(serviceProvider, view).ExtractMethod(extractInput).Wait(); if (expected.IsError) { Assert.AreEqual(expected.Text, extractInput.FailureReason); Assert.AreEqual(input, view.TextBuffer.CurrentSnapshot.GetText()); } else { Assert.AreEqual(null, extractInput.FailureReason); Assert.AreEqual(expected.Text, view.TextBuffer.CurrentSnapshot.GetText()); } } }
private void ExtractMethodTest(string input, object extract, TestResult expected, string scopeName = null, string targetName = "g", Version version = null, params string[] parameters) { Func<Span> textRange = () => { return GetSelectionSpan(input, extract); }; ExtractMethodTest(input, textRange, expected, scopeName, targetName, version, parameters); }
/// <summary> /// Updates given results with parent info if results are greater than 1. /// Add parent results as first result in updated result. /// </summary> /// <param name="results">Results.</param> /// <param name="parentResult">Parent results.</param> /// <returns>Updated results which contains parent result as first result. All other results contains parent result info.</returns> private List <UTF.TestResult> UpdateResultsWithParentInfo(List <UTF.TestResult> results, UTF.TestResult parentResult) { // Return results in case there are no results. if (!results.Any()) { return(results); } // UpdatedResults contain parent result at first position and remaining results has parent info updated. var updatedResults = new List <UTF.TestResult>(); updatedResults.Add(parentResult); foreach (var result in results) { result.ExecutionId = Guid.NewGuid(); result.ParentExecId = parentResult.ExecutionId; parentResult.InnerResultsCount++; updatedResults.Add(result); } return(updatedResults); }
internal UnitTestResult[] RunTestMethod() { Debug.Assert(this.test != null, "Test should not be null."); Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null."); List <UTF.TestResult> results = new List <UTF.TestResult>(); var isDataDriven = false; // Parent result. Added in properties bag only when results are greater than 1. var parentResultWatch = new Stopwatch(); parentResultWatch.Start(); var parentResult = new UTF.TestResult { Outcome = UTF.UnitTestOutcome.InProgress, ExecutionId = Guid.NewGuid() }; if (this.testMethodInfo.TestMethodOptions.Executor != null) { UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false); if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) { isDataDriven = true; Stopwatch watch = new Stopwatch(); watch.Start(); try { IEnumerable <object> dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext); if (dataRows == null) { watch.Stop(); var inconclusiveResult = new UTF.TestResult(); inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; inconclusiveResult.Duration = watch.Elapsed; results.Add(inconclusiveResult); } else { try { int rowIndex = 0; foreach (object dataRow in dataRows) { watch.Reset(); watch.Start(); this.testContext.SetDataRow(dataRow); UTF.TestResult[] testResults; try { testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { testResults = new[] { new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } }; } watch.Stop(); foreach (var testResult in testResults) { testResult.DatarowIndex = rowIndex; testResult.Duration = watch.Elapsed; } rowIndex++; results.AddRange(testResults); } } finally { this.testContext.SetDataConnection(null); this.testContext.SetDataRow(null); } } } catch (Exception ex) { watch.Stop(); var failedResult = new UTF.TestResult(); failedResult.Outcome = UTF.UnitTestOutcome.Error; failedResult.TestFailureException = ex; failedResult.Duration = watch.Elapsed; results.Add(failedResult); } } else { UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes <Attribute>(false)?.Where(a => a is UTF.ITestDataSource).OfType <UTF.ITestDataSource>().ToArray(); if (testDataSources != null && testDataSources.Length > 0) { isDataDriven = true; foreach (var testDataSource in testDataSources) { foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) { this.testMethodInfo.SetArguments(data); UTF.TestResult[] testResults; try { testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { testResults = new[] { new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } }; } foreach (var testResult in testResults) { testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); } results.AddRange(testResults); this.testMethodInfo.SetArguments(null); } } } else { try { results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)); } catch (Exception ex) { results.Add(new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }); } } } } else { PlatformServiceProvider.Instance.AdapterTraceLogger.LogError( "Not able to get executor for method {0}.{1}", this.testMethodInfo.TestClassName, this.testMethodInfo.TestMethodName); } parentResultWatch.Stop(); parentResult.Duration = parentResultWatch.Elapsed; // Get aggregate outcome. var aggregateOutcome = this.GetAggregateOutcome(results); this.testContext.SetOutcome(aggregateOutcome); // Set a result in case no result is present. if (!results.Any()) { results.Add(new UTF.TestResult() { Outcome = aggregateOutcome, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult) }); } // In case of data driven, set parent info in results. if (isDataDriven) { parentResult.Outcome = aggregateOutcome; results = this.UpdateResultsWithParentInfo(results, parentResult); } return(results.ToArray().ToUnitTestResults()); }
private void CheckAsyncResult(TestResult result, Action done) { if (result.IsComplete) { if ((dynamic)(result.ErrorMessage) && result.ErrorMessage.length > 0) { ConsoleProxy.log("Test Failed: " + result.ErrorMessage); ClientHost.Service.FailTest(result.ErrorMessage, done); } else { ConsoleProxy.log("Test Passed"); ClientHost.Service.PassTest(done); } } else { var handler = this; window.setTimeout((Action)(() => { handler.CheckAsyncResult(result, done); }), 500); } }
public void DeleteShouldReturn400BadRequestIfTheKeyIsInvalid() { // Arrange const string ResourceKey = "badkey"; // using key of type string to force bad request // Act using (var client = CreateHttpClient()) { var deleteRequest = new HttpRequestMessage( HttpMethod.Delete, string.Format("{0}/{1}", ServiceUri, ResourceKey)); AddIfMatchHeader(deleteRequest, null); var response = client.Send(deleteRequest); var result = new TestResult(response); // Assert Assert.AreEqual(HttpStatusCode.BadRequest, result.Response.StatusCode); } }
public void RecordResult(TestResult testResult) { this.results.Add(testResult); this.called = true; }
private bool ExecuteDataSourceBasedTests(List <UTF.TestResult> results) { var isDataDriven = false; UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false); if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) { isDataDriven = true; Stopwatch watch = new Stopwatch(); watch.Start(); try { IEnumerable <object> dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext); if (dataRows == null) { watch.Stop(); var inconclusiveResult = new UTF.TestResult(); inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; inconclusiveResult.Duration = watch.Elapsed; results.Add(inconclusiveResult); } else { try { int rowIndex = 0; foreach (object dataRow in dataRows) { UTF.TestResult[] testResults = this.ExecuteTestWithDataRow(dataRow, rowIndex++); results.AddRange(testResults); } } finally { this.testContext.SetDataConnection(null); this.testContext.SetDataRow(null); } } } catch (Exception ex) { watch.Stop(); var failedResult = new UTF.TestResult(); failedResult.Outcome = UTF.UnitTestOutcome.Error; failedResult.TestFailureException = ex; failedResult.Duration = watch.Elapsed; results.Add(failedResult); } } else { UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes <Attribute>(false)?.Where(a => a is UTF.ITestDataSource).OfType <UTF.ITestDataSource>().ToArray(); if (testDataSources != null && testDataSources.Length > 0) { isDataDriven = true; foreach (var testDataSource in testDataSources) { foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) { try { var testResults = this.ExecuteTestWithDataSource(testDataSource, data); results.AddRange(testResults); } finally { this.testMethodInfo.SetArguments(null); } } } } } return(isDataDriven); }
private static Task<HttpResponseMessage> ExecuteAsync( TestResult result = null) { return CreateSubjectUnderTest(result) .ExecuteAsync(It.IsAny<CancellationToken>()); }
private static OkResult<TestResult> CreateSubjectUnderTest( TestResult result = null) { return new OkResult<TestResult>( result ?? ObjectMother.Create<TestResult>()); }
/// <summary> /// Run a data driven test. Test case is executed once for each data row. /// </summary> /// <param name="testContext"> /// The test Context. /// </param> /// <param name="testMethodInfo"> /// The test Method Info. /// </param> /// <param name="testMethod"> /// The test Method. /// </param> /// <param name="executor"> /// The default test method executor. /// </param> /// <returns> /// The results after running all the data driven tests. /// </returns> public UTF.TestResult[] RunDataDrivenTest(UTF.TestContext testContext, UTF.ITestMethod testMethodInfo, ITestMethod testMethod, UTF.TestMethodAttribute executor) { Stopwatch watch = new Stopwatch(); watch.Start(); // Figure out where (as well as the current directory) we could look for data files // for unit tests this means looking at the the location of the test itself List <string> dataFolders = new List <string>(); dataFolders.Add(Path.GetDirectoryName(new Uri(testMethodInfo.MethodInfo.Module.Assembly.CodeBase).LocalPath)); List <UTF.TestResult> dataRowResults = new List <UTF.TestResult>(); // Connect to data source. TestDataConnectionFactory factory = new TestDataConnectionFactory(); string providerNameInvariant; string connectionString; string tableName; UTF.DataAccessMethod dataAccessMethod; try { this.GetConnectionProperties(testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false)[0], out providerNameInvariant, out connectionString, out tableName, out dataAccessMethod); } catch (Exception ex) { watch.Stop(); var result = new UTF.TestResult(); result.Outcome = UTF.UnitTestOutcome.Failed; result.TestFailureException = ex; result.Duration = watch.Elapsed; return(new UTF.TestResult[] { result }); } try { using (TestDataConnection connection = factory.Create(providerNameInvariant, connectionString, dataFolders)) { DataTable table = connection.ReadTable(tableName, null); DataRow[] rows = table.Select(); Debug.Assert(rows != null, "rows should not be null."); if (rows.Length == 0) { watch.Stop(); var inconclusiveResult = new UTF.TestResult(); inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; inconclusiveResult.Duration = watch.Elapsed; return(new UTF.TestResult[] { inconclusiveResult }); } IEnumerable <int> permutation = this.GetPermutation(dataAccessMethod, rows.Length); TestContextImplementation testContextImpl = testContext as TestContextImplementation; try { testContextImpl.SetDataConnection(connection.Connection); // For each data row... foreach (int rowIndex in permutation) { watch.Reset(); watch.Start(); testContextImpl.SetDataRow(rows[rowIndex]); UTF.TestResult[] currentResult = new UTF.TestResult[1]; try { currentResult = executor.Execute(testMethodInfo); } catch (Exception ex) { currentResult[0].Outcome = UTF.UnitTestOutcome.Failed; // Trace whole exception but do not show call stack to the user, only show message. EqtTrace.ErrorIf(EqtTrace.IsErrorEnabled, "Unit Test Adapter threw exception: {0}", ex); } currentResult[0].DatarowIndex = rowIndex; watch.Stop(); currentResult[0].Duration = watch.Elapsed; Debug.Assert(currentResult[0] != null, "current result should not be null."); dataRowResults.Add(currentResult[0]); // Clear the testContext's internal string writer to start afresh for the next datarow iteration testContextImpl.ClearMessages(); } } finally { testContextImpl.SetDataConnection(null); testContextImpl.SetDataRow(null); } } } catch (Exception ex) { string message = ExceptionExtensions.GetExceptionMessage(ex); UTF.TestResult failedResult = new UTF.TestResult(); failedResult.Outcome = UTF.UnitTestOutcome.Error; failedResult.TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ErrorDataConnectionFailed, ex.Message), ex); return(new UTF.TestResult[] { failedResult }); } return(dataRowResults.ToArray()); }
private static void SkippedTest(string key, TestResult testResult, IssueModel issue) { issue.Status = Status.Untested; IssueManager.AddIssue(issue); }