public void ExecuteAsync_Sets_ObjectContent_Value_As_Content()
{
var result = new TestResult();
Assert.AreSame(
result,
(ExecuteAsync(result).Result.Content as ObjectContent<TestResult>).Value);
}
private static void FailedTest(string key, TestResult testResult, IssueModel issue)
{
JiraInfoProvider.SaveStackTrace(key, GetStackTrace(testResult.TestFailureException));
issue.Summary = testResult.TestFailureException.Message;
issue.Status = Status.Failed;
IssueManager.AddIssue(issue);
}
public static void SendTestResult(string key, TestResult testResult)
{
var issueModel = new IssueModel()
{
Key = key,
Time = testResult.Duration.Milliseconds,
IsTestComplete = true
};
switch (testResult.Outcome)
{
case UnitTestOutcome.Failed:
FailedTest(key, testResult, issueModel);
break;
case UnitTestOutcome.Passed:
PassedTest(key, testResult, issueModel);
break;
default:
SkippedTest(key, testResult, issueModel);
break;
}
}
internal UnitTestResult[] RunTestMethod()
{
Debug.Assert(this.test != null, "Test should not be null.");
Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null.");
List <UTF.TestResult> results = new List <UTF.TestResult>();
if (this.testMethodInfo.TestMethodOptions.Executor != null)
{
UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false);
if (dataSourceAttribute != null && dataSourceAttribute.Length == 1)
{
Stopwatch watch = new Stopwatch();
watch.Start();
try
{
IEnumerable <object> dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext);
if (dataRows == null)
{
watch.Stop();
var inconclusiveResult = new UTF.TestResult();
inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive;
inconclusiveResult.Duration = watch.Elapsed;
results.Add(inconclusiveResult);
}
else
{
try
{
int rowIndex = 0;
foreach (object dataRow in dataRows)
{
watch.Reset();
watch.Start();
this.testContext.SetDataRow(dataRow);
UTF.TestResult[] testResults;
try
{
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
testResults = new[]
{
new UTF.TestResult()
{
TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex)
}
};
}
watch.Stop();
foreach (var testResult in testResults)
{
testResult.DatarowIndex = rowIndex;
testResult.Duration = watch.Elapsed;
}
rowIndex++;
results.AddRange(testResults);
}
}
finally
{
this.testContext.SetDataConnection(null);
this.testContext.SetDataRow(null);
}
}
}
catch (Exception ex)
{
watch.Stop();
var failedResult = new UTF.TestResult();
failedResult.Outcome = UTF.UnitTestOutcome.Error;
failedResult.TestFailureException = ex;
failedResult.Duration = watch.Elapsed;
results.Add(failedResult);
}
}
else
{
UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes <Attribute>(true)?.Where(a => a is UTF.ITestDataSource).OfType <UTF.ITestDataSource>().ToArray();
if (testDataSources != null && testDataSources.Length > 0)
{
foreach (var testDataSource in testDataSources)
{
foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
{
this.testMethodInfo.SetArguments(data);
UTF.TestResult[] testResults;
try
{
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
testResults = new[]
{
new UTF.TestResult()
{
TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex)
}
};
}
foreach (var testResult in testResults)
{
testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
}
results.AddRange(testResults);
this.testMethodInfo.SetArguments(null);
}
}
}
else
{
try
{
results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo));
}
catch (Exception ex)
{
results.Add(new UTF.TestResult()
{
TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex)
});
}
}
}
}
else
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(
"Not able to get executor for method {0}.{1}",
this.testMethodInfo.TestClassName,
this.testMethodInfo.TestMethodName);
}
if (results != null && results.Count > 0)
{
// aggregate for data driven tests
UTF.UnitTestOutcome aggregateOutcome = UTF.UnitTestOutcome.Passed;
foreach (var result in results)
{
if (result.Outcome != UTF.UnitTestOutcome.Passed)
{
if (aggregateOutcome != UTF.UnitTestOutcome.Failed)
{
if (result.Outcome == UTF.UnitTestOutcome.Failed ||
aggregateOutcome != UTF.UnitTestOutcome.Timeout)
{
aggregateOutcome = result.Outcome;
}
}
}
}
this.testContext.SetOutcome(aggregateOutcome);
}
else
{
this.testContext.SetOutcome(UTF.UnitTestOutcome.Unknown);
results.Add(new UTF.TestResult()
{
Outcome = UTF.UnitTestOutcome.Unknown, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult)
});
}
return(results.ToArray().ToUnitTestResults());
}
private void ExtractMethodTest(string input, Func<Span> extract, TestResult expected, string scopeName = null, string targetName = "g", Version version = null, params string[] parameters) {
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version ?? new Version(2, 7));
var serviceProvider = PythonToolsTestUtilities.CreateMockServiceProvider();
using (var analyzer = new VsProjectAnalyzer(serviceProvider, fact)) {
var buffer = new MockTextBuffer(input, "Python", "C:\\fob.py");
var view = new MockTextView(buffer);
buffer.AddProperty(typeof(VsProjectAnalyzer), analyzer);
analyzer.MonitorTextBufferAsync(buffer).Wait();
var extractInput = new ExtractMethodTestInput(true, scopeName, targetName, parameters ?? new string[0]);
view.Selection.Select(
new SnapshotSpan(view.TextBuffer.CurrentSnapshot, extract()),
false
);
new MethodExtractor(serviceProvider, view).ExtractMethod(extractInput).Wait();
if (expected.IsError) {
Assert.AreEqual(expected.Text, extractInput.FailureReason);
Assert.AreEqual(input, view.TextBuffer.CurrentSnapshot.GetText());
} else {
Assert.AreEqual(null, extractInput.FailureReason);
Assert.AreEqual(expected.Text, view.TextBuffer.CurrentSnapshot.GetText());
}
}
}
private void ExtractMethodTest(string input, object extract, TestResult expected, string scopeName = null, string targetName = "g", Version version = null, params string[] parameters) {
Func<Span> textRange = () => {
return GetSelectionSpan(input, extract);
};
ExtractMethodTest(input, textRange, expected, scopeName, targetName, version, parameters);
}
/// <summary>
/// Updates given results with parent info if results are greater than 1.
/// Add parent results as first result in updated result.
/// </summary>
/// <param name="results">Results.</param>
/// <param name="parentResult">Parent results.</param>
/// <returns>Updated results which contains parent result as first result. All other results contains parent result info.</returns>
private List <UTF.TestResult> UpdateResultsWithParentInfo(List <UTF.TestResult> results, UTF.TestResult parentResult)
{
// Return results in case there are no results.
if (!results.Any())
{
return(results);
}
// UpdatedResults contain parent result at first position and remaining results has parent info updated.
var updatedResults = new List <UTF.TestResult>();
updatedResults.Add(parentResult);
foreach (var result in results)
{
result.ExecutionId = Guid.NewGuid();
result.ParentExecId = parentResult.ExecutionId;
parentResult.InnerResultsCount++;
updatedResults.Add(result);
}
return(updatedResults);
}
internal UnitTestResult[] RunTestMethod()
{
Debug.Assert(this.test != null, "Test should not be null.");
Debug.Assert(this.testMethodInfo.TestMethod != null, "Test method should not be null.");
List <UTF.TestResult> results = new List <UTF.TestResult>();
var isDataDriven = false;
// Parent result. Added in properties bag only when results are greater than 1.
var parentResultWatch = new Stopwatch();
parentResultWatch.Start();
var parentResult = new UTF.TestResult
{
Outcome = UTF.UnitTestOutcome.InProgress,
ExecutionId = Guid.NewGuid()
};
if (this.testMethodInfo.TestMethodOptions.Executor != null)
{
UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false);
if (dataSourceAttribute != null && dataSourceAttribute.Length == 1)
{
isDataDriven = true;
Stopwatch watch = new Stopwatch();
watch.Start();
try
{
IEnumerable <object> dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext);
if (dataRows == null)
{
watch.Stop();
var inconclusiveResult = new UTF.TestResult();
inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive;
inconclusiveResult.Duration = watch.Elapsed;
results.Add(inconclusiveResult);
}
else
{
try
{
int rowIndex = 0;
foreach (object dataRow in dataRows)
{
watch.Reset();
watch.Start();
this.testContext.SetDataRow(dataRow);
UTF.TestResult[] testResults;
try
{
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
testResults = new[]
{
new UTF.TestResult()
{
TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex)
}
};
}
watch.Stop();
foreach (var testResult in testResults)
{
testResult.DatarowIndex = rowIndex;
testResult.Duration = watch.Elapsed;
}
rowIndex++;
results.AddRange(testResults);
}
}
finally
{
this.testContext.SetDataConnection(null);
this.testContext.SetDataRow(null);
}
}
}
catch (Exception ex)
{
watch.Stop();
var failedResult = new UTF.TestResult();
failedResult.Outcome = UTF.UnitTestOutcome.Error;
failedResult.TestFailureException = ex;
failedResult.Duration = watch.Elapsed;
results.Add(failedResult);
}
}
else
{
UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes <Attribute>(false)?.Where(a => a is UTF.ITestDataSource).OfType <UTF.ITestDataSource>().ToArray();
if (testDataSources != null && testDataSources.Length > 0)
{
isDataDriven = true;
foreach (var testDataSource in testDataSources)
{
foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
{
this.testMethodInfo.SetArguments(data);
UTF.TestResult[] testResults;
try
{
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
testResults = new[]
{
new UTF.TestResult()
{
TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex)
}
};
}
foreach (var testResult in testResults)
{
testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
}
results.AddRange(testResults);
this.testMethodInfo.SetArguments(null);
}
}
}
else
{
try
{
results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo));
}
catch (Exception ex)
{
results.Add(new UTF.TestResult()
{
TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex)
});
}
}
}
}
else
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(
"Not able to get executor for method {0}.{1}",
this.testMethodInfo.TestClassName,
this.testMethodInfo.TestMethodName);
}
parentResultWatch.Stop();
parentResult.Duration = parentResultWatch.Elapsed;
// Get aggregate outcome.
var aggregateOutcome = this.GetAggregateOutcome(results);
this.testContext.SetOutcome(aggregateOutcome);
// Set a result in case no result is present.
if (!results.Any())
{
results.Add(new UTF.TestResult()
{
Outcome = aggregateOutcome, TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult)
});
}
// In case of data driven, set parent info in results.
if (isDataDriven)
{
parentResult.Outcome = aggregateOutcome;
results = this.UpdateResultsWithParentInfo(results, parentResult);
}
return(results.ToArray().ToUnitTestResults());
}
private void CheckAsyncResult(TestResult result, Action done)
{
if (result.IsComplete)
{
if ((dynamic)(result.ErrorMessage) && result.ErrorMessage.length > 0)
{
ConsoleProxy.log("Test Failed: " + result.ErrorMessage);
ClientHost.Service.FailTest(result.ErrorMessage, done);
}
else
{
ConsoleProxy.log("Test Passed");
ClientHost.Service.PassTest(done);
}
}
else
{
var handler = this;
window.setTimeout((Action)(() =>
{ handler.CheckAsyncResult(result, done); }), 500);
}
}
public void DeleteShouldReturn400BadRequestIfTheKeyIsInvalid()
{
// Arrange
const string ResourceKey = "badkey";
// using key of type string to force bad request
// Act
using (var client = CreateHttpClient())
{
var deleteRequest = new HttpRequestMessage(
HttpMethod.Delete, string.Format("{0}/{1}", ServiceUri, ResourceKey));
AddIfMatchHeader(deleteRequest, null);
var response = client.Send(deleteRequest);
var result = new TestResult(response);
// Assert
Assert.AreEqual(HttpStatusCode.BadRequest, result.Response.StatusCode);
}
}
public void RecordResult(TestResult testResult)
{
this.results.Add(testResult);
this.called = true;
}
private bool ExecuteDataSourceBasedTests(List <UTF.TestResult> results)
{
var isDataDriven = false;
UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false);
if (dataSourceAttribute != null && dataSourceAttribute.Length == 1)
{
isDataDriven = true;
Stopwatch watch = new Stopwatch();
watch.Start();
try
{
IEnumerable <object> dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext);
if (dataRows == null)
{
watch.Stop();
var inconclusiveResult = new UTF.TestResult();
inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive;
inconclusiveResult.Duration = watch.Elapsed;
results.Add(inconclusiveResult);
}
else
{
try
{
int rowIndex = 0;
foreach (object dataRow in dataRows)
{
UTF.TestResult[] testResults = this.ExecuteTestWithDataRow(dataRow, rowIndex++);
results.AddRange(testResults);
}
}
finally
{
this.testContext.SetDataConnection(null);
this.testContext.SetDataRow(null);
}
}
}
catch (Exception ex)
{
watch.Stop();
var failedResult = new UTF.TestResult();
failedResult.Outcome = UTF.UnitTestOutcome.Error;
failedResult.TestFailureException = ex;
failedResult.Duration = watch.Elapsed;
results.Add(failedResult);
}
}
else
{
UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes <Attribute>(false)?.Where(a => a is UTF.ITestDataSource).OfType <UTF.ITestDataSource>().ToArray();
if (testDataSources != null && testDataSources.Length > 0)
{
isDataDriven = true;
foreach (var testDataSource in testDataSources)
{
foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
{
try
{
var testResults = this.ExecuteTestWithDataSource(testDataSource, data);
results.AddRange(testResults);
}
finally
{
this.testMethodInfo.SetArguments(null);
}
}
}
}
}
return(isDataDriven);
}
private static Task<HttpResponseMessage> ExecuteAsync(
TestResult result = null)
{
return CreateSubjectUnderTest(result)
.ExecuteAsync(It.IsAny<CancellationToken>());
}
private static OkResult<TestResult> CreateSubjectUnderTest(
TestResult result = null)
{
return new OkResult<TestResult>(
result ?? ObjectMother.Create<TestResult>());
}
/// <summary>
/// Run a data driven test. Test case is executed once for each data row.
/// </summary>
/// <param name="testContext">
/// The test Context.
/// </param>
/// <param name="testMethodInfo">
/// The test Method Info.
/// </param>
/// <param name="testMethod">
/// The test Method.
/// </param>
/// <param name="executor">
/// The default test method executor.
/// </param>
/// <returns>
/// The results after running all the data driven tests.
/// </returns>
public UTF.TestResult[] RunDataDrivenTest(UTF.TestContext testContext, UTF.ITestMethod testMethodInfo, ITestMethod testMethod, UTF.TestMethodAttribute executor)
{
Stopwatch watch = new Stopwatch();
watch.Start();
// Figure out where (as well as the current directory) we could look for data files
// for unit tests this means looking at the the location of the test itself
List <string> dataFolders = new List <string>();
dataFolders.Add(Path.GetDirectoryName(new Uri(testMethodInfo.MethodInfo.Module.Assembly.CodeBase).LocalPath));
List <UTF.TestResult> dataRowResults = new List <UTF.TestResult>();
// Connect to data source.
TestDataConnectionFactory factory = new TestDataConnectionFactory();
string providerNameInvariant;
string connectionString;
string tableName;
UTF.DataAccessMethod dataAccessMethod;
try
{
this.GetConnectionProperties(testMethodInfo.GetAttributes <UTF.DataSourceAttribute>(false)[0], out providerNameInvariant, out connectionString, out tableName, out dataAccessMethod);
}
catch (Exception ex)
{
watch.Stop();
var result = new UTF.TestResult();
result.Outcome = UTF.UnitTestOutcome.Failed;
result.TestFailureException = ex;
result.Duration = watch.Elapsed;
return(new UTF.TestResult[] { result });
}
try
{
using (TestDataConnection connection = factory.Create(providerNameInvariant, connectionString, dataFolders))
{
DataTable table = connection.ReadTable(tableName, null);
DataRow[] rows = table.Select();
Debug.Assert(rows != null, "rows should not be null.");
if (rows.Length == 0)
{
watch.Stop();
var inconclusiveResult = new UTF.TestResult();
inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive;
inconclusiveResult.Duration = watch.Elapsed;
return(new UTF.TestResult[] { inconclusiveResult });
}
IEnumerable <int> permutation = this.GetPermutation(dataAccessMethod, rows.Length);
TestContextImplementation testContextImpl = testContext as TestContextImplementation;
try
{
testContextImpl.SetDataConnection(connection.Connection);
// For each data row...
foreach (int rowIndex in permutation)
{
watch.Reset();
watch.Start();
testContextImpl.SetDataRow(rows[rowIndex]);
UTF.TestResult[] currentResult = new UTF.TestResult[1];
try
{
currentResult = executor.Execute(testMethodInfo);
}
catch (Exception ex)
{
currentResult[0].Outcome = UTF.UnitTestOutcome.Failed;
// Trace whole exception but do not show call stack to the user, only show message.
EqtTrace.ErrorIf(EqtTrace.IsErrorEnabled, "Unit Test Adapter threw exception: {0}", ex);
}
currentResult[0].DatarowIndex = rowIndex;
watch.Stop();
currentResult[0].Duration = watch.Elapsed;
Debug.Assert(currentResult[0] != null, "current result should not be null.");
dataRowResults.Add(currentResult[0]);
// Clear the testContext's internal string writer to start afresh for the next datarow iteration
testContextImpl.ClearMessages();
}
}
finally
{
testContextImpl.SetDataConnection(null);
testContextImpl.SetDataRow(null);
}
}
}
catch (Exception ex)
{
string message = ExceptionExtensions.GetExceptionMessage(ex);
UTF.TestResult failedResult = new UTF.TestResult();
failedResult.Outcome = UTF.UnitTestOutcome.Error;
failedResult.TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ErrorDataConnectionFailed, ex.Message), ex);
return(new UTF.TestResult[] { failedResult });
}
return(dataRowResults.ToArray());
}
private static void SkippedTest(string key, TestResult testResult, IssueModel issue)
{
issue.Status = Status.Untested;
IssueManager.AddIssue(issue);
}
