public void TestCancel() {
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterATests.Union(TestInfo.TestAdapterBTests).ToArray();
var testCases = expectedTests.Select(tr => tr.TestCase);
var thread = new System.Threading.Thread(o => {
executor.RunTests(testCases, runContext, recorder);
});
thread.Start();
// One of the tests being run is hard coded to take 10 secs
Assert.IsTrue(thread.IsAlive);
System.Threading.Thread.Sleep(100);
executor.Cancel();
System.Threading.Thread.Sleep(100);
// It should take less than 10 secs to cancel
// Depending on which assemblies are loaded, it may take some time
// to obtain the interpreters service.
Assert.IsTrue(thread.Join(10000));
System.Threading.Thread.Sleep(100);
Assert.IsFalse(thread.IsAlive);
// Canceled test cases do not get recorded
Assert.IsTrue(recorder.Results.Count < expectedTests.Length);
}
public void TestRunAll() {
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
executor.RunTests(new[] { TestInfo.TestAdapterLibProjectFilePath, TestInfo.TestAdapterAProjectFilePath, TestInfo.TestAdapterBProjectFilePath }, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests) {
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult, expectedResult.TestCase.FullyQualifiedName + " not found in results");
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
private static void PrintTestResults(MockTestExecutionRecorder recorder)
{
foreach (var message in recorder.Messages)
{
Console.WriteLine(message);
}
foreach (var result in recorder.Results)
{
Console.WriteLine("Test: " + result.TestCase.FullyQualifiedName);
Console.WriteLine("Result: " + result.Outcome);
foreach (var msg in result.Messages)
{
Console.WriteLine("Message " + msg.Category + ":");
Console.WriteLine(msg.Text);
}
Console.WriteLine("");
}
}
public void Run() {
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests) {
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestPassInTestList()
{
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = Enumerable.Repeat(TestInfo.TestAdapterATests, 10).SelectMany();
var runContext = CreateRunContext(expectedTests, dryRun: true);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
AssertUtil.ArrayEquals(
expectedTests.Select(t => t.TestCase.FullyQualifiedName).ToList(),
recorder.Results.Select(t => t.TestCase.FullyQualifiedName).ToList()
);
}
internal static void PrintTestResults(MockTestExecutionRecorder recorder)
{
foreach (var message in recorder.Messages)
{
Console.WriteLine(message);
}
foreach (var result in recorder.Results)
{
Console.WriteLine("Test: {0}", result.TestCase.FullyQualifiedName);
Console.WriteLine("Result: {0}", result.Outcome);
Console.WriteLine("Duration: {0}ms", result.Duration.TotalMilliseconds);
foreach (var msg in result.Messages)
{
Console.WriteLine("Message {0}:", msg.Category);
Console.WriteLine(msg.Text);
}
Console.WriteLine("");
}
}
public void TestEnvironment()
{
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = new[] { TestInfo.EnvironmentTestSuccess };
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(new[] { TestInfo.TestAdapterEnvironmentProject }, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests)
{
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult, expectedResult.TestCase.FullyQualifiedName + " not found in results");
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void Run()
{
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests)
{
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestRunAll()
{
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
var runContext = CreateRunContext(expectedTests, Version.InterpreterPath);
executor.RunTests(expectedTests.Select(ti => ti.SourceCodeFilePath), runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests)
{
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
Assert.IsTrue(actualResult.Duration >= expectedResult.MinDuration);
}
}
public void TestEnvironment()
{
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = new[] { TestInfo.EnvironmentTestSuccess };
var runContext = CreateRunContext(expectedTests, Version.InterpreterPath);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests)
{
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestRunAll()
{
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
executor.RunTests(new[] { TestInfo.TestAdapterLibProjectFilePath, TestInfo.TestAdapterAProjectFilePath, TestInfo.TestAdapterBProjectFilePath }, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests)
{
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult, expectedResult.TestCase.FullyQualifiedName + " not found in results");
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestMultiprocessing()
{
PythonPaths.Python27_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterMultiprocessingTests;
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests)
{
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestExtensionReference()
{
// This test uses a 32-bit Python 2.7 .pyd
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = new[] { TestInfo.ExtensionReferenceTestSuccess };
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests)
{
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestInheritance()
{
// TODO: Figure out the proper fix to make this test pass.
// There's a confusion between source file path and class file path.
// Note that the equivalent manual test in IDE works fine.
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterBInheritanceTests;
var runContext = CreateRunContext(expectedTests, Version.InterpreterPath);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests)
{
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestLoadError()
{
// A load error is when unittest module fails to load the test (prior to running it)
// For example, if the file where the test is defined has an unhandled ImportError.
// We check that this only causes the tests that can't be loaded to fail,
// all other tests in the test run which can be loaded successfully will be run.
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.GetTestAdapterLoadErrorTests(ImportErrorFormat);
var runContext = CreateRunContext(expectedTests, Version.InterpreterPath);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests)
{
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
if (expectedResult.ContainedErrorMessage != null)
{
Assert.IsNotNull(actualResult.ErrorMessage);
Assert.IsTrue(
actualResult.ErrorMessage.Contains(expectedResult.ContainedErrorMessage),
string.Format("Error message did not contain expected text: {0}", expectedResult.ContainedErrorMessage)
);
}
else
{
Assert.IsNull(actualResult.ErrorMessage);
}
}
}
public void TestCancel()
{
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterATests.Union(TestInfo.TestAdapterBTests).ToArray();
var testCases = expectedTests.Select(tr => tr.TestCase);
var thread = new System.Threading.Thread(o => {
executor.RunTests(testCases, runContext, recorder);
});
thread.Start();
// One of the tests being run is hard coded to take 10 secs
Assert.IsTrue(thread.IsAlive);
System.Threading.Thread.Sleep(100);
executor.Cancel();
System.Threading.Thread.Sleep(100);
// It should take less than 10 secs to cancel
// Depending on which assemblies are loaded, it may take some time
// to obtain the interpreters service.
Assert.IsTrue(thread.Join(10000));
System.Threading.Thread.Sleep(100);
Assert.IsFalse(thread.IsAlive);
// Canceled test cases do not get recorded
Assert.IsTrue(recorder.Results.Count < expectedTests.Length);
}
private static void TestLoadError(string projectName)
{
// A load error is when unittest module fails to load the test (prior to running it)
// For example, if the file where the test is defined has an unhandled ImportError.
// We check that this only causes the tests that can't be loaded to fail,
// all other tests in the test run which can be loaded successfully will be run.
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.GetTestAdapterLoadErrorTests(projectName);
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests)
{
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestRunAll() {
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
var runContext = CreateRunContext(expectedTests);
executor.RunTests(expectedTests.Select(ti => ti.SourceCodeFilePath), runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestExtensionReference() {
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = new[] { TestInfo.ExtensionReferenceTestSuccess };
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(new[] { TestInfo.TestAdapterExtensionReferenceProject }, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests) {
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult, expectedResult.TestCase.FullyQualifiedName + " not found in results");
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
private static void PrintTestResults(MockTestExecutionRecorder recorder) {
foreach (var message in recorder.Messages) {
Console.WriteLine(message);
}
foreach (var result in recorder.Results) {
Console.WriteLine("Test: " + result.TestCase.FullyQualifiedName);
Console.WriteLine("Result: " + result.Outcome);
foreach(var msg in result.Messages) {
Console.WriteLine("Message " + msg.Category + ":");
Console.WriteLine(msg.Text);
}
Console.WriteLine("");
}
}
public void TestExtensionReference() {
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = new[] { TestInfo.ExtensionReferenceTestSuccess };
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestLoadError() {
PythonPaths.Python27_x64.AssertInstalled();
// A load error is when unittest module fails to load the test (prior to running it)
// For example, if the file where the test is defined has an unhandled ImportError.
// We check that this only causes the tests that can't be loaded to fail,
// all other tests in the test run which can be loaded successfully will be run.
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterLoadErrorTests;
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestPassInTestList() {
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = Enumerable.Repeat(TestInfo.TestAdapterATests, 10).SelectMany();
var runContext = CreateRunContext(expectedTests, dryRun: true);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
AssertUtil.ArrayEquals(
expectedTests.Select(t => t.TestCase.FullyQualifiedName).ToList(),
recorder.Results.Select(t => t.TestCase.FullyQualifiedName).ToList()
);
}
public void TestInheritance() {
// TODO: Figure out the proper fix to make this test pass.
// There's a confusion between source file path and class file path.
// Note that the equivalent manual test in IDE works fine.
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterBInheritanceTests;
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestMultiprocessing() {
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterMultiprocessingTests;
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
