public void TestRun() {
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests) {
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestExtensionReference() {
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = new[] { TestInfo.ExtensionReferenceTestSuccess };
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestLoadError() {
PythonPaths.Python27_x64.AssertInstalled();
// A load error is when unittest module fails to load the test (prior to running it)
// For example, if the file where the test is defined has an unhandled ImportError.
// We check that this only causes the tests that can't be loaded to fail,
// all other tests in the test run which can be loaded successfully will be run.
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterLoadErrorTests;
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestInheritance() {
// TODO: Figure out the proper fix to make this test pass.
// There's a confusion between source file path and class file path.
// Note that the equivalent manual test in IDE works fine.
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterBInheritanceTests;
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestCancel() {
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterATests.Union(TestInfo.TestAdapterBTests).ToArray();
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
var thread = new System.Threading.Thread(o => {
executor.RunTests(testCases, runContext, recorder);
});
thread.Start();
// One of the tests being run is hard coded to take 10 secs
Assert.IsTrue(thread.IsAlive);
System.Threading.Thread.Sleep(100);
executor.Cancel();
System.Threading.Thread.Sleep(100);
// It should take less than 10 secs to cancel
// Depending on which assemblies are loaded, it may take some time
// to obtain the interpreters service.
Assert.IsTrue(thread.Join(10000));
System.Threading.Thread.Sleep(100);
Assert.IsFalse(thread.IsAlive);
// Canceled test cases do not get recorded
Assert.IsTrue(recorder.Results.Count < expectedTests.Length);
}
public void TestRunAll() {
PythonPaths.Python27_x64.AssertInstalled();
PythonPaths.Python33_x64.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
var runContext = CreateRunContext(expectedTests);
executor.RunTests(expectedTests.Select(ti => ti.SourceCodeFilePath), runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestExtensionReference() {
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = new[] { TestInfo.ExtensionReferenceTestSuccess };
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(new[] { TestInfo.TestAdapterExtensionReferenceProject }, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests) {
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult, expectedResult.TestCase.FullyQualifiedName + " not found in results");
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestMultiprocessing() {
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var runContext = new MockRunContext();
var expectedTests = TestInfo.TestAdapterMultiprocessingTests;
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(new[] { TestInfo.TestAdapterMultiprocessingProjectFilePath }, runContext, recorder);
PrintTestResults(recorder.Results);
foreach (var expectedResult in expectedTests) {
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.IsNotNull(actualResult, expectedResult.TestCase.FullyQualifiedName + " not found in results");
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestMultiprocessing() {
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = TestInfo.TestAdapterMultiprocessingTests;
var runContext = CreateRunContext(expectedTests);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
var resultNames = recorder.Results.Select(tr => tr.TestCase.FullyQualifiedName).ToSet();
foreach (var expectedResult in expectedTests) {
AssertUtil.ContainsAtLeast(resultNames, expectedResult.TestCase.FullyQualifiedName);
var actualResult = recorder.Results.SingleOrDefault(tr => tr.TestCase.FullyQualifiedName == expectedResult.TestCase.FullyQualifiedName);
Assert.AreEqual(expectedResult.Outcome, actualResult.Outcome, expectedResult.TestCase.FullyQualifiedName + " had incorrect result");
}
}
public void TestPassInTestList() {
PythonPaths.Python27.AssertInstalled();
var executor = new TestExecutor();
var recorder = new MockTestExecutionRecorder();
var expectedTests = Enumerable.Repeat(TestInfo.TestAdapterATests, 10).SelectMany();
var runContext = CreateRunContext(expectedTests, dryRun: true);
var testCases = expectedTests.Select(tr => tr.TestCase);
executor.RunTests(testCases, runContext, recorder);
PrintTestResults(recorder);
AssertUtil.ArrayEquals(
expectedTests.Select(t => t.TestCase.FullyQualifiedName).ToList(),
recorder.Results.Select(t => t.TestCase.FullyQualifiedName).ToList()
);
}
