public void TestTagsToTraits()
{
// Initialize a mock sink to keep track of the discovered tests.
MockTestCaseDiscoverySink testSink = new MockTestCaseDiscoverySink();
// Discover tests from the reference project.
TestDiscoverer discoverer = new TestDiscoverer();
discoverer.DiscoverTests(Common.ReferenceExeList,
new MockDiscoveryContext(),
new MockMessageLogger(),
testSink);
// Get the test with tags.
TestCase tagsTest = testSink.Tests.Where(test => test.DisplayName == "With tags").First();
// The tags should be present in the test as traits.
string traits = tagsTest.Traits
.Select(trait => trait.Name)
.Aggregate("", (acc, add) => acc + "," + add);
Assert.AreEqual(2, tagsTest.Traits.Count());
Assert.IsTrue(tagsTest.Traits.Any(trait => trait.Name == "tag"), traits);
Assert.IsTrue(tagsTest.Traits.Any(trait => trait.Name == "neat"), traits);
}
private static void RunLocally(IFrameworkHandle frameworkHandle, IGrouping <string, TestCase> source)
{
var discoverer = new TestDiscoverer();
var tests2 = discoverer.LoadFromSources(new[] { source.Key },
x => frameworkHandle.SendMessage(TestMessageLevel.Informational, x));
if (!tests2.Any())
{
return;
}
var runner = new TestRunner(discoverer);
runner.Load(new[] { tests2[0].Type.Assembly }).GetAwaiter().GetResult();
foreach (var testCase in source)
{
var testClassName = (string)testCase.GetPropertyValue(Constants.TestClassProperty);
var testName = (string)testCase.GetPropertyValue(Constants.TestMethodProperty);
var method = discoverer.Get(testClassName, testName);
if (method == null)
{
frameworkHandle.RecordResult(new TestResult(testCase)
{
Outcome = TestOutcome.NotFound
});
continue;
}
RunTestMethod(frameworkHandle, testCase, runner, method);
}
runner.Shutdown().GetAwaiter().GetResult();
}
public void TestDiscoverTestsComplex()
{
QmlTestRunnerMock.Setup(m => m.Execute(It.IsRegex("-functions"), It.IsAny <IDiscoveryContext>())).Returns(
new QmlTestRunnerResult("", FunctionsComplex, 0));
TestDiscoverer.DiscoverTests(new string[] { Path }, null, FrameworkHandleMock.Object, DiscoverySinkMock.Object);
DiscoverySinkMock.Verify(m => m.SendTestCase(It.IsAny <TestCase>()), Times.Exactly(5));
}
/// <summary>
/// Initializes the grid data
/// </summary>
private void InitializeGrid()
{
var testDiscoverer = new TestDiscoverer();
this.TestCases = testDiscoverer.GetTests();
this.TestCases = this.TestCases.OrderBy(t => t.Priority).ThenBy(p => p.Severity).ToList();
this.TestCasesRows = this.TestCases;
}
public void DiscoverTests_UsingSilverlightUnitTestDll_FindsTests()
{
TestDiscoverer discoverer = new TestDiscoverer();
string thisAssemblyFilePath = Assembly.GetExecutingAssembly().Location;
string thisAssemblyPath = Path.GetDirectoryName(thisAssemblyFilePath);
string thisProjectBinPath = Path.GetDirectoryName(thisAssemblyPath);
string thisProjectPath = Path.GetDirectoryName(thisProjectBinPath);
string thisSolutionPath = Path.GetDirectoryName(thisProjectPath);
Assert.IsNotNull(thisSolutionPath);
string silverlightUnitTestProjectPath = Path.Combine(thisSolutionPath, "SilverlightUnitTest");
string silverlightUnitTestDllFilePath = Path.Combine(silverlightUnitTestProjectPath, "bin", ConfigurationName, "SilverlightUnitTest.dll");
Assert.IsTrue(File.Exists(silverlightUnitTestDllFilePath), $"File doesn't exist: {silverlightUnitTestDllFilePath}");
List <string> testAssemblies = new List <string>
{
silverlightUnitTestDllFilePath
};
Mock <IDiscoveryContext> mockDiscoveryContext = new Mock <IDiscoveryContext>();
ConsoleMessageLogger consoleMessageLogger = new ConsoleMessageLogger();
Mock <ITestCaseDiscoverySink> mockTestCaseDiscoverySink = new Mock <ITestCaseDiscoverySink>();
List <TestCase> discoveredTestCases = new List <TestCase>();
mockTestCaseDiscoverySink.Setup(x => x.SendTestCase(It.IsAny <TestCase>())).Callback <TestCase>((testCase) =>
{
discoveredTestCases.Add(testCase);
});
discoverer.DiscoverTests(
testAssemblies,
mockDiscoveryContext.Object,
consoleMessageLogger,
mockTestCaseDiscoverySink.Object);
List <string> expectedTestCaseDisplayNames = new List <string>
{
"SilverlightUnitTest.AsyncTest.EnqueueTestComplete_ShouldPass",
"SilverlightUnitTest.AsyncTest.Assert_Fail_ShouldFail",
"SilverlightUnitTest.AsyncTest.ReturnTask_ShouldPass",
"SilverlightUnitTest.AsyncTest.ReturnTask_WithAssertFail_ShouldFail",
"SilverlightUnitTest.AsyncTest.AwaitTask_ShouldPass",
"SilverlightUnitTest.AsyncTest.AwaitTask_WithAssertFail_ShouldFail",
"SilverlightUnitTest.UnitTest1.TestMethod1"
};
List <string> discoveredTestCaseNames = discoveredTestCases.Select(x => x.FullyQualifiedName).ToList();
string message = string.Join(Environment.NewLine, discoveredTestCaseNames);
CollectionAssert.AreEqual(expectedTestCaseDisplayNames, discoveredTestCaseNames, message);
}
public void TestDiscoverTestsError()
{
QmlTestRunnerMock.Setup(m => m.Execute(It.IsRegex("-functions"), It.IsAny <IDiscoveryContext>())).Returns(
new QmlTestRunnerResult("", FunctionsError, 1));
TestDiscoverer.DiscoverTests(new string[] { Path }, null, FrameworkHandleMock.Object, DiscoverySinkMock.Object);
DiscoverySinkMock.Verify(m => m.SendTestCase(It.IsAny <TestCase>()), Times.Never());
FrameworkHandleMock.Verify(m => m.SendMessage(TestMessageLevel.Error, It.IsAny <string>()),
Times.Once());
}
public void TestDiscoverTestsSimple()
{
QmlTestRunnerMock.Setup(m => m.Execute(It.IsRegex("-functions"), It.IsAny <IDiscoveryContext>())).Returns(
new QmlTestRunnerResult("", FunctionsSimple, 0));
TestDiscoverer.DiscoverTests(new string[] { Path }, null, FrameworkHandleMock.Object, DiscoverySinkMock.Object);
DiscoverySinkMock.Verify(m => m.SendTestCase(It.Is <TestCase>(tc =>
tc.FullyQualifiedName == "simpletest::test_simple" &&
tc.CodeFilePath == Path &&
tc.ExecutorUri == QmlTestExecutor.ExecutorUri &&
tc.Source == Path
)), Times.Once);
}
public void DiscoversAllTests()
{
// Initialize a mock sink to keep track of the discovered tests.
MockTestCaseDiscoverySink testSink = new MockTestCaseDiscoverySink();
// Discover tests from the reference project.
TestDiscoverer discoverer = new TestDiscoverer();
discoverer.DiscoverTests(Common.ReferenceExeList,
new MockDiscoveryContext(),
new MockMessageLogger(),
testSink);
// There is a known number of test cases in the reference project.
Assert.AreEqual(Common.ReferenceTestCount, testSink.Tests.Count);
}
public void TestDiscover()
{
var ctx = new MockDiscoveryContext(new MockRunSettings(_runSettings));
var sink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
const string projectPath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\PythonApplication107.pyproj";
const string testFilePath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\test1.py";
new TestDiscoverer().DiscoverTests(
new[] { testFilePath },
ctx,
logger,
sink
);
PrintTestCases(sink.Tests);
var expectedTests = new[] {
TestInfo.FromRelativePaths("Test_test1", "test_A", projectPath, testFilePath, 17, TestOutcome.Passed),
TestInfo.FromRelativePaths("Test_test1", "test_B", projectPath, testFilePath, 21, TestOutcome.Passed),
TestInfo.FromRelativePaths("Test_test2", "test_C", projectPath, testFilePath, 48, TestOutcome.Passed)
};
Assert.AreEqual(expectedTests.Length, sink.Tests.Count);
foreach (var expectedTest in expectedTests)
{
var expectedFullyQualifiedName = TestDiscoverer.MakeFullyQualifiedTestName(expectedTest.RelativeClassFilePath, expectedTest.ClassName, expectedTest.MethodName);
var actualTestCase = sink.Tests.SingleOrDefault(tc => tc.FullyQualifiedName == expectedFullyQualifiedName);
Assert.IsNotNull(actualTestCase, expectedFullyQualifiedName);
Assert.AreEqual(expectedTest.MethodName, actualTestCase.DisplayName, expectedFullyQualifiedName);
Assert.AreEqual(new Uri(TestExecutor.ExecutorUriString), actualTestCase.ExecutorUri);
Assert.AreEqual(expectedTest.SourceCodeLineNumber, actualTestCase.LineNumber, expectedFullyQualifiedName);
Assert.IsTrue(IsSameFile(expectedTest.SourceCodeFilePath, actualTestCase.CodeFilePath), expectedFullyQualifiedName);
sink.Tests.Remove(actualTestCase);
}
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
PrintTestCases(sink.Tests);
}
// Tests that filters in runsettings are obeyed.
public void FiltersTestExecutables()
{
// Initialize a mock sink to keep track of the discovered tests.
MockTestCaseDiscoverySink testSink = new MockTestCaseDiscoverySink();
// Configure a mock context.
var context = new MockDiscoveryContext();
var provider = new CatchSettingsProvider();
provider.Settings = new CatchAdapterSettings();
provider.Settings.TestExeInclude.Add(@"ReferenceCatchProject\.exe");
context.MockSettings.Provider = provider;
// Discover tests from the reference project and from anon-existent exe.
// The non-existent exe should get filtered out and cause no trouble.
TestDiscoverer discoverer = new TestDiscoverer();
List <string> exeList = new List <string>();
exeList.AddRange(Common.ReferenceExeList);
exeList.Add("nonsense.exe");
discoverer.DiscoverTests(Common.ReferenceExeList,
context,
new MockMessageLogger(),
testSink);
// There is a known number of test cases in the reference project.
Assert.AreEqual(Common.ReferenceTestCount, testSink.Tests.Count);
// Clear the sink.
testSink = new MockTestCaseDiscoverySink();
// Filter all exes.
provider.Settings.TestExeInclude.Clear();
provider.Settings.TestExeInclude.Add("laksjdlkjalsdjasljd");
// Discover again.
discoverer.DiscoverTests(Common.ReferenceExeList,
context,
new MockMessageLogger(),
testSink);
// There should be no tests, as nothing matches the filter.
Assert.AreEqual(testSink.Tests.Count, 0);
}
public void RunTest(RunTests instruction, TestResultReporter reporter)
{
var name = new AssemblyName(instruction.AssemblyName);
var assembly = Assembly.Load(name);
try
{
var discoverer = new TestDiscoverer();
discoverer.Load(assembly, instruction.Source);
WrapRun(instruction, reporter, discoverer, assembly);
}
catch (Exception ex)
{
reporter.AddResult(new TestResult(instruction.TestCases[0])
{
ErrorStackTrace = ex.StackTrace,
ErrorMessage = ex.Message,
Result = TestOutcome.Failed,
});
}
}
public void DiscoversNoTests()
{
// Initialize a mock sink to keep track of the discovered tests.
MockTestCaseDiscoverySink testSink = new MockTestCaseDiscoverySink();
TestDiscoverer discoverer = new TestDiscoverer();
var cd = System.IO.Directory.GetCurrentDirectory();
// Unfortunately it doesn't get copied with the DeployItemAttribute, no idea why.
System.IO.File.WriteAllText(@"nonecatchexe.cmd", @"@echo Non Catch Output line");
// Returns an unexpected first line.
discoverer.DiscoverTests(new List <String>()
{
@"nonecatchexe.cmd"
},
new MockDiscoveryContext(),
new MockMessageLogger(),
testSink);
// Zero test cases should be registered.
Assert.AreEqual(0, testSink.Tests.Count);
}
static async Task Main(string[] args)
{
var discoverer = new TestDiscoverer();
discoverer.Load(new[] { typeof(IncidentWrapper).Assembly });
var runner = new TestRunner(discoverer);
runner.Load(new[] { typeof(IncidentWrapper).Assembly }).GetAwaiter().GetResult();
var result = runner.RunAll().GetAwaiter().GetResult();
var faulty = result.Where(x => !x.IsSuccess).ToList();
var cmd = new RunTests
{
AssemblyName = typeof(EnvironmentTests).Assembly.GetName().Name,
AssemblyPath = Path.GetDirectoryName(typeof(EnvironmentTests).Assembly.Location),
//TestCases = new[]
//{
// new TestCaseToRun("Coderr.IntegrationTests.Core.TestCases.EnvironmentTests.Clearing_environment_should_remove_all_incidents_in_it")
// {
// TestClassName = "EnvironmentTests",
// TestMethodName = "Clearing_environment_should_remove_all_incidents_in_it"
// },
//},
Source = typeof(EnvironmentTests).Assembly.Location
};
var receiver = new ConsoleReceiver();
using (var coordinator = new AppDomainRunner("TesTSuite1"))
{
coordinator.Create();
coordinator.AddDirectory(
@"C:\src\1tcompany\coderr\oss\Coderr.Server\src\Tools\Coderr.IntegrationTests\Coderr.IntegrationTests\bin\Debug\net461");
coordinator.RunTests(cmd, receiver);
}
}
public void TestCaseLinesCorrect()
{
// Initialize a mock sink to keep track of the discovered tests.
MockTestCaseDiscoverySink testSink = new MockTestCaseDiscoverySink();
// Discover tests from the reference project.
TestDiscoverer discoverer = new TestDiscoverer();
discoverer.DiscoverTests(Common.ReferenceExeList,
new MockDiscoveryContext(),
new MockMessageLogger(),
testSink);
// The reference project test cases are on these lines.
var linesOfCases = new Dictionary <string, int>();
linesOfCases.Add("No tags", 6);
linesOfCases.Add("With tags", 16);
linesOfCases.Add("Has failure", 24);
foreach (var test in testSink.Tests)
{
// Process only tests we have hard coded here.
if (linesOfCases.ContainsKey(test.FullyQualifiedName))
{
// Check that the line number is correct.
Assert.AreEqual(linesOfCases[test.FullyQualifiedName], test.LineNumber, test.FullyQualifiedName);
// Remove the case so we can check all were handled.
linesOfCases.Remove(test.FullyQualifiedName);
}
}
// Make sure all the cases we wanted got checked.
Assert.AreEqual(linesOfCases.Count, 0, String.Format("Unhandled cases: {0}", linesOfCases.ToString()));
}
private static void WrapRun(RunTests instruction, TestResultReporter reporter, TestDiscoverer discoverer,
Assembly assembly)
{
var runner = new TestRunner(discoverer);
runner.Load(new[] { assembly }).GetAwaiter().GetResult();
var result = new List <TestResult>();
foreach (var testCase in instruction.TestCases)
{
reporter.Started(testCase);
var tc = discoverer.Find(testCase.TestClassName);
var method = tc?.Methods.FirstOrDefault(x => x.Name == testCase.TestMethodName);
if (tc == null || method == null)
{
reporter.Ended(testCase, TestOutcome.NotFound);
reporter.AddResult(new TestResult(testCase)
{
Result = TestOutcome.NotFound
});
}
var testResult = runner.Run(tc, method).GetAwaiter().GetResult();
TestOutcome outcome;
if (testResult.IsIgnored)
{
outcome = TestOutcome.Ignored;
}
else if (testResult.IsSuccess)
{
outcome = TestOutcome.Success;
}
else
{
outcome = TestOutcome.Failed;
}
reporter.Ended(testCase, outcome);
reporter.AddResult(new TestResult(testCase)
{
Result = outcome,
ErrorStackTrace = testResult.Exception?.StackTrace,
ErrorMessage = testResult.Exception?.Message
});
}
}
