public void Will_call_test_suite_finished_with_final_result()
{
var runner = new TestableTestRunner();
var testCallback = new MockTestMethodRunnerCallback();
var context = runner.SetupTestContext();
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.TestRunnerJsName)).Returns("jsPath");
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", testCallback.Object);
testCallback.Verify(x => x.TestSuiteFinished(It.IsAny <TestCaseSummary>()));
}
public void Will_not_clean_up_test_context_if_debug_mode()
{
var runner = new TestableTestRunner();
var context = runner.SetupTestContext();
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.TestRunnerJsName)).Returns("jsPath");
runner.ClassUnderTest.EnableDebugMode();
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html");
runner.Mock <ITestContextBuilder>().Verify(x => x.CleanupContext(context), Times.Never());
}
public void Will_capture_message_logged_via_jasmine_log()
{
var testRunner = TestRunner.Create();
TestCaseSummary result = null;
TestUtils.RunAsJasmineVersionOne(() =>
{
result = testRunner.RunTests(@"JS\Test\jasmineLog.js", new ExceptionThrowingRunnerCallback());
});
Assert.Equal("hello", result.Logs.Single().Message);
}
public void Will_run_tests_from_a_folder_and_a_file()
{
var testRunner = TestRunner.Create();
TestCaseSummary result =
testRunner.RunTests(new List <string> {
@"JS\Test\basic-qunit.js", @"JS\Test\SubFolder"
},
new ExceptionThrowingRunnerCallback());
Assert.Equal(1, result.FailedCount);
Assert.Equal(5, result.PassedCount);
Assert.Equal(6, result.TotalCount);
}
public void Will_not_clean_up_test_context_if_open_in_browser_is_set()
{
var runner = new TestableTestRunner();
var context = runner.SetupTestContext();
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.TestRunnerJsName)).Returns("jsPath");
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", new TestOptions {
TestLaunchMode = TestLaunchMode.FullBrowser
});
runner.Mock <ITestContextBuilder>().Verify(x => x.CleanupContext(context), Times.Never());
}
public void Will_encode_file_path_of_test_file_with_space()
{
if (ChutzpahTestSettingsFile.ForceWebServerMode)
{
return;
}
var testRunner = TestRunner.Create();
TestCaseSummary result = testRunner.RunTests(@"JS\Test\PathEncoding\With Space+\pathEncoding.js", new ExceptionThrowingRunnerCallback());
Assert.Equal(0, result.FailedCount);
Assert.Equal(1, result.PassedCount);
Assert.Equal(1, result.TotalCount);
}
public void Will_run_multiple_files_and_aggregate_results()
{
var testRunner = TestRunner.Create();
var tests = new List <string>
{
@"JS\Test\basic-qunit.js",
@"JS\Test\basic-qunit.html"
};
TestCaseSummary result = testRunner.RunTests(tests, new ExceptionThrowingRunnerCallback());
Assert.Equal(2, result.FailedCount);
Assert.Equal(6, result.PassedCount);
Assert.Equal(8, result.TotalCount);
}
public void Will_run_test_which_logs_object_to_jasmine_log()
{
var testRunner = TestRunner.Create();
TestCaseSummary result = null;
TestUtils.RunAsJasmineVersionOne(() =>
{
result = testRunner.RunTests(@"JS\Test\jasmineLog.js", new ExceptionThrowingRunnerCallback());
});
Assert.Equal(0, result.FailedCount);
Assert.Equal(1, result.PassedCount);
Assert.Equal(1, result.TotalCount);
}
public void Will_clean_up_test_context()
{
var runner = new TestableTestRunner();
var context = new TestContext {
TestHarnessPath = @"D:\harnessPath.html"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.IsAny <PathInfo>(), It.IsAny <TestOptions>(), out context)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.TestRunnerJsName)).Returns("jsPath");
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html");
runner.Mock <ITestContextBuilder>().Verify(x => x.CleanupContext(context));
}
public void Will_call_test_suite_started()
{
var runner = new TestableTestRunner();
var testCallback = new MockTestMethodRunnerCallback();
var context = new TestContext {
TestHarnessPath = @"D:\harnessPath.html"
};
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.IsAny <PathInfo>(), It.IsAny <TestOptions>(), out context)).Returns(true);
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", testCallback.Object);
testCallback.Verify(x => x.TestSuiteStarted());
}
public void Will_execute_nothing_if_test_file_has_infinite_loop()
{
var testRunner = TestRunner.Create();
TestCaseSummary result = testRunner.RunTests(new List <string> {
@"JS\Test\spinWait.js"
}, new TestOptions {
TestFileTimeoutMilliseconds = 500
});
Assert.Equal(1, result.Errors.Count);
Assert.Equal(0, result.FailedCount);
Assert.Equal(0, result.PassedCount);
Assert.Equal(0, result.TotalCount);
}
public void Transform(TestCaseSummary testFileSummary, string outFile)
{
if (testFileSummary == null)
{
throw new ArgumentNullException("testFileSummary");
}
if (string.IsNullOrEmpty(outFile))
{
throw new ArgumentNullException("outFile");
}
var result = Transform(testFileSummary);
File.WriteAllText(outFile, result);
}
public void Will_run_jasmine_require_html_test_where_test_file_uses_requirejs_command()
{
var testRunner = TestRunner.Create();
TestCaseSummary result = null;
TestUtils.RunAsJasmineVersionOne(() =>
{
result = testRunner.RunTests(@"JS\Test\requirejs\jasmine-test.html", new ExceptionThrowingRunnerCallback());
});
Assert.Equal(0, result.FailedCount);
Assert.Equal(2, result.PassedCount);
Assert.Equal(2, result.TotalCount);
}
public void Will_execute_nothing_if_test_takes_longer_than_timeout()
{
var testRunner = TestRunner.Create();
TestCaseSummary result = testRunner.RunTests(new List <string> {
@"JS\Test\timeoutTest.js"
}, new TestOptions {
TestFileTimeoutMilliseconds = 500
});
Assert.Equal(1, result.Errors.Count);
Assert.Equal(0, result.FailedCount);
Assert.Equal(0, result.PassedCount);
Assert.Equal(0, result.TotalCount);
}
public virtual void Transform(TestCaseSummary testFileSummary, string outFile)
{
if (testFileSummary == null)
{
throw new ArgumentNullException("testFileSummary");
}
else if (string.IsNullOrEmpty(outFile))
{
throw new ArgumentNullException("outFile");
}
var result = Transform(testFileSummary);
fileSystem.WriteAllText(outFile, result, Encoding);
}
public void Will_execute_test_if_test_takes_less_than_timeout()
{
var testRunner = TestRunner.Create();
TestCaseSummary result = testRunner.RunTests(new List <string> {
@"JS\Test\timeoutTest.js"
},
new TestOptions {
TestFileTimeoutMilliseconds = 1500
},
new ExceptionThrowingRunnerCallback());
Assert.Equal(0, result.FailedCount);
Assert.Equal(1, result.PassedCount);
Assert.Equal(1, result.TotalCount);
}
public void Will_execute_nothing_if_test_takes_longer_than_timeout_from_settings_file()
{
var testRunner = TestRunner.Create();
// The time out from test options will get overridden by the one from the file
TestCaseSummary result = testRunner.RunTests(new List <string> {
@"JS\Test\TimeoutSetting\timeoutTest.js"
}, new TestOptions {
TestFileTimeoutMilliseconds = 3500
});
Assert.Equal(1, result.Errors.Count);
Assert.Equal(0, result.FailedCount);
Assert.Equal(0, result.PassedCount);
Assert.Equal(0, result.TotalCount);
}
public void Will_open_test_file_in_browser_when_given_flag()
{
var runner = new TestableTestRunner();
var context = runner.SetupTestContext(testRunnerPath: "testRunner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.TestRunnerJsName)).Returns("runner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("testRunner.js")).Returns("runner.js");
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", new TestOptions {
TestLaunchMode = TestLaunchMode.FullBrowser
});
runner.Mock <IProcessHelper>().Verify(x => x.LaunchFileInBrowser(@"D:\harnessPath.html", It.IsAny <string>(), It.IsAny <IDictionary <string, string> >()));
}
public void Will_run_multiple_js_files_and_aggregate_results()
{
var testRunner = TestRunner.Create();
ChutzpahTracer.Enabled = true;
ChutzpahTracer.AddConsoleListener();
var tests = new List <string>
{
@"JS\Test\basic-qunit.js",
@"JS\Test\basic-jasmine.js"
};
TestCaseSummary result = testRunner.RunTests(tests, new ExceptionThrowingRunnerCallback());
Assert.Equal(2, result.FailedCount);
Assert.Equal(6, result.PassedCount);
Assert.Equal(8, result.TotalCount);
}
public void Will_run_multiple_test_files_and_return_results()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context1 = new TestContext {
TestHarnessPath = @"D:\harnessPath1.html", TestRunner = "runner1"
};
var context2 = new TestContext {
TestHarnessPath = @"D:\harnessPath2.htm", TestRunner = "runner2"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.Is <PathInfo>(f => f.FullPath == @"path\tests1.html"), It.IsAny <TestOptions>(), out context1)).Returns(true);
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.Is <PathInfo>(f => f.FullPath == @"path\tests2.html"), It.IsAny <TestOptions>(), out context2)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests1.html")).Returns(@"D:\path\tests1.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests2.htm")).Returns(@"D:\path\tests2.htm");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner1")).Returns("jsPath1");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner2")).Returns("jsPath2");
var args1 = TestableTestRunner.BuildArgs("jsPath1", "file:///D:/harnessPath1.html", "execution", Constants.DefaultTestFileTimeout);
var args2 = TestableTestRunner.BuildArgs("jsPath2", "file:///D:/harnessPath2.htm", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args1, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>(0, summary));
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args2, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>(0, summary));
runner.Mock <IFileProbe>()
.Setup(x => x.FindScriptFiles(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
}, It.IsAny <TestingMode>()))
.Returns(new List <PathInfo> {
new PathInfo {
FullPath = @"path\tests1.html"
}, new PathInfo {
FullPath = @"path\tests2.html"
}
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
});
Assert.Equal(2, res.TotalCount);
}
public void Will_run_test_file_and_return_test_results_model()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = runner.SetupTestContext(testPaths: new[] { @"path\tests.html" }, harnessPath: @"harnessPath", testRunnerPath: "runner");
runner.Mock <ITestExecutionProvider>()
.Setup(x => x.Execute(It.IsAny <TestOptions>(), It.IsAny <TestContext>(), TestExecutionMode.Execution, It.IsAny <ITestMethodRunnerCallback>()))
.Returns(new List <TestFileSummary> {
summary
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html");
Assert.Equal(1, res.TotalCount);
}
public override void TestSuiteFinished(TestContext context, TestCaseSummary testResultsSummary)
{
var statusBarText = "";
if (testResultsSummary.SkippedCount > 0)
{
statusBarText = string.Format("{0} passed, {1} failed, {2} skipped, {3} total", testResultsSummary.PassedCount, testResultsSummary.FailedCount, testResultsSummary.SkippedCount, testResultsSummary.TotalCount);
}
else
{
statusBarText = string.Format("{0} passed, {1} failed, {2} total", testResultsSummary.PassedCount, testResultsSummary.FailedCount, testResultsSummary.TotalCount);
}
var text = string.Format("========== Total Tests: {0} ==========\n", statusBarText);
testPane.OutputString(text);
SetStatusBarMessage(statusBarText);
}
public void Will_run_multiple_test_files_and_return_results()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context1 = runner.SetupTestContext(harnessPath: @"harnessPath1", testRunnerPath: "runner1", testPaths: new [] { @"path\tests1.html" });
var context2 = runner.SetupTestContext(harnessPath: @"harnessPath2", testRunnerPath: "runner2", testPaths: new [] { @"path\tests2.html" });
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests1.html")).Returns(@"D:\path\tests1.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests2.htm")).Returns(@"D:\path\tests2.htm");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner1")).Returns("jsPath1");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner2")).Returns("jsPath2");
var args1 = TestableTestRunner.BuildArgs("jsPath1", "harnessPath1", "execution", Constants.DefaultTestFileTimeout);
var args2 = TestableTestRunner.BuildArgs("jsPath2", "harnessPath2", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args1, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args2, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
runner.Mock <IFileProbe>()
.Setup(x => x.FindScriptFiles(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
}))
.Returns(new List <PathInfo> {
new PathInfo {
FullPath = @"path\tests1.html"
}, new PathInfo {
FullPath = @"path\tests2.html"
}
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
});
Assert.Equal(2, res.TotalCount);
}
public void Will_call_process_transforms()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var testCallback = new MockTestMethodRunnerCallback();
var context = runner.SetupTestContext(testRunnerPath: "testRunner.js");
runner.Mock <ITestExecutionProvider>()
.Setup(x => x.Execute(It.IsAny <TestOptions>(), It.IsAny <TestContext>(), TestExecutionMode.Execution, It.IsAny <ITestMethodRunnerCallback>()))
.Returns(new List <TestFileSummary> {
summary
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(new[] { @"path\tests.html" }, testCallback.Object);
runner.Mock <ITransformProcessor>().Verify(x => x.ProcessTransforms(It.Is <IEnumerable <TestContext> >(c => c.Count() == 1 && c.Single() == context), res));
}
public void Will_open_test_file_in_browser_when_given_flag()
{
var runner = new TestableTestRunner();
var context = new TestContext {
TestHarnessPath = @"D:\harnessPath.html", TestRunner = "testRunner.js"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.IsAny <PathInfo>(), It.IsAny <TestOptions>(), out context)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.TestRunnerJsName)).Returns("runner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("testRunner.js")).Returns("runner.js");
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", new TestOptions {
OpenInBrowser = true
});
runner.Mock <IProcessHelper>().Verify(x => x.LaunchFileInBrowser(@"D:\harnessPath.html"));
}
public override string Transform(TestCaseSummary testFileSummary)
{
if (testFileSummary == null)
{
throw new ArgumentNullException("testFileSummary");
}
var builder = new StringBuilder();
builder.AppendLine(@"<?xml version=""1.0"" encoding=""UTF-8"" ?>");
builder.AppendLine(@"<testsuites>");
foreach (TestFileSummary file in testFileSummary.TestFileSummaries)
{
builder.AppendLine(
string.Format(@" <testsuite name=""{0}"" tests=""{1}"" failures=""{2}"" time=""{3}"">",
Encode(file.Path), file.Tests.Count, file.Tests.Count(x => !x.Passed), file.TimeTaken));
;
foreach (TestCase test in file.Tests)
{
if (test.Passed)
{
builder.AppendLine(string.Format(@" <testcase name=""{0}"" time=""{1}"" />",
Encode(test.GetDisplayName()), test.TimeTaken));
}
else
{
TestResult failureCase = test.TestResults.FirstOrDefault(x => !x.Passed);
if (failureCase != null)
{
string failureMessage = failureCase.GetFailureMessage();
builder.AppendLine(string.Format(@" <testcase name=""{0}"" time=""{1}"">",
Encode(test.GetDisplayName()), test.TimeTaken));
builder.AppendLine(string.Format(@" <failure message=""{0}""></failure>",
Encode(failureMessage)));
builder.AppendLine(string.Format(@" </testcase>"));
}
}
}
builder.AppendLine(@" </testsuite>");
}
builder.AppendLine(@"</testsuites>");
return(builder.ToString());
}
public override void Transform(TestCaseSummary testFileSummary, string outFile)
{
if (testFileSummary == null)
{
throw new ArgumentNullException("testFileSummary");
}
if (string.IsNullOrEmpty(outFile))
{
throw new ArgumentNullException("outFile");
}
if (testFileSummary.CoverageObject == null)
{
return;
}
CoverageOutputGenerator.WriteHtmlFile(outFile, testFileSummary.CoverageObject);
}
public override string Transform(TestCaseSummary testFileSummary)
{
if (testFileSummary == null)
{
throw new ArgumentNullException("testFileSummary");
}
else if (testFileSummary.CoverageObject == null)
{
return(string.Empty);
}
var sb = new StringBuilder();
foreach (var coverageData in testFileSummary.CoverageObject.Values)
{
AppendCoverageForFile(sb, coverageData);
}
return(sb.ToString());
}
private XmlElement AddTestResultsRoot(TestCaseSummary summary, XmlDocument document)
{
var testResults = document.CreateElement("test-results");
testResults.SetAttribute("name", "Chutzpah Test Results");
testResults.SetAttribute("total", summary.TotalCount.ToString());
testResults.SetAttribute("errors", summary.Errors.Count.ToString());
testResults.SetAttribute("failures", summary.FailedCount.ToString());
testResults.SetAttribute("not-run", (summary.TotalCount - summary.Errors.Count - summary.FailedCount - summary.PassedCount).ToString());
testResults.SetAttribute("inconclusive", "0");
testResults.SetAttribute("ignored", "0");
testResults.SetAttribute("skipped", (summary.TotalCount - summary.Errors.Count - summary.FailedCount - summary.PassedCount).ToString());
testResults.SetAttribute("invalid", "0");
testResults.SetAttribute("date", DateTime.Now.ToString("yyyy-MM-dd"));
testResults.SetAttribute("time", DateTime.Now.ToString("HH:mm:ss"));
document.AppendChild(testResults);
return(testResults);
}
private static TestCaseSummary GetTestCaseSummary()
{
var toReturn = new TestCaseSummary();
toReturn.CoverageObject = new CoverageData();
toReturn.CoverageObject["/no/coverage"] = new CoverageFileData
{
FilePath = "/no/coverage",
LineExecutionCounts = null
};
toReturn.CoverageObject["/some/lines"] = new CoverageFileData
{
FilePath = "/some/lines",
LineExecutionCounts = new int?[] { 1, 2, null, 5, 0 }
};
return(toReturn);
}
