public void Will_record_unknown_exception_from_test_runner()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var testCallback = new MockTestMethodRunnerCallback();
var context = new TestContext {
TestHarnessPath = @"D:\harnessPath.html", TestRunner = "testRunner.js"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.IsAny <PathInfo>(), It.IsAny <TestOptions>(), out context)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("testRunner.js")).Returns("runner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns(TestRunner.HeadlessBrowserName);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(TestRunner.HeadlessBrowserName, TestableTestRunner.ExecutionPhantomArgs, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>((int)TestProcessExitCode.Unknown, summary));
TestCaseSummary res = runner.ClassUnderTest.RunTests(new[] { @"path\tests.html" }, testCallback.Object);
testCallback.Verify(x => x.FileError(It.IsAny <TestError>()));
Assert.Equal(1, res.TotalCount);
Assert.Equal(1, res.Errors.Count);
}
public void Will_run_test_files_found_from_given_folder_path()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = new TestContext {
TestHarnessPath = @"D:\harnessPath.html", TestRunner = "runner"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.IsAny <PathInfo>(), It.IsAny <TestOptions>(), out context)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner")).Returns("jsPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>()
.Setup(x => x.FindScriptFiles(new List <string> {
@"path\testFolder"
}, It.IsAny <TestingMode>()))
.Returns(new List <PathInfo> {
new PathInfo {
FullPath = @"path\tests.html"
}
});
var args = TestableTestRunner.BuildArgs("jsPath", "file:///D:/harnessPath.html", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput("browserPath", args, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>(0, summary));
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\testFolder");
Assert.Equal(1, res.TotalCount);
}
public void Will_add_exception_to_errors_and_move_to_next_test_file()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var testCallback = new MockTestMethodRunnerCallback();
var exception = new Exception();
var context = runner.SetupTestContext(testRunnerPath: "testRunner.js", testPaths: new [] { @"path\tests1.html" });
runner.SetupTestContext(testRunnerPath: "testRunner.js", testPaths: new [] { @"path\tests2.html" }, @throw: true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests1.html")).Throws(exception);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests2.html")).Returns(@"D:\path\tests2.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("testRunner.js")).Returns("runner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
var args = TestableTestRunner.BuildArgs("runner.js", "file:///D:/harnessPath.html", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput("browserPath", args, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
TestCaseSummary res = runner.ClassUnderTest.RunTests(new[] { @"path\tests1.html", @"path\tests2.html" }, testCallback.Object);
testCallback.Verify(x => x.FileError(It.IsAny <TestError>()));
Assert.Equal(1, res.Errors.Count);
Assert.Equal(1, res.TotalCount);
}
public void Will_pass_timeout_option_to_test_runner()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = new TestContext {
TestHarnessPath = @"D:\harnessPath.html", TestRunner = "runner"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.IsAny <PathInfo>(), It.IsAny <TestOptions>(), out context)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner")).Returns("jsPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
var args = TestableTestRunner.BuildArgs("jsPath", "file:///D:/harnessPath.html", "execution", 5000);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput("browserPath", args, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>(0, summary));
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", new TestOptions {
TestFileTimeoutMilliseconds = 5000
});
Assert.Equal(1, res.TotalCount);
}
public void Will_throw_if_test_files_collection_is_null()
{
var runner = new TestableTestRunner();
var ex = Record.Exception(() => runner.ClassUnderTest.RunTests((IEnumerable <string>)null)) as ArgumentNullException;
Assert.NotNull(ex);
}
public static async void FailureInQueueOfTestStarting_DoesNotQueueTestFinished_DoesNotInvokeTest()
{
var messages = new List <IMessageSinkMessage>();
var messageBus = Substitute.For <IMessageBus>();
messageBus
.QueueMessage(null)
.ReturnsForAnyArgs(callInfo =>
{
var msg = callInfo.Arg <IMessageSinkMessage>();
messages.Add(msg);
if (msg is ITestStarting)
{
throw new InvalidOperationException();
}
return(true);
});
var runner = TestableTestRunner.Create(messageBus);
await Assert.ThrowsAsync <InvalidOperationException>(() => runner.RunAsync());
var starting = Assert.Single(messages);
Assert.IsAssignableFrom <ITestStarting>(starting);
Assert.False(runner.InvokeTestAsync_Called);
}
public void Will_use_timeout_from_context_if_exists()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = runner.SetupTestContext(testRunnerPath: "runner");
context.TestFileSettings.TestFileTimeout = 6000;
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner")).Returns("jsPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
var args = TestableTestRunner.BuildArgs("jsPath", "file:///D:/harnessPath.html", "execution", 6000);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput("browserPath", args, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", new TestOptions {
TestFileTimeoutMilliseconds = 5000
});
Assert.Equal(1, res.TotalCount);
}
public static async void Messages()
{
var messageBus = new SpyMessageBus();
var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m);
var result = await runner.RunAsync();
Assert.Equal(21.12m, result.Time);
Assert.False(runner.TokenSource.IsCancellationRequested);
Assert.Collection(messageBus.Messages,
msg =>
{
var testStarting = Assert.IsAssignableFrom <ITestStarting>(msg);
Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, testStarting.TestCollection);
Assert.Same(runner.TestCase, testStarting.TestCase);
Assert.Equal("Display Name", testStarting.Test.DisplayName);
},
msg => { }, // Pass/fail/skip, will be tested elsewhere
msg =>
{
var testFinished = Assert.IsAssignableFrom <ITestFinished>(msg);
Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, testFinished.TestCollection);
Assert.Same(runner.TestCase, testFinished.TestCase);
Assert.Equal("Display Name", testFinished.Test.DisplayName);
Assert.Equal(21.12m, testFinished.ExecutionTime);
Assert.Empty(testFinished.Output);
}
);
}
public static TestableTestRunner CreateForAssemblyWithConfigFile(string configFilename)
{
TestableTestRunner result = CreateForAssembly();
result.Executor.ConfigFilename = configFilename;
return(result);
}
public void Will_run_test_files_found_from_chutzpah_json_files()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = runner.SetupTestContext(testRunnerPath: "runner");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner")).Returns("jsPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\chutzpah.json")).Returns(@"D:\path\chutzpah.json");
runner.Mock <IFileProbe>()
.Setup(x => x.FindScriptFiles(It.IsAny <ChutzpahTestSettingsFile>()))
.Returns(new List <PathInfo> {
new PathInfo {
FullPath = @"path\tests.html"
}
});
var args = TestableTestRunner.BuildArgs("jsPath", "file:///D:/harnessPath.html", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput("browserPath", args, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\chutzpah.json");
Assert.Equal(1, res.TotalCount);
runner.Mock <IFileProbe>()
.Verify(x => x.FindScriptFiles(It.IsAny <List <string> >()), Times.Never());
}
public void ValuesArePassedToExecutorWrapper()
{
TestableTestRunner runner = TestableTestRunner.CreateForAssembly();
runner.RunClass("foo");
Assert.Equal("foo", runner.Executor.RunClass_Type);
}
public static TestableTestRunner CreateForTests(string xml)
{
TestableTestRunner result = Create();
XmlNode xmlNode = xml == null ? null : MakeXmlNode(xml);
result.Executor.RunTests__Result = xmlNode;
return(result);
}
public void Will_return_null_given_empty_path()
{
var runner = new TestableTestRunner();
var result = runner.ClassUnderTest.GetTestContext(string.Empty);
Assert.Null(result);
}
public void NullAssemblyReturnsNoTests()
{
TestableTestRunner runner = TestableTestRunner.CreateForAssembly(null);
TestRunnerResult result = runner.RunAssembly();
Assert.Equal(TestRunnerResult.NoTests, result);
}
public void Will_batch_test_files_with_same_context_given_testFileBatching_setting_is_enabled()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context1 = runner.SetupTestContext(harnessPath: @"D:\harnessPath1.html", testRunnerPath: "runner1", testPaths: new[] { @"path\tests1.js", @"path\tests2.js" });
var context2 = runner.SetupTestContext(harnessPath: @"D:\harnessPath2.htm", testRunnerPath: "runner2", testPaths: new[] { @"path2\tests1.js", @"path2\tests2.js" });
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner1")).Returns("jsPath1");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner2")).Returns("jsPath2");
var args1 = TestableTestRunner.BuildArgs("jsPath1", "file:///D:/harnessPath1.html", "execution", Constants.DefaultTestFileTimeout);
var args2 = TestableTestRunner.BuildArgs("jsPath2", "file:///D:/harnessPath2.htm", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args1, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args2, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
runner.Mock <IFileProbe>()
.Setup(x => x.FindScriptFiles(new List <string> {
@"path\tests1a.js", @"path\tests2a.js", @"path2\tests1a.js", @"path2\tests2a.js"
}))
.Returns(new List <PathInfo> {
new PathInfo {
FullPath = @"path\tests1.js"
},
new PathInfo {
FullPath = @"path\tests2.js"
},
new PathInfo {
FullPath = @"path2\tests1.js"
},
new PathInfo {
FullPath = @"path2\tests2.js"
},
});
var settingsForPath = new ChutzpahTestSettingsFile {
SettingsFileDirectory = "path", EnableTestFileBatching = true
}.InheritFromDefault();
var settingsForPath2 = new ChutzpahTestSettingsFile {
SettingsFileDirectory = "path2", EnableTestFileBatching = true
}.InheritFromDefault();
runner.Mock <IChutzpahTestSettingsService>().Setup(x => x.FindSettingsFile(It.Is <string>(p => p.Contains(@"path\")), It.IsAny <ChutzpahSettingsFileEnvironments>())).Returns(settingsForPath);
runner.Mock <IChutzpahTestSettingsService>().Setup(x => x.FindSettingsFile(It.Is <string>(p => p.Contains(@"path2\")), It.IsAny <ChutzpahSettingsFileEnvironments>())).Returns(settingsForPath2);
TestCaseSummary res = runner.ClassUnderTest.RunTests(new List <string> {
@"path\tests1a.js", @"path\tests2a.js", @"path2\tests1a.js", @"path2\tests2a.js"
});
Assert.Equal(2, res.TotalCount);
}
public void Will_clean_up_test_context()
{
var runner = new TestableTestRunner();
var context = runner.SetupTestContext();
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html");
runner.Mock <ITestContextBuilder>().Verify(x => x.CleanupContext(context));
}
public void EmptyAssemblyNodeReturnsNoTests()
{
const string xml = @"<assembly name='C:\foo\bar.dll' run-date='2008-09-20' run-time='12:34:56' time='0.000' total='0' passed='0' failed='0' skipped='0'/>";
TestableTestRunner runner = TestableTestRunner.CreateForAssembly(xml);
TestRunnerResult result = runner.RunAssembly();
Assert.Equal(TestRunnerResult.NoTests, result);
}
public void ValuesArePassedToExecutorWrapper()
{
TestableTestRunner runner = TestableTestRunner.CreateForAssembly();
runner.RunTest("foo", "bar");
Assert.Equal("foo", runner.Executor.RunTest_Type);
Assert.Equal("bar", runner.Executor.RunTest_Method);
}
public void Will_call_test_suite_finished_with_final_result()
{
var runner = new TestableTestRunner();
var testCallback = new MockTestMethodRunnerCallback();
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html", testCallback.Object);
testCallback.Verify(x => x.TestSuiteFinished(null, It.IsAny <TestCaseSummary>()));
}
public void Will_clean_test_context()
{
var runner = new TestableTestRunner();
var context = new TestContext();
runner.ClassUnderTest.CleanTestContext(context);
runner.Mock <ITestContextBuilder>().Verify(x => x.CleanupContext(context));
}
public void CallsAssemblyStart()
{
var runner = TestableTestRunner.CreateForAssembly();
runner.RunAssembly();
runner.Logger.Verify(l => l.AssemblyStart(runner.Executor.AssemblyFilename,
runner.Executor.ConfigFilename,
runner.Executor.XunitVersion));
}
public void Will_true_if_test_file()
{
var runner = new TestableTestRunner();
runner.Mock <ITestContextBuilder>().Setup(x => x.IsTestFile("a.js")).Returns(true);
var result = runner.ClassUnderTest.IsTestFile("a.js");
Assert.True(result);
}
public void CallsAssemblyStartWithConfigFilename()
{
TestableTestRunner runner = TestableTestRunner.CreateForAssemblyWithConfigFile(@"C:\biff\baz.config");
runner.RunAssembly();
runner.Logger.Verify(l => l.AssemblyStart(@"C:\foo\bar.dll",
@"C:\biff\baz.config",
@"abcd"));
}
public void CallsTransformerWhenAssemblyIsFinished()
{
TestableTestRunner runner = TestableTestRunner.CreateForAssembly();
StubTransformer transformer = new StubTransformer();
runner.RunAssembly(new IResultXmlTransform[] { transformer });
Assert.True(transformer.Transform__Called);
Assert.Equal(runner.Executor.RunAssembly__Result.OuterXml, transformer.Transform_Xml);
}
public void Will_throw_if_test_runner_js_does_not_exist()
{
var runner = new TestableTestRunner();
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.TestRunnerJsName)).Returns((string)null);
var ex = Record.Exception(() => runner.ClassUnderTest.RunTests("someFile")) as FileNotFoundException;
Assert.NotNull(ex);
}
public void Will_throw_if_headless_browser_does_not_exist()
{
TestableTestRunner runner = new TestableTestRunner();
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns((string)null);
var ex = Record.Exception(() => runner.ClassUnderTest.DiscoverTests("someFile")) as FileNotFoundException;
Assert.NotNull(ex);
}
public void Will_false_if_not_test_file()
{
var runner = new TestableTestRunner();
runner.Mock <ITestContextBuilder>().Setup(x => x.IsTestFile("a.js", null)).Returns(false);
var result = runner.ClassUnderTest.IsTestFile("a.js", null);
Assert.False(result);
}
public void Will_get_test_context()
{
var runner = new TestableTestRunner();
var context = new TestContext();
runner.Mock <ITestContextBuilder>().Setup(x => x.BuildContext("a.js", It.IsAny <TestOptions>())).Returns(context);
var res = runner.ClassUnderTest.GetTestContext("a.js");
Assert.Equal(context, res);
}
public static async void Cancellation_AllOthers_CallsExtensibilityMethods(Type messageTypeToCancelOn, bool shouldTestPass, string skipReason = null)
{
var messageBus = new SpyMessageBus(msg => !(messageTypeToCancelOn.IsAssignableFrom(msg.GetType())));
var runner = TestableTestRunner.Create(messageBus, skipReason: skipReason, lambda: () => Assert.True(shouldTestPass));
await runner.RunAsync();
Assert.True(runner.TokenSource.IsCancellationRequested);
Assert.True(runner.AfterTestStarting_Called);
Assert.True(runner.BeforeTestFinished_Called);
}
public static async void Cancellation_TestCleanupFailure_SetsCancellationToken()
{
var messageBus = new SpyMessageBus(msg => !(msg is ITestCleanupFailure));
var runner = TestableTestRunner.Create(messageBus);
runner.BeforeTestFinished_Callback = aggregator => aggregator.Add(new Exception());
await runner.RunAsync();
Assert.True(runner.TokenSource.IsCancellationRequested);
}