public void Will_run_multiple_test_files_and_return_results()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context1 = new TestContext {
TestHarnessPath = @"D:\harnessPath1.html", TestRunner = "runner1"
};
var context2 = new TestContext {
TestHarnessPath = @"D:\harnessPath2.htm", TestRunner = "runner2"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.Is <PathInfo>(f => f.FullPath == @"path\tests1.html"), It.IsAny <TestOptions>(), out context1)).Returns(true);
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.Is <PathInfo>(f => f.FullPath == @"path\tests2.html"), It.IsAny <TestOptions>(), out context2)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests1.html")).Returns(@"D:\path\tests1.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests2.htm")).Returns(@"D:\path\tests2.htm");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner1")).Returns("jsPath1");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner2")).Returns("jsPath2");
var args1 = TestableTestRunner.BuildArgs("jsPath1", "file:///D:/harnessPath1.html", "execution", Constants.DefaultTestFileTimeout);
var args2 = TestableTestRunner.BuildArgs("jsPath2", "file:///D:/harnessPath2.htm", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args1, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>(0, summary));
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args2, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>(0, summary));
runner.Mock <IFileProbe>()
.Setup(x => x.FindScriptFiles(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
}, It.IsAny <TestingMode>()))
.Returns(new List <PathInfo> {
new PathInfo {
FullPath = @"path\tests1.html"
}, new PathInfo {
FullPath = @"path\tests2.html"
}
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
});
Assert.Equal(2, res.TotalCount);
}
public void Will_fire_file_finished_event()
{
var reader = new TestableTestCaseStreamReader();
var json = JsonStreamEvents.FileStartEventJson + JsonStreamEvents.TestStartEventJson + JsonStreamEvents.TestDoneEventJson + JsonStreamEvents.FileDoneEventJson;
var context = reader.BuildContext("file");
var stream = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(json)));
var processStream = new ProcessStreamStringSource(new Mock <IProcessWrapper>().Object, stream, 1000);
var callback = new Mock <ITestMethodRunnerCallback>();
TestFileSummary result = null;
callback.Setup(x => x.FileFinished("file", It.IsAny <TestFileSummary>())).Callback <string, TestFileSummary>((f, t) => result = t);;
reader.ClassUnderTest.Read(processStream, new TestOptions(), context, callback.Object);
Assert.NotNull(result);
Assert.Equal(10, result.TimeTaken);
}
public void Will_run_test_file_and_return_test_results_model()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = runner.SetupTestContext(testPaths: new[] { @"path\tests.html" }, harnessPath: @"harnessPath", testRunnerPath: "runner");
runner.Mock <ITestExecutionProvider>()
.Setup(x => x.Execute(It.IsAny <TestOptions>(), It.IsAny <TestContext>(), TestExecutionMode.Execution, It.IsAny <ITestMethodRunnerCallback>()))
.Returns(new List <TestFileSummary> {
summary
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html");
Assert.Equal(1, res.TotalCount);
}
public void Will_run_multiple_test_files_and_return_results()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context1 = runner.SetupTestContext(harnessPath: @"harnessPath1", testRunnerPath: "runner1", testPaths: new [] { @"path\tests1.html" });
var context2 = runner.SetupTestContext(harnessPath: @"harnessPath2", testRunnerPath: "runner2", testPaths: new [] { @"path\tests2.html" });
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests1.html")).Returns(@"D:\path\tests1.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests2.htm")).Returns(@"D:\path\tests2.htm");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner1")).Returns("jsPath1");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner2")).Returns("jsPath2");
var args1 = TestableTestRunner.BuildArgs("jsPath1", "harnessPath1", "execution", Constants.DefaultTestFileTimeout);
var args2 = TestableTestRunner.BuildArgs("jsPath2", "harnessPath2", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args1, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(It.IsAny <string>(), args2, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
runner.Mock <IFileProbe>()
.Setup(x => x.FindScriptFiles(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
}))
.Returns(new List <PathInfo> {
new PathInfo {
FullPath = @"path\tests1.html"
}, new PathInfo {
FullPath = @"path\tests2.html"
}
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(new List <string> {
@"path\tests1a.html", @"path\tests2a.htm"
});
Assert.Equal(2, res.TotalCount);
}
public void Will_fire_file_finished_event()
{
var reader = new TestableTestCaseStreamReader();
var json = @"#_#FileDone#_# {""type"":""FileDone"",""timetaken"":10,""failed"":1,""passed"":2}";
var context = new TestContext {
InputTestFile = "file"
};
var stream = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(json)));
var processStream = new ProcessStream(new Mock <IProcessWrapper>().Object, stream);
var callback = new Mock <ITestMethodRunnerCallback>();
TestFileSummary result = null;
callback.Setup(x => x.FileFinished("file", It.IsAny <TestFileSummary>())).Callback <string, TestFileSummary>((f, t) => result = t);;
reader.ClassUnderTest.Read(processStream, new TestOptions(), context, callback.Object, false);
Assert.NotNull(result);
Assert.Equal(10, result.TimeTaken);
}
public void Will_call_process_transforms()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var testCallback = new MockTestMethodRunnerCallback();
var context = runner.SetupTestContext(testRunnerPath: "testRunner.js");
runner.Mock <ITestExecutionProvider>()
.Setup(x => x.Execute(It.IsAny <TestOptions>(), It.IsAny <TestContext>(), TestExecutionMode.Execution, It.IsAny <ITestMethodRunnerCallback>()))
.Returns(new List <TestFileSummary> {
summary
});
TestCaseSummary res = runner.ClassUnderTest.RunTests(new[] { @"path\tests.html" }, testCallback.Object);
runner.Mock <ITransformProcessor>().Verify(x => x.ProcessTransforms(It.Is <IEnumerable <TestContext> >(c => c.Count() == 1 && c.Single() == context), res));
}
public void Will_call_process_transforms()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var testCallback = new MockTestMethodRunnerCallback();
var context = runner.SetupTestContext(testRunnerPath: "testRunner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("testRunner.js")).Returns("runner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns(TestRunner.HeadlessBrowserName);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(TestRunner.HeadlessBrowserName, TestableTestRunner.ExecutionPhantomArgs, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >((int)TestProcessExitCode.Unknown, new List <TestFileSummary> {
summary
}));
TestCaseSummary res = runner.ClassUnderTest.RunTests(new[] { @"path\tests.html" }, testCallback.Object);
runner.Mock <ITransformProcessor>().Verify(x => x.ProcessTransforms(It.Is <IEnumerable <TestContext> >(c => c.Count() == 1 && c.Single() == context), res));
}
public void Will_run_test_file_and_return_test_results_model()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = runner.SetupTestContext(testPaths: new [] { @"path\tests.html" }, harnessPath: @"D:\harnessPath.html", testRunnerPath: "runner");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner")).Returns("jsPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
var args = TestableTestRunner.BuildArgs("jsPath", "file:///D:/harnessPath.html", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput("browserPath", args, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >(0, new List <TestFileSummary> {
summary
}));
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html");
Assert.Equal(1, res.TotalCount);
}
public void Will_record_unknown_exception_from_test_runner()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var testCallback = new MockTestMethodRunnerCallback();
var context = runner.SetupTestContext(testRunnerPath: "testRunner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("testRunner.js")).Returns("runner.js");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns(TestRunner.HeadlessBrowserName);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput(TestRunner.HeadlessBrowserName, TestableTestRunner.ExecutionPhantomArgs, It.IsAny <Func <ProcessStream, IList <TestFileSummary> > >()))
.Returns(new ProcessResult <IList <TestFileSummary> >((int)TestProcessExitCode.Unknown, new List <TestFileSummary> {
summary
}));
TestCaseSummary res = runner.ClassUnderTest.RunTests(new[] { @"path\tests.html" }, testCallback.Object);
testCallback.Verify(x => x.FileError(It.IsAny <TestError>()));
Assert.Equal(1, res.TotalCount);
Assert.Equal(1, res.Errors.Count);
}
public void Will_run_test_file_and_return_test_results_model()
{
var runner = new TestableTestRunner();
var summary = new TestFileSummary("somePath");
summary.AddTestCase(new TestCase());
var context = new TestContext {
TestHarnessPath = @"D:\harnessPath.html", TestRunner = "runner"
};
runner.Mock <ITestContextBuilder>().Setup(x => x.TryBuildContext(It.Is <PathInfo>(f => f.FullPath == @"path\tests.html"), It.IsAny <TestOptions>(), out context)).Returns(true);
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath("runner")).Returns("jsPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(TestRunner.HeadlessBrowserName)).Returns("browserPath");
runner.Mock <IFileProbe>().Setup(x => x.FindFilePath(@"path\tests.html")).Returns(@"D:\path\tests.html");
var args = TestableTestRunner.BuildArgs("jsPath", "file:///D:/harnessPath.html", "execution", Constants.DefaultTestFileTimeout);
runner.Mock <IProcessHelper>()
.Setup(x => x.RunExecutableAndProcessOutput("browserPath", args, It.IsAny <Func <ProcessStream, TestFileSummary> >()))
.Returns(new ProcessResult <TestFileSummary>(0, summary));
TestCaseSummary res = runner.ClassUnderTest.RunTests(@"path\tests.html");
Assert.Equal(1, res.TotalCount);
}
private static TestCaseSummary BuildTestCaseSummary()
{
var summary = new TestCaseSummary();
var fileSummary = new TestFileSummary("path1")
{
TimeTaken = 1500
};
fileSummary.AddTestCase(new TestCase
{
ModuleName = "module1",
TestName = "test1",
TestResults = new List <TestResult> {
new TestResult {
Passed = false, Message = "some failure"
}
},
TimeTaken = 1000
});
fileSummary.AddTestCase(new TestCase
{
ModuleName = "module1",
TestName = "test2",
TestResults = new List <TestResult> {
new TestResult {
Passed = true
}
},
TimeTaken = 500
});
var fileSummary2 = new TestFileSummary("path>2")
{
TimeTaken = 2000
};
fileSummary2.AddTestCase(new TestCase
{
TestName = "test3",
TestResults = new List <TestResult> {
new TestResult {
Passed = true
}
},
TimeTaken = 1000
});
fileSummary2.AddTestCase(new TestCase
{
TestName = "test<4",
TestResults = new List <TestResult> {
new TestResult {
Passed = false, Message = "bad<failure"
}
},
TimeTaken = 1000
});
summary.Append(fileSummary);
summary.Append(fileSummary2);
return(summary);
}
public virtual void FileFinished(string fileName, TestFileSummary testResultsSummary)
{
}
private TestFileSummary ReadFromStream(StreamReader stream, TestContext testContext, TestOptions testOptions, ITestMethodRunnerCallback callback, bool debugEnabled)
{
var referencedFile = testContext.ReferencedFiles.SingleOrDefault(x => x.IsFileUnderTest);
var testIndex = 0;
var summary = new TestFileSummary(testContext.InputTestFile);
var codeCoverageEnabled = (!testContext.TestFileSettings.EnableCodeCoverage.HasValue && testOptions.CoverageOptions.Enabled) ||
(testContext.TestFileSettings.EnableCodeCoverage.HasValue && testContext.TestFileSettings.EnableCodeCoverage.Value);
if (codeCoverageEnabled)
{
summary.CoverageObject = new CoverageData();
}
string line;
while ((line = stream.ReadLine()) != null)
{
if (debugEnabled)
{
Console.WriteLine(line);
}
var match = prefixRegex.Match(line);
if (!match.Success)
{
continue;
}
var type = match.Groups["type"].Value;
var json = match.Groups["json"].Value;
// Only update last event timestamp if it is an important event.
// Log and error could happen even though no test progress is made
if (!type.Equals("Log") && !type.Equals("Error"))
{
lastTestEvent = DateTime.Now;
}
try
{
JsTestCase jsTestCase = null;
switch (type)
{
case "FileStart":
callback.FileStarted(testContext.InputTestFile);
break;
case "CoverageObject":
var jsCov = jsonSerializer.Deserialize <JsCoverage>(json);
summary.CoverageObject = coverageEngine.DeserializeCoverageObject(jsCov.Object, testContext);
break;
case "FileDone":
var jsFileDone = jsonSerializer.Deserialize <JsFileDone>(json);
summary.TimeTaken = jsFileDone.TimeTaken;
callback.FileFinished(testContext.InputTestFile, summary);
break;
case "TestStart":
jsTestCase = jsonSerializer.Deserialize <JsTestCase>(json);
jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
callback.TestStarted(jsTestCase.TestCase);
break;
case "TestDone":
jsTestCase = jsonSerializer.Deserialize <JsTestCase>(json);
jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
AddLineNumber(referencedFile, testIndex, jsTestCase);
testIndex++;
callback.TestFinished(jsTestCase.TestCase);
summary.AddTestCase(jsTestCase.TestCase);
break;
case "Log":
var log = jsonSerializer.Deserialize <JsLog>(json);
// This is an internal log message
if (log.Log.Message.StartsWith(internalLogPrefix))
{
ChutzpahTracer.TraceInformation("Phantom Log - {0}", log.Log.Message.Substring(internalLogPrefix.Length).Trim());
break;
}
log.Log.InputTestFile = testContext.InputTestFile;
callback.FileLog(log.Log);
summary.Logs.Add(log.Log);
break;
case "Error":
var error = jsonSerializer.Deserialize <JsError>(json);
error.Error.InputTestFile = testContext.InputTestFile;
callback.FileError(error.Error);
summary.Errors.Add(error.Error);
ChutzpahTracer.TraceError("Eror recieved from Phantom {0}", error.Error.Message);
break;
}
}
catch (SerializationException e)
{
// Ignore malformed json and move on
ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line);
}
}
return(summary);
}
public virtual void FileFinished(TestContext context, TestFileSummary testResultsSummary)
{
}
public override void FileFinished(string fileName, TestFileSummary testResultsSummary)
{
var text = string.Format("{0} passed, {1} failed, {2} total (chutzpah).\n\n", testResultsSummary.PassedCount, testResultsSummary.FailedCount, testResultsSummary.TotalCount);
testPane.OutputString(text);
}
