Exemplo n.º 1
0
        private void BuildTestContexts(
            TestOptions options,
            IEnumerable <PathInfo> scriptPaths,
            ParallelOptions parallelOptions,
            CancellationTokenSource cancellationSource,
            int resultCount,
            ConcurrentBag <TestContext> testContexts,
            ITestMethodRunnerCallback callback,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(scriptPaths, parallelOptions, testFile =>
            {
                ChutzpahTracer.TraceInformation("Building test context for {0}", testFile.FullPath);

                try
                {
                    if (cancellationSource.IsCancellationRequested)
                    {
                        return;
                    }
                    TestContext testContext;

                    resultCount++;
                    if (testContextBuilder.TryBuildContext(testFile, options, out testContext))
                    {
                        testContexts.Add(testContext);
                    }
                    else
                    {
                        ChutzpahTracer.TraceWarning("Unable to build test context for {0}", testFile.FullPath);
                    }

                    // Limit the number of files we can scan to attempt to build a context for
                    // This is important in the case of folder scanning where many JS files may not be
                    // test files.
                    if (resultCount >= options.FileSearchLimit)
                    {
                        ChutzpahTracer.TraceError("File search limit hit!!!");
                        cancellationSource.Cancel();
                    }
                }
                catch (Exception e)
                {
                    var error = new TestError
                    {
                        InputTestFile = testFile.FullPath,
                        Message       = e.ToString()
                    };

                    overallSummary.Errors.Add(error);
                    callback.FileError(error);

                    ChutzpahTracer.TraceError(e, "Error during building test context for {0}", testFile.FullPath);
                }
                finally
                {
                    ChutzpahTracer.TraceInformation("Finished building test context for {0}", testFile.FullPath);
                }
            });
        }
Exemplo n.º 2
0
        private static void HandleTestProcessExitCode(int exitCode, string inputTestFile, IList <TestError> errors, ITestMethodRunnerCallback callback)
        {
            string errorMessage = null;

            switch ((TestProcessExitCode)exitCode)
            {
            case TestProcessExitCode.AllPassed:
            case TestProcessExitCode.SomeFailed:
                return;

            case TestProcessExitCode.Timeout:
                errorMessage = "Timeout occurred when executing test file";
                break;

            default:
                errorMessage = "Unknown error occurred when executing test file. Received exit code of " + exitCode;
                break;
            }

            if (!string.IsNullOrEmpty(errorMessage))
            {
                var error = new TestError
                {
                    InputTestFile = inputTestFile,
                    Message       = errorMessage
                };

                errors.Add(error);

                callback.FileError(error);
                ChutzpahTracer.TraceError("Headless browser returned with an error: {0}", errorMessage);
            }
        }
        private void FireErrorOutput(TestContext testContext, ITestMethodRunnerCallback callback, StreamingTestFileContext testFileContext, JsRunnerOutput jsRunnerOutput)
        {
            var error = jsRunnerOutput as JsError;

            error.Error.InputTestFile = testFileContext.ReferencedFile.Path;
            error.Error.PathFromTestSettingsDirectory = testFileContext.ReferencedFile.PathFromTestSettingsDirectory;
            callback.FileError(testContext, error.Error);
            testFileContext.TestFileSummary.Errors.Add(error.Error);

            if (testFileContext.TestContext.TestFileSettings.CreateFailedTestForFileError.GetValueOrDefault())
            {
                var fileErrorTest = new TestCase();
                fileErrorTest.InputTestFile = testFileContext.ReferencedFile.Path;
                fileErrorTest.PathFromTestSettingsDirectory = testFileContext.ReferencedFile.PathFromTestSettingsDirectory;
                fileErrorTest.TestName = string.Format("!! File Error #{0} - Error encountered outside of test case execution !!", testFileContext.TestFileSummary.Errors.Count);
                fileErrorTest.TestResults.Add(new TestResult {
                    Passed = false, StackTrace = error.Error.StackAsString ?? error.Error.FormatStackObject(), Message = error.Error.Message
                });
                callback.TestStarted(testContext, fileErrorTest);
                callback.TestFinished(testContext, fileErrorTest);

                testFileContext.TestFileSummary.AddTestCase(fileErrorTest);
            }

            ChutzpahTracer.TraceError("Error received from Phantom {0}", error.Error.Message);
        }
 public void FileError(TestContext context, TestError error)
 {
     lock (sync)
     {
         nestedCallback.FileError(context, error);
     }
 }
Exemplo n.º 5
0
 public void FileError(TestError error)
 {
     lock (sync)
     {
         nestedCallback.FileError(error);
     }
 }
Exemplo n.º 6
0
        private void FireErrorOutput(ITestMethodRunnerCallback callback, TestFileContext testFileContext, JsRunnerOutput jsRunnerOutput)
        {
            var error = jsRunnerOutput as JsError;

            error.Error.InputTestFile = testFileContext.ReferencedFile.Path;
            callback.FileError(error.Error);
            testFileContext.TestFileSummary.Errors.Add(error.Error);

            ChutzpahTracer.TraceError("Eror recieved from Phantom {0}", error.Error.Message);
        }
Exemplo n.º 7
0
        private void ExecuteTestContexts(
            TestOptions options,
            TestExecutionMode testExecutionMode,
            ITestMethodRunnerCallback callback,
            ConcurrentBag <TestContext> testContexts,
            ParallelOptions parallelOptions,
            string headlessBrowserPath,
            ConcurrentQueue <TestFileSummary> testFileSummaries,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(
                testContexts,
                parallelOptions,
                testContext =>
            {
                ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);

                try
                {
                    testHarnessBuilder.CreateTestHarness(testContext, options);

                    if (options.OpenInBrowser)
                    {
                        ChutzpahTracer.TraceInformation(
                            "Launching test harness '{0}' for file '{1}' in a browser",
                            testContext.TestHarnessPath,
                            testContext.InputTestFile);
                        process.LaunchFileInBrowser(testContext.TestHarnessPath);
                    }
                    else
                    {
                        ChutzpahTracer.TraceInformation(
                            "Invoking headless browser on test harness '{0}' for file '{1}'",
                            testContext.TestHarnessPath,
                            testContext.InputTestFile);

                        var testSummary = InvokeTestRunner(
                            headlessBrowserPath,
                            options,
                            testContext,
                            testExecutionMode,
                            callback);

                        ChutzpahTracer.TraceInformation(
                            "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
                            testContext.TestHarnessPath,
                            testContext.InputTestFile,
                            testSummary.PassedCount,
                            testSummary.FailedCount,
                            testSummary.Errors.Count);

                        ChutzpahTracer.TraceInformation(
                            "Finished running headless browser on test harness '{0}' for file '{1}'",
                            testContext.TestHarnessPath,
                            testContext.InputTestFile);
                        testFileSummaries.Enqueue(testSummary);
                    }
                }
                catch (Exception e)
                {
                    var error = new TestError
                    {
                        InputTestFile = testContext.InputTestFile,
                        Message       = e.ToString()
                    };

                    overallSummary.Errors.Add(error);
                    callback.FileError(error);

                    ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.InputTestFile);
                }
                finally
                {
                    ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.InputTestFile, testExecutionMode);
                }
            });


            // Clean up test context
            foreach (var testContext in testContexts)
            {
                // Don't clean up context if in debug mode
                if (!m_debugEnabled && !options.OpenInBrowser)
                {
                    try
                    {
                        ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.InputTestFile);
                        testContextBuilder.CleanupContext(testContext);
                    }
                    catch (Exception e)
                    {
                        ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.InputTestFile);
                    }
                }
            }
        }
Exemplo n.º 8
0
        private void ExecuteTestContexts(
            TestOptions options,
            TestExecutionMode testExecutionMode,
            ITestMethodRunnerCallback callback,
            ConcurrentBag <TestContext> testContexts,
            ParallelOptions parallelOptions,
            string headlessBrowserPath,
            ConcurrentQueue <TestFileSummary> testFileSummaries,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(
                testContexts,
                parallelOptions,
                testContext =>
            {
                ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);

                try
                {
                    try
                    {
                        testHarnessBuilder.CreateTestHarness(testContext, options);
                    }
                    catch (IOException)
                    {
                        // Mark this creation failed so we do not try to clean it up later
                        // This is to work around a bug in TestExplorer that runs chutzpah in parallel on
                        // the same files
                        // TODO(mmanela): Re-evalute if this is needed once they fix that bug
                        testContext.TestHarnessCreationFailed = true;
                        ChutzpahTracer.TraceWarning("Marking test harness creation failed for harness {0} and test file {1}", testContext.TestHarnessPath, testContext.FirstInputTestFile);
                        throw;
                    }

                    if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
                    {
                        ChutzpahTracer.TraceInformation(
                            "Launching test harness '{0}' for file '{1}' in a browser",
                            testContext.TestHarnessPath,
                            testContext.FirstInputTestFile);

                        // Allow override from command line.
                        var browserArgs = testContext.TestFileSettings.BrowserArguments;
                        if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
                        {
                            var path    = BrowserPathHelper.GetBrowserPath(options.BrowserName);
                            browserArgs = new Dictionary <string, string>
                            {
                                { Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
                            };
                        }

                        process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
                    }
                    else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
                    {
                        ChutzpahTracer.TraceInformation(
                            "Invoking headless browser on test harness '{0}' for file '{1}'",
                            testContext.TestHarnessPath,
                            testContext.FirstInputTestFile);

                        var testSummaries = InvokeTestRunner(
                            headlessBrowserPath,
                            options,
                            testContext,
                            testExecutionMode,
                            callback);

                        foreach (var testSummary in testSummaries)
                        {
                            ChutzpahTracer.TraceInformation(
                                "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
                                testContext.TestHarnessPath,
                                testSummary.Path,
                                testSummary.PassedCount,
                                testSummary.FailedCount,
                                testSummary.Errors.Count);

                            ChutzpahTracer.TraceInformation(
                                "Finished running headless browser on test harness '{0}' for file '{1}'",
                                testContext.TestHarnessPath,
                                testSummary.Path);

                            testFileSummaries.Enqueue(testSummary);
                        }
                    }
                    else if (options.TestLaunchMode == TestLaunchMode.Custom)
                    {
                        if (options.CustomTestLauncher == null)
                        {
                            throw new ArgumentNullException("TestOptions.CustomTestLauncher");
                        }
                        ChutzpahTracer.TraceInformation(
                            "Launching custom test on test harness '{0}' for file '{1}'",
                            testContext.TestHarnessPath,
                            testContext.FirstInputTestFile);
                        options.CustomTestLauncher.LaunchTest(testContext);
                    }
                    else
                    {
                        Debug.Assert(false);
                    }
                }
                catch (Exception e)
                {
                    var error = new TestError
                    {
                        InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
                        Message       = e.ToString()
                    };

                    overallSummary.Errors.Add(error);
                    callback.FileError(error);

                    ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
                }
                finally
                {
                    ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
                }
            });


            // Clean up test context
            foreach (var testContext in testContexts)
            {
                // Don't clean up context if in debug mode
                if (!m_debugEnabled &&
                    !testContext.TestHarnessCreationFailed &&
                    options.TestLaunchMode != TestLaunchMode.FullBrowser &&
                    options.TestLaunchMode != TestLaunchMode.Custom)
                {
                    try
                    {
                        ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
                        testContextBuilder.CleanupContext(testContext);
                    }
                    catch (Exception e)
                    {
                        ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
                    }
                }
            }
        }
Exemplo n.º 9
0
        private static void HandleTestProcessExitCode(int exitCode, string inputTestFile, IList<TestError> errors, ITestMethodRunnerCallback callback)
        {
            string errorMessage = null;

            switch ((TestProcessExitCode)exitCode)
            {
                case TestProcessExitCode.AllPassed:
                case TestProcessExitCode.SomeFailed:
                    return;
                case TestProcessExitCode.Timeout:
                    errorMessage = "Timeout occurred when executing test file";
                    break;
                default:
                    errorMessage = "Unknown error occurred when executing test file. Received exit code of " + exitCode;
                    break;
            }

            if (!string.IsNullOrEmpty(errorMessage))
            {
                var error = new TestError
                {
                    InputTestFile = inputTestFile,
                    Message = errorMessage
                };

                errors.Add(error);

                callback.FileError(error);
                ChutzpahTracer.TraceError("Headless browser returned with an error: {0}", errorMessage);
            }
        }
Exemplo n.º 10
0
        private void BuildTestContexts(
            TestOptions options,
            List<List<PathInfo>> scriptPathGroups,
            ParallelOptions parallelOptions,
            CancellationTokenSource cancellationSource,
            int resultCount,
            ConcurrentBag<TestContext> testContexts,
            ITestMethodRunnerCallback callback,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(scriptPathGroups, parallelOptions, testFiles =>
            {
                var pathString = string.Join(",", testFiles.Select(x => x.FullPath));
                ChutzpahTracer.TraceInformation("Building test context for {0}", pathString);

                try
                {
                    if (cancellationSource.IsCancellationRequested) return;
                    TestContext testContext;

                    resultCount++;
                    if (testContextBuilder.TryBuildContext(testFiles, options, out testContext))
                    {
                        testContexts.Add(testContext);
                    }
                    else
                    {
                        ChutzpahTracer.TraceWarning("Unable to build test context for {0}", pathString);
                    }

                    // Limit the number of files we can scan to attempt to build a context for
                    // This is important in the case of folder scanning where many JS files may not be
                    // test files.
                    if (resultCount >= options.FileSearchLimit)
                    {
                        ChutzpahTracer.TraceError("File search limit hit!!!");
                        cancellationSource.Cancel();
                    }
                }
                catch (Exception e)
                {
                    var error = new TestError
                    {
                        InputTestFile = testFiles.Select(x => x.FullPath).FirstOrDefault(),
                        Message = e.ToString()
                    };

                    overallSummary.Errors.Add(error);
                    callback.FileError(error);

                    ChutzpahTracer.TraceError(e, "Error during building test context for {0}", pathString);
                }
                finally
                {
                    ChutzpahTracer.TraceInformation("Finished building test context for {0}", pathString);
                }
            });
        }
Exemplo n.º 11
0
        private void ExecuteTestContexts(
            TestOptions options,
            TestExecutionMode testExecutionMode,
            ITestMethodRunnerCallback callback,
            ConcurrentBag<TestContext> testContexts,
            ParallelOptions parallelOptions,
            string headlessBrowserPath,
            ConcurrentQueue<TestFileSummary> testFileSummaries,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(
                testContexts,
                parallelOptions,
                testContext =>
                {
                    ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);

                    try
                    {
                        testHarnessBuilder.CreateTestHarness(testContext, options);

                        if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
                        {
                            ChutzpahTracer.TraceInformation(
                                "Launching test harness '{0}' for file '{1}' in a browser",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);

                            // Allow override from command line.
                            var browserArgs = testContext.TestFileSettings.BrowserArguments;
                            if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
                            {
                                var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
                                browserArgs = new Dictionary<string, string>
                                {
                                    { Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
                                };
                            }
                                                   
                            process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
                        }
                        else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
                        {
                            ChutzpahTracer.TraceInformation(
                                "Invoking headless browser on test harness '{0}' for file '{1}'",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);

                            var testSummaries = InvokeTestRunner(
                                headlessBrowserPath,
                                options,
                                testContext,
                                testExecutionMode,
                                callback);

                            foreach (var testSummary in testSummaries)
                            {

                                ChutzpahTracer.TraceInformation(
                                    "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
                                    testContext.TestHarnessPath,
                                    testSummary.Path,
                                    testSummary.PassedCount,
                                    testSummary.FailedCount,
                                    testSummary.Errors.Count);

                                ChutzpahTracer.TraceInformation(
                                    "Finished running headless browser on test harness '{0}' for file '{1}'",
                                    testContext.TestHarnessPath,
                                    testSummary.Path);

                                testFileSummaries.Enqueue(testSummary);
                            }
                        }
                        else if (options.TestLaunchMode == TestLaunchMode.Custom)
                        {
                            if (options.CustomTestLauncher == null)
                            {
                                throw new ArgumentNullException("TestOptions.CustomTestLauncher");
                            }
                            ChutzpahTracer.TraceInformation(
                                "Launching custom test on test harness '{0}' for file '{1}'",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);
                            options.CustomTestLauncher.LaunchTest(testContext);
                        }
                        else
                        {
                            Debug.Assert(false);
                        }
                    }
                    catch (Exception e)
                    {
                        var error = new TestError
                        {
                            InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
                            Message = e.ToString()
                        };

                        overallSummary.Errors.Add(error);
                        callback.FileError(error);

                        ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
                    }
                    finally
                    {
                        ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
                    }
                });


            // Clean up test context
            foreach (var testContext in testContexts)
            {
                // Don't clean up context if in debug mode
                if (!m_debugEnabled
                    && options.TestLaunchMode != TestLaunchMode.FullBrowser
                    && options.TestLaunchMode != TestLaunchMode.Custom)
                {
                    try
                    {
                        ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
                        testContextBuilder.CleanupContext(testContext);

                    }
                    catch (Exception e)
                    {
                        ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
                    }
                }
            }
        }
Exemplo n.º 12
0
        private TestFileSummary ReadFromStream(StreamReader stream, TestContext testContext, TestOptions testOptions, ITestMethodRunnerCallback callback, bool debugEnabled)
        {
            var referencedFile = testContext.ReferencedFiles.SingleOrDefault(x => x.IsFileUnderTest);
            var testIndex      = 0;
            var summary        = new TestFileSummary(testContext.InputTestFile);


            var codeCoverageEnabled = (!testContext.TestFileSettings.EnableCodeCoverage.HasValue && testOptions.CoverageOptions.Enabled) ||
                                      (testContext.TestFileSettings.EnableCodeCoverage.HasValue && testContext.TestFileSettings.EnableCodeCoverage.Value);

            if (codeCoverageEnabled)
            {
                summary.CoverageObject = new CoverageData();
            }

            string line;

            while ((line = stream.ReadLine()) != null)
            {
                if (debugEnabled)
                {
                    Console.WriteLine(line);
                }

                var match = prefixRegex.Match(line);
                if (!match.Success)
                {
                    continue;
                }
                var type = match.Groups["type"].Value;
                var json = match.Groups["json"].Value;

                // Only update last event timestamp if it is an important event.
                // Log and error could happen even though no test progress is made
                if (!type.Equals("Log") && !type.Equals("Error"))
                {
                    lastTestEvent = DateTime.Now;
                }

                try
                {
                    JsTestCase jsTestCase = null;
                    switch (type)
                    {
                    case "FileStart":
                        callback.FileStarted(testContext.InputTestFile);
                        break;

                    case "CoverageObject":
                        var jsCov = jsonSerializer.Deserialize <JsCoverage>(json);
                        summary.CoverageObject = coverageEngine.DeserializeCoverageObject(jsCov.Object, testContext);
                        break;

                    case "FileDone":
                        var jsFileDone = jsonSerializer.Deserialize <JsFileDone>(json);
                        summary.TimeTaken = jsFileDone.TimeTaken;
                        callback.FileFinished(testContext.InputTestFile, summary);
                        break;

                    case "TestStart":
                        jsTestCase = jsonSerializer.Deserialize <JsTestCase>(json);
                        jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
                        callback.TestStarted(jsTestCase.TestCase);
                        break;

                    case "TestDone":
                        jsTestCase = jsonSerializer.Deserialize <JsTestCase>(json);
                        jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
                        AddLineNumber(referencedFile, testIndex, jsTestCase);
                        testIndex++;
                        callback.TestFinished(jsTestCase.TestCase);
                        summary.AddTestCase(jsTestCase.TestCase);
                        break;

                    case "Log":
                        var log = jsonSerializer.Deserialize <JsLog>(json);

                        // This is an internal log message
                        if (log.Log.Message.StartsWith(internalLogPrefix))
                        {
                            ChutzpahTracer.TraceInformation("Phantom Log - {0}", log.Log.Message.Substring(internalLogPrefix.Length).Trim());
                            break;
                        }

                        log.Log.InputTestFile = testContext.InputTestFile;
                        callback.FileLog(log.Log);
                        summary.Logs.Add(log.Log);
                        break;

                    case "Error":

                        var error = jsonSerializer.Deserialize <JsError>(json);
                        error.Error.InputTestFile = testContext.InputTestFile;
                        callback.FileError(error.Error);
                        summary.Errors.Add(error.Error);

                        ChutzpahTracer.TraceError("Eror recieved from Phantom {0}", error.Error.Message);

                        break;
                    }
                }
                catch (SerializationException e)
                {
                    // Ignore malformed json and move on
                    ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line);
                }
            }

            return(summary);
        }
Exemplo n.º 13
0
        private void FireErrorOutput(ITestMethodRunnerCallback callback, StreamingTestFileContext testFileContext, JsRunnerOutput jsRunnerOutput)
        {
            var error = jsRunnerOutput as JsError;

            error.Error.InputTestFile = testFileContext.ReferencedFile.Path;
            callback.FileError(error.Error);
            testFileContext.TestFileSummary.Errors.Add(error.Error);

            ChutzpahTracer.TraceError("Eror recieved from Phantom {0}", error.Error.Message);
        }
Exemplo n.º 14
0
        private void ExecuteTestContexts(
            TestOptions options,
            TestExecutionMode testExecutionMode,
            ITestMethodRunnerCallback callback,
            ConcurrentBag<TestContext> testContexts,
            ParallelOptions parallelOptions,
            string headlessBrowserPath,
            ConcurrentQueue<TestFileSummary> testFileSummaries,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(
                testContexts,
                parallelOptions,
                testContext =>
                {
                    ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);

                    try
                    {
                        testHarnessBuilder.CreateTestHarness(testContext, options);

                        if (options.OpenInBrowser)
                        {
                            ChutzpahTracer.TraceInformation(
                                "Launching test harness '{0}' for file '{1}' in a browser",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);
                            process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName);
                        }
                        else
                        {
                            ChutzpahTracer.TraceInformation(
                                "Invoking headless browser on test harness '{0}' for file '{1}'",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);

                            var testSummaries = InvokeTestRunner(
                                headlessBrowserPath,
                                options,
                                testContext,
                                testExecutionMode,
                                callback);

                            foreach (var testSummary in testSummaries)
                            {

                                ChutzpahTracer.TraceInformation(
                                    "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
                                    testContext.TestHarnessPath,
                                    testSummary.Path,
                                    testSummary.PassedCount,
                                    testSummary.FailedCount,
                                    testSummary.Errors.Count);

                                ChutzpahTracer.TraceInformation(
                                    "Finished running headless browser on test harness '{0}' for file '{1}'",
                                    testContext.TestHarnessPath,
                                    testSummary.Path);

                                testFileSummaries.Enqueue(testSummary);
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        var error = new TestError
                        {
                            InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
                            Message = e.ToString()
                        };

                        overallSummary.Errors.Add(error);
                        callback.FileError(error);

                        ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
                    }
                    finally
                    {
                        ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
                    }
                });

            // Clean up test context
            foreach (var testContext in testContexts)
            {
                // Don't clean up context if in debug mode
                if (!m_debugEnabled && !options.OpenInBrowser)
                {
                    try
                    {
                        ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
                        testContextBuilder.CleanupContext(testContext);
                    }
                    catch (Exception e)
                    {
                        ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
                    }
                }
            }
        }
Exemplo n.º 15
0
        private TestFileSummary ReadFromStream(StreamReader stream, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled)
        {
            var referencedFile = testContext.ReferencedJavaScriptFiles.SingleOrDefault(x => x.IsFileUnderTest);
            var testIndex = 0;
            var summary = new TestFileSummary(testContext.InputTestFile);
            string line;
            while ((line = stream.ReadLine()) != null)
            {
                if (debugEnabled) Console.WriteLine(line);

                var match = prefixRegex.Match(line);
                if (!match.Success) continue;
                var type = match.Groups["type"].Value;
                var json = match.Groups["json"].Value;

                // Only update last event timestamp if it is an important event.
                // Log and error could happen even though no test progress is made
                if (!type.Equals("Log") && !type.Equals("Error"))
                {
                    lastTestEvent = DateTime.Now;
                }

                try
                {
                    JsTestCase jsTestCase = null;
                    switch (type)
                    {
                        case "FileStart":
                            callback.FileStarted(testContext.InputTestFile);
                            break;

                        case "CoverageObject":
                            var jsCov = jsonSerializer.Deserialize<JsCoverage>(json);
                            summary.CoverageObject = coverageEngine.DeserializeCoverageObject(jsCov.Object, testContext);
                            break;

                        case "FileDone":
                            var jsFileDone = jsonSerializer.Deserialize<JsFileDone>(json);
                            summary.TimeTaken = jsFileDone.TimeTaken;
                            callback.FileFinished(testContext.InputTestFile, summary);
                            break;

                        case "TestStart":
                            jsTestCase = jsonSerializer.Deserialize<JsTestCase>(json);
                            jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
                            callback.TestStarted(jsTestCase.TestCase);
                            break;

                        case "TestDone":
                            jsTestCase = jsonSerializer.Deserialize<JsTestCase>(json);
                            jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
                            AddLineNumber(referencedFile, testIndex, jsTestCase);
                            testIndex++;
                            callback.TestFinished(jsTestCase.TestCase);
                            summary.AddTestCase(jsTestCase.TestCase);
                            break;

                        case "Log":
                            var log = jsonSerializer.Deserialize<JsLog>(json);

                            // This is an internal log message
                            if (log.Log.Message.StartsWith(internalLogPrefix))
                            {
                                ChutzpahTracer.TraceInformation("Phantom Log - {0}",log.Log.Message.Substring(internalLogPrefix.Length).Trim());
                                break;
                            }

                            log.Log.InputTestFile = testContext.InputTestFile;
                            callback.FileLog(log.Log);
                            summary.Logs.Add(log.Log);
                            break;

                        case "Error":
                            var error = jsonSerializer.Deserialize<JsError>(json);
                            error.Error.InputTestFile = testContext.InputTestFile;
                            callback.FileError(error.Error);
                            summary.Errors.Add(error.Error);
                            break;
                    }
                }
                catch (SerializationException e)
                {
                    // Ignore malformed json and move on
                    ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line);
                }
            }

            return summary;
        }