public IList<TestFileSummary> Read(ProcessStream processStream, TestOptions testOptions, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled) { if (processStream == null) throw new ArgumentNullException("processStream"); if (testOptions == null) throw new ArgumentNullException("testOptions"); if (testContext == null) throw new ArgumentNullException("testContext"); lastTestEvent = DateTime.Now; var timeout = (testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500; // Add buffer to timeout to account for serialization var readerTask = Task<IList<TestFileSummary>>.Factory.StartNew(() => ReadFromStream(processStream.StreamReader, testContext, testOptions, callback, debugEnabled)); while (readerTask.Status == TaskStatus.WaitingToRun || (readerTask.Status == TaskStatus.Running && (DateTime.Now - lastTestEvent).TotalMilliseconds < timeout)) { Thread.Sleep(100); } if (readerTask.IsCompleted) { ChutzpahTracer.TraceInformation("Finished reading stream from test file '{0}'", testContext.FirstInputTestFile); return readerTask.Result; } else { // We timed out so kill the process and return an empty test file summary ChutzpahTracer.TraceError("Test file '{0}' timed out after running for {1} milliseconds", testContext.FirstInputTestFile, (DateTime.Now - lastTestEvent).TotalMilliseconds); processStream.TimedOut = true; processStream.KillProcess(); return testContext.ReferencedFiles.Where(x => x.IsFileUnderTest).Select(file => new TestFileSummary(file.Path)).ToList(); } }
public IList<TestFileSummary> Read(ProcessStream processStream, TestOptions testOptions, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled) { if (processStream == null) throw new ArgumentNullException("processStream"); if (testOptions == null) throw new ArgumentNullException("testOptions"); if (testContext == null) throw new ArgumentNullException("testContext"); lastTestEvent = DateTime.Now; var timeout = (testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500; // Add buffer to timeout to account for serialization var codeCoverageEnabled = (!testContext.TestFileSettings.EnableCodeCoverage.HasValue && testOptions.CoverageOptions.Enabled) || (testContext.TestFileSettings.EnableCodeCoverage.HasValue && testContext.TestFileSettings.EnableCodeCoverage.Value); var streamingTestFileContexts = testContext.ReferencedFiles .Where(x => x.IsFileUnderTest) .Select(x => new StreamingTestFileContext(x, testContext, codeCoverageEnabled)) .ToList(); var deferredEvents = new List<Action<StreamingTestFileContext>>(); var readerTask = Task<IList<TestFileSummary>>.Factory.StartNew(() => ReadFromStream(processStream.StreamReader, testContext, testOptions, streamingTestFileContexts, deferredEvents, callback, debugEnabled)); while (readerTask.Status == TaskStatus.WaitingToRun || (readerTask.Status == TaskStatus.Running && (DateTime.Now - lastTestEvent).TotalMilliseconds < timeout)) { Thread.Sleep(100); } if (readerTask.IsCompleted) { ChutzpahTracer.TraceInformation("Finished reading stream from test file '{0}'", testContext.FirstInputTestFile); return readerTask.Result; } else { // Since we times out make sure we play the deferred events so we do not lose errors // We will just attach these events to the first test context at this point since we do // not know where they belong PlayDeferredEvents(streamingTestFileContexts.FirstOrDefault(), deferredEvents); // We timed out so kill the process and return an empty test file summary ChutzpahTracer.TraceError("Test file '{0}' timed out after running for {1} milliseconds", testContext.FirstInputTestFile, (DateTime.Now - lastTestEvent).TotalMilliseconds); processStream.TimedOut = true; processStream.KillProcess(); return testContext.ReferencedFiles.Where(x => x.IsFileUnderTest).Select(file => new TestFileSummary(file.Path)).ToList(); } }
public void CreateTestHarness(TestContext testContext, TestOptions options) { if (!string.IsNullOrEmpty(testContext.TestHarnessPath)) { // If we already have a test harness path then this means we executed on an existing html file or url // So we dont need to generate the harness return; } SetupAmdPathsIfNeeded(testContext.TestFileSettings, testContext.ReferencedFiles.ToList(), testContext.TestHarnessDirectory); string testFilePathHash = hasher.Hash(string.Join(",",testContext.InputTestFiles)); string testHtmlFilePath = Path.Combine(testContext.TestHarnessDirectory, string.Format(Constants.ChutzpahTemporaryFileFormat, testFilePathHash, "test.html")); testContext.TemporaryFiles.Add(testHtmlFilePath); var templatePath = GetTestHarnessTemplatePath(testContext.FrameworkDefinition, testContext.TestFileSettings); string testHtmlTemplate = fileSystem.GetText(templatePath); var harness = new TestHarness(testContext.TestFileSettings, options, testContext.ReferencedFiles, fileSystem); if (testContext.CoverageEngine != null) { testContext.CoverageEngine.PrepareTestHarnessForCoverage(harness, testContext.FrameworkDefinition, testContext.TestFileSettings); } var kvps = testContext.ReferencedFiles .Where(x => x.IsFileUnderTest) .SelectMany(x => x.FrameworkReplacements); var frameworkReplacements = new Dictionary<string, string>(); foreach (var pair in kvps) { frameworkReplacements[pair.Key] = pair.Value; } string testHtmlText = harness.CreateHtmlText(testHtmlTemplate, frameworkReplacements); fileSystem.Save(testHtmlFilePath, testHtmlText); testContext.TestHarnessPath = testHtmlFilePath; }
public void LaunchTest(TestContext testContext) { // Start IE. ProcessStartInfo startInfo = new ProcessStartInfo() { UseShellExecute = true, FileName = BrowserPathHelper.GetBrowserPath("ie"), Arguments = string.Format("-noframemerging -suspended -debug {0}", FileProbe.GenerateFileUrl(testContext.TestHarnessPath)) // -noframemerging // This is what VS does when launching the script debugger. // Unsure whether strictly necessary. // -suspended // This is what VS does when launching the script debugger. // Seems to cause IE to suspend all threads which is what we want. // -debug // This is what VS does when launching the script debugger. // Not sure exactly what it does. }; Process ieMainProcess = Process.Start(startInfo); // Get child 'tab' process spawned by IE. // We need to wait a few ms for IE to open the process. Process ieTabProcess = null; for (int i = 0;; ++i) { System.Threading.Thread.Sleep(10); ieTabProcess = ProcessExtensions.FindFirstChildProcessOf(ieMainProcess.Id); if (ieTabProcess != null) { break; } if (i > 400) { // 400 * 10 = 4s timeout throw new InvalidOperationException("Timeout waiting for Internet Explorer child process to start."); } } // Debug the script in that tab process. DteHelpers.DebugAttachToProcess(ieTabProcess.Id, "script"); // Resume the threads in the IE process which where started off suspended. ieTabProcess.Resume(); }
public void CleanTestContext(TestContext context) { testContextBuilder.CleanupContext(context); }
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode) { string runnerArgs; var testModeStr = testExecutionMode.ToString().ToLowerInvariant(); var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout; runnerArgs = string.Format("--ignore-ssl-errors=true --proxy-type=none \"{0}\" {1} {2} {3} {4} {5}", runnerPath, fileUrl, testModeStr, timeout, context.TestFileSettings.IgnoreResourceLoadingErrors.Value, context.TestFileSettings.UserAgent); return runnerArgs; }
private IList<TestFileSummary> InvokeTestRunner(string headlessBrowserPath, TestOptions options, TestContext testContext, TestExecutionMode testExecutionMode, ITestMethodRunnerCallback callback) { string runnerPath = fileProbe.FindFilePath(testContext.TestRunner); string fileUrl = BuildHarnessUrl(testContext.TestHarnessPath, testContext.IsRemoteHarness); string runnerArgs = BuildRunnerArgs(options, testContext, fileUrl, runnerPath, testExecutionMode); Func<ProcessStream, IList<TestFileSummary>> streamProcessor = processStream => testCaseStreamReaderFactory.Create().Read(processStream, options, testContext, callback, m_debugEnabled); var processResult = process.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor); HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.Select(x => x.Errors).FirstOrDefault(), callback); return processResult.Model; }
public bool TryBuildContext(PathInfo file, TestOptions options, out TestContext context) { context = BuildContext(file, options); return context != null; }
public void CleanupContext(TestContext context) { if (context == null) throw new ArgumentNullException("context"); foreach (var file in context.TemporaryFiles) { try { fileSystem.DeleteFile(file); } catch (IOException) { // Supress exception } } }
public bool TryBuildContext(IEnumerable<string> files, TestOptions options, out TestContext context) { context = BuildContext(files, options); return context != null; }
public StreamingTestFileContext(ReferencedFile referencedFile, TestContext testContext, bool coverageEnabled) { SeenTests = new HashSet<Tuple<string, string>>(); ReferencedFile = referencedFile; TestContext = testContext; TestFileSummary = new TestFileSummary(referencedFile.Path); if (coverageEnabled) { TestFileSummary.CoverageObject = new CoverageData(); } }
private IList<TestFileSummary> ReadFromStream(StreamReader stream, TestContext testContext, TestOptions testOptions, IList<StreamingTestFileContext> streamingTestFileContexts, IList<Action<StreamingTestFileContext>> deferredEvents, ITestMethodRunnerCallback callback, bool debugEnabled) { var testIndex = 0; string line; StreamingTestFileContext currentTestFileContext = null; if (streamingTestFileContexts.Count == 1) { currentTestFileContext = streamingTestFileContexts.First(); } while ((line = stream.ReadLine()) != null) { if (debugEnabled) Console.WriteLine(line); var match = prefixRegex.Match(line); if (!match.Success) continue; var type = match.Groups["type"].Value; var json = match.Groups["json"].Value; // Only update last event timestamp if it is an important event. // Log and error could happen even though no test progress is made if (!type.Equals("Log") && !type.Equals("Error")) { lastTestEvent = DateTime.Now; } try { switch (type) { case "FileStart": FireFileStarted(callback, testContext); break; case "CoverageObject": var jsCov = jsonSerializer.Deserialize<JsCoverage>(json); if (currentTestFileContext == null) { AddDeferredEvent((fileContext) => FireCoverageObject(callback, fileContext, jsCov), deferredEvents); } else { FireCoverageObject(callback, currentTestFileContext, jsCov); } break; case "FileDone": var jsFileDone = jsonSerializer.Deserialize<JsFileDone>(json); FireFileFinished(callback, testContext.InputTestFilesString, streamingTestFileContexts, jsFileDone); break; case "TestStart": var jsTestCaseStart = jsonSerializer.Deserialize<JsTestCase>(json); StreamingTestFileContext newContext = null; var testName = jsTestCaseStart.TestCase.TestName.Trim(); var moduleName = (jsTestCaseStart.TestCase.ModuleName ?? "").Trim(); var fileContexts = GetFileMatches(testName, streamingTestFileContexts); if (fileContexts.Count == 0 && currentTestFileContext == null) { // If there are no matches and not file context has been used yet // then just choose the first context newContext = streamingTestFileContexts[0]; } else if (fileContexts.Count == 0) { // If there is already a current context and no matches we just keep using that context // unless this test name has been used already in the current context. In that case // move to the next one that hasn't seen this file yet var testAlreadySeenInCurrentContext = currentTestFileContext.HasTestBeenSeen(moduleName, testName); if (testAlreadySeenInCurrentContext) { newContext = streamingTestFileContexts.FirstOrDefault(x => !x.HasTestBeenSeen(moduleName, testName)) ?? currentTestFileContext; } } else if (fileContexts.Count > 1) { // If we found the test has more than one file match // try to choose the best match, otherwise just choose the first one // If we have no file context yet take the first one if (currentTestFileContext == null) { newContext = fileContexts.First(); } else { // In this case we have an existing file context so we need to // 1. Check to see if this test has been seen already on that context // if so we need to try the next file context that matches it // 2. If it is not seen yet in the current context and the current context // is one of the matches then keep using it var testAlreadySeenInCurrentContext = currentTestFileContext.HasTestBeenSeen(moduleName, testName); var currentContextInFileMatches = fileContexts.Any(x => x == currentTestFileContext); if (!testAlreadySeenInCurrentContext && currentContextInFileMatches) { // Keep the current context newContext = currentTestFileContext; } else { // Either take first not used context OR the first one newContext = fileContexts.Where(x => !x.IsUsed).FirstOrDefault() ?? fileContexts.First(); } } } else if (fileContexts.Count == 1) { // We found a unique match newContext = fileContexts[0]; } if (newContext != null && newContext != currentTestFileContext) { currentTestFileContext = newContext; testIndex = 0; } currentTestFileContext.IsUsed = true; currentTestFileContext.MarkTestSeen(moduleName, testName); PlayDeferredEvents(currentTestFileContext, deferredEvents); jsTestCaseStart.TestCase.InputTestFile = currentTestFileContext.ReferencedFile.Path; callback.TestStarted(jsTestCaseStart.TestCase); ChutzpahTracer.TraceInformation("Test Case Started:'{0}'", jsTestCaseStart.TestCase.GetDisplayName()); break; case "TestDone": var jsTestCaseDone = jsonSerializer.Deserialize<JsTestCase>(json); var currentTestIndex = testIndex; FireTestFinished(callback, currentTestFileContext, jsTestCaseDone, currentTestIndex); testIndex++; break; case "Log": var log = jsonSerializer.Deserialize<JsLog>(json); if (currentTestFileContext != null) { FireLogOutput(callback, currentTestFileContext, log); } else { AddDeferredEvent((fileContext) => FireLogOutput(callback, fileContext, log), deferredEvents); } break; case "Error": var error = jsonSerializer.Deserialize<JsError>(json); if (currentTestFileContext != null) { FireErrorOutput(callback, currentTestFileContext, error); } else { AddDeferredEvent((fileContext) => FireErrorOutput(callback, fileContext, error), deferredEvents); } break; } } catch (SerializationException e) { // Ignore malformed json and move on ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line); } } return streamingTestFileContexts.Select(x => x.TestFileSummary).ToList(); }
private void FireFileStarted(ITestMethodRunnerCallback callback, TestContext testContext) { callback.FileStarted(testContext.InputTestFilesString); }
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestRunnerMode testRunnerMode) { string runnerArgs; var testModeStr = testRunnerMode.ToString().ToLowerInvariant(); var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds; if (timeout.HasValue && timeout > 0) { runnerArgs = string.Format("--proxy-type=none \"{0}\" {1} {2} {3}", runnerPath, fileUrl, testModeStr, timeout); } else { runnerArgs = string.Format("--proxy-type=none \"{0}\" {1} {2}", runnerPath, fileUrl, testModeStr); } return runnerArgs; }
private TestFileSummary ReadFromStream(StreamReader stream, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled) { var referencedFile = testContext.ReferencedJavaScriptFiles.SingleOrDefault(x => x.IsFileUnderTest); var testIndex = 0; var summary = new TestFileSummary(testContext.InputTestFile); string line; while ((line = stream.ReadLine()) != null) { if (debugEnabled) Console.WriteLine(line); var match = prefixRegex.Match(line); if (!match.Success) continue; var type = match.Groups["type"].Value; var json = match.Groups["json"].Value; // Only update last event timestamp if it is an important event. // Log and error could happen even though no test progress is made if (!type.Equals("Log") && !type.Equals("Error")) { lastTestEvent = DateTime.Now; } try { JsTestCase jsTestCase = null; switch (type) { case "FileStart": callback.FileStarted(testContext.InputTestFile); break; case "CoverageObject": var jsCov = jsonSerializer.Deserialize<JsCoverage>(json); summary.CoverageObject = coverageEngine.DeserializeCoverageObject(jsCov.Object, testContext); break; case "FileDone": var jsFileDone = jsonSerializer.Deserialize<JsFileDone>(json); summary.TimeTaken = jsFileDone.TimeTaken; callback.FileFinished(testContext.InputTestFile, summary); break; case "TestStart": jsTestCase = jsonSerializer.Deserialize<JsTestCase>(json); jsTestCase.TestCase.InputTestFile = testContext.InputTestFile; callback.TestStarted(jsTestCase.TestCase); break; case "TestDone": jsTestCase = jsonSerializer.Deserialize<JsTestCase>(json); jsTestCase.TestCase.InputTestFile = testContext.InputTestFile; AddLineNumber(referencedFile, testIndex, jsTestCase); testIndex++; callback.TestFinished(jsTestCase.TestCase); summary.AddTestCase(jsTestCase.TestCase); break; case "Log": var log = jsonSerializer.Deserialize<JsLog>(json); // This is an internal log message if (log.Log.Message.StartsWith(internalLogPrefix)) { ChutzpahTracer.TraceInformation("Phantom Log - {0}",log.Log.Message.Substring(internalLogPrefix.Length).Trim()); break; } log.Log.InputTestFile = testContext.InputTestFile; callback.FileLog(log.Log); summary.Logs.Add(log.Log); break; case "Error": var error = jsonSerializer.Deserialize<JsError>(json); error.Error.InputTestFile = testContext.InputTestFile; callback.FileError(error.Error); summary.Errors.Add(error.Error); break; } } catch (SerializationException e) { // Ignore malformed json and move on ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line); } } return summary; }