public IList<TestFileSummary> Read(ProcessStream processStream, TestOptions testOptions, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled)
{
if (processStream == null) throw new ArgumentNullException("processStream");
if (testOptions == null) throw new ArgumentNullException("testOptions");
if (testContext == null) throw new ArgumentNullException("testContext");
lastTestEvent = DateTime.Now;
var timeout = (testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500; // Add buffer to timeout to account for serialization
var readerTask = Task<IList<TestFileSummary>>.Factory.StartNew(() => ReadFromStream(processStream.StreamReader, testContext, testOptions, callback, debugEnabled));
while (readerTask.Status == TaskStatus.WaitingToRun
|| (readerTask.Status == TaskStatus.Running && (DateTime.Now - lastTestEvent).TotalMilliseconds < timeout))
{
Thread.Sleep(100);
}
if (readerTask.IsCompleted)
{
ChutzpahTracer.TraceInformation("Finished reading stream from test file '{0}'", testContext.FirstInputTestFile);
return readerTask.Result;
}
else
{
// We timed out so kill the process and return an empty test file summary
ChutzpahTracer.TraceError("Test file '{0}' timed out after running for {1} milliseconds", testContext.FirstInputTestFile, (DateTime.Now - lastTestEvent).TotalMilliseconds);
processStream.TimedOut = true;
processStream.KillProcess();
return testContext.ReferencedFiles.Where(x => x.IsFileUnderTest).Select(file => new TestFileSummary(file.Path)).ToList();
}
}
public IList<TestFileSummary> Read(ProcessStream processStream, TestOptions testOptions, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled)
{
if (processStream == null) throw new ArgumentNullException("processStream");
if (testOptions == null) throw new ArgumentNullException("testOptions");
if (testContext == null) throw new ArgumentNullException("testContext");
lastTestEvent = DateTime.Now;
var timeout = (testContext.TestFileSettings.TestFileTimeout ?? testOptions.TestFileTimeoutMilliseconds) + 500; // Add buffer to timeout to account for serialization
var codeCoverageEnabled = (!testContext.TestFileSettings.EnableCodeCoverage.HasValue && testOptions.CoverageOptions.Enabled)
|| (testContext.TestFileSettings.EnableCodeCoverage.HasValue && testContext.TestFileSettings.EnableCodeCoverage.Value);
var streamingTestFileContexts = testContext.ReferencedFiles
.Where(x => x.IsFileUnderTest)
.Select(x => new StreamingTestFileContext(x, testContext, codeCoverageEnabled))
.ToList();
var deferredEvents = new List<Action<StreamingTestFileContext>>();
var readerTask = Task<IList<TestFileSummary>>.Factory.StartNew(() => ReadFromStream(processStream.StreamReader, testContext, testOptions, streamingTestFileContexts, deferredEvents, callback, debugEnabled));
while (readerTask.Status == TaskStatus.WaitingToRun
|| (readerTask.Status == TaskStatus.Running && (DateTime.Now - lastTestEvent).TotalMilliseconds < timeout))
{
Thread.Sleep(100);
}
if (readerTask.IsCompleted)
{
ChutzpahTracer.TraceInformation("Finished reading stream from test file '{0}'", testContext.FirstInputTestFile);
return readerTask.Result;
}
else
{
// Since we times out make sure we play the deferred events so we do not lose errors
// We will just attach these events to the first test context at this point since we do
// not know where they belong
PlayDeferredEvents(streamingTestFileContexts.FirstOrDefault(), deferredEvents);
// We timed out so kill the process and return an empty test file summary
ChutzpahTracer.TraceError("Test file '{0}' timed out after running for {1} milliseconds", testContext.FirstInputTestFile, (DateTime.Now - lastTestEvent).TotalMilliseconds);
processStream.TimedOut = true;
processStream.KillProcess();
return testContext.ReferencedFiles.Where(x => x.IsFileUnderTest).Select(file => new TestFileSummary(file.Path)).ToList();
}
}
public void CreateTestHarness(TestContext testContext, TestOptions options)
{
if (!string.IsNullOrEmpty(testContext.TestHarnessPath))
{
// If we already have a test harness path then this means we executed on an existing html file or url
// So we dont need to generate the harness
return;
}
SetupAmdPathsIfNeeded(testContext.TestFileSettings, testContext.ReferencedFiles.ToList(), testContext.TestHarnessDirectory);
string testFilePathHash = hasher.Hash(string.Join(",",testContext.InputTestFiles));
string testHtmlFilePath = Path.Combine(testContext.TestHarnessDirectory, string.Format(Constants.ChutzpahTemporaryFileFormat, testFilePathHash, "test.html"));
testContext.TemporaryFiles.Add(testHtmlFilePath);
var templatePath = GetTestHarnessTemplatePath(testContext.FrameworkDefinition, testContext.TestFileSettings);
string testHtmlTemplate = fileSystem.GetText(templatePath);
var harness = new TestHarness(testContext.TestFileSettings, options, testContext.ReferencedFiles, fileSystem);
if (testContext.CoverageEngine != null)
{
testContext.CoverageEngine.PrepareTestHarnessForCoverage(harness, testContext.FrameworkDefinition, testContext.TestFileSettings);
}
var kvps = testContext.ReferencedFiles
.Where(x => x.IsFileUnderTest)
.SelectMany(x => x.FrameworkReplacements);
var frameworkReplacements = new Dictionary<string, string>();
foreach (var pair in kvps)
{
frameworkReplacements[pair.Key] = pair.Value;
}
string testHtmlText = harness.CreateHtmlText(testHtmlTemplate, frameworkReplacements);
fileSystem.Save(testHtmlFilePath, testHtmlText);
testContext.TestHarnessPath = testHtmlFilePath;
}
public void LaunchTest(TestContext testContext)
{
// Start IE.
ProcessStartInfo startInfo = new ProcessStartInfo()
{
UseShellExecute = true,
FileName = BrowserPathHelper.GetBrowserPath("ie"),
Arguments = string.Format("-noframemerging -suspended -debug {0}", FileProbe.GenerateFileUrl(testContext.TestHarnessPath))
// -noframemerging
// This is what VS does when launching the script debugger.
// Unsure whether strictly necessary.
// -suspended
// This is what VS does when launching the script debugger.
// Seems to cause IE to suspend all threads which is what we want.
// -debug
// This is what VS does when launching the script debugger.
// Not sure exactly what it does.
};
Process ieMainProcess = Process.Start(startInfo);
// Get child 'tab' process spawned by IE.
// We need to wait a few ms for IE to open the process.
Process ieTabProcess = null;
for (int i = 0;; ++i) {
System.Threading.Thread.Sleep(10);
ieTabProcess = ProcessExtensions.FindFirstChildProcessOf(ieMainProcess.Id);
if (ieTabProcess != null) {
break; }
if (i > 400) { // 400 * 10 = 4s timeout
throw new InvalidOperationException("Timeout waiting for Internet Explorer child process to start."); }
}
// Debug the script in that tab process.
DteHelpers.DebugAttachToProcess(ieTabProcess.Id, "script");
// Resume the threads in the IE process which where started off suspended.
ieTabProcess.Resume();
}
public void CleanTestContext(TestContext context)
{
testContextBuilder.CleanupContext(context);
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestExecutionMode testExecutionMode)
{
string runnerArgs;
var testModeStr = testExecutionMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds ?? Constants.DefaultTestFileTimeout;
runnerArgs = string.Format("--ignore-ssl-errors=true --proxy-type=none \"{0}\" {1} {2} {3} {4} {5}",
runnerPath,
fileUrl,
testModeStr,
timeout,
context.TestFileSettings.IgnoreResourceLoadingErrors.Value,
context.TestFileSettings.UserAgent);
return runnerArgs;
}
private IList<TestFileSummary> InvokeTestRunner(string headlessBrowserPath,
TestOptions options,
TestContext testContext,
TestExecutionMode testExecutionMode,
ITestMethodRunnerCallback callback)
{
string runnerPath = fileProbe.FindFilePath(testContext.TestRunner);
string fileUrl = BuildHarnessUrl(testContext.TestHarnessPath, testContext.IsRemoteHarness);
string runnerArgs = BuildRunnerArgs(options, testContext, fileUrl, runnerPath, testExecutionMode);
Func<ProcessStream, IList<TestFileSummary>> streamProcessor =
processStream => testCaseStreamReaderFactory.Create().Read(processStream, options, testContext, callback, m_debugEnabled);
var processResult = process.RunExecutableAndProcessOutput(headlessBrowserPath, runnerArgs, streamProcessor);
HandleTestProcessExitCode(processResult.ExitCode, testContext.FirstInputTestFile, processResult.Model.Select(x => x.Errors).FirstOrDefault(), callback);
return processResult.Model;
}
public bool TryBuildContext(PathInfo file, TestOptions options, out TestContext context)
{
context = BuildContext(file, options);
return context != null;
}
public void CleanupContext(TestContext context)
{
if (context == null) throw new ArgumentNullException("context");
foreach (var file in context.TemporaryFiles)
{
try
{
fileSystem.DeleteFile(file);
}
catch (IOException)
{
// Supress exception
}
}
}
public bool TryBuildContext(IEnumerable<string> files, TestOptions options, out TestContext context)
{
context = BuildContext(files, options);
return context != null;
}
public StreamingTestFileContext(ReferencedFile referencedFile, TestContext testContext, bool coverageEnabled)
{
SeenTests = new HashSet<Tuple<string, string>>();
ReferencedFile = referencedFile;
TestContext = testContext;
TestFileSummary = new TestFileSummary(referencedFile.Path);
if (coverageEnabled)
{
TestFileSummary.CoverageObject = new CoverageData();
}
}
private IList<TestFileSummary> ReadFromStream(StreamReader stream, TestContext testContext, TestOptions testOptions, IList<StreamingTestFileContext> streamingTestFileContexts, IList<Action<StreamingTestFileContext>> deferredEvents, ITestMethodRunnerCallback callback, bool debugEnabled)
{
var testIndex = 0;
string line;
StreamingTestFileContext currentTestFileContext = null;
if (streamingTestFileContexts.Count == 1)
{
currentTestFileContext = streamingTestFileContexts.First();
}
while ((line = stream.ReadLine()) != null)
{
if (debugEnabled) Console.WriteLine(line);
var match = prefixRegex.Match(line);
if (!match.Success) continue;
var type = match.Groups["type"].Value;
var json = match.Groups["json"].Value;
// Only update last event timestamp if it is an important event.
// Log and error could happen even though no test progress is made
if (!type.Equals("Log") && !type.Equals("Error"))
{
lastTestEvent = DateTime.Now;
}
try
{
switch (type)
{
case "FileStart":
FireFileStarted(callback, testContext);
break;
case "CoverageObject":
var jsCov = jsonSerializer.Deserialize<JsCoverage>(json);
if (currentTestFileContext == null)
{
AddDeferredEvent((fileContext) => FireCoverageObject(callback, fileContext, jsCov), deferredEvents);
}
else
{
FireCoverageObject(callback, currentTestFileContext, jsCov);
}
break;
case "FileDone":
var jsFileDone = jsonSerializer.Deserialize<JsFileDone>(json);
FireFileFinished(callback, testContext.InputTestFilesString, streamingTestFileContexts, jsFileDone);
break;
case "TestStart":
var jsTestCaseStart = jsonSerializer.Deserialize<JsTestCase>(json);
StreamingTestFileContext newContext = null;
var testName = jsTestCaseStart.TestCase.TestName.Trim();
var moduleName = (jsTestCaseStart.TestCase.ModuleName ?? "").Trim();
var fileContexts = GetFileMatches(testName, streamingTestFileContexts);
if (fileContexts.Count == 0 && currentTestFileContext == null)
{
// If there are no matches and not file context has been used yet
// then just choose the first context
newContext = streamingTestFileContexts[0];
}
else if (fileContexts.Count == 0)
{
// If there is already a current context and no matches we just keep using that context
// unless this test name has been used already in the current context. In that case
// move to the next one that hasn't seen this file yet
var testAlreadySeenInCurrentContext = currentTestFileContext.HasTestBeenSeen(moduleName, testName);
if (testAlreadySeenInCurrentContext)
{
newContext = streamingTestFileContexts.FirstOrDefault(x => !x.HasTestBeenSeen(moduleName, testName)) ?? currentTestFileContext;
}
}
else if (fileContexts.Count > 1)
{
// If we found the test has more than one file match
// try to choose the best match, otherwise just choose the first one
// If we have no file context yet take the first one
if (currentTestFileContext == null)
{
newContext = fileContexts.First();
}
else
{
// In this case we have an existing file context so we need to
// 1. Check to see if this test has been seen already on that context
// if so we need to try the next file context that matches it
// 2. If it is not seen yet in the current context and the current context
// is one of the matches then keep using it
var testAlreadySeenInCurrentContext = currentTestFileContext.HasTestBeenSeen(moduleName, testName);
var currentContextInFileMatches = fileContexts.Any(x => x == currentTestFileContext);
if (!testAlreadySeenInCurrentContext && currentContextInFileMatches)
{
// Keep the current context
newContext = currentTestFileContext;
}
else
{
// Either take first not used context OR the first one
newContext = fileContexts.Where(x => !x.IsUsed).FirstOrDefault() ?? fileContexts.First();
}
}
}
else if (fileContexts.Count == 1)
{
// We found a unique match
newContext = fileContexts[0];
}
if (newContext != null && newContext != currentTestFileContext)
{
currentTestFileContext = newContext;
testIndex = 0;
}
currentTestFileContext.IsUsed = true;
currentTestFileContext.MarkTestSeen(moduleName, testName);
PlayDeferredEvents(currentTestFileContext, deferredEvents);
jsTestCaseStart.TestCase.InputTestFile = currentTestFileContext.ReferencedFile.Path;
callback.TestStarted(jsTestCaseStart.TestCase);
ChutzpahTracer.TraceInformation("Test Case Started:'{0}'", jsTestCaseStart.TestCase.GetDisplayName());
break;
case "TestDone":
var jsTestCaseDone = jsonSerializer.Deserialize<JsTestCase>(json);
var currentTestIndex = testIndex;
FireTestFinished(callback, currentTestFileContext, jsTestCaseDone, currentTestIndex);
testIndex++;
break;
case "Log":
var log = jsonSerializer.Deserialize<JsLog>(json);
if (currentTestFileContext != null)
{
FireLogOutput(callback, currentTestFileContext, log);
}
else
{
AddDeferredEvent((fileContext) => FireLogOutput(callback, fileContext, log), deferredEvents);
}
break;
case "Error":
var error = jsonSerializer.Deserialize<JsError>(json);
if (currentTestFileContext != null)
{
FireErrorOutput(callback, currentTestFileContext, error);
}
else
{
AddDeferredEvent((fileContext) => FireErrorOutput(callback, fileContext, error), deferredEvents);
}
break;
}
}
catch (SerializationException e)
{
// Ignore malformed json and move on
ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line);
}
}
return streamingTestFileContexts.Select(x => x.TestFileSummary).ToList();
}
private void FireFileStarted(ITestMethodRunnerCallback callback, TestContext testContext)
{
callback.FileStarted(testContext.InputTestFilesString);
}
private static string BuildRunnerArgs(TestOptions options, TestContext context, string fileUrl, string runnerPath, TestRunnerMode testRunnerMode)
{
string runnerArgs;
var testModeStr = testRunnerMode.ToString().ToLowerInvariant();
var timeout = context.TestFileSettings.TestFileTimeout ?? options.TestFileTimeoutMilliseconds;
if (timeout.HasValue && timeout > 0)
{
runnerArgs = string.Format("--proxy-type=none \"{0}\" {1} {2} {3}",
runnerPath,
fileUrl,
testModeStr,
timeout);
}
else
{
runnerArgs = string.Format("--proxy-type=none \"{0}\" {1} {2}", runnerPath, fileUrl, testModeStr);
}
return runnerArgs;
}
private TestFileSummary ReadFromStream(StreamReader stream, TestContext testContext, ITestMethodRunnerCallback callback, bool debugEnabled)
{
var referencedFile = testContext.ReferencedJavaScriptFiles.SingleOrDefault(x => x.IsFileUnderTest);
var testIndex = 0;
var summary = new TestFileSummary(testContext.InputTestFile);
string line;
while ((line = stream.ReadLine()) != null)
{
if (debugEnabled) Console.WriteLine(line);
var match = prefixRegex.Match(line);
if (!match.Success) continue;
var type = match.Groups["type"].Value;
var json = match.Groups["json"].Value;
// Only update last event timestamp if it is an important event.
// Log and error could happen even though no test progress is made
if (!type.Equals("Log") && !type.Equals("Error"))
{
lastTestEvent = DateTime.Now;
}
try
{
JsTestCase jsTestCase = null;
switch (type)
{
case "FileStart":
callback.FileStarted(testContext.InputTestFile);
break;
case "CoverageObject":
var jsCov = jsonSerializer.Deserialize<JsCoverage>(json);
summary.CoverageObject = coverageEngine.DeserializeCoverageObject(jsCov.Object, testContext);
break;
case "FileDone":
var jsFileDone = jsonSerializer.Deserialize<JsFileDone>(json);
summary.TimeTaken = jsFileDone.TimeTaken;
callback.FileFinished(testContext.InputTestFile, summary);
break;
case "TestStart":
jsTestCase = jsonSerializer.Deserialize<JsTestCase>(json);
jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
callback.TestStarted(jsTestCase.TestCase);
break;
case "TestDone":
jsTestCase = jsonSerializer.Deserialize<JsTestCase>(json);
jsTestCase.TestCase.InputTestFile = testContext.InputTestFile;
AddLineNumber(referencedFile, testIndex, jsTestCase);
testIndex++;
callback.TestFinished(jsTestCase.TestCase);
summary.AddTestCase(jsTestCase.TestCase);
break;
case "Log":
var log = jsonSerializer.Deserialize<JsLog>(json);
// This is an internal log message
if (log.Log.Message.StartsWith(internalLogPrefix))
{
ChutzpahTracer.TraceInformation("Phantom Log - {0}",log.Log.Message.Substring(internalLogPrefix.Length).Trim());
break;
}
log.Log.InputTestFile = testContext.InputTestFile;
callback.FileLog(log.Log);
summary.Logs.Add(log.Log);
break;
case "Error":
var error = jsonSerializer.Deserialize<JsError>(json);
error.Error.InputTestFile = testContext.InputTestFile;
callback.FileError(error.Error);
summary.Errors.Add(error.Error);
break;
}
}
catch (SerializationException e)
{
// Ignore malformed json and move on
ChutzpahTracer.TraceError(e, "Recieved malformed json from Phantom in this line: '{0}'", line);
}
}
return summary;
}