private void ProcessTransforms(ChutzpahTestSettingsFile settings, TestCaseSummary overallSummary, TransformResult results)
        {
            // Do this here per settings file in case an individual transformer has any associated state
            // - we want them fresh
            var knownTransforms =
                transformerProvider
                .GetTransformers(fileSystem)
                .ToDictionary(x => x.Name, x => x, StringComparer.InvariantCultureIgnoreCase);

            foreach (var transformConfig in settings.Transforms)
            {
                SummaryTransformer transform = null;
                if (knownTransforms.TryGetValue(transformConfig.Name, out transform))
                {
                    var outputPath = transformConfig.Path;
                    if (!fileSystem.IsPathRooted(outputPath) && !string.IsNullOrWhiteSpace(transformConfig.SettingsFileDirectory))
                    {
                        outputPath = fileSystem.GetFullPath(Path.Combine(transformConfig.SettingsFileDirectory, outputPath));
                    }

                    // TODO: In future, this would ideally split out the summary to just those parts
                    // relevant to the files associated with the settings file being handled
                    transform.Transform(overallSummary, outputPath);

                    results.AddResult(transform.Name, outputPath);
                }
            }
        }
 public void TestSuiteFinished(TestCaseSummary testResultsSummary)
 {
     lock (sync)
     {
         nestedCallback.TestSuiteFinished(testResultsSummary);
     }
 }
        private static TestCaseSummary GetTestCaseSummary()
        {
            var testCaseSummary = new TestCaseSummary();
            testCaseSummary.CoverageObject = new CoverageData();

            testCaseSummary.CoverageObject["/no/coverage"] = new CoverageFileData
            {
                FilePath = "/no/coverage",
                LineExecutionCounts = null
            };

            testCaseSummary.CoverageObject["/three/lines/two/covered"] = new CoverageFileData
            {
                FilePath = "/three/lines/two/covered",
                LineExecutionCounts = new int?[] { null, 2, null, 5, 0 }
            };

            testCaseSummary.CoverageObject["/four/lines/four/covered"] = new CoverageFileData
            {
                FilePath = "/four/lines/four/covered",
                LineExecutionCounts = new int?[] { null, 2, 3, 4, 5 }
            };

            return testCaseSummary;
        }
        public override void TestSuiteFinished(TestCaseSummary testResultsSummary)
        {

            if (testResultsSummary.CoverageObject != null && testResultsSummary.CoverageObject.Any())
            {
                Console.WriteLine();
                PrintCodeCoverageResults(testResultsSummary.CoverageObject);
            }

            if (showFailureReport)
            {
                PrintErrorReport(testResultsSummary);
            }


            Console.WriteLine();
            var seconds = testResultsSummary.TotalRuntime / 1000.0;

            if (testResultsSummary.SkippedCount > 0)
            {
                Console.WriteLine("=== {0} total, {1} failed, {2} skipped, took {3:n} seconds ===", testResultsSummary.TotalCount, testResultsSummary.FailedCount, testResultsSummary.SkippedCount, seconds);
            }
            else
            {
                Console.WriteLine("=== {0} total, {1} failed, took {2:n} seconds ===", testResultsSummary.TotalCount, testResultsSummary.FailedCount, seconds);
            }

            base.TestSuiteFinished(testResultsSummary);
        }
        private static TestCaseSummary BuildTestCaseSummary()
        {
            var summary = new TestCaseSummary();
            var fileSummary = new TestFileSummary("path1"){ TimeTaken = 1500};
            fileSummary.AddTestCase(new TestCase
            {
                ModuleName = "module1",
                TestName = "test1",
                TestResults = new List<TestResult> { new TestResult { Passed = false, Message = "some failure" } }
            });
            fileSummary.AddTestCase(new TestCase
            {
                ModuleName = "module1",
                TestName = "test2",
                TestResults = new List<TestResult> { new TestResult { Passed = true } }
            });

            var fileSummary2 = new TestFileSummary("path>2") { TimeTaken = 2000 };
            fileSummary2.AddTestCase(new TestCase
            {
                TestName = "test3",
                TestResults = new List<TestResult> { new TestResult { Passed = true } }
            });
            fileSummary2.AddTestCase(new TestCase
            {
                TestName = "test<4",
                TestResults = new List<TestResult> { new TestResult { Passed = false, Message = "bad<failure" } }
            });

            summary.Append(fileSummary);
            summary.Append(fileSummary2);
            return summary;
        }
        private void PrintErrorReport(TestCaseSummary testResultsSummary)
        {
            var failedTests = (from testResult in testResultsSummary.Tests 
                              where !testResult.ResultsAllPassed
                              select testResult).ToList();

            var fileErrors = testResultsSummary.Errors;

            if (failedTests.Count > 0 || fileErrors.Count > 0)
            {

                Console.WriteLine("\n\n--- Failure Report :: {0} Failed Tests, {1} File Errors ---", failedTests.Count, fileErrors.Count);

                foreach (var fileError in fileErrors)
                {
                    FileError(fileError);    
                }

                foreach (var result in failedTests)
                {
                    TestFailed(result);
                }


                Console.WriteLine("-----------------------------------------");
            }
        }
        public override string Transform(TestCaseSummary testFileSummary)
        {
            if (testFileSummary == null) throw new ArgumentNullException("testFileSummary");

            var document = new System.Xml.XmlDocument();
            var testResults = AddTestResultsRoot(testFileSummary, document);

            foreach (var testFile in testFileSummary.TestFileSummaries)
            {
                var testSuite = AddTestSuite(document, testResults, testFile.Path,
                    testFile.PassedCount,
                    testFile.TimeTaken / 1000m,
                    testFile.PassedCount + testFile.FailedCount == testFile.TotalCount,
                    testFile.PassedCount == testFile.TotalCount);
                var results = testSuite.FirstChild;

                foreach (var group in testFile.TestGroups)
                {
                    Int32 total = group.Value.Count;
                    Int32 passed = group.Value.Count(t => t.ResultsAllPassed);
                    decimal time = group.Value.Sum(t => t.TimeTaken) / 1000m;

                    var groupSuite = AddTestSuite(document, results, group.Key, passed, time, total > 0, total == passed);
                    foreach (var test in group.Value)
                    {
                        AddTestCase(test, groupSuite.FirstChild, document);
                    }
                }
            }

            return GenerateReportXml(document);
        }
 public override void TestSuiteFinished(TestCaseSummary testResultsSummary)
 {
     var statusBarText = string.Format("{0} passed, {1} failed, {2} total", testResultsSummary.PassedCount, testResultsSummary.FailedCount, testResultsSummary.TotalCount);
     var text = string.Format("========== Total Tests: {0} passed, {1} failed, {2} total ==========\n", testResultsSummary.PassedCount, testResultsSummary.FailedCount, testResultsSummary.TotalCount);
     testPane.OutputString(text);
     SetStatusBarMessage(statusBarText);
 }
Example #9
0
        public void Transform(TestCaseSummary testFileSummary, string outFile)
        {
            if (testFileSummary == null) throw new ArgumentNullException("testFileSummary");
            if(string.IsNullOrEmpty(outFile)) throw new ArgumentNullException("outFile");

            var result = Transform(testFileSummary);
            File.WriteAllText(outFile, result);
        }
        public override void TestSuiteFinished(TestCaseSummary testResultsSummary)
        {
            if (testResultsSummary.CoverageObject != null)
            {
                WriteCoverageFiles(testResultsSummary.CoverageObject);
            }

            base.TestSuiteFinished(testResultsSummary);
        }
Example #11
0
        public void ProcessTransforms(IEnumerable<TestContext> testContexts, TestCaseSummary overallSummary)
        {
            var allTestSettings = testContexts.Select(x => x.TestFileSettings).Distinct();

            foreach (var settings in allTestSettings)
            {
                if (settings.Transforms != null && settings.Transforms.Any())
                {
                    ProcessTransforms(settings, overallSummary);
                }
            }
        }
        public virtual void Transform(TestCaseSummary testFileSummary, string outFile)
        {
            if (testFileSummary == null)
            {
                throw new ArgumentNullException("testFileSummary");
            }
            else if (string.IsNullOrEmpty(outFile))
            {
                throw new ArgumentNullException("outFile");
            }

            var result = Transform(testFileSummary);
            fileSystem.WriteAllText(outFile, result, Encoding);
        }
        public TransformResult ProcessTransforms(IEnumerable<TestContext> testContexts, TestCaseSummary overallSummary)
        {
            var results = new TransformResult();
            var allTestSettings = testContexts.Select(x => x.TestFileSettings).Distinct();

            foreach (var settings in allTestSettings)
            {
                if (settings.Transforms != null && settings.Transforms.Any())
                {
                    ProcessTransforms(settings, overallSummary, results);
                }
            }

            return results;
        }
Example #14
0
        public override void TestSuiteFinished(TestCaseSummary testResultsSummary)
        {
            base.TestSuiteFinished(testResultsSummary);

            if (!runContext.IsDataCollectionEnabled || testResultsSummary.CoverageObject == null)
            {
                return;
            }

            var directory = runContext.SolutionDirectory;
            var coverageHtmlFile = CoverageOutputGenerator.WriteHtmlFile(directory, testResultsSummary.CoverageObject);
            var processHelper = new ProcessHelper();

            processHelper.LaunchFileInBrowser(coverageHtmlFile);
        }
Example #15
0
        public override string Transform(TestCaseSummary testFileSummary)
        {
            if (testFileSummary == null)
            {
                throw new ArgumentNullException("testFileSummary");
            }
            else if (testFileSummary.CoverageObject == null)
            {
                return string.Empty;
            }

            var sb = new StringBuilder();
            foreach (var coverageData in testFileSummary.CoverageObject.Values)
            {
                AppendCoverageForFile(sb, coverageData);
            }

            return sb.ToString();
        }
        private XmlElement AddTestResultsRoot(TestCaseSummary summary, XmlDocument document)
        {
            var testResults = document.CreateElement("test-results");
            testResults.SetAttribute("name", "Chutzpah Test Results");

            testResults.SetAttribute("total", summary.TotalCount.ToString());
            testResults.SetAttribute("errors", summary.Errors.Count.ToString());
            testResults.SetAttribute("failures", summary.FailedCount.ToString());
            testResults.SetAttribute("not-run", (summary.TotalCount - summary.Errors.Count - summary.FailedCount - summary.PassedCount).ToString());
            testResults.SetAttribute("inconclusive", "0");
            testResults.SetAttribute("ignored", "0");
            testResults.SetAttribute("skipped", (summary.TotalCount - summary.Errors.Count - summary.FailedCount - summary.PassedCount).ToString());
            testResults.SetAttribute("invalid", "0");
            testResults.SetAttribute("date", DateTime.Now.ToString("yyyy-MM-dd"));
            testResults.SetAttribute("time", DateTime.Now.ToString("HH:mm:ss"));

            document.AppendChild(testResults);
            return testResults;
        }
        private static TestCaseSummary GetTestCaseSummary()
        {
            var toReturn = new TestCaseSummary();
            toReturn.CoverageObject = new CoverageData();

            toReturn.CoverageObject["/no/coverage"] = new CoverageFileData
            {
                FilePath = "/no/coverage",
                LineExecutionCounts = null
            };

            toReturn.CoverageObject["/some/lines"] = new CoverageFileData
            {
                FilePath = "/some/lines",
                LineExecutionCounts = new int?[] { 1, 2, null, 5, 0 }
            };

            return toReturn;
        }
        public override void Transform(TestCaseSummary testFileSummary, string outFile)
        {
            if (testFileSummary == null)
            {
                throw new ArgumentNullException("testFileSummary");
            }

            if (string.IsNullOrEmpty(outFile))
            {
                throw new ArgumentNullException("outFile");
            }

            if (testFileSummary.CoverageObject == null)
            {
                return;
            }

            CoverageOutputGenerator.WriteHtmlFile(outFile, testFileSummary.CoverageObject);
        }
        public override string Transform(TestCaseSummary testFileSummary)
        {
            if (testFileSummary == null)
            {
                throw new ArgumentNullException("testFileSummary");
            }

            GetOverallStats(testFileSummary.CoverageObject);
            var builder = new StringBuilder();
            builder.AppendLine(@"<?xml version=""1.0"" encoding=""UTF-8"" ?>");
            builder.AppendLine(@"<report>");
            AppendOverallStats(builder, testFileSummary.CoverageObject);
            builder.AppendLine(@" <data>");
            builder.AppendLine(@"  <all name=""all classes"">");
            AppendOverallCoverage(builder);
            AppendCoverageBySourceFile(builder, testFileSummary.CoverageObject);
            builder.AppendLine(@"  </all>");
            builder.AppendLine(@" </data>");
            builder.AppendLine(@"</report>");
            return builder.ToString();
        }
        public override string Transform(TestCaseSummary testFileSummary)
        {
            if (testFileSummary == null) throw new ArgumentNullException("testFileSummary");

            var builder = new StringBuilder();
            builder.AppendLine(@"<?xml version=""1.0"" encoding=""UTF-8"" ?>");
            builder.AppendLine(@"<testsuites>");
            foreach (TestFileSummary file in testFileSummary.TestFileSummaries)
            {
                builder.AppendLine(
                    string.Format(@"  <testsuite name=""{0}"" tests=""{1}"" failures=""{2}"" time=""{3}"">",
                                  Encode(file.Path), file.Tests.Count, file.Tests.Count(x => !x.ResultsAllPassed), ConvertMillisecondsToSeconds(file.TimeTaken)));
                ;
                foreach (TestCase test in file.Tests)
                {
                    if (test.ResultsAllPassed)
                    {
                        builder.AppendLine(string.Format(@"    <testcase name=""{0}"" time=""{1}"" />",
                                           Encode(test.GetDisplayName()), ConvertMillisecondsToSeconds(test.TimeTaken)));
                    }
                    else
                    {
                        TestResult failureCase = test.TestResults.FirstOrDefault(x => !x.Passed);
                        if (failureCase != null)
                        {
                            string failureMessage = failureCase.GetFailureMessage();
                            builder.AppendLine(string.Format(@"    <testcase name=""{0}"" time=""{1}"">",
                                                             Encode(test.GetDisplayName()), ConvertMillisecondsToSeconds(test.TimeTaken)));
                            builder.AppendLine(string.Format(@"      <failure message=""{0}""></failure>",
                                                             Encode(failureMessage)));
                            builder.AppendLine(string.Format(@"    </testcase>"));
                        }
                    }
                }
                builder.AppendLine(@"  </testsuite>");
            }
            builder.AppendLine(@"</testsuites>");
            return builder.ToString();
        }
Example #21
0
        public override void TestSuiteFinished(TestCaseSummary testResultsSummary)
        {
            base.TestSuiteFinished(testResultsSummary);

            if(!runContext.IsDataCollectionEnabled || testResultsSummary.CoverageObject == null)
            {
                return;
            }

            try
            {
                // If we do not have a solutiondirectory, we assume that we are running in tfs build
                // In that case we only write to the testrundirectory and do not open a browser
                if(string.IsNullOrEmpty(runContext.SolutionDirectory))
                {
                    ChutzpahTracer.TraceInformation("Chutzpah runs in TFSBuild, writing coverage file to {0}", runContext.TestRunDirectory);

                    var directory = runContext.TestRunDirectory;
                    CoverageOutputGenerator.WriteHtmlFile(directory, testResultsSummary.CoverageObject);
                    CoverageOutputGenerator.WriteJsonFile(directory, testResultsSummary.CoverageObject);

                }
                else
                {
                    ChutzpahTracer.TraceInformation("Chutzpah runs not in TFSBuild opening coverage file in browser");

                    var directory = runContext.SolutionDirectory;
                    var coverageHtmlFile = CoverageOutputGenerator.WriteHtmlFile(directory, testResultsSummary.CoverageObject);
                    var processHelper = new ProcessHelper();

                    processHelper.LaunchFileInBrowser(coverageHtmlFile);
                }
            }
            catch(Exception e)
            {
                frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format("Error while writing coverage output: {0}", e));
            }
        }
Example #22
0
        private TestCaseSummary ProcessTestPaths(IEnumerable<string> testPaths,
                                                 TestOptions options,
                                                 TestExecutionMode testExecutionMode,
                                                 ITestMethodRunnerCallback callback)
        {

            options.TestExecutionMode = testExecutionMode;

            stopWatch.Start();
            string headlessBrowserPath = fileProbe.FindFilePath(HeadlessBrowserName);
            if (testPaths == null)
                throw new ArgumentNullException("testPaths");
            if (headlessBrowserPath == null)
                throw new FileNotFoundException("Unable to find headless browser: " + HeadlessBrowserName);
            if (fileProbe.FindFilePath(TestRunnerJsName) == null)
                throw new FileNotFoundException("Unable to find test runner base js file: " + TestRunnerJsName);

            var overallSummary = new TestCaseSummary();

            // Concurrent list to collect test contexts
            var testContexts = new ConcurrentBag<TestContext>();

            // Concurrent collection used to gather the parallel results from
            var testFileSummaries = new ConcurrentQueue<TestFileSummary>();
            var resultCount = 0;
            var cancellationSource = new CancellationTokenSource();


            // Given the input paths discover the potential test files
            var scriptPaths = FindTestFiles(testPaths, options);

            // Group the test files by their chutzpah.json files. Then check if those settings file have batching mode enabled.
            // If so, we keep those tests in a group together to be used in one context
            // Otherwise, we put each file in its own test group so each get their own context
            var testRunConfiguration = BuildTestRunConfiguration(scriptPaths, options);

            ConfigureTracing(testRunConfiguration);

            var parallelism = testRunConfiguration.MaxDegreeOfParallelism.HasValue
                                ? Math.Min(options.MaxDegreeOfParallelism, testRunConfiguration.MaxDegreeOfParallelism.Value)
                                : options.MaxDegreeOfParallelism;

            var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = parallelism, CancellationToken = cancellationSource.Token };
            
            ChutzpahTracer.TraceInformation("Chutzpah run started in mode {0} with parallelism set to {1}", testExecutionMode, parallelOptions.MaxDegreeOfParallelism);

            // Build test contexts in parallel given a list of files each
            BuildTestContexts(options, testRunConfiguration.TestGroups, parallelOptions, cancellationSource, resultCount, testContexts, callback, overallSummary);


            // Compile the test contexts
            if (!PerformBatchCompile(callback, testContexts))
            {
                return overallSummary;
            }

            // Build test harness for each context and execute it in parallel
            ExecuteTestContexts(options, testExecutionMode, callback, testContexts, parallelOptions, headlessBrowserPath, testFileSummaries, overallSummary);


            // Gather TestFileSummaries into TaseCaseSummary
            foreach (var fileSummary in testFileSummaries)
            {
                overallSummary.Append(fileSummary);
            }

            stopWatch.Stop();
            overallSummary.SetTotalRunTime((int)stopWatch.Elapsed.TotalMilliseconds);

            overallSummary.TransformResult = transformProcessor.ProcessTransforms(testContexts, overallSummary);

            // Clear the settings file cache since in VS Chutzpah is not unloaded from memory.
            // If we don't clear then the user can never update the file.
            testSettingsService.ClearCache();


            ChutzpahTracer.TraceInformation(
                "Chutzpah run finished with {0} passed, {1} failed and {2} errors",
                overallSummary.PassedCount,
                overallSummary.FailedCount,
                overallSummary.Errors.Count);

            return overallSummary;
        }
Example #23
0
        public override string Transform(TestCaseSummary testFileSummary)
        {
            if(testFileSummary == null) throw new ArgumentNullException("testFileSummary");

            testRun = new TestRunType
            {
                id = Guid.NewGuid().ToString(),
                name = "Chutzpah_JS_UnitTest_" + DateTime.Now.ToString("yy-MMM-dd hh:mm:ss zz")
            };

            var windowsIdentity = System.Security.Principal.WindowsIdentity.GetCurrent();
            if (windowsIdentity != null)
                testRun.runUser = windowsIdentity.Name;

            testRun.Items = new object[]
            {
                new TestRunTypeResultSummary(),
                new ResultsType(),
                new TestDefinitionType(),
                new TestEntriesType1(),
                new TestRunTypeTestLists(),
                new TestRunTypeTimes(),
                new TestSettingsType
                {
                    name = "Default",
                    id = Guid.NewGuid().ToString(),
                    Execution = new TestSettingsTypeExecution
                    {
                        TestTypeSpecific = new TestSettingsTypeExecutionTestTypeSpecific{}
                    }
                }
            };
            // Time taken is current time
            testRun.Items.GetInstance<TestRunTypeTimes>(VSTSExtensions.TestRunItemType.Times).creation = DateTime.Now.AddSeconds(-testFileSummary.TimeTaken).ToString("O");
            testRun.Items.GetInstance<TestRunTypeTimes>(VSTSExtensions.TestRunItemType.Times).start = DateTime.Now.AddSeconds(-testFileSummary.TimeTaken).ToString("O");
            testRun.Items.GetInstance<TestRunTypeTimes>(VSTSExtensions.TestRunItemType.Times).queuing = DateTime.Now.AddSeconds(-testFileSummary.TimeTaken).ToString("O");

            var testList = new TestListType
            {
                name = "Results Not in a List",
                id = Guid.NewGuid().ToString()
            };

            var testTypeId = Guid.NewGuid();

            var currentTestCases = testFileSummary.Tests.Select(x => new VSTSTestCase().UpdateWith(x));

            var testsHelper = currentTestCases.ToList();
            testRun.Items.GetInstance<TestRunTypeTimes>(VSTSExtensions.TestRunItemType.Times).finish = DateTime.Now.ToString("O");
            testRun.Items.GetInstance<TestRunTypeResultSummary>(VSTSExtensions.TestRunItemType.ResultSummary).outcome = testsHelper.Count(x => x.Passed) == testsHelper.Count() ? "Passed" : "Failed";

            var counter = new CountersType
            {
                aborted = 0,
                completed = 0,
                disconnected = 0,
                error = 0,
                passed = testsHelper.Count(x => x.Passed),
                executed = testsHelper.Count,
                failed = testsHelper.Count(x => !x.Passed),
                total = testsHelper.Count,
                inProgress = 0,
                pending = 0,
                warning = 0,
                notExecuted = 0,
                notRunnable = 0,
                passedButRunAborted = 0,
                inconclusive = 0,
                timeout = 0
            };

            // total attribute is not written if totalSpecified is false
            counter.totalSpecified = true;

            testRun.Items.GetInstance<TestRunTypeResultSummary>(VSTSExtensions.TestRunItemType.ResultSummary).Items = new object[]
            {
                counter
            };

            testRun.Items.GetInstance<TestDefinitionType>(VSTSExtensions.TestRunItemType.TestDefinition).Items = testsHelper
                .Select(
                    (testCase) => new UnitTestType
                    {
                        id = testCase.Id.ToString(),
                        name = testCase.TestName,
                        storage = testCase.InputTestFile,
                        Items = new[]
                        {
                            new BaseTestTypeExecution
                            {
                                id= testCase.ExecutionId.ToString()
                            }
                        },
                        TestMethod = new UnitTestTypeTestMethod
                        {
                            adapterTypeName = "Microsoft.VisualStudio.TestTools.TestTypes.Unit.UnitTestAdapter",
                            className = Path.GetFileNameWithoutExtension(testCase.InputTestFile),
                            codeBase = testCase.InputTestFile,
                            name = testCase.TestName
                        }
                    }).ToArray();

            testRun.Items.GetInstance<TestDefinitionType>(VSTSExtensions.TestRunItemType.TestDefinition).ItemsElementName = testsHelper
                    .Select(
                        (testCase) => ItemsChoiceType4.UnitTest).ToArray();

            testRun.Items.GetInstance<TestRunTypeTestLists>(VSTSExtensions.TestRunItemType.TestLists).TestList = new[]
            {
                testList,
                // This has to be hard-coded.
                new TestListType
                {
                    name = "All Loaded Results",
                    id = "19431567-8539-422a-85d7-44ee4e166bda"
                }
            };

            testRun.Items.GetInstance<TestEntriesType1>(VSTSExtensions.TestRunItemType.TestEntries).TestEntry = testsHelper.Select(testCase => new TestEntryType
            {
                testId = testCase.Id.ToString(),
                executionId = testCase.ExecutionId.ToString(),
                testListId = testList.id
            }).ToArray();

            testRun.Items.GetInstance<ResultsType>(VSTSExtensions.TestRunItemType.Results).Items = testsHelper.Select((testCase) =>
            {
                var unitTestResultType = new UnitTestResultType
                {
                    executionId = testCase.ExecutionId.ToString(),
                    testId = testCase.Id.ToString(),
                    testName = testCase.TestName,

                    computerName = Environment.MachineName,
                    duration = new TimeSpan(0,0,testCase.TimeTaken).ToString("c"),
                    // I tried adding this to StandardConsoleRunner, but it demanded too many changes.
                    // Setting start to the creation date.
                    startTime = DateTime.Now.AddSeconds(-testFileSummary.TimeTaken).ToString("O"),
                    // Setting end time to creation date + time taken to run this test.
                    endTime = DateTime.Now.AddSeconds((-testFileSummary.TimeTaken) + testCase.TimeTaken).ToString("O"),
                    // This is for specific test type.
                    testType = "13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b",
                    outcome = testCase.Passed ? "Passed" : "Failed",
                    testListId = testList.id,

                };

                if (!testCase.Passed)
                {
                    unitTestResultType.Items = new[]
                        {
                            new OutputType()
                            {
                                ErrorInfo = new OutputTypeErrorInfo{Message = testCase.exception !=null ? testCase.exception.ToString(): string.Join(",",testCase.TestResults.Where(x=>!x.Passed).Select(x=>x.Message))}
                            }
                        };
                }
                return unitTestResultType;

            }).ToArray();

            testRun.Items.GetInstance<ResultsType>(VSTSExtensions.TestRunItemType.Results).ItemsElementName =
                testsHelper.Select(testCase => ItemsChoiceType3.UnitTestResult).ToArray();

            var stringStream = new StringWriter();
            var xs = new XmlSerializer(typeof(TestRunType));
            xs.Serialize(stringStream,testRun);

            return stringStream.ToString();
        }
 public override void TestSuiteFinished(TestCaseSummary summary)
 {
     base.TestSuiteFinished(summary);
     WriteLine("testSuiteFinished name='{0}'", Escape(testSuiteName));
 }
Example #25
0
 private static void ProcessTestSummaryTransformers(CommandLine commandLine, TestCaseSummary testResultsSummary)
 {
     var transformers = new SummaryTransformerProvider().GetTransformers(new FileSystemWrapper());
     foreach (var transformer in transformers.Where(x => commandLine.UnmatchedArguments.ContainsKey(x.Name)))
     {
         var path = commandLine.UnmatchedArguments[transformer.Name];
         transformer.Transform(testResultsSummary, path);
     }
 }
Example #26
0
 public virtual void TestSuiteFinished(TestCaseSummary testResultsSummary) { }
Example #27
0
        private void BuildTestContexts(
            TestOptions options,
            List<List<PathInfo>> scriptPathGroups,
            ParallelOptions parallelOptions,
            CancellationTokenSource cancellationSource,
            int resultCount,
            ConcurrentBag<TestContext> testContexts,
            ITestMethodRunnerCallback callback,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(scriptPathGroups, parallelOptions, testFiles =>
            {
                var pathString = string.Join(",", testFiles.Select(x => x.FullPath));
                ChutzpahTracer.TraceInformation("Building test context for {0}", pathString);

                try
                {
                    if (cancellationSource.IsCancellationRequested) return;
                    TestContext testContext;

                    resultCount++;
                    if (testContextBuilder.TryBuildContext(testFiles, options, out testContext))
                    {
                        testContexts.Add(testContext);
                    }
                    else
                    {
                        ChutzpahTracer.TraceWarning("Unable to build test context for {0}", pathString);
                    }

                    // Limit the number of files we can scan to attempt to build a context for
                    // This is important in the case of folder scanning where many JS files may not be
                    // test files.
                    if (resultCount >= options.FileSearchLimit)
                    {
                        ChutzpahTracer.TraceError("File search limit hit!!!");
                        cancellationSource.Cancel();
                    }
                }
                catch (Exception e)
                {
                    var error = new TestError
                    {
                        InputTestFile = testFiles.Select(x => x.FullPath).FirstOrDefault(),
                        Message = e.ToString()
                    };

                    overallSummary.Errors.Add(error);
                    callback.FileError(error);

                    ChutzpahTracer.TraceError(e, "Error during building test context for {0}", pathString);
                }
                finally
                {
                    ChutzpahTracer.TraceInformation("Finished building test context for {0}", pathString);
                }
            });
        }
        public override void TestSuiteFinished(TestCaseSummary summary)
        {
            base.TestSuiteFinished(summary);

            Console.WriteLine("##teamcity[testSuiteFinished name='{0}']",Escape(ChutzpahJavascriptTestSuiteName));
        }
 // We are overwritting the default Transform above so we do not need this method
 public override string Transform(TestCaseSummary testFileSummary)
 {
     throw new NotImplementedException();
 }
Example #30
0
        private void ExecuteTestContexts(
            TestOptions options,
            TestExecutionMode testExecutionMode,
            ITestMethodRunnerCallback callback,
            ConcurrentBag<TestContext> testContexts,
            ParallelOptions parallelOptions,
            string headlessBrowserPath,
            ConcurrentQueue<TestFileSummary> testFileSummaries,
            TestCaseSummary overallSummary)
        {
            Parallel.ForEach(
                testContexts,
                parallelOptions,
                testContext =>
                {
                    ChutzpahTracer.TraceInformation("Start test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);

                    try
                    {
                        testHarnessBuilder.CreateTestHarness(testContext, options);

                        if (options.TestLaunchMode == TestLaunchMode.FullBrowser)
                        {
                            ChutzpahTracer.TraceInformation(
                                "Launching test harness '{0}' for file '{1}' in a browser",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);

                            // Allow override from command line.
                            var browserArgs = testContext.TestFileSettings.BrowserArguments;
                            if (!string.IsNullOrWhiteSpace(options.BrowserArgs))
                            {
                                var path = BrowserPathHelper.GetBrowserPath(options.BrowserName);
                                browserArgs = new Dictionary<string, string>
                                {
                                    { Path.GetFileNameWithoutExtension(path), options.BrowserArgs }
                                };
                            }
                                                   
                            process.LaunchFileInBrowser(testContext.TestHarnessPath, options.BrowserName, browserArgs);
                        }
                        else if (options.TestLaunchMode == TestLaunchMode.HeadlessBrowser)
                        {
                            ChutzpahTracer.TraceInformation(
                                "Invoking headless browser on test harness '{0}' for file '{1}'",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);

                            var testSummaries = InvokeTestRunner(
                                headlessBrowserPath,
                                options,
                                testContext,
                                testExecutionMode,
                                callback);

                            foreach (var testSummary in testSummaries)
                            {

                                ChutzpahTracer.TraceInformation(
                                    "Test harness '{0}' for file '{1}' finished with {2} passed, {3} failed and {4} errors",
                                    testContext.TestHarnessPath,
                                    testSummary.Path,
                                    testSummary.PassedCount,
                                    testSummary.FailedCount,
                                    testSummary.Errors.Count);

                                ChutzpahTracer.TraceInformation(
                                    "Finished running headless browser on test harness '{0}' for file '{1}'",
                                    testContext.TestHarnessPath,
                                    testSummary.Path);

                                testFileSummaries.Enqueue(testSummary);
                            }
                        }
                        else if (options.TestLaunchMode == TestLaunchMode.Custom)
                        {
                            if (options.CustomTestLauncher == null)
                            {
                                throw new ArgumentNullException("TestOptions.CustomTestLauncher");
                            }
                            ChutzpahTracer.TraceInformation(
                                "Launching custom test on test harness '{0}' for file '{1}'",
                                testContext.TestHarnessPath,
                                testContext.FirstInputTestFile);
                            options.CustomTestLauncher.LaunchTest(testContext);
                        }
                        else
                        {
                            Debug.Assert(false);
                        }
                    }
                    catch (Exception e)
                    {
                        var error = new TestError
                        {
                            InputTestFile = testContext.InputTestFiles.FirstOrDefault(),
                            Message = e.ToString()
                        };

                        overallSummary.Errors.Add(error);
                        callback.FileError(error);

                        ChutzpahTracer.TraceError(e, "Error during test execution of {0}", testContext.FirstInputTestFile);
                    }
                    finally
                    {
                        ChutzpahTracer.TraceInformation("Finished test run for {0} in {1} mode", testContext.FirstInputTestFile, testExecutionMode);
                    }
                });


            // Clean up test context
            foreach (var testContext in testContexts)
            {
                // Don't clean up context if in debug mode
                if (!m_debugEnabled
                    && options.TestLaunchMode != TestLaunchMode.FullBrowser
                    && options.TestLaunchMode != TestLaunchMode.Custom)
                {
                    try
                    {
                        ChutzpahTracer.TraceInformation("Cleaning up test context for {0}", testContext.FirstInputTestFile);
                        testContextBuilder.CleanupContext(testContext);

                    }
                    catch (Exception e)
                    {
                        ChutzpahTracer.TraceError(e, "Error cleaning up test context for {0}", testContext.FirstInputTestFile);
                    }
                }
            }
        }