Beispiel #1
0
        internal static void StartAllureLogging(ITest test, string testUuid, string testContainerUuid,
                                                TestFixture fixture)
        {
            var testFullNameForLog = GenerateFullNameWithParameters(test, test.FullName);
            var ourFixture         = new TestResultContainer
            {
                uuid = testContainerUuid,
                name = test.ClassName
            };

            AllureLifecycle.Instance.StartTestContainer(ourFixture);
            var testResult = new TestResult
            {
                uuid     = testUuid,
                name     = test.Name,
                fullName = testFullNameForLog,
                labels   = new List <Label>
                {
                    Label.Thread(),
                Label.Host(),
                Label.TestClass(test.ClassName),
                    Label.TestMethod(test.MethodName),
                    Label.Package(test.ClassName)
                },
                historyId     = testFullNameForLog,
                statusDetails = new StatusDetails()
            };

            AllureLifecycle.Instance.StartTestCase(testResult);
            AddInfoInTestCase(test, testUuid, fixture);
        }
        void HandleTestResults(TestResultContainer summary)
        {
            Exception ex = summary.Exception;

            if (ex != null)
            {
                TestContext.WriteLine("System error occured while checking target.");
            }
            TestResults result = summary.TestResults;

            if (result == null)
            {
                TestContext.WriteLine("No test results information available");
            }
            else
            {
                TestContext.WriteLine(
                    "AppName = {0}, testname = {1}, Browser = {2},OS = {3} viewport = {4}x{5}, matched = {6},mismatched = {7}, missing = {8}, aborted = {9}\n",
                    result.AppName,
                    result.Name,
                    result.HostApp,
                    result.HostOS,
                    result.HostDisplaySize.Width,
                    result.HostDisplaySize.Height,
                    result.Matches,
                    result.Mismatches,
                    result.Missing,
                    (result.IsAborted ? "aborted" : "no"));
            }
        }
Beispiel #3
0
        public override void BeforeTest(ITest test)
        {
            var fixture = new TestResultContainer
            {
                uuid = test.Id,
                name = test.ClassName
            };

            Allure.StartTestContainer(fixture);
        }
Beispiel #4
0
        public void BeforeTest(ITest suite)
        {
            _ignoredContainerId = suite.Id + "-ignored";
            var fixture = new TestResultContainer
            {
                uuid = _ignoredContainerId,
                name = suite.ClassName
            };

            AllureLifecycle.Instance.StartTestContainer(fixture);
        }
Beispiel #5
0
        private void StartTestContainer(string fullTestName)
        {
            _testContainerId = Guid.NewGuid().ToString();
            var container = new TestResultContainer
            {
                uuid = _testContainerId,
                name = fullTestName,
            };

            _allureLifecycle.StartTestContainer(container);
        }
Beispiel #6
0
        //[OneTimeSetUp]
        public void AllureBeforeAllTestsInClass()
        {
            var nunitFixture = TestExecutionContext.CurrentContext.CurrentTest;

            var fixture = new TestResultContainer
            {
                uuid = nunitFixture.Id,
                name = nunitFixture.ClassName
            };

            Allure.StartTestContainer(fixture);
        }
        private void StartTestContainer()
        {
            const string testContainerInterfix = "-tc-";

            _containerGuid = string.Concat(Guid.NewGuid().ToString(), testContainerInterfix, _test.UniqueID);
            var container = new TestResultContainer
            {
                uuid = _containerGuid,
                name = _test.DisplayName
            };

            AllureLifecycle.StartTestContainer(container);
        }
        private void StartTestContainer()
        {
            var testFixture = GetTestFixture(_test);

            _containerGuid = string.Concat(Guid.NewGuid().ToString(), "-tc-", testFixture.Id);
            var container = new TestResultContainer
            {
                uuid = _containerGuid,
                name = _test.FullName
            };

            AllureLifecycle.StartTestContainer(container);
        }
Beispiel #9
0
        public void SetupForAllure()
        {
            Environment.CurrentDirectory = Path.GetDirectoryName(GetType().Assembly.Location);
            var nunitFixture = TestExecutionContext.CurrentContext.CurrentTest;

            var fixture = new TestResultContainer
            {
                uuid = nunitFixture.Id,
                name = nunitFixture.ClassName
            };

            Allure.StartTestContainer(fixture);
        }
Beispiel #10
0
        internal static TestResultContainer GetTestResultContainer()
        {
            var trc = new TestResultContainer();

            trc.uuid        = Guid.NewGuid().ToString("N");
            trc.name        = trc.uuid;
            trc.description = trc.uuid;
            trc.links       = new List <Link>()
            {
                Link.AddIssue(name: new Random().Next(999999).ToString(), url: "/bug"),
                Link.AddTms(name: new Random().Next(999999).ToString(), url: "/test")
            };
            return(trc);
        }
Beispiel #11
0
        internal static TestResultContainer StartTestContainer(FeatureContext featureContext,
                                                               ScenarioContext scenarioContext)
        {
            var containerId = GetFeatureContainerId(featureContext?.FeatureInfo);

            var scenarioContainer = new TestResultContainer
            {
                uuid = NewId()
            };

            AllureLifecycle.Instance.StartTestContainer(containerId, scenarioContainer);
            scenarioContext?.Set(scenarioContainer);
            featureContext?.Get <HashSet <TestResultContainer> >().Add(scenarioContainer);
            return(scenarioContainer);
        }
        public BaseTest(OneTimeSetupTearDownFixture testSuite, ITestOutputHelper output)
        {
            _output    = (TestOutputHelper)output;
            _testSuite = testSuite;

            testContainer = new TestResultContainer();

            var type       = _output.GetType();
            var testMember = type.GetField("test", BindingFlags.Instance | BindingFlags.NonPublic);
            var test       = (ITest)testMember.GetValue(_output);

            testContainer.uuid        = test.TestCase.UniqueID;
            testContainer.description = test.TestCase.UniqueID;

            _testSuite.lifeCycle.StartTestContainer(testContainer);

            _testSuite.driver.Manage().Window.Maximize();
        }
Beispiel #13
0
        public static string addTestContainer(testContainer testContainer)
        {
            var trc = new TestResultContainer();

            if (string.IsNullOrWhiteSpace(testContainer.id))
            {
                testContainer.id = Guid.NewGuid().ToString("N");
            }
            trc.uuid = testContainer.id;
            if (string.IsNullOrWhiteSpace(testContainer.name))
            {
                testContainer.name = "Container Name " + trc.uuid;
            }
            trc.name = testContainer.name;
            if (!string.IsNullOrWhiteSpace(testContainer.description))
            {
                testContainer.description = "Container Description " + trc.uuid;
            }
            trc.description = testContainer.description;
            if (testContainer.listLinks != null)
            {
                if (testContainer.listLinks.Count > 0)
                {
                    trc.links = new List <Link>();
                    for (int i = 0; i < testContainer.listLinks.Count; i++)
                    {
                        if (testContainer.listLinks[i].isIssue)
                        {
                            Link.Issue(name: testContainer.listLinks[i].name, url: testContainer.listLinks[i].url);
                        }
                        else
                        {
                            Link.Tms(name: testContainer.listLinks[i].name, url: testContainer.listLinks[i].url);
                        }
                    }
                }
            }

            instance.StartTestContainer(trc);
            return(trc.uuid);
        }
 public void Write(TestResultContainer testResult)
 {
     Write(testResult, AllureConstants.TestResultContainerFileSuffix);
 }
        public override object InvokeBinding(IBinding binding, IContextManager contextManager, object[] arguments,
                                             ITestTracer testTracer, out TimeSpan duration)
        {
            var hook = binding as HookBinding;

            // process hook
            if (hook != null)
            {
                var featureContainerId = AllureHelper.GetFeatureContainerId(contextManager.FeatureContext?.FeatureInfo);

                switch (hook.HookType)
                {
                case HookType.BeforeFeature:
                    if (hook.HookOrder == int.MinValue)
                    {
                        // starting point
                        var featureContainer = new TestResultContainer
                        {
                            uuid = AllureHelper.GetFeatureContainerId(contextManager.FeatureContext?.FeatureInfo)
                        };
                        allure.StartTestContainer(featureContainer);

                        contextManager.FeatureContext.Set(new HashSet <TestResultContainer>());
                        contextManager.FeatureContext.Set(new HashSet <TestResult>());

                        return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
                    }
                    else
                    {
                        try
                        {
                            this.StartFixture(hook, featureContainerId);
                            var result = base.InvokeBinding(binding, contextManager, arguments, testTracer,
                                                            out duration);
                            allure.StopFixture(x => x.status = Status.passed);
                            return(result);
                        }
                        catch (Exception ex)
                        {
                            allure.StopFixture(x => x.status = Status.broken);

                            // if BeforeFeature is failed execution is stopped. We need to create, update, stop and write everything here.

                            // create fake scenario container
                            var scenarioContainer =
                                AllureHelper.StartTestContainer(contextManager.FeatureContext, null);

                            // start fake scenario
                            var scenario = AllureHelper.StartTestCase(scenarioContainer.uuid,
                                                                      contextManager.FeatureContext, null);

                            // update, stop and write
                            allure
                            .StopTestCase(x =>
                            {
                                x.status        = Status.broken;
                                x.statusDetails = AllureHelper.GetStatusDetails(ex);
                            })
                            .WriteTestCase(scenario.uuid)
                            .StopTestContainer(scenarioContainer.uuid)
                            .WriteTestContainer(scenarioContainer.uuid)
                            .StopTestContainer(featureContainerId)
                            .WriteTestContainer(featureContainerId);

                            throw;
                        }
                    }

                case HookType.BeforeStep:
                case HookType.AfterStep:
                {
                    var scenario = AllureHelper.GetCurrentTestCase(contextManager.ScenarioContext);

                    try
                    {
                        return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
                    }
                    catch (Exception ex)
                    {
                        allure
                        .UpdateTestCase(scenario.uuid,
                                        x =>
                            {
                                x.status        = Status.broken;
                                x.statusDetails = AllureHelper.GetStatusDetails(ex);
                            });
                        throw;
                    }
                }

                case HookType.BeforeScenario:
                case HookType.AfterScenario:
                    if (hook.HookOrder == int.MinValue || hook.HookOrder == int.MaxValue)
                    {
                        return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
                    }
                    else
                    {
                        var scenarioContainer = AllureHelper.GetCurrentTestConainer(contextManager.ScenarioContext);

                        try
                        {
                            this.StartFixture(hook, scenarioContainer.uuid);
                            var result = base.InvokeBinding(binding, contextManager, arguments, testTracer,
                                                            out duration);
                            allure.StopFixture(x => x.status = Status.passed);
                            return(result);
                        }
                        catch (Exception ex)
                        {
                            allure.StopFixture(x => x.status = Status.broken);

                            // get or add new scenario
                            var scenario = AllureHelper.GetCurrentTestCase(contextManager.ScenarioContext) ??
                                           AllureHelper.StartTestCase(scenarioContainer.uuid,
                                                                      contextManager.FeatureContext, contextManager.ScenarioContext);

                            allure.UpdateTestCase(scenario.uuid,
                                                  x =>
                            {
                                x.status        = Status.broken;
                                x.statusDetails = AllureHelper.GetStatusDetails(ex);
                            });
                            throw;
                        }
                    }

                case HookType.AfterFeature:
                    if (hook.HookOrder == int.MaxValue)
                    // finish point
                    {
                        WriteScenarios(contextManager);
                        allure
                        .StopTestContainer(featureContainerId)
                        .WriteTestContainer(featureContainerId);

                        return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
                    }
                    else
                    {
                        try
                        {
                            StartFixture(hook, featureContainerId);
                            var result = base.InvokeBinding(binding, contextManager, arguments, testTracer,
                                                            out duration);
                            allure.StopFixture(x => x.status = Status.passed);
                            return(result);
                        }
                        catch (Exception ex)
                        {
                            var scenario = contextManager.FeatureContext.Get <HashSet <TestResult> >().Last();
                            allure
                            .StopFixture(x => x.status = Status.broken)
                            .UpdateTestCase(scenario.uuid,
                                            x =>
                            {
                                x.status        = Status.broken;
                                x.statusDetails = AllureHelper.GetStatusDetails(ex);
                            });

                            WriteScenarios(contextManager);

                            allure
                            .StopTestContainer(featureContainerId)
                            .WriteTestContainer(featureContainerId);

                            throw;
                        }
                    }

                case HookType.BeforeScenarioBlock:
                case HookType.AfterScenarioBlock:
                case HookType.BeforeTestRun:
                case HookType.AfterTestRun:
                default:
                    return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
                }
            }
            else
            {
                return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
            }
        }
Beispiel #16
0
 public void Write(TestResultContainer testResult)
 {
     this.Write(testResult, AllureConstants.TEST_RESULT_CONTAINER_FILE_SUFFIX);
 }
 public void Write(TestResultContainer testResult)
 {
     LinkHelper.UpdateLinks(testResult.links, configuration.Links);
     Write(testResult, AllureConstants.TEST_RESULT_CONTAINER_FILE_SUFFIX);
 }
        public TestResultContainer Call(object state)
        {
            try
            {
                logger_.Log("enter - type: {0} ; name: {1}", TaskType, RunningTest.TestName);
                testResults_ = null;
                switch (TaskType)
                {
                case TaskType.Open:
                    logger_.Log("Task.run opening task");
                    if (renderResult_ != null)
                    {
                        string        userAgent  = renderResult_.UserAgent;
                        RectangleSize deviceSize = renderResult_.DeviceSize;
                        logger_.Verbose("setting device size: {0}", deviceSize);
                        EyesConnector.SetUserAgent(userAgent);
                        EyesConnector.SetDeviceSize(deviceSize);
                    }
                    else
                    {
                        // We are in exception mode - trying to do eyes.open() without first render
                        RenderBrowserInfo browserInfo = RunningTest.BrowserInfo;
                        EyesConnector.SetDeviceSize(new Size(browserInfo.Width, browserInfo.Height));
                    }
                    EyesConnector.Open(configuration_);
                    break;

                case TaskType.Check:
                    logger_.Log("Task.run check task");
                    try
                    {
                        string          imageLocation = renderResult_.ImageLocation;
                        string          domLocation   = renderResult_.DomLocation;
                        VGRegion[]      vgRegions     = renderResult_.SelectorRegions;
                        IList <IRegion> regions       = new List <IRegion>();
                        if (vgRegions != null)
                        {
                            foreach (VGRegion reg in vgRegions)
                            {
                                if (!(string.IsNullOrWhiteSpace(reg.Error)))
                                {
                                    logger_.Log("Warning: region error: {0}", reg.Error);
                                }
                                else
                                {
                                    regions.Add(reg);
                                }
                            }
                        }
                        logger_.Verbose(renderResult_.ToString());
                        if (imageLocation == null)
                        {
                            logger_.Log("CHECKING IMAGE WITH NULL LOCATION!");
                        }
                        Location location = null;
                        if (regionSelectors_.Count > 0 && regions.Count > 0)
                        {
                            VisualGridSelector[] targetSelector = regionSelectors_[regionSelectors_.Count - 1];
                            if (targetSelector.Length > 0 && "target".Equals(targetSelector[0].Category))
                            {
                                location = regions[regions.Count - 1].Location;
                            }
                        }
                        EyesConnector.MatchWindow(configuration_, imageLocation, domLocation, checkSettings_, regions,
                                                  regionSelectors_, location, renderResult_, source_);
                    }
                    catch (WebException we)
                    {
                        Stream stream            = we.Response.GetResponseStream();
                        byte[] responseBodyBytes = CommonUtils.ReadToEnd(stream);
                        string responseBodyStr   = Encoding.UTF8.GetString(responseBodyBytes);
                        logger_.Log($"Error: {we}\n{responseBodyStr}");
                    }
                    catch (Exception e)
                    {
                        logger_.Log("Error: " + e);
                    }
                    break;

                case TaskType.Close:
                    logger_.Log("Task.run close task");
                    try
                    {
                        testResults_ = EyesConnector.Close(true, configuration_);
                    }
                    catch (TestFailedException tfe)
                    {
                        logger_.Log("Test Failed: " + tfe);
                        Exception    = tfe;
                        testResults_ = tfe.TestResults;
                    }
                    catch (Exception e)
                    {
                        logger_.Log("Error: " + e);
                        Exception = e;
                    }
                    break;

                case TaskType.Abort:
                    logger_.Log("Task.run abort task");
                    testResults_ = EyesConnector.AbortIfNotClosed();
                    break;
                }
                TestResultContainer testResultContainer = new TestResultContainer(testResults_, BrowserInfo, Exception);
                NotifySuccessAllListeners();
                return(testResultContainer);
            }
            catch (Exception e)
            {
                logger_.Log("Error: " + e);
                NotifyFailureAllListeners(e);
            }
            finally
            {
                logger_.Verbose("marking {0} task as complete: {1}", TaskType, RunningTest.TestName);
                IsTaskComplete = true;
                //call the callback
            }
            return(null);
        }
        protected override TestResultsSummary GetAllTestResultsImpl(bool shouldThrowException)
        {
            Logger.Verbose("enter");
            Dictionary <IVisualGridEyes, ICollection <Task <TestResultContainer> > > allFutures = new Dictionary <IVisualGridEyes, ICollection <Task <TestResultContainer> > >();

            List <IVisualGridEyes> allEyes;

            lock (allEyes_)
            {
                allEyes = new List <IVisualGridEyes>(allEyes_);
            }

            foreach (IVisualGridEyes eyes in allEyes)
            {
                ICollection <Task <TestResultContainer> > futureList = eyes.Close();
                Logger.Verbose("adding a {0} items list to allFutures.", futureList.Count);
                if (allFutures.TryGetValue(eyes, out ICollection <Task <TestResultContainer> > futures) && futures != null)
                {
                    foreach (Task <TestResultContainer> future in futures)
                    {
                        futureList.Add(future);
                    }
                }
                allFutures.Add(eyes, futureList);
            }

            Exception exception = null;

            NotifyAllServices();
            List <TestResultContainer> allResults = new List <TestResultContainer>();

            Logger.Verbose("trying to call future.get on {0} future lists.", allFutures.Count);
            foreach (KeyValuePair <IVisualGridEyes, ICollection <Task <TestResultContainer> > > entry in allFutures)
            {
                ICollection <Task <TestResultContainer> > value = entry.Value;
                IVisualGridEyes key = entry.Key;
                key.TestResults.Clear();
                Logger.Verbose("trying to call future.get on {0} futures of {1}", value.Count, key);
                foreach (Task <TestResultContainer> future in value)
                {
                    Logger.Verbose("calling future.get on {0}", key);
                    TestResultContainer testResultContainer = null;
                    try
                    {
                        if (Task.WhenAny(future, Task.Delay(waitForResultTimeout_)).Result == future)
                        {
                            testResultContainer = future.Result;
                            if (testResultContainer.Exception != null && exception == null)
                            {
                                exception = testResultContainer.Exception;
                            }
                            allResults.Add(testResultContainer);
                            key.TestResults.Add(testResultContainer);
                        }
                    }
                    catch (Exception e)
                    {
                        Logger.Log("Error: " + e);
                        if (exception == null)
                        {
                            exception = e;
                        }
                    }
                    Logger.Verbose("got TestResultContainer: {0}", testResultContainer);
                }
            }

            //lock (allEyes_) allEyes_.Clear();
            //eyesToOpenList_.Clear();
            //renderingTaskList_.Clear();

            StopServices();
            NotifyAllServices();

            if (shouldThrowException && exception != null)
            {
                Logger.Log("Error: " + exception);
                throw exception;
            }
            Logger.Verbose("exit");
            TaskScheduler.UnobservedTaskException -= TaskScheduler_UnobservedTaskException;
            return(new TestResultsSummary(allResults));
        }
        public void TestChangeConfigAfterOpen(bool useVisualGrid)
        {
            IWebDriver driver = SeleniumUtils.CreateChromeDriver();

            try
            {
                EyesRunner runner = useVisualGrid ? (EyesRunner) new VisualGridRunner(10) : new ClassicRunner();
                Eyes       eyes   = new Eyes(runner);

                Configuration conf = new Configuration();

                conf.SetAppName("app").SetTestName("test");
                conf.SetBatch(TestDataProvider.BatchInfo);

                conf.SetAccessibilityValidation(null).SetIgnoreDisplacements(false);
                eyes.SetConfiguration(conf);

                eyes.Open(driver);

                eyes.CheckWindow();

                conf.SetAccessibilityValidation(new AccessibilitySettings(AccessibilityLevel.AAA, AccessibilityGuidelinesVersion.WCAG_2_1)).SetIgnoreDisplacements(true);
                eyes.SetConfiguration(conf);

                eyes.CheckWindow();

                conf.SetAccessibilityValidation(new AccessibilitySettings(AccessibilityLevel.AA, AccessibilityGuidelinesVersion.WCAG_2_0)).SetMatchLevel(MatchLevel.Layout);
                eyes.SetConfiguration(conf);

                eyes.CheckWindow();

                eyes.Close(false);
                TestResultsSummary resultsSummary = runner.GetAllTestResults(false);

                Assert.AreEqual(1, resultsSummary.Count);

                TestResultContainer resultsContainer = resultsSummary[0];
                TestResults         results          = resultsContainer.TestResults;

                SessionResults sessionResults = TestUtils.GetSessionResults(eyes.ApiKey, results);

                string browserInfo = resultsContainer.BrowserInfo?.ToString();

                Assert.IsNull(sessionResults.StartInfo.DefaultMatchSettings.AccessibilitySettings, browserInfo);
                Assert.IsFalse(sessionResults.StartInfo.DefaultMatchSettings.IgnoreDisplacements, browserInfo);
                Assert.AreEqual(MatchLevel.Strict, sessionResults.StartInfo.DefaultMatchSettings.MatchLevel, browserInfo);

                Assert.AreEqual(3, sessionResults.ActualAppOutput.Length, browserInfo);

                Assert.IsNull(sessionResults.ActualAppOutput[0].ImageMatchSettings.AccessibilitySettings, browserInfo);

                Assert.AreEqual(AccessibilityLevel.AAA, sessionResults.ActualAppOutput[1].ImageMatchSettings.AccessibilitySettings.Level, browserInfo);
                Assert.AreEqual(AccessibilityGuidelinesVersion.WCAG_2_1, sessionResults.ActualAppOutput[1].ImageMatchSettings.AccessibilitySettings.GuidelinesVersion, browserInfo);
                Assert.IsTrue(sessionResults.ActualAppOutput[1].ImageMatchSettings.IgnoreDisplacements, browserInfo);
                Assert.AreEqual(MatchLevel.Strict, sessionResults.ActualAppOutput[1].ImageMatchSettings.MatchLevel, browserInfo);

                Assert.AreEqual(AccessibilityLevel.AA, sessionResults.ActualAppOutput[2].ImageMatchSettings.AccessibilitySettings.Level, browserInfo);
                Assert.AreEqual(AccessibilityGuidelinesVersion.WCAG_2_0, sessionResults.ActualAppOutput[2].ImageMatchSettings.AccessibilitySettings.GuidelinesVersion, browserInfo);
                Assert.IsTrue(sessionResults.ActualAppOutput[2].ImageMatchSettings.IgnoreDisplacements, browserInfo);
                Assert.AreEqual(MatchLevel.Layout2, sessionResults.ActualAppOutput[2].ImageMatchSettings.MatchLevel, browserInfo);
            }
            finally
            {
                driver.Quit();
            }
        }