internal static void StartAllureLogging(ITest test, string testUuid, string testContainerUuid, TestFixture fixture) { var testFullNameForLog = GenerateFullNameWithParameters(test, test.FullName); var ourFixture = new TestResultContainer { uuid = testContainerUuid, name = test.ClassName }; AllureLifecycle.Instance.StartTestContainer(ourFixture); var testResult = new TestResult { uuid = testUuid, name = test.Name, fullName = testFullNameForLog, labels = new List <Label> { Label.Thread(), Label.Host(), Label.TestClass(test.ClassName), Label.TestMethod(test.MethodName), Label.Package(test.ClassName) }, historyId = testFullNameForLog, statusDetails = new StatusDetails() }; AllureLifecycle.Instance.StartTestCase(testResult); AddInfoInTestCase(test, testUuid, fixture); }
void HandleTestResults(TestResultContainer summary) { Exception ex = summary.Exception; if (ex != null) { TestContext.WriteLine("System error occured while checking target."); } TestResults result = summary.TestResults; if (result == null) { TestContext.WriteLine("No test results information available"); } else { TestContext.WriteLine( "AppName = {0}, testname = {1}, Browser = {2},OS = {3} viewport = {4}x{5}, matched = {6},mismatched = {7}, missing = {8}, aborted = {9}\n", result.AppName, result.Name, result.HostApp, result.HostOS, result.HostDisplaySize.Width, result.HostDisplaySize.Height, result.Matches, result.Mismatches, result.Missing, (result.IsAborted ? "aborted" : "no")); } }
public override void BeforeTest(ITest test) { var fixture = new TestResultContainer { uuid = test.Id, name = test.ClassName }; Allure.StartTestContainer(fixture); }
public void BeforeTest(ITest suite) { _ignoredContainerId = suite.Id + "-ignored"; var fixture = new TestResultContainer { uuid = _ignoredContainerId, name = suite.ClassName }; AllureLifecycle.Instance.StartTestContainer(fixture); }
private void StartTestContainer(string fullTestName) { _testContainerId = Guid.NewGuid().ToString(); var container = new TestResultContainer { uuid = _testContainerId, name = fullTestName, }; _allureLifecycle.StartTestContainer(container); }
//[OneTimeSetUp] public void AllureBeforeAllTestsInClass() { var nunitFixture = TestExecutionContext.CurrentContext.CurrentTest; var fixture = new TestResultContainer { uuid = nunitFixture.Id, name = nunitFixture.ClassName }; Allure.StartTestContainer(fixture); }
private void StartTestContainer() { const string testContainerInterfix = "-tc-"; _containerGuid = string.Concat(Guid.NewGuid().ToString(), testContainerInterfix, _test.UniqueID); var container = new TestResultContainer { uuid = _containerGuid, name = _test.DisplayName }; AllureLifecycle.StartTestContainer(container); }
private void StartTestContainer() { var testFixture = GetTestFixture(_test); _containerGuid = string.Concat(Guid.NewGuid().ToString(), "-tc-", testFixture.Id); var container = new TestResultContainer { uuid = _containerGuid, name = _test.FullName }; AllureLifecycle.StartTestContainer(container); }
public void SetupForAllure() { Environment.CurrentDirectory = Path.GetDirectoryName(GetType().Assembly.Location); var nunitFixture = TestExecutionContext.CurrentContext.CurrentTest; var fixture = new TestResultContainer { uuid = nunitFixture.Id, name = nunitFixture.ClassName }; Allure.StartTestContainer(fixture); }
internal static TestResultContainer GetTestResultContainer() { var trc = new TestResultContainer(); trc.uuid = Guid.NewGuid().ToString("N"); trc.name = trc.uuid; trc.description = trc.uuid; trc.links = new List <Link>() { Link.AddIssue(name: new Random().Next(999999).ToString(), url: "/bug"), Link.AddTms(name: new Random().Next(999999).ToString(), url: "/test") }; return(trc); }
internal static TestResultContainer StartTestContainer(FeatureContext featureContext, ScenarioContext scenarioContext) { var containerId = GetFeatureContainerId(featureContext?.FeatureInfo); var scenarioContainer = new TestResultContainer { uuid = NewId() }; AllureLifecycle.Instance.StartTestContainer(containerId, scenarioContainer); scenarioContext?.Set(scenarioContainer); featureContext?.Get <HashSet <TestResultContainer> >().Add(scenarioContainer); return(scenarioContainer); }
public BaseTest(OneTimeSetupTearDownFixture testSuite, ITestOutputHelper output) { _output = (TestOutputHelper)output; _testSuite = testSuite; testContainer = new TestResultContainer(); var type = _output.GetType(); var testMember = type.GetField("test", BindingFlags.Instance | BindingFlags.NonPublic); var test = (ITest)testMember.GetValue(_output); testContainer.uuid = test.TestCase.UniqueID; testContainer.description = test.TestCase.UniqueID; _testSuite.lifeCycle.StartTestContainer(testContainer); _testSuite.driver.Manage().Window.Maximize(); }
public static string addTestContainer(testContainer testContainer) { var trc = new TestResultContainer(); if (string.IsNullOrWhiteSpace(testContainer.id)) { testContainer.id = Guid.NewGuid().ToString("N"); } trc.uuid = testContainer.id; if (string.IsNullOrWhiteSpace(testContainer.name)) { testContainer.name = "Container Name " + trc.uuid; } trc.name = testContainer.name; if (!string.IsNullOrWhiteSpace(testContainer.description)) { testContainer.description = "Container Description " + trc.uuid; } trc.description = testContainer.description; if (testContainer.listLinks != null) { if (testContainer.listLinks.Count > 0) { trc.links = new List <Link>(); for (int i = 0; i < testContainer.listLinks.Count; i++) { if (testContainer.listLinks[i].isIssue) { Link.Issue(name: testContainer.listLinks[i].name, url: testContainer.listLinks[i].url); } else { Link.Tms(name: testContainer.listLinks[i].name, url: testContainer.listLinks[i].url); } } } } instance.StartTestContainer(trc); return(trc.uuid); }
public void Write(TestResultContainer testResult) { Write(testResult, AllureConstants.TestResultContainerFileSuffix); }
public override object InvokeBinding(IBinding binding, IContextManager contextManager, object[] arguments, ITestTracer testTracer, out TimeSpan duration) { var hook = binding as HookBinding; // process hook if (hook != null) { var featureContainerId = AllureHelper.GetFeatureContainerId(contextManager.FeatureContext?.FeatureInfo); switch (hook.HookType) { case HookType.BeforeFeature: if (hook.HookOrder == int.MinValue) { // starting point var featureContainer = new TestResultContainer { uuid = AllureHelper.GetFeatureContainerId(contextManager.FeatureContext?.FeatureInfo) }; allure.StartTestContainer(featureContainer); contextManager.FeatureContext.Set(new HashSet <TestResultContainer>()); contextManager.FeatureContext.Set(new HashSet <TestResult>()); return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration)); } else { try { this.StartFixture(hook, featureContainerId); var result = base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration); allure.StopFixture(x => x.status = Status.passed); return(result); } catch (Exception ex) { allure.StopFixture(x => x.status = Status.broken); // if BeforeFeature is failed execution is stopped. We need to create, update, stop and write everything here. // create fake scenario container var scenarioContainer = AllureHelper.StartTestContainer(contextManager.FeatureContext, null); // start fake scenario var scenario = AllureHelper.StartTestCase(scenarioContainer.uuid, contextManager.FeatureContext, null); // update, stop and write allure .StopTestCase(x => { x.status = Status.broken; x.statusDetails = AllureHelper.GetStatusDetails(ex); }) .WriteTestCase(scenario.uuid) .StopTestContainer(scenarioContainer.uuid) .WriteTestContainer(scenarioContainer.uuid) .StopTestContainer(featureContainerId) .WriteTestContainer(featureContainerId); throw; } } case HookType.BeforeStep: case HookType.AfterStep: { var scenario = AllureHelper.GetCurrentTestCase(contextManager.ScenarioContext); try { return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration)); } catch (Exception ex) { allure .UpdateTestCase(scenario.uuid, x => { x.status = Status.broken; x.statusDetails = AllureHelper.GetStatusDetails(ex); }); throw; } } case HookType.BeforeScenario: case HookType.AfterScenario: if (hook.HookOrder == int.MinValue || hook.HookOrder == int.MaxValue) { return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration)); } else { var scenarioContainer = AllureHelper.GetCurrentTestConainer(contextManager.ScenarioContext); try { this.StartFixture(hook, scenarioContainer.uuid); var result = base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration); allure.StopFixture(x => x.status = Status.passed); return(result); } catch (Exception ex) { allure.StopFixture(x => x.status = Status.broken); // get or add new scenario var scenario = AllureHelper.GetCurrentTestCase(contextManager.ScenarioContext) ?? AllureHelper.StartTestCase(scenarioContainer.uuid, contextManager.FeatureContext, contextManager.ScenarioContext); allure.UpdateTestCase(scenario.uuid, x => { x.status = Status.broken; x.statusDetails = AllureHelper.GetStatusDetails(ex); }); throw; } } case HookType.AfterFeature: if (hook.HookOrder == int.MaxValue) // finish point { WriteScenarios(contextManager); allure .StopTestContainer(featureContainerId) .WriteTestContainer(featureContainerId); return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration)); } else { try { StartFixture(hook, featureContainerId); var result = base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration); allure.StopFixture(x => x.status = Status.passed); return(result); } catch (Exception ex) { var scenario = contextManager.FeatureContext.Get <HashSet <TestResult> >().Last(); allure .StopFixture(x => x.status = Status.broken) .UpdateTestCase(scenario.uuid, x => { x.status = Status.broken; x.statusDetails = AllureHelper.GetStatusDetails(ex); }); WriteScenarios(contextManager); allure .StopTestContainer(featureContainerId) .WriteTestContainer(featureContainerId); throw; } } case HookType.BeforeScenarioBlock: case HookType.AfterScenarioBlock: case HookType.BeforeTestRun: case HookType.AfterTestRun: default: return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration)); } } else { return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration)); } }
public void Write(TestResultContainer testResult) { this.Write(testResult, AllureConstants.TEST_RESULT_CONTAINER_FILE_SUFFIX); }
public void Write(TestResultContainer testResult) { LinkHelper.UpdateLinks(testResult.links, configuration.Links); Write(testResult, AllureConstants.TEST_RESULT_CONTAINER_FILE_SUFFIX); }
public TestResultContainer Call(object state) { try { logger_.Log("enter - type: {0} ; name: {1}", TaskType, RunningTest.TestName); testResults_ = null; switch (TaskType) { case TaskType.Open: logger_.Log("Task.run opening task"); if (renderResult_ != null) { string userAgent = renderResult_.UserAgent; RectangleSize deviceSize = renderResult_.DeviceSize; logger_.Verbose("setting device size: {0}", deviceSize); EyesConnector.SetUserAgent(userAgent); EyesConnector.SetDeviceSize(deviceSize); } else { // We are in exception mode - trying to do eyes.open() without first render RenderBrowserInfo browserInfo = RunningTest.BrowserInfo; EyesConnector.SetDeviceSize(new Size(browserInfo.Width, browserInfo.Height)); } EyesConnector.Open(configuration_); break; case TaskType.Check: logger_.Log("Task.run check task"); try { string imageLocation = renderResult_.ImageLocation; string domLocation = renderResult_.DomLocation; VGRegion[] vgRegions = renderResult_.SelectorRegions; IList <IRegion> regions = new List <IRegion>(); if (vgRegions != null) { foreach (VGRegion reg in vgRegions) { if (!(string.IsNullOrWhiteSpace(reg.Error))) { logger_.Log("Warning: region error: {0}", reg.Error); } else { regions.Add(reg); } } } logger_.Verbose(renderResult_.ToString()); if (imageLocation == null) { logger_.Log("CHECKING IMAGE WITH NULL LOCATION!"); } Location location = null; if (regionSelectors_.Count > 0 && regions.Count > 0) { VisualGridSelector[] targetSelector = regionSelectors_[regionSelectors_.Count - 1]; if (targetSelector.Length > 0 && "target".Equals(targetSelector[0].Category)) { location = regions[regions.Count - 1].Location; } } EyesConnector.MatchWindow(configuration_, imageLocation, domLocation, checkSettings_, regions, regionSelectors_, location, renderResult_, source_); } catch (WebException we) { Stream stream = we.Response.GetResponseStream(); byte[] responseBodyBytes = CommonUtils.ReadToEnd(stream); string responseBodyStr = Encoding.UTF8.GetString(responseBodyBytes); logger_.Log($"Error: {we}\n{responseBodyStr}"); } catch (Exception e) { logger_.Log("Error: " + e); } break; case TaskType.Close: logger_.Log("Task.run close task"); try { testResults_ = EyesConnector.Close(true, configuration_); } catch (TestFailedException tfe) { logger_.Log("Test Failed: " + tfe); Exception = tfe; testResults_ = tfe.TestResults; } catch (Exception e) { logger_.Log("Error: " + e); Exception = e; } break; case TaskType.Abort: logger_.Log("Task.run abort task"); testResults_ = EyesConnector.AbortIfNotClosed(); break; } TestResultContainer testResultContainer = new TestResultContainer(testResults_, BrowserInfo, Exception); NotifySuccessAllListeners(); return(testResultContainer); } catch (Exception e) { logger_.Log("Error: " + e); NotifyFailureAllListeners(e); } finally { logger_.Verbose("marking {0} task as complete: {1}", TaskType, RunningTest.TestName); IsTaskComplete = true; //call the callback } return(null); }
protected override TestResultsSummary GetAllTestResultsImpl(bool shouldThrowException) { Logger.Verbose("enter"); Dictionary <IVisualGridEyes, ICollection <Task <TestResultContainer> > > allFutures = new Dictionary <IVisualGridEyes, ICollection <Task <TestResultContainer> > >(); List <IVisualGridEyes> allEyes; lock (allEyes_) { allEyes = new List <IVisualGridEyes>(allEyes_); } foreach (IVisualGridEyes eyes in allEyes) { ICollection <Task <TestResultContainer> > futureList = eyes.Close(); Logger.Verbose("adding a {0} items list to allFutures.", futureList.Count); if (allFutures.TryGetValue(eyes, out ICollection <Task <TestResultContainer> > futures) && futures != null) { foreach (Task <TestResultContainer> future in futures) { futureList.Add(future); } } allFutures.Add(eyes, futureList); } Exception exception = null; NotifyAllServices(); List <TestResultContainer> allResults = new List <TestResultContainer>(); Logger.Verbose("trying to call future.get on {0} future lists.", allFutures.Count); foreach (KeyValuePair <IVisualGridEyes, ICollection <Task <TestResultContainer> > > entry in allFutures) { ICollection <Task <TestResultContainer> > value = entry.Value; IVisualGridEyes key = entry.Key; key.TestResults.Clear(); Logger.Verbose("trying to call future.get on {0} futures of {1}", value.Count, key); foreach (Task <TestResultContainer> future in value) { Logger.Verbose("calling future.get on {0}", key); TestResultContainer testResultContainer = null; try { if (Task.WhenAny(future, Task.Delay(waitForResultTimeout_)).Result == future) { testResultContainer = future.Result; if (testResultContainer.Exception != null && exception == null) { exception = testResultContainer.Exception; } allResults.Add(testResultContainer); key.TestResults.Add(testResultContainer); } } catch (Exception e) { Logger.Log("Error: " + e); if (exception == null) { exception = e; } } Logger.Verbose("got TestResultContainer: {0}", testResultContainer); } } //lock (allEyes_) allEyes_.Clear(); //eyesToOpenList_.Clear(); //renderingTaskList_.Clear(); StopServices(); NotifyAllServices(); if (shouldThrowException && exception != null) { Logger.Log("Error: " + exception); throw exception; } Logger.Verbose("exit"); TaskScheduler.UnobservedTaskException -= TaskScheduler_UnobservedTaskException; return(new TestResultsSummary(allResults)); }
public void TestChangeConfigAfterOpen(bool useVisualGrid) { IWebDriver driver = SeleniumUtils.CreateChromeDriver(); try { EyesRunner runner = useVisualGrid ? (EyesRunner) new VisualGridRunner(10) : new ClassicRunner(); Eyes eyes = new Eyes(runner); Configuration conf = new Configuration(); conf.SetAppName("app").SetTestName("test"); conf.SetBatch(TestDataProvider.BatchInfo); conf.SetAccessibilityValidation(null).SetIgnoreDisplacements(false); eyes.SetConfiguration(conf); eyes.Open(driver); eyes.CheckWindow(); conf.SetAccessibilityValidation(new AccessibilitySettings(AccessibilityLevel.AAA, AccessibilityGuidelinesVersion.WCAG_2_1)).SetIgnoreDisplacements(true); eyes.SetConfiguration(conf); eyes.CheckWindow(); conf.SetAccessibilityValidation(new AccessibilitySettings(AccessibilityLevel.AA, AccessibilityGuidelinesVersion.WCAG_2_0)).SetMatchLevel(MatchLevel.Layout); eyes.SetConfiguration(conf); eyes.CheckWindow(); eyes.Close(false); TestResultsSummary resultsSummary = runner.GetAllTestResults(false); Assert.AreEqual(1, resultsSummary.Count); TestResultContainer resultsContainer = resultsSummary[0]; TestResults results = resultsContainer.TestResults; SessionResults sessionResults = TestUtils.GetSessionResults(eyes.ApiKey, results); string browserInfo = resultsContainer.BrowserInfo?.ToString(); Assert.IsNull(sessionResults.StartInfo.DefaultMatchSettings.AccessibilitySettings, browserInfo); Assert.IsFalse(sessionResults.StartInfo.DefaultMatchSettings.IgnoreDisplacements, browserInfo); Assert.AreEqual(MatchLevel.Strict, sessionResults.StartInfo.DefaultMatchSettings.MatchLevel, browserInfo); Assert.AreEqual(3, sessionResults.ActualAppOutput.Length, browserInfo); Assert.IsNull(sessionResults.ActualAppOutput[0].ImageMatchSettings.AccessibilitySettings, browserInfo); Assert.AreEqual(AccessibilityLevel.AAA, sessionResults.ActualAppOutput[1].ImageMatchSettings.AccessibilitySettings.Level, browserInfo); Assert.AreEqual(AccessibilityGuidelinesVersion.WCAG_2_1, sessionResults.ActualAppOutput[1].ImageMatchSettings.AccessibilitySettings.GuidelinesVersion, browserInfo); Assert.IsTrue(sessionResults.ActualAppOutput[1].ImageMatchSettings.IgnoreDisplacements, browserInfo); Assert.AreEqual(MatchLevel.Strict, sessionResults.ActualAppOutput[1].ImageMatchSettings.MatchLevel, browserInfo); Assert.AreEqual(AccessibilityLevel.AA, sessionResults.ActualAppOutput[2].ImageMatchSettings.AccessibilitySettings.Level, browserInfo); Assert.AreEqual(AccessibilityGuidelinesVersion.WCAG_2_0, sessionResults.ActualAppOutput[2].ImageMatchSettings.AccessibilitySettings.GuidelinesVersion, browserInfo); Assert.IsTrue(sessionResults.ActualAppOutput[2].ImageMatchSettings.IgnoreDisplacements, browserInfo); Assert.AreEqual(MatchLevel.Layout2, sessionResults.ActualAppOutput[2].ImageMatchSettings.MatchLevel, browserInfo); } finally { driver.Quit(); } }