internal static void StartAllureLogging(ITest test, string testUuid, string testContainerUuid,
TestFixture fixture)
{
var testFullNameForLog = GenerateFullNameWithParameters(test, test.FullName);
var ourFixture = new TestResultContainer
{
uuid = testContainerUuid,
name = test.ClassName
};
AllureLifecycle.Instance.StartTestContainer(ourFixture);
var testResult = new TestResult
{
uuid = testUuid,
name = test.Name,
fullName = testFullNameForLog,
labels = new List <Label>
{
Label.Thread(),
Label.Host(),
Label.TestClass(test.ClassName),
Label.TestMethod(test.MethodName),
Label.Package(test.ClassName)
},
historyId = testFullNameForLog,
statusDetails = new StatusDetails()
};
AllureLifecycle.Instance.StartTestCase(testResult);
AddInfoInTestCase(test, testUuid, fixture);
}
void HandleTestResults(TestResultContainer summary)
{
Exception ex = summary.Exception;
if (ex != null)
{
TestContext.WriteLine("System error occured while checking target.");
}
TestResults result = summary.TestResults;
if (result == null)
{
TestContext.WriteLine("No test results information available");
}
else
{
TestContext.WriteLine(
"AppName = {0}, testname = {1}, Browser = {2},OS = {3} viewport = {4}x{5}, matched = {6},mismatched = {7}, missing = {8}, aborted = {9}\n",
result.AppName,
result.Name,
result.HostApp,
result.HostOS,
result.HostDisplaySize.Width,
result.HostDisplaySize.Height,
result.Matches,
result.Mismatches,
result.Missing,
(result.IsAborted ? "aborted" : "no"));
}
}
public override void BeforeTest(ITest test)
{
var fixture = new TestResultContainer
{
uuid = test.Id,
name = test.ClassName
};
Allure.StartTestContainer(fixture);
}
public void BeforeTest(ITest suite)
{
_ignoredContainerId = suite.Id + "-ignored";
var fixture = new TestResultContainer
{
uuid = _ignoredContainerId,
name = suite.ClassName
};
AllureLifecycle.Instance.StartTestContainer(fixture);
}
private void StartTestContainer(string fullTestName)
{
_testContainerId = Guid.NewGuid().ToString();
var container = new TestResultContainer
{
uuid = _testContainerId,
name = fullTestName,
};
_allureLifecycle.StartTestContainer(container);
}
//[OneTimeSetUp]
public void AllureBeforeAllTestsInClass()
{
var nunitFixture = TestExecutionContext.CurrentContext.CurrentTest;
var fixture = new TestResultContainer
{
uuid = nunitFixture.Id,
name = nunitFixture.ClassName
};
Allure.StartTestContainer(fixture);
}
private void StartTestContainer()
{
const string testContainerInterfix = "-tc-";
_containerGuid = string.Concat(Guid.NewGuid().ToString(), testContainerInterfix, _test.UniqueID);
var container = new TestResultContainer
{
uuid = _containerGuid,
name = _test.DisplayName
};
AllureLifecycle.StartTestContainer(container);
}
private void StartTestContainer()
{
var testFixture = GetTestFixture(_test);
_containerGuid = string.Concat(Guid.NewGuid().ToString(), "-tc-", testFixture.Id);
var container = new TestResultContainer
{
uuid = _containerGuid,
name = _test.FullName
};
AllureLifecycle.StartTestContainer(container);
}
public void SetupForAllure()
{
Environment.CurrentDirectory = Path.GetDirectoryName(GetType().Assembly.Location);
var nunitFixture = TestExecutionContext.CurrentContext.CurrentTest;
var fixture = new TestResultContainer
{
uuid = nunitFixture.Id,
name = nunitFixture.ClassName
};
Allure.StartTestContainer(fixture);
}
internal static TestResultContainer GetTestResultContainer()
{
var trc = new TestResultContainer();
trc.uuid = Guid.NewGuid().ToString("N");
trc.name = trc.uuid;
trc.description = trc.uuid;
trc.links = new List <Link>()
{
Link.AddIssue(name: new Random().Next(999999).ToString(), url: "/bug"),
Link.AddTms(name: new Random().Next(999999).ToString(), url: "/test")
};
return(trc);
}
internal static TestResultContainer StartTestContainer(FeatureContext featureContext,
ScenarioContext scenarioContext)
{
var containerId = GetFeatureContainerId(featureContext?.FeatureInfo);
var scenarioContainer = new TestResultContainer
{
uuid = NewId()
};
AllureLifecycle.Instance.StartTestContainer(containerId, scenarioContainer);
scenarioContext?.Set(scenarioContainer);
featureContext?.Get <HashSet <TestResultContainer> >().Add(scenarioContainer);
return(scenarioContainer);
}
public BaseTest(OneTimeSetupTearDownFixture testSuite, ITestOutputHelper output)
{
_output = (TestOutputHelper)output;
_testSuite = testSuite;
testContainer = new TestResultContainer();
var type = _output.GetType();
var testMember = type.GetField("test", BindingFlags.Instance | BindingFlags.NonPublic);
var test = (ITest)testMember.GetValue(_output);
testContainer.uuid = test.TestCase.UniqueID;
testContainer.description = test.TestCase.UniqueID;
_testSuite.lifeCycle.StartTestContainer(testContainer);
_testSuite.driver.Manage().Window.Maximize();
}
public static string addTestContainer(testContainer testContainer)
{
var trc = new TestResultContainer();
if (string.IsNullOrWhiteSpace(testContainer.id))
{
testContainer.id = Guid.NewGuid().ToString("N");
}
trc.uuid = testContainer.id;
if (string.IsNullOrWhiteSpace(testContainer.name))
{
testContainer.name = "Container Name " + trc.uuid;
}
trc.name = testContainer.name;
if (!string.IsNullOrWhiteSpace(testContainer.description))
{
testContainer.description = "Container Description " + trc.uuid;
}
trc.description = testContainer.description;
if (testContainer.listLinks != null)
{
if (testContainer.listLinks.Count > 0)
{
trc.links = new List <Link>();
for (int i = 0; i < testContainer.listLinks.Count; i++)
{
if (testContainer.listLinks[i].isIssue)
{
Link.Issue(name: testContainer.listLinks[i].name, url: testContainer.listLinks[i].url);
}
else
{
Link.Tms(name: testContainer.listLinks[i].name, url: testContainer.listLinks[i].url);
}
}
}
}
instance.StartTestContainer(trc);
return(trc.uuid);
}
public void Write(TestResultContainer testResult)
{
Write(testResult, AllureConstants.TestResultContainerFileSuffix);
}
public override object InvokeBinding(IBinding binding, IContextManager contextManager, object[] arguments,
ITestTracer testTracer, out TimeSpan duration)
{
var hook = binding as HookBinding;
// process hook
if (hook != null)
{
var featureContainerId = AllureHelper.GetFeatureContainerId(contextManager.FeatureContext?.FeatureInfo);
switch (hook.HookType)
{
case HookType.BeforeFeature:
if (hook.HookOrder == int.MinValue)
{
// starting point
var featureContainer = new TestResultContainer
{
uuid = AllureHelper.GetFeatureContainerId(contextManager.FeatureContext?.FeatureInfo)
};
allure.StartTestContainer(featureContainer);
contextManager.FeatureContext.Set(new HashSet <TestResultContainer>());
contextManager.FeatureContext.Set(new HashSet <TestResult>());
return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
}
else
{
try
{
this.StartFixture(hook, featureContainerId);
var result = base.InvokeBinding(binding, contextManager, arguments, testTracer,
out duration);
allure.StopFixture(x => x.status = Status.passed);
return(result);
}
catch (Exception ex)
{
allure.StopFixture(x => x.status = Status.broken);
// if BeforeFeature is failed execution is stopped. We need to create, update, stop and write everything here.
// create fake scenario container
var scenarioContainer =
AllureHelper.StartTestContainer(contextManager.FeatureContext, null);
// start fake scenario
var scenario = AllureHelper.StartTestCase(scenarioContainer.uuid,
contextManager.FeatureContext, null);
// update, stop and write
allure
.StopTestCase(x =>
{
x.status = Status.broken;
x.statusDetails = AllureHelper.GetStatusDetails(ex);
})
.WriteTestCase(scenario.uuid)
.StopTestContainer(scenarioContainer.uuid)
.WriteTestContainer(scenarioContainer.uuid)
.StopTestContainer(featureContainerId)
.WriteTestContainer(featureContainerId);
throw;
}
}
case HookType.BeforeStep:
case HookType.AfterStep:
{
var scenario = AllureHelper.GetCurrentTestCase(contextManager.ScenarioContext);
try
{
return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
}
catch (Exception ex)
{
allure
.UpdateTestCase(scenario.uuid,
x =>
{
x.status = Status.broken;
x.statusDetails = AllureHelper.GetStatusDetails(ex);
});
throw;
}
}
case HookType.BeforeScenario:
case HookType.AfterScenario:
if (hook.HookOrder == int.MinValue || hook.HookOrder == int.MaxValue)
{
return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
}
else
{
var scenarioContainer = AllureHelper.GetCurrentTestConainer(contextManager.ScenarioContext);
try
{
this.StartFixture(hook, scenarioContainer.uuid);
var result = base.InvokeBinding(binding, contextManager, arguments, testTracer,
out duration);
allure.StopFixture(x => x.status = Status.passed);
return(result);
}
catch (Exception ex)
{
allure.StopFixture(x => x.status = Status.broken);
// get or add new scenario
var scenario = AllureHelper.GetCurrentTestCase(contextManager.ScenarioContext) ??
AllureHelper.StartTestCase(scenarioContainer.uuid,
contextManager.FeatureContext, contextManager.ScenarioContext);
allure.UpdateTestCase(scenario.uuid,
x =>
{
x.status = Status.broken;
x.statusDetails = AllureHelper.GetStatusDetails(ex);
});
throw;
}
}
case HookType.AfterFeature:
if (hook.HookOrder == int.MaxValue)
// finish point
{
WriteScenarios(contextManager);
allure
.StopTestContainer(featureContainerId)
.WriteTestContainer(featureContainerId);
return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
}
else
{
try
{
StartFixture(hook, featureContainerId);
var result = base.InvokeBinding(binding, contextManager, arguments, testTracer,
out duration);
allure.StopFixture(x => x.status = Status.passed);
return(result);
}
catch (Exception ex)
{
var scenario = contextManager.FeatureContext.Get <HashSet <TestResult> >().Last();
allure
.StopFixture(x => x.status = Status.broken)
.UpdateTestCase(scenario.uuid,
x =>
{
x.status = Status.broken;
x.statusDetails = AllureHelper.GetStatusDetails(ex);
});
WriteScenarios(contextManager);
allure
.StopTestContainer(featureContainerId)
.WriteTestContainer(featureContainerId);
throw;
}
}
case HookType.BeforeScenarioBlock:
case HookType.AfterScenarioBlock:
case HookType.BeforeTestRun:
case HookType.AfterTestRun:
default:
return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
}
}
else
{
return(base.InvokeBinding(binding, contextManager, arguments, testTracer, out duration));
}
}
public void Write(TestResultContainer testResult)
{
this.Write(testResult, AllureConstants.TEST_RESULT_CONTAINER_FILE_SUFFIX);
}
public void Write(TestResultContainer testResult)
{
LinkHelper.UpdateLinks(testResult.links, configuration.Links);
Write(testResult, AllureConstants.TEST_RESULT_CONTAINER_FILE_SUFFIX);
}
public TestResultContainer Call(object state)
{
try
{
logger_.Log("enter - type: {0} ; name: {1}", TaskType, RunningTest.TestName);
testResults_ = null;
switch (TaskType)
{
case TaskType.Open:
logger_.Log("Task.run opening task");
if (renderResult_ != null)
{
string userAgent = renderResult_.UserAgent;
RectangleSize deviceSize = renderResult_.DeviceSize;
logger_.Verbose("setting device size: {0}", deviceSize);
EyesConnector.SetUserAgent(userAgent);
EyesConnector.SetDeviceSize(deviceSize);
}
else
{
// We are in exception mode - trying to do eyes.open() without first render
RenderBrowserInfo browserInfo = RunningTest.BrowserInfo;
EyesConnector.SetDeviceSize(new Size(browserInfo.Width, browserInfo.Height));
}
EyesConnector.Open(configuration_);
break;
case TaskType.Check:
logger_.Log("Task.run check task");
try
{
string imageLocation = renderResult_.ImageLocation;
string domLocation = renderResult_.DomLocation;
VGRegion[] vgRegions = renderResult_.SelectorRegions;
IList <IRegion> regions = new List <IRegion>();
if (vgRegions != null)
{
foreach (VGRegion reg in vgRegions)
{
if (!(string.IsNullOrWhiteSpace(reg.Error)))
{
logger_.Log("Warning: region error: {0}", reg.Error);
}
else
{
regions.Add(reg);
}
}
}
logger_.Verbose(renderResult_.ToString());
if (imageLocation == null)
{
logger_.Log("CHECKING IMAGE WITH NULL LOCATION!");
}
Location location = null;
if (regionSelectors_.Count > 0 && regions.Count > 0)
{
VisualGridSelector[] targetSelector = regionSelectors_[regionSelectors_.Count - 1];
if (targetSelector.Length > 0 && "target".Equals(targetSelector[0].Category))
{
location = regions[regions.Count - 1].Location;
}
}
EyesConnector.MatchWindow(configuration_, imageLocation, domLocation, checkSettings_, regions,
regionSelectors_, location, renderResult_, source_);
}
catch (WebException we)
{
Stream stream = we.Response.GetResponseStream();
byte[] responseBodyBytes = CommonUtils.ReadToEnd(stream);
string responseBodyStr = Encoding.UTF8.GetString(responseBodyBytes);
logger_.Log($"Error: {we}\n{responseBodyStr}");
}
catch (Exception e)
{
logger_.Log("Error: " + e);
}
break;
case TaskType.Close:
logger_.Log("Task.run close task");
try
{
testResults_ = EyesConnector.Close(true, configuration_);
}
catch (TestFailedException tfe)
{
logger_.Log("Test Failed: " + tfe);
Exception = tfe;
testResults_ = tfe.TestResults;
}
catch (Exception e)
{
logger_.Log("Error: " + e);
Exception = e;
}
break;
case TaskType.Abort:
logger_.Log("Task.run abort task");
testResults_ = EyesConnector.AbortIfNotClosed();
break;
}
TestResultContainer testResultContainer = new TestResultContainer(testResults_, BrowserInfo, Exception);
NotifySuccessAllListeners();
return(testResultContainer);
}
catch (Exception e)
{
logger_.Log("Error: " + e);
NotifyFailureAllListeners(e);
}
finally
{
logger_.Verbose("marking {0} task as complete: {1}", TaskType, RunningTest.TestName);
IsTaskComplete = true;
//call the callback
}
return(null);
}
protected override TestResultsSummary GetAllTestResultsImpl(bool shouldThrowException)
{
Logger.Verbose("enter");
Dictionary <IVisualGridEyes, ICollection <Task <TestResultContainer> > > allFutures = new Dictionary <IVisualGridEyes, ICollection <Task <TestResultContainer> > >();
List <IVisualGridEyes> allEyes;
lock (allEyes_)
{
allEyes = new List <IVisualGridEyes>(allEyes_);
}
foreach (IVisualGridEyes eyes in allEyes)
{
ICollection <Task <TestResultContainer> > futureList = eyes.Close();
Logger.Verbose("adding a {0} items list to allFutures.", futureList.Count);
if (allFutures.TryGetValue(eyes, out ICollection <Task <TestResultContainer> > futures) && futures != null)
{
foreach (Task <TestResultContainer> future in futures)
{
futureList.Add(future);
}
}
allFutures.Add(eyes, futureList);
}
Exception exception = null;
NotifyAllServices();
List <TestResultContainer> allResults = new List <TestResultContainer>();
Logger.Verbose("trying to call future.get on {0} future lists.", allFutures.Count);
foreach (KeyValuePair <IVisualGridEyes, ICollection <Task <TestResultContainer> > > entry in allFutures)
{
ICollection <Task <TestResultContainer> > value = entry.Value;
IVisualGridEyes key = entry.Key;
key.TestResults.Clear();
Logger.Verbose("trying to call future.get on {0} futures of {1}", value.Count, key);
foreach (Task <TestResultContainer> future in value)
{
Logger.Verbose("calling future.get on {0}", key);
TestResultContainer testResultContainer = null;
try
{
if (Task.WhenAny(future, Task.Delay(waitForResultTimeout_)).Result == future)
{
testResultContainer = future.Result;
if (testResultContainer.Exception != null && exception == null)
{
exception = testResultContainer.Exception;
}
allResults.Add(testResultContainer);
key.TestResults.Add(testResultContainer);
}
}
catch (Exception e)
{
Logger.Log("Error: " + e);
if (exception == null)
{
exception = e;
}
}
Logger.Verbose("got TestResultContainer: {0}", testResultContainer);
}
}
//lock (allEyes_) allEyes_.Clear();
//eyesToOpenList_.Clear();
//renderingTaskList_.Clear();
StopServices();
NotifyAllServices();
if (shouldThrowException && exception != null)
{
Logger.Log("Error: " + exception);
throw exception;
}
Logger.Verbose("exit");
TaskScheduler.UnobservedTaskException -= TaskScheduler_UnobservedTaskException;
return(new TestResultsSummary(allResults));
}
public void TestChangeConfigAfterOpen(bool useVisualGrid)
{
IWebDriver driver = SeleniumUtils.CreateChromeDriver();
try
{
EyesRunner runner = useVisualGrid ? (EyesRunner) new VisualGridRunner(10) : new ClassicRunner();
Eyes eyes = new Eyes(runner);
Configuration conf = new Configuration();
conf.SetAppName("app").SetTestName("test");
conf.SetBatch(TestDataProvider.BatchInfo);
conf.SetAccessibilityValidation(null).SetIgnoreDisplacements(false);
eyes.SetConfiguration(conf);
eyes.Open(driver);
eyes.CheckWindow();
conf.SetAccessibilityValidation(new AccessibilitySettings(AccessibilityLevel.AAA, AccessibilityGuidelinesVersion.WCAG_2_1)).SetIgnoreDisplacements(true);
eyes.SetConfiguration(conf);
eyes.CheckWindow();
conf.SetAccessibilityValidation(new AccessibilitySettings(AccessibilityLevel.AA, AccessibilityGuidelinesVersion.WCAG_2_0)).SetMatchLevel(MatchLevel.Layout);
eyes.SetConfiguration(conf);
eyes.CheckWindow();
eyes.Close(false);
TestResultsSummary resultsSummary = runner.GetAllTestResults(false);
Assert.AreEqual(1, resultsSummary.Count);
TestResultContainer resultsContainer = resultsSummary[0];
TestResults results = resultsContainer.TestResults;
SessionResults sessionResults = TestUtils.GetSessionResults(eyes.ApiKey, results);
string browserInfo = resultsContainer.BrowserInfo?.ToString();
Assert.IsNull(sessionResults.StartInfo.DefaultMatchSettings.AccessibilitySettings, browserInfo);
Assert.IsFalse(sessionResults.StartInfo.DefaultMatchSettings.IgnoreDisplacements, browserInfo);
Assert.AreEqual(MatchLevel.Strict, sessionResults.StartInfo.DefaultMatchSettings.MatchLevel, browserInfo);
Assert.AreEqual(3, sessionResults.ActualAppOutput.Length, browserInfo);
Assert.IsNull(sessionResults.ActualAppOutput[0].ImageMatchSettings.AccessibilitySettings, browserInfo);
Assert.AreEqual(AccessibilityLevel.AAA, sessionResults.ActualAppOutput[1].ImageMatchSettings.AccessibilitySettings.Level, browserInfo);
Assert.AreEqual(AccessibilityGuidelinesVersion.WCAG_2_1, sessionResults.ActualAppOutput[1].ImageMatchSettings.AccessibilitySettings.GuidelinesVersion, browserInfo);
Assert.IsTrue(sessionResults.ActualAppOutput[1].ImageMatchSettings.IgnoreDisplacements, browserInfo);
Assert.AreEqual(MatchLevel.Strict, sessionResults.ActualAppOutput[1].ImageMatchSettings.MatchLevel, browserInfo);
Assert.AreEqual(AccessibilityLevel.AA, sessionResults.ActualAppOutput[2].ImageMatchSettings.AccessibilitySettings.Level, browserInfo);
Assert.AreEqual(AccessibilityGuidelinesVersion.WCAG_2_0, sessionResults.ActualAppOutput[2].ImageMatchSettings.AccessibilitySettings.GuidelinesVersion, browserInfo);
Assert.IsTrue(sessionResults.ActualAppOutput[2].ImageMatchSettings.IgnoreDisplacements, browserInfo);
Assert.AreEqual(MatchLevel.Layout2, sessionResults.ActualAppOutput[2].ImageMatchSettings.MatchLevel, browserInfo);
}
finally
{
driver.Quit();
}
}
