private List <FixtureResult> BuildFixtureResults(NUnitHelpMethodType type, TestFixture testFixture)
{
var fixtureResultsList = new HashSet <FixtureResult>();
var testResult = TestExecutionContext.CurrentContext.CurrentResult;
foreach (var method in GetNUnitHelpMethods(type, testFixture))
{
var fr = new FixtureResult {
name = method
};
if (testResult.StackTrace != null && testResult.StackTrace.Contains(method))
{
AllureLifecycle.UpdateTestCase(x => x.description += $"\n{method} {type} method failed\n");
fr.status = Status.failed;
fr.statusDetails.message = testResult.Message;
fr.statusDetails.trace = testResult.StackTrace;
if (type == NUnitHelpMethodType.SetUp)
{
_isSetupFailed = true;
}
}
else
{
fr.status = Status.passed;
}
fixtureResultsList.Add(fr);
}
return(fixtureResultsList.ToList());
}
/// <summary>
/// Reports the specified fixture running result to the specified XML element.
/// </summary>
/// <param name="result">The fixture running result.</param>
/// <param name="element">The XML element to which the result is reported.</param>
protected virtual void ReportFixture(FixtureResult result, XElement element)
{
var fixtureElement = new XElement("fixture",
new XAttribute("type", result.FixtureDescriptor.FixtureAttributeType),
new XAttribute("name", result.FixtureDescriptor.Name),
new XAttribute("fullName", result.FixtureDescriptor.FullName),
new XAttribute("description", result.FixtureDescriptor.Description),
new XAttribute("tag", result.FixtureDescriptor.Tag ?? string.Empty),
new XAttribute("benefit", result.FixtureDescriptor.Benefit ?? string.Empty),
new XAttribute("role", result.FixtureDescriptor.Role ?? string.Empty),
new XAttribute("feature", result.FixtureDescriptor.Feature ?? string.Empty),
new XAttribute("background", result.FixtureDescriptor.Background ?? string.Empty),
new XAttribute("status", result.Status),
new XAttribute("startTime", result.StartTime.GetValueOrDefault()),
new XAttribute("endTime", result.EndTime.GetValueOrDefault()),
new XAttribute("duration", result.Duration.GetValueOrDefault().TotalSeconds),
new XAttribute("formattedDescription", FixtureFormatter.FormatFixture(result.FixtureDescriptor))
);
if (result.Exception is not null)
{
fixtureElement.Add(new XElement("exception", result.Exception));
}
result.StepResults.ForEach(stepResult => ReportFixtureStep(stepResult, fixtureElement));
result.Results.ForEach(subResult => ReportFixture(subResult, fixtureElement));
element.Add(fixtureElement);
}
public void Verify_NJasmine_implementation(FixtureResult fixtureResult)
{
fixtureResult.succeeds();
fixtureResult.containsTrace(@"
running test 1
running test 2
TearDown NamespaceSetupB
");
}
/// <summary>
/// Reports the specified fixture running result.
/// </summary>
/// <param name="result">The fixture running result.</param>
/// <param name="level">The level of the fixture running result.</param>
protected override void Report(FixtureResult result, int level)
{
var documentRoot = Document?.Root;
if (documentRoot is null)
{
return;
}
ReportFixture(result, documentRoot);
}
public void Dispose()
{
var afterFixtureUUID = Guid.NewGuid().ToString("N");
afterFixture = new FixtureResult()
{
name = afterFixtureUUID,
description = afterFixtureUUID
};
StackTrace st = new StackTrace();
StackFrame sf = st.GetFrame(0);
MethodBase testName = sf.GetMethod();
string currentDate = DateTime.Now.ToString("ddMMyyyy");
ITakesScreenshot screenshotHandler = _testSuite.driver as ITakesScreenshot;
Screenshot screenshot = screenshotHandler.GetScreenshot();
string filePath = $@"{_testSuite.resultsDirectory }\{testName + currentDate}.png";
screenshot.SaveAsFile(filePath, ScreenshotImageFormat.Png);
var stepResultID = Guid.NewGuid().ToString("N");
var stepResult = new StepResult();
stepResult.name = stepResultID;
var testResultId = Guid.NewGuid().ToString("N");
var result = new TestResult();
result.uuid = testResultId;
_testSuite.lifeCycle.StartAfterFixture(testContainer.uuid, afterFixtureUUID, afterFixture)
.StartStep(stepResultID, stepResult)
.AddAttachment(testName.Name + currentDate, "image/png", filePath)
.AddScreenDiff(testResultId, filePath, filePath, filePath)
.StopStep(x => x.status = Status.none);
_testSuite.lifeCycle.AddAttachment(testName.Name + currentDate, "image/png", filePath);
_testSuite.lifeCycle.StopFixture(afterFixtureUUID);
_testSuite.lifeCycle.StopTestContainer(testContainer.uuid)
.WriteTestContainer(testContainer.uuid);
}
/// <summary>
/// Reports the title of the fixture with the specified fixture running result
/// and level of the fixture running result.
/// </summary>
/// <param name="result">The fixture running result.</param>
/// <param name="level">The level of the fixture running result.</param>
/// <returns>The next level of the specified fixture running result.</returns>
protected virtual int ReportFixtureTitle(FixtureResult result, int level)
{
var formattedDescription = FixtureFormatter.FormatFixture(result.FixtureDescriptor);
if (result.FixtureDescriptor.FixtureAttributeType == typeof(AssemblyFixtureAttribute) ||
result.FixtureDescriptor.FixtureAttributeType == typeof(NamespaceFixtureAttribute))
{
return(IsFullReport ? ReportFixtureTitle(formattedDescription, level, null) : level);
}
return(ReportFixtureTitle(formattedDescription, level, result.Status));
}
/// <summary>
/// Reports the specified fixture running result.
/// </summary>
/// <param name="result">The fixture running result.</param>
/// <param name="level">The level of the fixture running result.</param>
protected override void Report(FixtureResult result, int level)
{
var nextLevel = ReportFixtureTitle(result, level);
ReportFixtureStep(result, nextLevel);
result.Results.ForEach(subResult => Report(subResult, nextLevel));
if (result.FixtureDescriptor.FixtureAttributeType == typeof(AssemblyFixtureAttribute))
{
CarnaConsole.WriteLine();
}
}
/// <summary>
/// Reports the fixture step running result that is contained by the specified fixture running result
/// and the level of the fixture running result.
/// </summary>
/// <param name="result">The fixture running result.</param>
/// <param name="level">The level of the fixture running result.</param>
protected virtual void ReportFixtureStep(FixtureResult result, int level)
{
if (!StepVisible)
{
return;
}
var indent = Indent(level);
result.StepResults.ForEach(stepResult =>
{
ReportValue(JoinFormattedLines(FixtureFormatter.FormatFixtureStep(stepResult.Step), indent), stepResult.Status);
ReportStatus(stepResult.Status, true);
});
}
/// <summary>
/// Runs a fixture with the specified filter, step runner factory,
/// and a value that indicates whether to run a fixture in parallel.
/// </summary>
/// <param name="filter">
/// The filter that determines whether to run a fixture.
/// </param>
/// <param name="stepRunnerFactory">
/// The factory to create a step runner.
/// </param>
/// <param name="parallel">
/// <c>true</c> if a fixture is run in parallel; otherwise, <c>false</c>.
/// </param>
/// <returns>
/// The result of a fixture running if a fixture can be run; otherwise, <c>null</c>.
/// </returns>
protected virtual FixtureResult?Run(IFixtureFilter?filter, IFixtureStepRunnerFactory stepRunnerFactory, bool parallel)
{
if (!CanRun(filter))
{
return(null);
}
var startTime = DateTime.UtcNow;
OnFixtureRunning(new FixtureRunEventArgs(FixtureResult.Of(FixtureDescriptor).StartAt(startTime).Running()));
var result = RunCore(startTime, filter, stepRunnerFactory, parallel);
OnFixtureRun(new FixtureRunEventArgs(result));
return(result);
}
private FixtureResult RunCore(DateTime startTime, IFixtureFilter?filter, IFixtureStepRunnerFactory stepRunnerFactory, bool parallel)
{
var result = FixtureResult.Of(FixtureDescriptor).StartAt(startTime);
try
{
var aroundFixtureAttributes = RetrieveAroundFixtureAttributes().ToList();
aroundFixtureAttributes.ForEach(attribute => attribute.OnFixtureRunning(FixtureContext.Of(FixtureDescriptor)));
try
{
return(Run(startTime, filter, stepRunnerFactory, parallel));
}
finally
{
aroundFixtureAttributes.ForEach(attribute => attribute.OnFixtureRun(FixtureContext.Of(FixtureDescriptor)));
}
}
catch (Exception exc)
{
return(RecordEndTime(result).Failed(exc));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="FixtureRunEventArgs"/> class
/// with the specified fixture running result.
/// </summary>
/// <param name="result">The fixture running result.</param>
public FixtureRunEventArgs(FixtureResult result)
{
Result = result;
}
/// <summary> /// Reports the specified fixture running result. /// </summary> /// <param name="result">The fixture running result.</param> /// <param name="level">The level of the fixture running result.</param> protected abstract void Report(FixtureResult result, int level);
/// <summary> /// Readies a fixture state. /// </summary> protected virtual void Ready() => OnFixtureReady(new FixtureRunEventArgs(FixtureResult.Of(FixtureDescriptor).Ready()));
private List <FixtureResult> BuildFixtureResults(NUnitHelpMethodType type, TestFixture testFixture)
{
var fixtureResultsList = new HashSet <FixtureResult>();
var testResult = TestExecutionContext.CurrentContext.CurrentResult;
foreach (var method in GetNUnitHelpMethods(type, testFixture))
{
var fr = new FixtureResult {
name = method
};
if (fr.start == 0)
{
fr.start = DateTimeOffset.Now.ToUnixTimeMilliseconds();
}
if (fr.stop == 0)
{
fr.stop = fr.start;
}
if (testResult.StackTrace != null && testResult.StackTrace.Contains(method))
{
AllureLifecycle.UpdateTestCase(x => x.description += $"\n{method} {type} method failed\n");
fr.status = Status.failed;
fr.statusDetails.message = testResult.Message;
fr.statusDetails.trace = testResult.StackTrace;
if (type == NUnitHelpMethodType.SetUp)
{
_isSetupFailed = true;
}
}
else
{
fr.status = Status.passed;
if (type == NUnitHelpMethodType.OneTimeTearDown)
{
fr.statusDetails.message = "It's not possible to get status of OneTimeTearDown";
fr.statusDetails.trace = "See Allure.NUnit wiki";
fr.status = Status.none;
}
}
try
{
if (testFixture.HasChildren)
{
var properties = _test.Properties;
if (properties.ContainsKey(method))
{
var methodProperty = (SetUpTearDownHelper)properties.Get(method);
fr.start = methodProperty.StartTime;
fr.stop = methodProperty.EndTime;
fr.statusDetails.message = methodProperty.Exception?.Message;
fr.statusDetails.trace = methodProperty.Exception?.StackTrace;
if (!string.IsNullOrEmpty(methodProperty.CustomName))
{
fr.name = methodProperty.CustomName;
}
}
else
{
properties = GetTestFixture(_test).Properties;
if (properties.ContainsKey(method))
{
var methodProperty = (SetUpTearDownHelper)properties.Get(method);
fr.start = methodProperty.StartTime;
fr.stop = methodProperty.EndTime;
fr.statusDetails.message = methodProperty.Exception?.Message;
fr.statusDetails.trace = methodProperty.Exception?.StackTrace;
if (!string.IsNullOrEmpty(methodProperty.CustomName))
{
fr.name = methodProperty.CustomName;
}
}
}
}
}
catch (Exception)
{
//
}
fr.stage = Stage.finished;
fixtureResultsList.Add(fr);
}
return(fixtureResultsList.ToList());
}
protected void ProcessFlowRow(Parse theCurrentRow)
{
try {
string specialActionName = RuntimeDirect.MakeDirect(
Processor.ParseTree <Cell, MemberName>(new CellRange(theCurrentRow.Parts, 1)).ToString());
TypedValue result = Processor.Invoke(new FlowKeywords(this), specialActionName, theCurrentRow.Parts);
if (!result.IsValid)
{
result = Processor.Invoke(this, specialActionName, theCurrentRow.Parts);
}
if (!result.IsValid)
{
result = CellOperation.TryInvoke(this,
new CellRange(MethodCells(new CellRange(theCurrentRow.Parts))),
new CellRange(ParameterCells(new CellRange(theCurrentRow.Parts))),
theCurrentRow.Parts);
}
if (!result.IsValid)
{
if (theCurrentRow.Parts.Text.Length > 0)
{
var newFixture = Processor.ParseTree <Cell, Interpreter>(theCurrentRow);
var adapter = newFixture as MutableDomainAdapter;
if (adapter != null)
{
adapter.SetSystemUnderTest(SystemUnderTest);
}
ProcessRestOfTable(newFixture, theCurrentRow);
IHaveFinishedTable = true;
}
else
{
result.ThrowExceptionIfNotValid();
}
}
else
{
if (TestStatus.IsAbandoned)
{
TestStatus.MarkIgnore(theCurrentRow);
return;
}
object wrapResult = FixtureResult.Wrap(result.Value);
if (wrapResult is bool)
{
ColorMethodName(theCurrentRow.Parts, (bool)wrapResult);
}
else if (wrapResult is Fixture)
{
ProcessRestOfTable((Fixture)wrapResult, theCurrentRow);
IHaveFinishedTable = true;
return;
}
}
}
catch (IgnoredException) {}
catch (ParseException <Cell> e) {
TestStatus.MarkException(e.Subject, e);
IHaveFinishedTable = true;
}
catch (Exception e) {
TestStatus.MarkException(theCurrentRow.Parts, e);
IHaveFinishedTable = true;
}
}
/// <summary> /// Runs a fixture with the specified start time, filter, step runner factory, /// and a value that indicates whether to run a fixture in parallel. /// </summary> /// <param name="startTime">The start time at which a fixture is run.</param> /// <param name="filter"> /// The filter that determines whether to run a fixture. /// </param> /// <param name="stepRunnerFactory"> /// The factory to create a step runner. /// </param> /// <param name="parallel"> /// <c>true</c> if a fixture is run in parallel; otherwise, <c>false</c>. /// </param> /// <returns> /// The result of a fixture running. /// </returns> /// <exception cref="FixtureInstanceNotInstantiateException"> /// The instance of the fixture is not instantiate. /// </exception> protected override FixtureResult Run(DateTime startTime, IFixtureFilter?filter, IFixtureStepRunnerFactory stepRunnerFactory, bool parallel) => Run(stepRunnerFactory, FixtureResult.Of(FixtureDescriptor).StartAt(startTime));
public static void Run(string loc, string[] args)
{
MainArgs ma = MainArgs.ValueOf(args);
string excludes = ma.Get("exclude", null);
string includes = ma.Get("include", null);
string outfile = ma.Get("out", "TestResult.xml");
CoreExtensions.Host.InitializeService();
TestSuiteBuilder builder = new TestSuiteBuilder();
TestPackage testPackage = new TestPackage(loc);
RemoteTestRunner remoteTestRunner = new RemoteTestRunner();
remoteTestRunner.Load(testPackage);
TestSuite suite = builder.Build(testPackage);
TestSuite root = suite.Tests[0] as TestSuite;
List <TestFixture> fixtures = new List <TestFixture>();
ScanFixtures(root, fixtures);
Console.WriteLine("--------------- {0} TextFixtures --------------- \n", fixtures.Count);
//TestName testName = ((TestMethod) ((TestFixture) test.Tests[0]).Tests[0]).MethodName;
ITestFilter filter = null;
bool hasInclude = !string.IsNullOrEmpty(includes);
bool hasExclude = !string.IsNullOrEmpty(excludes);
if (hasInclude)
{
if (hasExclude)
{
// incldue+exclude; exclude first
filter =
new AndFilter(new ITestFilter[]
{ new NotFilter(new CategoryFilter(excludes.Split(','))), new SimpleNameFilter(includes.Split(',')), });
}
else
{
// include
filter = new SimpleNameFilter(includes.Split(','));
}
}
else // no include
{
if (hasExclude)
{
// Only exclude
filter = new NotFilter(new CategoryFilter(excludes.Split(',')));
}
else
{
// none
filter = new TrueFilter();
}
}
int succCnt = 0, failCnt = 0, errorCnt = 0, assertCnt = 0;
TestResult tr = new TestResult(new TestName());
RunEventListener eventLsn = new RunEventListener();
foreach (TestFixture tf in fixtures)
{
TestResult result = tf.Run(eventLsn, filter);
FixtureResult fr = null;
if (result.Results != null)
{
fr = new FixtureResult(result);
succCnt += fr.succCnt;
failCnt += fr.failCnt;
errorCnt += fr.errorCnt;
assertCnt += fr.assertCnt;
Console.WriteLine(" Done: " + fr.ToString());
}
else
{
Console.WriteLine(" Done: no result.");
}
tr.AddResult(result);
}
if (failCnt + errorCnt == 0)
{
Console.WriteLine(
@"=========================================
Test Success! Cases: {0}, asserts: {1}
=========================================",
succCnt, assertCnt);
}
else
{
Console.WriteLine(
@"=================================================================================
Test with errors: Cases: {0}, asserts: {4}, Succ: {1}, fail:{2}, error: {3}
=================================================================================",
succCnt + errorCnt + failCnt, succCnt, failCnt, errorCnt, assertCnt);
}
XmlResultWriter w = new XmlResultWriter(outfile);
w.SaveTestResult(tr);
Console.WriteLine("Result save to: {0}", outfile);
}