void LogFailureInformation(IFailureInformation info, Action <string> log = null, StringBuilder sb = null)
{
if (info == null)
{
return;
}
string message = ExceptionUtility.CombineMessages(info);
do_log($" Exception messages: {message}", log, sb);
string traces = ExceptionUtility.CombineStackTraces(info);
do_log($" Exception stack traces: {traces}", log, sb);
}
protected override bool Visit(ITestFailed testFailed)
{
var stackFrameInfo = GetStackFrameInfo(testFailed);
Log.LogError(null, null, null, stackFrameInfo.Item1, stackFrameInfo.Item2, 0, 0, 0, "{0}: {1}", Escape(testFailed.TestDisplayName), Escape(ExceptionUtility.CombineMessages(testFailed)));
var combinedStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
if (!String.IsNullOrWhiteSpace(combinedStackTrace))
{
Log.LogError(null, null, null, stackFrameInfo.Item1, stackFrameInfo.Item2, 0, 0, 0, "{0}", combinedStackTrace);
}
return(base.Visit(testFailed));
}
protected override bool Visit(ITestFailed failed)
{
TestRunState = TestRunState.Failure;
var testResult = failed.ToTdNetTestResult(TestState.Failed);
testResult.Message = ExceptionUtility.CombineMessages(failed);
testResult.StackTrace = ExceptionUtility.CombineStackTraces(failed);
TestListener.TestFinished(testResult);
//WriteOutput(name, output);
return(true);
}
protected override bool Visit(ITestFailed testFailed)
{
lock (consoleLock)
{
// TODO: Thread-safe way to figure out the default foreground color
Console.ForegroundColor = ConsoleColor.Red;
Console.Error.WriteLine(" {0} [FAIL]", Escape(testFailed.TestDisplayName));
Console.ForegroundColor = ConsoleColor.Gray;
Console.Error.WriteLine(" {0}", Escape(ExceptionUtility.CombineMessages(testFailed)));
WriteStackTrace(ExceptionUtility.CombineStackTraces(testFailed));
}
return(base.Visit(testFailed));
}
protected override bool Visit(ITestFailed testFailed)
{
AppVeyorLogger.UpdateTest(GetFinishedTestName(testFailed.Test.DisplayName), FrameworkName, assemblyFileName, "Failed",
Convert.ToInt64(testFailed.ExecutionTime * 1000), ExceptionUtility.CombineMessages(testFailed),
ExceptionUtility.CombineStackTraces(testFailed), testFailed.Output, null);
lock (consoleLock)
{
Console.WriteLine(" {0} [FAIL]", Escape(testFailed.Test.DisplayName));
Console.WriteLine(" {0}", Escape(ExceptionUtility.CombineMessages(testFailed)));
WriteStackTrace(ExceptionUtility.CombineStackTraces(testFailed));
}
return(base.Visit(testFailed));
}
void ReportError(string messageType, IFailureInformation failureInfo)
{
TestRunState = TestRunState.Failure;
var testResult = new TestResult
{
Name = String.Format("*** {0} ***", messageType),
State = TestState.Failed,
TimeSpan = TimeSpan.Zero,
TotalTests = 1,
Message = ExceptionUtility.CombineMessages(failureInfo),
StackTrace = ExceptionUtility.CombineStackTraces(failureInfo)
};
TestListener.TestFinished(testResult);
}
protected override bool Visit(ITestFailed testFailed)
{
lock (consoleLock)
{
// TODO: Thread-safe way to figure out the default foreground color
Program.SetConsoleForegroundColor(ConsoleColor.Red);
Console.Error.WriteLine(" {0} [FAIL]", Escape(testFailed.Test.DisplayName));
Program.SetConsoleForegroundColor(ConsoleColor.Gray);
Console.Error.WriteLine(" {0}", ExceptionUtility.CombineMessages(testFailed).Replace(Environment.NewLine, Environment.NewLine + " "));
WriteStackTrace(ExceptionUtility.CombineStackTraces(testFailed));
}
return(base.Visit(testFailed));
}
protected bool WriteError(string failureName, IFailureInformation failureInfo, IEnumerable <ITestCase> testCases)
{
foreach (var testCase in testCases)
{
var result = MakeVsTestResult(TestOutcome.Failed, testCase, testCase.DisplayName);
if (result != null)
{
result.ErrorMessage = String.Format("[{0}]: {1}", failureName, ExceptionUtility.CombineMessages(failureInfo));
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(failureInfo);
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
}
}
return(!cancelledThunk());
}
void ReportError(
string messageType,
_IErrorMetadata errorMetadata)
{
TestRunState = TestRunState.Failure;
var testResult = new TestResult
{
Name = $"*** {messageType} ***",
State = TestState.Failed,
TimeSpan = TimeSpan.Zero,
TotalTests = 1,
Message = ExceptionUtility.CombineMessages(errorMetadata),
StackTrace = ExceptionUtility.CombineStackTraces(errorMetadata)
};
TestListener.TestFinished(testResult);
}
protected override bool Visit(IErrorMessage error)
{
lock (consoleLock)
{
#if ASPNET50
Console.ForegroundColor = ConsoleColor.Red;
#endif
Console.Error.WriteLine(" {0} [FATAL]", Escape(error.ExceptionTypes[0]));
#if ASPNET50
Console.ForegroundColor = ConsoleColor.Gray;
#endif
Console.Error.WriteLine(" {0}", Escape(ExceptionUtility.CombineMessages(error)));
WriteStackTrace(ExceptionUtility.CombineStackTraces(error));
}
return(base.Visit(error));
}
private static Tuple <string, int> GetStackFrameInfo(IFailureInformation failureInfo)
{
var stackTraces = ExceptionUtility.CombineStackTraces(failureInfo);
if (stackTraces != null)
{
foreach (var frame in stackTraces.Split(new[] { Environment.NewLine }, 2, StringSplitOptions.RemoveEmptyEntries))
{
var match = stackFrameRegex.Match(frame);
if (match.Success)
{
return(Tuple.Create(match.Groups["file"].Value, Int32.Parse(match.Groups["line"].Value)));
}
}
}
return(Tuple.Create((string)null, 0));
}
protected override bool Visit(ITestFailed testFailed)
{
var result = MakeVsTestResult(TestOutcome.Failed, testFailed);
if (result != null)
{
result.ErrorMessage = ExceptionUtility.CombineMessages(testFailed);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
TryAndReport("RecordResult (Fail)", testFailed.TestCase, () => recorder.RecordResult(result));
}
else
{
logger.LogWarning(testFailed.TestCase, "(Fail) Could not find VS test case for {0} (ID = {1})", testFailed.TestCase.DisplayName, testFailed.TestCase.UniqueID);
}
return(!cancelledThunk());
}
public void NonXunitExceptionWithInnerExceptions()
{
Action innerTestCode = () => { throw new DivideByZeroException(); };
var inner = Record.Exception(innerTestCode);
Action outerTestCode = () => { throw new Exception("message", inner); };
var outer = Record.Exception(outerTestCode);
var failureInfo = new FailureInformation {
{ outer, -1 }, { inner, 0 }
};
var result = ExceptionUtility.CombineStackTraces(failureInfo);
Assert.Collection(result.Split(new[] { Environment.NewLine }, StringSplitOptions.None),
line => Assert.Contains("at ExceptionUtilityTests.CombineStackTraces", line),
line => Assert.Equal("----- Inner Stack Trace -----", line),
line => Assert.Contains("at ExceptionUtilityTests.CombineStackTraces", line)
);
}
void HandleTestFailed(MessageHandlerArgs <ITestFailed> args)
{
var testFailed = args.Message;
var result = MakeVsTestResult(TestOutcome.Failed, testFailed);
if (result != null)
{
result.ErrorMessage = ExceptionUtility.CombineMessages(testFailed);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
TryAndReport("RecordResult (Fail)", testFailed.TestCase, () => recorder.RecordResult(result));
}
else
{
logger.LogWarning(testFailed.TestCase, "(Fail) Could not find VS test case for {0} (ID = {1})", testFailed.TestCase.DisplayName, testFailed.TestCase.UniqueID);
}
HandleCancellation(args);
}
void WriteError(string failureName, IFailureInformation failureInfo, IEnumerable <ITestCase> testCases)
{
foreach (var testCase in testCases)
{
var result = MakeVsTestResult(TestOutcome.Failed, testCase, testCase.DisplayName);
if (result != null)
{
result.ErrorMessage = $"[{failureName}]: {ExceptionUtility.CombineMessages(failureInfo)}";
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(failureInfo);
TryAndReport("RecordEnd (Failure)", testCase, () => recorder.RecordEnd(result.TestCase, result.Outcome));
TryAndReport("RecordResult (Failure)", testCase, () => recorder.RecordResult(result));
}
else
{
logger.LogWarning(testCase, "(Failure) Could not find VS test case for {0} (ID = {1})", testCase.DisplayName, testCase.UniqueID);
}
}
}
/// <summary>
/// Logs an error message to the logger.
/// </summary>
/// <param name="failureType">The type of the failure</param>
/// <param name="errorMetadata">The failure information</param>
protected void LogError(
string failureType,
_IErrorMetadata errorMetadata)
{
Guard.ArgumentNotNull(nameof(failureType), failureType);
Guard.ArgumentNotNull(nameof(errorMetadata), errorMetadata);
var frameInfo = StackFrameInfo.FromErrorMetadata(errorMetadata);
lock (Logger.LockObject)
{
Logger.LogError(frameInfo, $" [{failureType}] {Escape(errorMetadata.ExceptionTypes.FirstOrDefault() ?? "(Unknown Exception Type)")}");
foreach (var messageLine in ExceptionUtility.CombineMessages(errorMetadata).Split(new[] { Environment.NewLine }, StringSplitOptions.None))
{
Logger.LogImportantMessage(frameInfo, $" {messageLine}");
}
LogStackTrace(frameInfo, ExceptionUtility.CombineStackTraces(errorMetadata));
}
}
async void MakeTestResultViewModel(ITestResultMessage testResult, TestState outcome)
{
var tcs = new TaskCompletionSource <TestResultViewModel>(TaskCreationOptions.RunContinuationsAsynchronously);
TestCaseViewModel testCase;
if (!testCases.TryGetValue(testResult.TestCase, out testCase))
{
// no matching reference, search by Unique ID as a fallback
testCase = testCases.FirstOrDefault(kvp => kvp.Key.UniqueID?.Equals(testResult.TestCase.UniqueID) ?? false).Value;
if (testCase == null)
{
return;
}
}
// Create the result VM on the UI thread as it updates properties
context.Post(_ =>
{
var result = new TestResultViewModel(testCase, testResult)
{
Duration = TimeSpan.FromSeconds((double)testResult.ExecutionTime)
};
if (outcome == TestState.Failed)
{
result.ErrorMessage = ExceptionUtility.CombineMessages((ITestFailed)testResult);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces((ITestFailed)testResult);
}
tcs.TrySetResult(result);
}, null);
var r = await tcs.Task;
listener.RecordResult(r); // bring it back to the threadpool thread
}
/// <summary>
/// Creates a stack frame info from error metadata.
/// </summary>
/// <param name="errorMetadata">The error to inspect</param>
/// <returns>The stack frame info</returns>
public static StackFrameInfo FromErrorMetadata(_IErrorMetadata?errorMetadata)
{
if (errorMetadata == null)
{
return(None);
}
var stackTraces = ExceptionUtility.CombineStackTraces(errorMetadata);
if (stackTraces != null)
{
foreach (var frame in stackTraces.Split(new[] { Environment.NewLine }, 2, StringSplitOptions.RemoveEmptyEntries))
{
var match = stackFrameRegex.Match(frame);
if (match.Success)
{
return(new StackFrameInfo(match.Groups["file"].Value, int.Parse(match.Groups["line"].Value)));
}
}
}
return(None);
}
internal void UpdateTestState(TestResultViewModel message)
{
TestResult = message;
Output = message.TestResultMessage.Output ?? string.Empty;
var msg = string.Empty;
var stackTrace = string.Empty;
var rs = Runners.RunStatus.NotRun;
if (message.TestResultMessage is ITestPassed)
{
Result = TestState.Passed;
msg = $"Success! {TestResult.Duration.TotalMilliseconds} ms";
rs = RunStatus.Ok;
}
if (message.TestResultMessage is ITestFailed)
{
Result = TestState.Failed;
var failedMessage = (ITestFailed)(message.TestResultMessage);
msg = ExceptionUtility.CombineMessages(failedMessage);
stackTrace = ExceptionUtility.CombineStackTraces(failedMessage);
rs = RunStatus.Failed;
}
if (message.TestResultMessage is ITestSkipped)
{
Result = TestState.Skipped;
var skipped = (ITestSkipped)(message.TestResultMessage);
msg = skipped.Reason;
rs = RunStatus.Skipped;
}
Message = msg;
StackTrace = stackTrace;
RunStatus = rs;
}
public static int Main(string[] args)
{
string host = null;
int port = 0;
// First argument is the connection string
if (args [0].StartsWith("tcp:"))
{
var parts = args [0].Split(':');
if (parts.Length != 3)
{
throw new Exception();
}
host = parts [1];
port = Int32.Parse(parts [2]);
args = args.Skip(1).ToArray();
}
// Make sure the TLS subsystem including the DependencyInjector is initialized.
// This would normally happen on system startup in
// `xamarin-macios/src/ObjcRuntime/Runtime.cs`.
MonoTlsProviderFactory.Initialize();
#if XUNIT_RUNNER
var writer = new TcpWriter(host, port);
var assemblyFileName = args[0];
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new DiagnosticTextWriterMessageSink(writer);
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink);
writer.WriteLine($"Discovering tests for {assemblyFileName}");
controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
writer.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); };
testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); };
testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); };
testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(discoverySink.TestCases, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
writer.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(writer.RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
#else
MonoSdksTextUI runner;
TcpWriter writer = null;
string resultsXml = null;
if (host != null)
{
Console.WriteLine($"Connecting to harness at {host}:{port}.");
resultsXml = Path.GetTempFileName();
args = args.Concat(new string[] { "-format:xunit", $"-result:{resultsXml}" }).ToArray();
writer = new TcpWriter(host, port);
runner = new MonoSdksTextUI(writer);
}
else
{
runner = new MonoSdksTextUI();
}
runner.Execute(args);
if (resultsXml != null)
{
writer.WriteLine($"STARTRESULTXML");
using (var resultsXmlStream = File.OpenRead(resultsXml)) resultsXmlStream.CopyTo(writer.RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
return(runner.Failure ? 1 : 0);
#endif
}
public int Run(string assemblyFileName, bool printXml, XunitFilters filters)
{
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Console.WriteLine($"Discovering tests for {assemblyFileName}");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
var threadpoolPump = typeof(ThreadPool).GetMethod("PumpThreadPool", BindingFlags.NonPublic | BindingFlags.Static);
if (threadpoolPump != null)
{
while (!resultsSink.Finished.WaitOne(0))
{
threadpoolPump.Invoke(this, null);
}
}
else
{
resultsSink.Finished.WaitOne();
}
if (printXml)
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.OpenStandardOutput());
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits)
{
WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''");
Log("Starting tests...");
var filters = new XunitFilters();
foreach (var trait in excludedTraits ?? Array.Empty <string>())
{
ParseEqualSeparatedArgument(filters.ExcludedTraits, trait);
}
var configuration = new TestAssemblyConfiguration
{
ShadowCopy = false,
ParallelizeAssembly = false,
ParallelizeTestCollections = false,
MaxParallelThreads = 1,
PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(
AppDomainSupport.Denied,
new NullSourceInformationProvider(),
assemblyFileName,
configFileName: null,
shadowCopy: false,
shadowCopyFolder: null,
diagnosticMessageSink: diagnosticSink,
verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Log($"Discovering tests for {assemblyFileName}...");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Log($"Discovery finished.");
Log("");
var summarySink = new DelegatingExecutionSummarySink(
testSink,
() => false,
(completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); };
testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); };
testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
Console.WriteLine(resultsXml.ToString());
Log("");
Log("Test results (Base64 encoded):");
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString()));
Log(base64, id: "results");
return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0);
}
public async Task <int> Run(string assemblyFileName, bool printXml, XunitFilters filters)
{
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}
if (printXml)
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.OpenStandardOutput());
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
static XElement CreateFailureElement(_IErrorMetadata errorMetadata) =>
new XElement("failure",
new XAttribute("exception-type", errorMetadata.ExceptionTypes[0] ?? "<unknown type>"),
new XElement("message", new XCData(XmlEscape(ExceptionUtility.CombineMessages(errorMetadata)))),
new XElement("stack-trace", new XCData(ExceptionUtility.CombineStackTraces(errorMetadata) ?? string.Empty))
);
/// <summary>
/// Send message about failed test in report portal.
/// </summary>
/// <param name="args"></param>
protected virtual void HandleFailed(MessageHandlerArgs <ITestFailed> args)
{
lock (Logger.LockObject)
{
try
{
var testEvent = args.Message;
string key = testEvent.Test.TestCase.UniqueID;
ITestReporter testReporter = TestReporterDictionary[key];
testReporter.Log(new AddLogItemRequest
{
Level = LogLevel.Error,
Time = DateTime.UtcNow,
Text = $"{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"
});
if (!string.IsNullOrEmpty(args.Message.Output))
{
testReporter.Log(new AddLogItemRequest
{
Level = LogLevel.Debug,
Time = DateTime.UtcNow,
Text = $"Test output:{Environment.NewLine}{args.Message.Output}"
});
}
testReporter.Finish(new FinishTestItemRequest()
{
EndTime = DateTime.UtcNow,
Status = Status.Failed
});
TestReporterDictionary.Remove(key);
}
catch (Exception exp)
{
Logger.LogError(exp.ToString());
}
}
}
void WriteError(string failureName, IFailureInformation failureInfo)
{
var stackFrameInfo = GetStackFrameInfo(failureInfo);
Log.LogError(null, null, null, stackFrameInfo.Item1, stackFrameInfo.Item2, 0, 0, 0, "[{0}] {1}", failureName, Escape(ExceptionUtility.CombineMessages(failureInfo)));
Log.LogError(null, null, null, stackFrameInfo.Item1, stackFrameInfo.Item2, 0, 0, 0, "{0}", ExceptionUtility.CombineStackTraces(failureInfo));
}
protected override bool Visit(ITestFailed testFailed)
{
logger.LogImportantMessage($"##teamcity[testFailed name='{Escape(displayNameFormatter.DisplayName(testFailed.Test))}' details='{Escape(ExceptionUtility.CombineMessages(testFailed))}|r|n{Escape(ExceptionUtility.CombineStackTraces(testFailed))}' flowId='{ToFlowId(testFailed.TestCollection.DisplayName)}']");
LogFinish(testFailed);
return(base.Visit(testFailed));
}
public static int Main(string[] args)
{
var arguments = new Stack <string> ();
string host = null;
int port = 0;
bool closeAfterTestRun;
bool failed;
for (var i = args.Length - 1; i >= 0; i--)
{
arguments.Push(args[i]);
}
// First argument is the connection string if we're driven by harness.exe, otherwise we're driven by UITests
if (arguments.Count > 0 && arguments.Peek().StartsWith("tcp:", StringComparison.Ordinal))
{
var parts = arguments.Pop().Split(':');
if (parts.Length != 3)
{
throw new Exception();
}
host = parts [1];
port = Int32.Parse(parts [2]);
closeAfterTestRun = true;
}
else
{
closeAfterTestRun = false;
}
// Make sure the TLS subsystem including the DependencyInjector is initialized.
// This would normally happen on system startup in
// `xamarin-macios/src/ObjcRuntime/Runtime.cs`.
MonoTlsProviderFactory.Initialize();
// some tests assert having a SynchronizationContext for MONOTOUCH, provide a default one
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
#if XUNIT_RUNNER
var writer = new TcpWriter(host, port);
var assemblyFileName = arguments.Pop();
var filters = XunitArgumentsParser.ParseArgumentsToFilter(arguments);
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new DiagnosticTextWriterMessageSink(writer);
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink);
Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}...");
writer.WriteLine($"Discovering tests for {assemblyFileName}");
controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
writer.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); };
testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); };
testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); };
testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(resultsXmlPath);
if (host != null)
{
writer.WriteLine($"STARTRESULTXML");
resultsXml.Save(((TcpWriter)writer).RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
#else
MonoSdksTextUI runner;
TextWriter writer = null;
string resultsXmlPath = Path.GetTempFileName();
string assemblyFileName = arguments.Peek();
if (File.Exists("nunit-excludes.txt"))
{
var excludes = File.ReadAllLines("nunit-excludes.txt");
arguments.Push("-exclude:" + String.Join(",", excludes));
}
arguments.Push("-labels");
arguments.Push("-format:xunit");
arguments.Push($"-result:{resultsXmlPath}");
if (host != null)
{
Console.WriteLine($"Connecting to harness at {host}:{port}.");
writer = new TcpWriter(host, port);
}
else
{
writer = ConsoleWriter.Out;
}
Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}...");
runner = new MonoSdksTextUI(writer);
runner.Execute(arguments.ToArray());
if (host != null)
{
writer.WriteLine($"STARTRESULTXML");
using (var resultsXmlStream = File.OpenRead(resultsXmlPath)) resultsXmlStream.CopyTo(((TcpWriter)writer).RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
failed = runner.Failure;
#endif
Interop.mono_sdks_ui_set_test_summary_message($"Summary: {(failed ? "Failed" : "Succeeded")} for {assemblyFileName}.");
if (!closeAfterTestRun)
{
Thread.Sleep(Int32.MaxValue);
}
return(failed ? 1 : 0);
}
protected virtual void HandleTestFailed(MessageHandlerArgs <ITestFailed> args)
{
var testFailed = args.Message;
logger.LogImportantMessage($"##teamcity[testFailed name='{Escape(displayNameFormatter.DisplayName(testFailed.Test))}' details='{Escape(ExceptionUtility.CombineMessages(testFailed))}|r|n{Escape(ExceptionUtility.CombineStackTraces(testFailed))}' flowId='{ToFlowId(testFailed.TestCollection.DisplayName)}']");
LogFinish(testFailed);
}
protected override bool Visit(IErrorMessage error)
{
var stackFrameInfo = GetStackFrameInfo(error);
Log.LogError(null, null, null, stackFrameInfo.Item1, stackFrameInfo.Item2, 0, 0, 0, "{0}", Escape(ExceptionUtility.CombineMessages(error)));
Log.LogError(null, null, null, stackFrameInfo.Item1, stackFrameInfo.Item2, 0, 0, 0, "{0}", ExceptionUtility.CombineStackTraces(error));
return(base.Visit(error));
}