static int RunTests(string [] original_args)
{
Console.WriteLine("Running tests");
var options = ApplicationOptions.Current;
// we generate the logs in two different ways depending if the generate xml flag was
// provided. If it was, we will write the xml file to the tcp writer if present, else
// we will write the normal console output using the LogWriter
var logger = new LogWriter(Console.Out);
logger.MinimumLogLevel = MinimumLogLevel.Info;
var testAssemblies = GetTestAssemblies();
TestRunner runner;
if (RegisterType.IsXUnit)
{
runner = new XUnitTestRunner(logger);
}
else
{
runner = new NUnitTestRunner(logger);
}
runner.Run(testAssemblies.ToList());
using (var writer = new StreamWriter(options.ResultFile)) {
runner.WriteResultsToFile(writer);
}
logger.Info($"Xml result can be found {options.ResultFile}");
logger.Info($"Tests run: {runner.TotalTests} Passed: {runner.PassedTests} Inconclusive: {runner.InconclusiveTests} Failed: {runner.FailedTests} Ignored: {runner.SkippedTests}");
return(runner.FailedTests != 0 ? 1 : 0);
}
public async Task ShouldSignIn()
{
var runner = new XUnitTestRunner(new TestClientFactory(ClientType.DirectLine).GetTestClient(), _logger);
var signInUrl = string.Empty;
// Execute the first part of the conversation.
await runner.RunTestAsync(Path.Combine(_transcriptsFolder, "ShouldSignIn1.transcript"));
// Obtain the signIn url.
await runner.AssertReplyAsync(activity =>
{
Assert.Equal(ActivityTypes.Message, activity.Type);
Assert.True(activity.Attachments.Count > 0);
var card = JsonConvert.DeserializeObject <SigninCard>(JsonConvert.SerializeObject(activity.Attachments.FirstOrDefault().Content));
signInUrl = card.Buttons[0].Value?.ToString();
Assert.False(string.IsNullOrEmpty(signInUrl));
});
// Execute the SignIn.
await runner.ClientSignInAsync(signInUrl);
// Execute the rest of the conversation.
await runner.RunTestAsync(Path.Combine(_transcriptsFolder, "ShouldSignIn2.transcript"));
}
private List <string> SelectTestsAsync(Assembly assembly, IReadOnlyCollection <Type> testTypesToRun)
{
var listResults = new List <string>();
var results = new RunTestResults
{
TestRunTime = DateTimeOffset.UtcNow
};
void OnTestFailed(TestFailedInfo info)
{
results.TestsFailCount++;
results.FailedTests.Add(new TestResultInfo
{
TestName = info.TestDisplayName, ExceptionMessage = info.ExceptionMessage, Output = info.Output
});
}
void OnTestPassed(TestPassedInfo info)
{
results.TestsPassCount++;
results.PassedTests.Add(new TestResultInfo {
TestName = info.TestDisplayName, Output = info.Output
});
}
void OnTestStarting(TestStartingInfo info)
{
results.TotalTestCount++;
}
void OnTestSkipped(TestSkippedInfo info)
{
results.TestsSkipCount++;
}
using (var runner = new XUnitTestRunner(assembly.Location))
{
runner.OnTestPassed = OnTestPassed;
runner.OnTestFailed = OnTestFailed;
runner.OnTestStarting = OnTestStarting;
runner.OnTestSkipped = OnTestSkipped;
// Run selected tests.
foreach (var type in testTypesToRun)
{
ExecuteTestsAsync(runner, type.FullName);
}
foreach (var x in results.PassedTests)
{
listResults.Add($"TestName: { x.TestName} -> Output: Passed");
}
foreach (var x in results.FailedTests)
{
listResults.Add($"TestName: { x.TestName} -> Output: Failed -> Exception: {x.ExceptionMessage}");
}
}
return(listResults);
}
public void SetUp()
{
_bus = MockRepository.GenerateMock <IMessageBus>();
_configuration = MockRepository.GenerateMock <IConfiguration>();
_assemblyReader = MockRepository.GenerateMock <IAssemblyPropertyReader>();
_fsService = MockRepository.GenerateMock <IFileSystemService>();
_runner = new XUnitTestRunner(_bus, _configuration, _assemblyReader, _fsService);
}
public static async Task Main(string[] args)
{
var command = new RootCommand(".NET test runner for Taurus")
{
NUnitTestRunner.GetNUnitCommand(),
XUnitTestRunner.GetNUnitCommand()
};
await command.InvokeAsync(args);
}
protected override TestRunner GetTestRunner(LogWriter logWriter)
{
var runner = new XUnitTestRunner(logWriter)
{
MaxParallelThreads = MaxParallelThreads
};
ConfigureRunnerFilters(runner, ApplicationOptions.Current);
return(runner);
}
public async override void ViewDidLoad()
{
base.ViewDidLoad();
var options = ApplicationOptions.Current;
TcpTextWriter writer = null;
if (!string.IsNullOrEmpty(options.HostName))
{
writer = new TcpTextWriter(options.HostName, options.HostPort);
}
// we generate the logs in two different ways depending if the generate xml flag was
// provided. If it was, we will write the xml file to the tcp writer if present, else
// we will write the normal console output using the LogWriter
var logger = (writer == null || options.EnableXml) ? new LogWriter() : new LogWriter(writer);
logger.MinimumLogLevel = MinimumLogLevel.Info;
var testAssemblies = GetTestAssemblies();
Xamarin.iOS.UnitTests.TestRunner runner;
if (RegisterType.IsXUnit)
{
runner = new XUnitTestRunner(logger);
}
else
{
runner = new NUnitTestRunner(logger);
}
var skippedTests = await IgnoreFileParser.ParseContentFilesAsync(NSBundle.MainBundle.BundlePath);
if (skippedTests.Any())
{
// ensure that we skip those tests that have been passed via the ignore files
runner.SkipTests(skippedTests);
}
runner.Run((IList <TestAssemblyInfo>)testAssemblies);
if (options.EnableXml)
{
runner.WriteResultsToFile(writer);
logger.Info("Xml file was written to the tcp listener.");
}
else
{
string resultsFilePath = runner.WriteResultsToFile();
logger.Info($"Xml result can be found {resultsFilePath}");
}
logger.Info($"Tests run: {runner.TotalTests} Passed: {runner.PassedTests} Inconclusive: {runner.InconclusiveTests} Failed: {runner.FailedTests} Ignored: {runner.SkippedTests}");
if (options.TerminateAfterExecution)
{
TerminateWithSuccess();
}
}
public async Task ManualTest()
{
var runner = new XUnitTestRunner(new TestClientFactory(ClientType.DirectLine).GetTestClient(), _logger);
await runner.SendActivityAsync(new Activity(ActivityTypes.ConversationUpdate));
await runner.AssertReplyAsync(activity =>
{
Assert.Equal(ActivityTypes.Message, activity.Type);
Assert.Equal("Hello and welcome!", activity.Text);
});
}
public async Task RunTestCases(TestCaseDataObject testData)
{
var testCase = testData.GetObject <TestCase>();
Logger.LogInformation(JsonConvert.SerializeObject(testCase, Formatting.Indented));
var options = TestClientOptions[testCase.HostBot];
var runner = new XUnitTestRunner(new TestClientFactory(testCase.ChannelId, options, Logger).GetTestClient(), TestRequestTimeout, ThinkTime, Logger);
var testParams = new Dictionary <string, string>
{
{ "DeliveryMode", testCase.DeliveryMode },
{ "TargetSkill", testCase.TargetSkill }
};
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, testCase.Script), testParams);
}
public async Task RunTestCases(TestCaseDataObject testData)
{
var userId = string.Empty;
var url = string.Empty;
var testCase = testData.GetObject <TestCase>();
Logger.LogInformation(JsonConvert.SerializeObject(testCase, Formatting.Indented));
var options = TestClientOptions[testCase.HostBot];
var runner = new XUnitTestRunner(new TestClientFactory(testCase.ChannelId, options, Logger).GetTestClient(), TestRequestTimeout, ThinkTime, Logger);
var testParamsStart = new Dictionary <string, string>
{
{ "DeliveryMode", testCase.DeliveryMode },
{ "TargetSkill", testCase.TargetSkill }
};
// Execute the first part of the conversation.
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, testCase.Script), testParamsStart);
await runner.AssertReplyAsync(activity =>
{
Assert.Equal(ActivityTypes.Message, activity.Type);
Assert.Contains("Navigate to http", activity.Text);
var message = activity.Text.Split(" ");
url = message[2];
userId = url.Split("user="******"UserId", userId },
{ "TargetSkill", testCase.TargetSkill }
};
// Execute the rest of the conversation passing the messageId.
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, "ProactiveEnd.json"), testParamsEnd);
}
protected override void ConfigureFilters(XUnitTestRunner runner)
{
base.ConfigureFilters(runner);
if (runner == null)
{
throw new ArgumentNullException(nameof(runner));
}
HashSet <string> excludedTestNames = null;
using (var s = typeof(XUnitInstrumentation).Assembly.GetManifestResourceStream("xunit-excluded-tests.txt")) {
using (var sr = new StreamReader(s, Encoding.UTF8)) {
excludedTestNames = LoadExcludedTests(sr);
}
}
// Known filters for CoreFX tests
var filters = new List <XUnitFilter> {
// From the Mono runtime (https://github.com/mono/mono/blob/master/mcs/build/tests.make#L255)
new XUnitFilter("category", "failing", true),
new XUnitFilter("category", "nonnetcoreapptests", true),
new XUnitFilter("category", "outerloop", true),
new XUnitFilter("Benchmark", "true", true),
// From some failing corefx tests
new XUnitFilter("category", "nonlinuxtests", true),
new XUnitFilter("category", "nonmonotests", true),
new XUnitFilter("category", "nonnetfxtests", true),
#if !DEBUG // aka "Release"
new XUnitFilter("category", "nonuapaottests", true),
#endif // !DEBUG
};
if (excludedTestNames != null && excludedTestNames.Count > 0)
{
foreach (string typeName in excludedTestNames)
{
filters.Add(new XUnitFilter(typeName, true));
}
}
runner.SetFilters(filters);
}
public async Task RunTestCases(TestCaseDataObject testData)
{
var testGuid = Guid.NewGuid().ToString();
var fileName = $"TestFile-{testGuid}.txt";
var testCase = testData.GetObject <TestCase>();
Logger.LogInformation(JsonConvert.SerializeObject(testCase, Formatting.Indented));
var options = TestClientOptions[testCase.HostBot];
var runner = new XUnitTestRunner(new TestClientFactory(testCase.ChannelId, options, Logger).GetTestClient(), TestRequestTimeout, ThinkTime, Logger);
// Execute the first part of the conversation.
var testParams = new Dictionary <string, string>
{
{ "DeliveryMode", testCase.DeliveryMode },
{ "TargetSkill", testCase.TargetSkill },
{ "FileName", fileName },
{ "TestGuid", testGuid }
};
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, testCase.Script), testParams);
// Create a new file to upload.
await using var stream = File.Create(Directory.GetCurrentDirectory() + $"/Skills/FileUpload/{fileName}");
await using var writer = new StreamWriter(stream);
await writer.WriteLineAsync($"GUID:{testGuid}");
writer.Close();
// Upload file.
await using var file = File.OpenRead(Directory.GetCurrentDirectory() + $"/Skills/FileUpload/{fileName}");
await runner.UploadAsync(file);
// Execute the rest of the conversation.
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, "FileUpload2.json"), testParams);
}
public async Task RunTestCases(TestCaseDataObject testData)
{
var signInUrl = string.Empty;
var testCase = testData.GetObject <TestCase>();
Logger.LogInformation(JsonConvert.SerializeObject(testCase, Formatting.Indented));
var options = TestClientOptions[testCase.HostBot];
var runner = new XUnitTestRunner(new TestClientFactory(testCase.ChannelId, options, Logger).GetTestClient(), TestRequestTimeout, Logger);
var testParams = new Dictionary <string, string>
{
{ "DeliveryMode", testCase.DeliveryMode },
{ "TargetSkill", testCase.TargetSkill }
};
// Execute the first part of the conversation.
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, testCase.Script), testParams);
await runner.AssertReplyAsync(activity =>
{
Assert.Equal(ActivityTypes.Message, activity.Type);
Assert.True(activity.Attachments.Count > 0);
var card = JsonConvert.DeserializeObject <SigninCard>(JsonConvert.SerializeObject(activity.Attachments.FirstOrDefault().Content));
signInUrl = card.Buttons[0].Value?.ToString();
Assert.False(string.IsNullOrEmpty(signInUrl));
});
// Execute the SignIn.
await runner.ClientSignInAsync(signInUrl);
// Execute the rest of the conversation passing the messageId.
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, "SignIn2.json"), testParams);
}
public async Task RunScripts(string transcript)
{
var runner = new XUnitTestRunner(new TestClientFactory(ClientType.DirectLine).GetTestClient(), _logger);
await runner.RunTestAsync(Path.Combine(_transcriptsFolder, transcript));
}
public async Task DialogSkillShouldConnect()
{
var runner = new XUnitTestRunner(new TestClientFactory(Channels.Directline, TestClientOptions[HostBot.EchoHostBot], Logger).GetTestClient(), TestRequestTimeout, Logger);
await runner.RunTestAsync(Path.Combine(_testScriptsFolder, "DialogSkill.json"));
}
public override async Task RunAsync()
{
var options = ApplicationOptions.Current;
TcpTextWriter writer = null;
if (!string.IsNullOrEmpty(options.HostName))
{
try
{
writer = new TcpTextWriter(options.HostName, options.HostPort);
}
catch (Exception ex)
{
Console.WriteLine("Network error: Cannot connect to {0}:{1}: {2}. Continuing on console.", options.HostName, options.HostPort, ex);
writer = null; // will default to the console
}
}
// we generate the logs in two different ways depending if the generate xml flag was
// provided. If it was, we will write the xml file to the tcp writer if present, else
// we will write the normal console output using the LogWriter
var logger = (writer == null || options.EnableXml) ? new LogWriter(Device) : new LogWriter(Device, writer);
logger.MinimumLogLevel = MinimumLogLevel.Info;
var testAssemblies = GetTestAssemblies();
TestRunner runner;
switch (TestRunner)
{
case TestRunnerType.NUnit:
throw new NotImplementedException();
default:
runner = new XUnitTestRunner(logger)
{
MaxParallelThreads = MaxParallelThreads
};
break;
}
if (!string.IsNullOrEmpty(IgnoreFilesDirectory))
{
var categories = await IgnoreFileParser.ParseTraitsContentFileAsync(IgnoreFilesDirectory, TestRunner == TestRunnerType.Xunit);
// add category filters if they have been added
runner.SkipCategories(categories);
var skippedTests = await IgnoreFileParser.ParseContentFilesAsync(IgnoreFilesDirectory);
if (skippedTests.Any())
{
// ensure that we skip those tests that have been passed via the ignore files
runner.SkipTests(skippedTests);
}
}
// if we have ignore files, ignore those tests
await runner.Run(testAssemblies).ConfigureAwait(false);
Core.TestRunner.Jargon jargon = Core.TestRunner.Jargon.NUnitV3;
switch (options.XmlVersion)
{
case XmlVersion.NUnitV2:
jargon = Core.TestRunner.Jargon.NUnitV2;
break;
case XmlVersion.NUnitV3:
default: // nunitv3 gives os the most amount of possible details
jargon = Core.TestRunner.Jargon.NUnitV3;
break;
}
if (options.EnableXml)
{
runner.WriteResultsToFile(writer ?? Console.Out, jargon);
logger.Info("Xml file was written to the tcp listener.");
}
else
{
string resultsFilePath = runner.WriteResultsToFile(jargon);
logger.Info($"Xml result can be found {resultsFilePath}");
}
logger.Info($"Tests run: {runner.TotalTests} Passed: {runner.PassedTests} Inconclusive: {runner.InconclusiveTests} Failed: {runner.FailedTests} Ignored: {runner.FilteredTests}");
if (options.TerminateAfterExecution)
{
TerminateWithSuccess();
}
}
