public void InitializeTest()
{
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Simulator_iOS64,
Mock.Of <IHarness> (),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug");
Assert.AreEqual(appName, appRunner.AppInformation.AppName);
Assert.AreEqual(appPath, appRunner.AppInformation.AppPath);
Assert.AreEqual(appPath, appRunner.AppInformation.LaunchAppPath);
Assert.AreEqual(appName, appRunner.AppInformation.BundleIdentifier);
}
public static int Main(string[] args)
{
var serviceProvider =
new ServiceCollection()
.AddSingleton <IConfigurationService, ConfigurationService>()
.AddSingleton <App>()
.AddSingleton <App.Apps>()
.AddSingleton <App.Backup>()
.AddSingleton <App.Config>()
.AddSingleton <App.Content>()
.AddSingleton <App.Schemas>()
.AddSingleton <App.Sync>()
.AddSingleton <App.Twitter>()
.AddSingleton <ISynchronizer, AppSynchronizer>()
.AddSingleton <ISynchronizer, ContentsSynchronizer>()
.AddSingleton <ISynchronizer, RulesSynchronizer>()
.AddSingleton <ISynchronizer, SchemasSynchronizer>()
.AddSingleton <ISynchronizer, WorkflowsSynchronizer>()
.AddSingleton <ILogger, ConsoleLogger>()
.AddSingleton <Synchronizer>()
.BuildServiceProvider();
try
{
var appRunner =
new AppRunner <App>()
.UseMicrosoftDependencyInjection(serviceProvider)
.UseFluentValidation(true);
return(appRunner.Run(args));
}
catch (Exception ex)
{
Console.WriteLine("ERROR: {0}", ex.Message);
return(-1);
}
}
public static AppRunner Expect <TEx>(this AppRunner appRunner,
Action <TEx>?assertEx = null)
where TEx : Exception
{
const int exceptionAsserted = int.MinValue + 1234;
return(appRunner
.UseErrorHandler((ctx, ex) =>
{
if (ex is not TEx)
{
throw new AssertFailedException($"exception {ex.GetType()} is not of type {typeof(TEx)}");
}
assertEx?.Invoke((TEx)ex);
return exceptionAsserted;
})
.AssertAfterRun(r =>
{
if (r.ExitCode != exceptionAsserted)
{
throw new AssertFailedException($"exception {typeof(TEx)} was not raised in the test run. {nameof(appRunner.UseErrorHandler)}");
}
}));
}
public void InstallWhenNoDevicesTest()
{
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Device_iOS,
Mock.Of <IHarness> (),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug");
devices.Setup(x => x.ConnectedDevices).Returns(new IHardwareDevice [0]);
Assert.ThrowsAsync <NoDeviceFoundException> (
async() => await appRunner.InstallAsync(new CancellationToken()),
"Install requires connected devices");
}
internal static AppRunner UsePrompting(
AppRunner appRunner,
Func <CommandContext, IPrompter> prompterOverride = null,
bool promptForMissingArguments = true,
Func <CommandContext, IArgument, string> argumentPromptTextOverride = null,
Predicate <IArgument> argumentFilter = null)
{
return(appRunner.Configure(c =>
{
prompterOverride = prompterOverride
?? c.Services.Get <Func <CommandContext, IPrompter> >()
?? (ctx =>
{
// create only one prompter per CommandContext
// in theory, an implementation could track prompts and values
var prompter = ctx.Services.Get <IPrompter>();
if (prompter == null)
{
prompter = new Prompter(ctx.Console);
ctx.Services.AddOrUpdate(prompter);
}
return prompter;
});
c.UseParameterResolver(ctx => prompterOverride(ctx));
if (promptForMissingArguments)
{
argumentFilter = argumentFilter ?? (a => a.Arity.RequiresAtLeastOne());
c.UseMiddleware(
(ctx, next) => PromptForMissingArguments(ctx, next,
new ArgumentPrompter(prompterOverride(ctx), argumentPromptTextOverride), argumentFilter),
MiddlewareSteps.ValuePromptMissingArguments);
}
}));
}
/// <summary>
/// The delegate to return the value for the <see cref="IArgument"/>
/// </summary>
/// <param name="appRunner"></param>
/// <param name="envVars">A dictionary containing the environment variables</param>
/// <param name="getKeysDelegates">
/// The delegates that will return the keys for an argument.
/// Uses <see cref="GetKeyFromAttribute"/> if none provided.
/// </param>
public static Func <IArgument, ArgumentDefault?> GetDefaultValue(
AppRunner appRunner, IDictionary?envVars = null,
params GetArgumentKeysDelegate[]?getKeysDelegates)
{
IDictionary?ev = envVars;
// AppConfig should never be null by the time this runs
IDictionary GetEnvVars()
{
if (appRunner.AppConfig is null)
{
throw new InvalidConfigurationException(
"AppRunner.AppConfig is null. This method has been called before AppRunner.Run was executed.");
}
return(ev ??= appRunner.AppConfig !.Environment.GetEnvironmentVariables());
}
return(GetValueFunc(Resources.A.ValueSource_EnvVar,
key => GetEnvVars().GetValueOrDefault <string>(key),
getKeysDelegates ?? new GetArgumentKeysDelegate[]
{
GetKeyFromAttribute
}));
}
public void List_ByDefault_ReturnsSuccess()
{
var mock = new Mock <IConnectorManager>();
mock.Setup(m => m.Verify(It.IsAny <CancellationToken>()))
.ReturnsAsync(true);
mock.Setup(m => m.List(It.IsAny <string>()))
.Returns(
() => new List <(string name, ConnectorData data)>
{
("Connector", new ConnectorData(new ConnectorSettings(), null))
}
);
var sp = GetDefaultServiceProvider(mock.Object);
var result = new AppRunner <EDRMethods>()
.UseMicrosoftDependencyInjection(sp)
.UseDefaultMiddleware()
.RunInMem("connector list filter");
result.ExitCode.Should().Be(0);
}
public MainWindow()
{
InitializeComponent();
var scopeBuilder = new SkitanaScopeBuilder();
scopeBuilder.RegisterSkiaRenderer()
.RegisterWpfInputServices(skiaElement)
.RegisterSkitanaAppFramework()
.RegisterAllControlsServices()
.RegisterSingleton(skiaElement);
var services = scopeBuilder.Build();
onCloseDisposable = services as IDisposable;
var factory = (IIoCFactory)services.GetService(typeof(IIoCFactory));
appRunner = factory.Create <WpfAppRunner>();
var app = factory.Create <AllControlsApp>();
cancellationTokenSource = new CancellationTokenSource();
appLoopTask = appRunner.RunAsync(app, cancellationTokenSource.Token);
}
public void UninstallToSimulatorTest()
{
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Simulator_iOS64,
Mock.Of <IHarness> (),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug");
appRunner.InitializeAsync().Wait();
var exception = Assert.ThrowsAsync <InvalidOperationException> (
async() => await appRunner.UninstallAsync(),
"Uninstall should not be allowed on a simulator");
}
internal static AppRunner AppendPipedInputToOperandList(AppRunner appRunner)
{
// -1 to ensure this middleware runs before any prompting so the value won't appear null
return(appRunner.Configure(c =>
c.UseMiddleware(InjectPipedInputToOperandList, MiddlewareSteps.PipedInput)));
}
public static AppRunner TrackingInvocations(this AppRunner runner)
{
return(runner.Configure(c => c.UseMiddleware(TrackingInvocationMiddleware,
new MiddlewareStep(MiddlewareStages.PostBindValuesPreInvoke, short.MaxValue))));
}
public void SubcommandsCannotFollowOperandsOfDefaultCommand()
{
var results = new AppRunner <DefaultApp>()
.Verify(new Scenario
{
When = { Args = "1 Do" },
private static void PrintContext(AppRunner appRunner, ILogger logger)
{
logger.WriteLine("");
logger.WriteLine(appRunner.ToString());
}
/// <summary>Run and verify the scenario expectations, output results to <see cref="Console"/></summary>
public static AppRunnerResult VerifyScenario(this AppRunner appRunner, IScenario scenario)
{
return(appRunner.VerifyScenario(new Logger(Console.WriteLine), scenario));
}
public async Task RunOnDeviceSuccessfullyTest()
{
var deviceSystemLog = new Mock <IFileBackedLog>();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
deviceSystemLog.SetupGet(x => x.Description).Returns(LogType.SystemLog.ToString());
SetupLogList(new[] { deviceSystemLog.Object });
_logs
.Setup(x => x.Create("device-" + DeviceName + "-mocked_timestamp.log", LogType.SystemLog.ToString(), It.IsAny <bool?>()))
.Returns(deviceSystemLog.Object);
var deviceLogCapturer = new Mock <IDeviceLogCapturer>();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>();
deviceLogCapturerFactory
.Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, DeviceName))
.Returns(deviceLogCapturer.Object);
var x = _logs.Object.First();
var appInformation = GetMockedAppBundleInfo();
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
deviceLogCapturerFactory.Object,
_mainLog.Object,
_logs.Object,
_helpers.Object,
new[] { "--appArg1=value1", "-g" });
var(deviceName, result) = await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
TimeSpan.FromSeconds(30));
// Verify
Assert.Equal(DeviceName, deviceName);
Assert.True(result.Succeeded);
var expectedArgs = GetExpectedDeviceMlaunchArgs();
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog>(),
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_hardwareDeviceLoader.VerifyAll();
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task RunOnSimulatorSuccessfullyTest()
{
_simulatorLoader
.Setup(x => x.FindSimulators(It.Is <TestTargetOs>(t => t.Platform == TestTarget.Simulator_tvOS), _mainLog.Object, It.IsAny <int>(), true, false))
.ReturnsAsync((_mockSimulator.Object, null));
var captureLog = new Mock <ICaptureLog>();
captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath);
captureLog.SetupGet(x => x.Description).Returns(LogType.SystemLog.ToString());
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, _mockSimulator.Object.Name + ".log"),
_mockSimulator.Object.SystemLog,
false,
LogType.SystemLog))
.Returns(captureLog.Object);
SetupLogList(new[] { captureLog.Object });
var appInformation = GetMockedAppBundleInfo();
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
new[] { "--appArg1=value1", "-g" });
var(deviceName, result) = await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Simulator_tvOS, null),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState : true);
// Verify
Assert.Equal(SimulatorDeviceName, deviceName);
Assert.True(result.Succeeded);
var expectedArgs = GetExpectedSimulatorMlaunchArgs();
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
_mainLog.Object,
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_simulatorLoader.VerifyAll();
captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce);
// When ensureCleanSimulatorState == true
_mockSimulator.Verify(x => x.PrepareSimulator(_mainLog.Object, AppBundleIdentifier));
_mockSimulator.Verify(x => x.KillEverything(_mainLog.Object));
}
static int Main(string[] args)
{
AppRunner <GodfatherChars> appRunner = new AppRunner <GodfatherChars>();
return(appRunner.Run(args));
}
public static void Start(string environmentName, string[] args, System.Drawing.Icon appIcon = null)
{
AppRunner.RunAsAdmin(args, new AppBootstrapper(environmentName, appIcon));
}
public async Task RunOnSimulatorSuccessfullyTest()
{
var harness = GetMockedHarness();
devices.Setup(x => x.ConnectedDevices).Returns(mockDevices.Reverse());
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
// Mock finding simulators
simulators
.Setup(x => x.LoadDevices(It.IsAny <ILog> (), false, false, false))
.Returns(Task.FromResult(true));
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
var simulator = new Mock <ISimulatorDevice> ();
simulator.SetupGet(x => x.Name).Returns("Test iPhone simulator");
simulator.SetupGet(x => x.UDID).Returns("58F21118E4D34FD69EAB7860BB9B38A0");
simulator.SetupGet(x => x.LogPath).Returns(simulatorLogPath);
simulator.SetupGet(x => x.SystemLog).Returns(Path.Combine(simulatorLogPath, "system.log"));
simulators
.Setup(x => x.FindSimulators(TestTarget.Simulator_iOS64, mainLog.Object, true, false))
.ReturnsAsync((simulator.Object, null));
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog> (x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new [] { "Some result here", "Tests run: 124", "Some result there" });
logs
.Setup(x => x.Create(It.IsAny <string> (), It.IsAny <string>(), It.IsAny <bool?> ()))
.Returns(listenerLogFile);
logs
.Setup(x => x.CreateFile(It.IsAny <string> (), It.IsAny <string> ()))
.Returns($"/path/to/log-{Guid.NewGuid ()}.log");
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var captureLog = new Mock <ICaptureLog> ();
captureLog.SetupGet(x => x.FullPath).Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory> ();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(logs.Object.Directory, simulator.Object.Name + ".log"),
simulator.Object.SystemLog,
true,
It.IsAny <string> ()))
.Returns(captureLog.Object);
var expectedArgs = $"--sdkroot {StringUtils.FormatArguments (xcodePath)} -v -v " +
$"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
$"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
$"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " +
$"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:127.0.0.1 " +
$"-setenv=NUNIT_HOSTNAME=127.0.0.1 -argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP " +
$"-argument=-app-arg:-hostport:{listenerPort} -setenv=NUNIT_HOSTPORT={listenerPort} " +
$"-setenv=env1=value1 -setenv=env2=value2 --launchsim {StringUtils.FormatArguments (appPath)} " +
$"--stdout=tty1 --stderr=tty1 --device=:v2:udid={simulator.Object.UDID}";
processManager
.Setup(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
mainLog.Object,
TimeSpan.FromMinutes(harness.Timeout * 2),
null,
It.IsAny <CancellationToken> ()))
.ReturnsAsync(new ProcessExecutionResult()
{
ExitCode = 0
});
var xmlResultFile = Path.ChangeExtension(testResultFilePath, "xml");
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory> (), // Use for devices only
testReporterFactory.Object,
TestTarget.Simulator_iOS64,
harness,
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2,
ensureCleanSimulatorState: true);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(0, result);
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Cancel(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Dispose(), Times.AtLeastOnce);
simulators.VerifyAll();
captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce);
captureLog.Verify(x => x.StopCapture(), Times.AtLeastOnce);
// When ensureCleanSimulatorState == true
simulator.Verify(x => x.PrepareSimulator(It.IsAny <ILog> (), appName));
simulator.Verify(x => x.KillEverything(mainLog.Object));
}
public static void Start(string environmentName, string[] args, Optional <Icon> appIcon)
{
AppRunner.RunAsAdmin(args, new AppBootstrapper(environmentName, appIcon), true);
}
static int Main(string[] args) => AppRunner.Run(args);
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
mainLog,
logs,
new Helpers(),
PassThroughArguments,
logCallback);
ProcessExecutionResult result;
(deviceName, result) = await appRunner.RunApp(
appBundleInfo,
target,
_arguments.Timeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
cancellationToken : cancellationToken);
if (!result.Succeeded)
{
if (result.TimedOut)
{
logger.LogError($"App run has timed out");
return(ExitCode.TIMED_OUT);
}
logger.LogError($"App run has failed. mlaunch exited with {result.ExitCode}");
return(ExitCode.APP_LAUNCH_FAILURE);
}
var systemLog = logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString());
if (systemLog == null)
{
logger.LogError("Application has finished but no system log found. Failed to determine the exit code!");
return(ExitCode.RETURN_CODE_NOT_SET);
}
var exitCode = new ExitCodeDetector().DetectExitCode(appBundleInfo, systemLog);
logger.LogInformation($"App run ended with {exitCode}");
if (_arguments.ExpectedExitCode != exitCode)
{
logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected");
return(ExitCode.GENERAL_FAILURE);
}
logger.LogInformation("Application has finished with exit code: " + exitCode +
(_arguments.ExpectedExitCode != 0 ? " (as expected)" : null));
return(ExitCode.SUCCESS);
}
private async Task <ExitCode> RunTest(
ILogger logger,
TestTarget target,
Logs logs,
ProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
ITunnelBore?tunnelBore,
CancellationToken cancellationToken = default)
{
logger.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
ILog mainLog = logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true);
int verbosity = GetMlaunchVerbosity(_arguments.Verbosity);
string?deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target, mainLog, cancellationToken);
}
catch (Exception e)
{
logger.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.IsSimulator())
{
logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? $" on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(_arguments.AppPackagePath, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
// use the knowledge base class to decide if the error is known, if it is, let the user know
// the failure reason
if (_errorKnowledgeBase.IsKnownInstallIssue(mainLog, out var errorMessage))
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode}): {errorMessage}");
}
else
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
}
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
logger.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? $"device '{deviceName}'" : target.AsString()));
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
mainLog,
logs,
new Helpers(),
useXmlOutput: true); // the cli ALWAYS will get the output as xml
try
{
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken);
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
if (_errorKnowledgeBase.IsKnownTestIssue(mainLog, out var failureMessage))
{
logger.LogError($"Application run failed:{Environment.NewLine}{failureMessage}");
}
else
{
logger.LogError($"Application run failed:{Environment.NewLine}{e}");
}
return(ExitCode.APP_CRASH);
}
finally
{
if (!target.IsSimulator() && deviceName != null)
{
logger.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
logger.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
logger.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
}
}
public void Verify <TApp>(
string args,
bool includeTransforms = false,
bool showsHelp = false,
bool showsTokens = false,
bool showsUnableToMapTokensToArgsMsg = false,
bool showsHelpRequestOnlyTokensAvailableMsg = false,
bool showsHintToUseParseT = false,
bool throwBeforeBind = false,
bool throwAtInvoke = false,
bool exitBeforeBind = false,
string expectedParseResult = null,
string expectedResult = null) where TApp : class
{
var parse = includeTransforms ? "[parse:t]" : "[parse]";
var contains = new List <string>();
var notContains = new List <string>();
void containsIf(bool yes, string value)
{
(yes ? contains : notContains).Add(value);
}
args = args ?? "";
containsIf(showsHelp, "Usage: dotnet testhost.dll ");
containsIf(args == "-h" || args.Contains(" -h") || args.Contains("-h "),
"Help requested. Only token transformations are available.");
containsIf(showsTokens, @"token transformations:
>>> from shell");
if (expectedParseResult != null)
{
contains.Add(expectedParseResult);
}
containsIf(showsUnableToMapTokensToArgsMsg,
"Unable to map tokens to arguments. Falling back to token transformations.");
containsIf(showsHelpRequestOnlyTokensAvailableMsg,
"Help requested. Only token transformations are available.");
containsIf(showsHintToUseParseT,
"Use [parse:T] to include token transformations");
var appRunner = new AppRunner <TApp>();
if (exitBeforeBind)
{
appRunner.Configure(c =>
c.UseMiddleware((ctx, next) => ExitCodes.Success,
MiddlewareStages.PostParseInputPreBindValues));
}
if (throwBeforeBind)
{
appRunner.Configure(c =>
c.UseMiddleware((ctx, next) =>
throw new Exception("throwBeforeBind exception"),
MiddlewareStages.PostParseInputPreBindValues));
}
if (throwAtInvoke)
{
appRunner.Configure(c =>
c.UseMiddleware((ctx, next) =>
throw new Exception("throwAtInvoke exception"),
MiddlewareStages.Invoke, int.MinValue));
}
appRunner.Configure(c =>
c.UseMiddleware((ctx, next) =>
throw new Exception("parse should exit before method invocation"),
MiddlewareStages.Invoke, int.MaxValue - 1));
var result = appRunner
.UseParseDirective()
.Verify(new Scenario
{
When = { Args = $"{parse} {args}" },
Then =
{
ExitCode = throwBeforeBind ? 1 : 0,
Output = expectedResult,
OutputContainsTexts = contains,
OutputNotContainsTexts = notContains
}
});
if (showsHelp && showsTokens)
{
// help before token transformations
var consoleOut = result.Console.OutText();
var helpIndex = consoleOut.IndexOf("Usage: dotnet testhost.dll ");
var tokensIndex = consoleOut.IndexOf("token transformations:");
tokensIndex.Should().BeLessThan(helpIndex);
}
}
public static void VerifyScenario(this AppRunner appRunner, ITestOutputHelper output, IScenario scenario)
{
appRunner.VerifyScenario(output.AsLogger(), scenario);
}
public void LongName_ShowsHelp()
{
var result = new AppRunner <App>().RunInMem("--help".SplitArgs());
result.CommandContext.ShowHelpOnExit.Should().BeTrue();
}
public async Task RunOnSimulatorWithNoAvailableSimulatorTest()
{
devices.Setup(x => x.ConnectedDevices).Returns(mockDevices.Reverse());
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
// Mock finding simulators
simulators
.Setup(x => x.LoadDevices(It.IsAny <ILog> (), false, false, false))
.Returns(Task.FromResult(true));
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
simulators
.Setup(x => x.FindSimulators(TestTarget.Simulator_tvOS, mainLog.Object, true, false))
.ReturnsAsync((null, null));
var listenerLogFile = new Mock <IFileBackedLog> ();
logs
.Setup(x => x.Create(It.IsAny <string> (), "TestLog", It.IsAny <bool> ()))
.Returns(listenerLogFile.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var captureLog = new Mock <ICaptureLog> ();
captureLog.SetupGet(x => x.FullPath).Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory> ();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(logs.Object.Directory, "tvos.log"),
"/path/to/simulator.log",
true,
It.IsAny <string> ()))
.Returns(captureLog.Object);
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory> (),
testReporterFactory.Object,
TestTarget.Simulator_tvOS,
GetMockedHarness(),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(1, result);
mainLog.Verify(x => x.WriteLine("Test run completed"), Times.Never);
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simulators.VerifyAll();
}
public void LongName_ShowsHelp()
{
var result = new AppRunner <App>().RunInMem("--help".SplitArgs(), _testOutputHelper);
result.OutputContains("Usage: dotnet testhost.dll [command]");
}
public async Task RunOnDeviceWithFailedTestsTest()
{
var harness = GetMockedHarness();
devices.Setup(d => d.FindDevice(It.IsAny <RunMode> (), It.IsAny <ILog> (), false, false)).ReturnsAsync(mockDevices[1]);
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), true, "Test iPad"))
.Returns(snapshotReporter.Object);
var deviceSystemLog = new Mock <IFileBackedLog> ();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog> (x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new [] { "Some result here", "[FAIL] This test failed", "Some result there", "Tests run: 3" });
logs
.Setup(x => x.Create(It.Is <string> (s => s.StartsWith("test-ios-")), "TestLog", It.IsAny <bool?> ()))
.Returns(listenerLogFile);
logs
.Setup(x => x.Create(It.Is <string> (s => s.StartsWith("device-Test iPad-")), "Device log", It.IsAny <bool?> ()))
.Returns(deviceSystemLog.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var deviceLogCapturer = new Mock <IDeviceLogCapturer> ();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory> ();
deviceLogCapturerFactory
.Setup(x => x.Create(mainLog.Object, deviceSystemLog.Object, "Test iPad"))
.Returns(deviceLogCapturer.Object);
var ips = new StringBuilder();
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList;
for (int i = 0; i < ipAddresses.Length; i++)
{
if (i > 0)
{
ips.Append(',');
}
ips.Append(ipAddresses [i].ToString());
}
var expectedArgs = $"-v -v -argument=-connection-mode -argument=none " +
$"-argument=-app-arg:-autostart -setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit " +
$"-setenv=NUNIT_AUTOEXIT=true -argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " +
$"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} " +
$"-argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{listenerPort} " +
$"-setenv=NUNIT_HOSTPORT={listenerPort} -setenv=env1=value1 -setenv=env2=value2 " +
$"--launchdev {StringUtils.FormatArguments (appPath)} --disable-memory-limits --wait-for-exit --devname \"Test iPad\"";
processManager
.Setup(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog> (),
TimeSpan.FromMinutes(harness.Timeout * 2),
null,
It.IsAny <CancellationToken> ()))
.ReturnsAsync(new ProcessExecutionResult()
{
ExitCode = 0
});
var xmlResultFile = Path.ChangeExtension(testResultFilePath, "xml");
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Failed, null)));
});
testReporter.Setup(r => r.Success).Returns(false);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
Mock.Of <ICaptureLogFactory> (), // Used for simulators only
deviceLogCapturerFactory.Object,
testReporterFactory.Object,
TestTarget.Device_iOS,
harness,
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(1, result);
processManager.VerifyAll();
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Cancel(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Dispose(), Times.AtLeastOnce);
snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task MultiProcessIdentificationTest(DiagnosticPortConnectionMode mode)
{
DiagnosticPortHelper.Generate(
mode,
out DiagnosticPortConnectionMode appConnectionMode,
out string diagnosticPortPath);
await using MonitorCollectRunner toolRunner = new(_outputHelper);
toolRunner.ConnectionMode = mode;
toolRunner.DiagnosticPortPath = diagnosticPortPath;
toolRunner.DisableAuthentication = true;
await toolRunner.StartAsync();
using HttpClient httpClient = await toolRunner.CreateHttpClientDefaultAddressAsync(_httpClientFactory);
ApiClient apiClient = new(_outputHelper, httpClient);
const int appCount = 3;
AppRunner[] appRunners = new AppRunner[appCount];
for (int i = 0; i < appCount; i++)
{
AppRunner runner = new(_outputHelper, Assembly.GetExecutingAssembly(), appId : i + 1);
runner.ConnectionMode = appConnectionMode;
runner.DiagnosticPortPath = diagnosticPortPath;
runner.ScenarioName = TestAppScenarios.AsyncWait.Name;
runner.Environment[ExpectedEnvVarName] = Guid.NewGuid().ToString("D");
appRunners[i] = runner;
}
IList <ProcessIdentifier> identifiers;
await appRunners.ExecuteAsync(async() =>
{
// Scope to only the processes that were launched by the test
IList <int> unmatchedPids = new List <int>();
foreach (AppRunner runner in appRunners)
{
unmatchedPids.Add(await runner.ProcessIdTask);
}
// Query for process identifiers
identifiers = (await apiClient.GetProcessesWithRetryAsync(
_outputHelper,
unmatchedPids.ToArray())).ToList();
Assert.NotNull(identifiers);
_outputHelper.WriteLine("Start enumerating discovered processes.");
foreach (ProcessIdentifier identifier in identifiers.ToList())
{
_outputHelper.WriteLine($"- PID: {identifier.Pid}");
_outputHelper.WriteLine($" UID: {identifier.Uid}");
_outputHelper.WriteLine($" Name: {identifier.Name}");
unmatchedPids.Remove(identifier.Pid);
}
_outputHelper.WriteLine("End enumerating discovered processes");
Assert.Empty(unmatchedPids);
Assert.Equal(appRunners.Length, identifiers.Count);
foreach (ProcessIdentifier processIdentifier in identifiers)
{
int pid = processIdentifier.Pid;
Guid uid = processIdentifier.Uid;
string name = processIdentifier.Name;
#if NET5_0_OR_GREATER
// CHECK 1: Get response for processes using PID, UID, and Name and check for consistency
List <ProcessInfo> processInfoQueriesCheck1 = new List <ProcessInfo>();
processInfoQueriesCheck1.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid));
// Only check with uid if it is non-empty; this can happen in connect mode if the ProcessInfo command fails
// to respond within the short period of time that is used to get the additional process information.
if (uid == Guid.Empty)
{
_outputHelper.WriteLine("Skipped uid-only check because it is empty GUID.");
}
else
{
processInfoQueriesCheck1.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, uid: uid));
}
VerifyProcessInfoEquality(processInfoQueriesCheck1);
#endif
// CHECK 2: Get response for requests using PID | PID and UID | PID, UID, and Name and check for consistency
List <ProcessInfo> processInfoQueriesCheck2 = new List <ProcessInfo>();
processInfoQueriesCheck2.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid));
processInfoQueriesCheck2.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid, uid: uid));
processInfoQueriesCheck2.Add(await apiClient.GetProcessWithRetryAsync(_outputHelper, pid: pid, uid: uid, name: name));
VerifyProcessInfoEquality(processInfoQueriesCheck2);
// CHECK 3: Get response for processes using PID and an unassociated (randomly generated) UID and ensure the proper exception is thrown
await VerifyInvalidRequestException(apiClient, pid, Guid.NewGuid(), null);
}
// CHECK 4: Get response for processes using invalid PID, UID, or Name and ensure the proper exception is thrown
await VerifyInvalidRequestException(apiClient, -1, null, null);
await VerifyInvalidRequestException(apiClient, null, Guid.NewGuid(), null);
await VerifyInvalidRequestException(apiClient, null, null, "");
// Verify each app instance is reported and shut them down.
foreach (AppRunner runner in appRunners)
{
Assert.True(runner.Environment.TryGetValue(ExpectedEnvVarName, out string expectedEnvVarValue));
await VerifyProcessAsync(apiClient, identifiers, await runner.ProcessIdTask, expectedEnvVarValue);
await runner.SendCommandAsync(TestAppScenarios.AsyncWait.Commands.Continue);
}
});
for (int i = 0; i < appCount; i++)
{
Assert.True(0 == appRunners[i].ExitCode, $"App {i} exit code is non-zero.");
}
// Query for process identifiers
identifiers = (await apiClient.GetProcessesAsync()).ToList();
Assert.NotNull(identifiers);
// Verify none of the apps are reported
List <int> runnerProcessIds = new(appCount);
for (int i = 0; i < appCount; i++)
{
runnerProcessIds.Add(await appRunners[i].ProcessIdTask);
}
foreach (ProcessIdentifier identifier in identifiers)
{
Assert.DoesNotContain(identifier.Pid, runnerProcessIds);
}
}
