public async Task EnsureFrameworksProduceIdenticalOutputForEachPageInNavViewAsync() { var genIdentities = AllPagesThatSupportSimpleTesting(); ExecutionEnvironment.CheckRunningAsAdmin(); WinAppDriverHelper.CheckIsInstalled(); var app1Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, "SplitView", "MVVMBasic", genIdentities); var errors = new List <string>(); foreach (var framework in GetAdditionalCsFrameworks()) { var app2Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, "SplitView", framework[0].ToString(), genIdentities); var pageExclusions = new Dictionary <string, string>(); pageExclusions.Add("Settings", "new ImageComparer.ExclusionArea(new Rectangle(480, 360, 450, 40), 1.25f)"); var testProjectDetails = SetUpTestProjectForAllNavViewPagesComparison(app1Details, app2Details, pageExclusions); var(testSuccess, testOutput) = RunWinAppDriverTests(testProjectDetails); // Note that failing tests will leave the projects behind, plus the apps and test certificates installed if (testSuccess) { UninstallAppx(app2Details.PackageFullName); RemoveCertificate(app2Details.CertificatePath); // Parent of images folder also contains the test project Fs.SafeDeleteDirectory(Path.Combine(testProjectDetails.imagesFolder, "..")); } else { errors.AddRange(testOutput); // A diff image is automatically created if the outputs differ if (Directory.GetFiles(testProjectDetails.imagesFolder, "DIFF-*.png").Any()) { errors.Add($"Failing test images in {testProjectDetails.imagesFolder}"); } } } var outputMessages = string.Join(Environment.NewLine, errors); if (outputMessages.Any()) { Assert.True(false, $"{Environment.NewLine}{Environment.NewLine}{outputMessages}"); } else { UninstallAppx(app1Details.PackageFullName); RemoveCertificate(app1Details.CertificatePath); } }
// There are tests with hardcoded projectType and framework values to make rerunning/debugging only some of the tests easier private async Task EnsureLanguageLaunchPageVisualsAreEquivalentAsync(string projectType, string framework, string page) { var genIdentities = new[] { page }; ExecutionEnvironment.CheckRunningAsAdmin(); WinAppDriverHelper.CheckIsInstalled(); var app1Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, projectType, framework, genIdentities, lastPageIsHome : true, createForScreenshots : true); var app2Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.VisualBasic, projectType, framework, genIdentities, lastPageIsHome : true, createForScreenshots : true); var noClickCount = 0; if (page == "wts.Page.Map") { noClickCount = 1; } else if (page == "wts.Page.Camera") { noClickCount = 2; } var longPause = page.EndsWith(".WebView") || page.EndsWith(".MediaPlayer"); var testProjectDetails = SetUpTestProjectForInitialScreenshotComparison(app1Details, app2Details, GetExclusionAreasForVisualEquivalencyTest(projectType, page), noClickCount, longPause); var(testSuccess, testOutput) = RunWinAppDriverTests(testProjectDetails); // Note that failing tests will leave the projects behind, plus the apps and test certificates installed if (testSuccess) { UninstallAppx(app1Details.PackageFullName); UninstallAppx(app2Details.PackageFullName); RemoveCertificate(app1Details.CertificatePath); RemoveCertificate(app2Details.CertificatePath); // Parent of images folder also contains the test project Fs.SafeDeleteDirectory(Path.Combine(testProjectDetails.imagesFolder, "..")); } var outputMessages = string.Join(Environment.NewLine, testOutput); // A diff image is automatically created if the outputs differ if (Directory.Exists(testProjectDetails.imagesFolder) && Directory.GetFiles(testProjectDetails.imagesFolder, "*.*-Diff.png").Any()) { Assert.True( testSuccess, $"Failing test images in {testProjectDetails.imagesFolder}{Environment.NewLine}{Environment.NewLine}{outputMessages}"); } else { Assert.True(testSuccess, outputMessages); } }
public WindowContentsTests() { this.testSettings = new TestSettings(); if (string.IsNullOrWhiteSpace(this.testSettings.SubscriptionKey) || this.testSettings.SubscriptionKey == "MUST-BE-SET") { Assert.Fail("Set subscription key in testsettings.json"); } // Need to be admin to save screenshots ExecutionEnvironment.CheckRunningAsAdmin(); }
public async Task RunBasicAccessibilityChecksAgainstEachPageWpfAsync() { // This test does not run against all combinations but relies on other tests to ensure output is the same for each combination. // In theory this means that there should be no need to repeat tests as they would find the same things. // As this is a slow test to run we do not want to increase overall execution time unnecessarily. ExecutionEnvironment.CheckRunningAsAdmin(); WinAppDriverHelper.CheckIsInstalled(); WinAppDriverHelper.StartIfNotRunning(); var pagesFoundWithIssues = 0; var rootFolder = $"{Path.GetPathRoot(Environment.CurrentDirectory)}UIT\\ALLY\\{DateTime.Now:dd_HHmmss}\\"; var reportFolder = Path.Combine(rootFolder, "Reports"); void ForOpenedPage(string projectName) { var processes = Process.GetProcessesByName(projectName); var config = Axe.Windows.Automation.Config.Builder.ForProcessId(processes[0].Id); config.WithOutputFileFormat(Axe.Windows.Automation.OutputFileFormat.A11yTest); config.WithOutputDirectory(reportFolder); var scanner = Axe.Windows.Automation.ScannerFactory.CreateScanner(config.Build()); try { var scanResults = scanner.Scan(); if (scanResults.ErrorCount > 0) { pagesFoundWithIssues++; } } catch (Axe.Windows.Automation.AxeWindowsAutomationException exc) { Assert.True(exc == null, exc.ToString()); } } try { var wpfTestPages = new List <string> { "wts.Wpf.Page.Blank", "wts.Wpf.Page.ListDetails", "wts.Wpf.Page.Settings", "wts.Wpf.Page.WebView", }; var appDetails = await SetUpWpfProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, ProjectTypes.SplitView, Frameworks.MVVMLight, wpfTestPages); var exePath = Path.Combine(appDetails.ProjectPath, appDetails.ProjectName, "bin", "Release", "netcoreapp3.1", $"{appDetails.ProjectName}.exe"); using (var appSession = WinAppDriverHelper.LaunchExe(exePath)) { appSession.Manage().Window.Maximize(); await Task.Delay(TimeSpan.FromSeconds(2)); var menuItems = appSession.FindElementsByClassName("ListBoxItem"); foreach (var menuItem in menuItems) { menuItem.Click(); await Task.Delay(TimeSpan.FromMilliseconds(1500)); // Allow page to load and animations to complete ForOpenedPage(appDetails.ProjectName); } } // Don't leave the app maximized in case we want to open the app again. // Some controls handle layout differently when the app is first opened maximized VirtualKeyboard.RestoreMaximizedWindow(); } finally { WinAppDriverHelper.StopIfRunning(); } Assert.True(0 == pagesFoundWithIssues, $"Accessibility issues found. For details, see reports in '{reportFolder}'."); }
private async Task EnsureCanNavigateToEveryPageWithoutErrorAsync(string framework, string language, string projectType) { var pageIdentities = AllTestablePages(); ExecutionEnvironment.CheckRunningAsAdmin(); WinAppDriverHelper.CheckIsInstalled(); WinAppDriverHelper.StartIfNotRunning(); VisualComparisonTestDetails appDetails = null; int pagesOpenedSuccessfully = 0; try { appDetails = await SetUpProjectForUiTestComparisonAsync(language, projectType, framework, pageIdentities); using (var appSession = WinAppDriverHelper.LaunchAppx(appDetails.PackageFamilyName)) { appSession.Manage().Window.Maximize(); await Task.Delay(TimeSpan.FromSeconds(2)); var menuItems = appSession.FindElementsByClassName("ListViewItem"); foreach (var menuItem in menuItems) { menuItem.Click(); Debug.WriteLine("Opened: " + menuItem.Text); await Task.Delay(TimeSpan.FromMilliseconds(1500)); // Allow page to load and animations to complete if (menuItem.Text == "Map") { // For location permission if (await ClickYesOnPopUpAsync(appSession)) { await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompt pagesOpenedSuccessfully++; } else { Assert.True(false, "Failed to click \"Yes\" on popup for Map permission."); } } else if (menuItem.Text == "Camera") { var cameraPermission = await ClickYesOnPopUpAsync(appSession); // For camera permission var microphonePermission = await ClickYesOnPopUpAsync(appSession); // For microphone permission if (cameraPermission && microphonePermission) { await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompts pagesOpenedSuccessfully++; } else { Assert.True(false, "Failed to click \"Yes\" on popups for Camera page permissions."); } } else { pagesOpenedSuccessfully++; } } // Don't leave the app maximized in case we want to open the app again. // Some controls handle layout differently when the app is first opened maximized VirtualKeyboard.RestoreMaximizedWindow(); } } finally { if (appDetails != null) { UninstallAppx(appDetails.PackageFullName); RemoveCertificate(appDetails.CertificatePath); } WinAppDriverHelper.StopIfRunning(); } var expectedPageCount = pageIdentities.Length + 1; // Add 1 for"Main page" added as well by default Assert.True(expectedPageCount == pagesOpenedSuccessfully, "Not all pages were opened successfully."); }
public async Task EnsureFrameworkLaunchPageVisualsAreEquivalentAsync(string projectType, string page, string[] frameworks) { var genIdentities = new[] { page }; var noClickCount = 0; if (page == "wts.Page.Map") { noClickCount = 1; } else if (page == "wts.Page.Camera") { noClickCount = 2; } ExecutionEnvironment.CheckRunningAsAdmin(); WinAppDriverHelper.CheckIsInstalled(); // MVVMBasic is considered the reference version. Compare generated apps with equivalent in other frameworks var refAppDetails = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, projectType, "MVVMBasic", genIdentities, lastPageIsHome : true, createForScreenshots : true); var otherProjDetails = new VisualComparisonTestDetails[frameworks.Length]; bool allTestsPassed = true; var outputMessages = string.Empty; for (int i = 0; i < frameworks.Length; i++) { string framework = frameworks[i]; otherProjDetails[i] = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, projectType, framework, genIdentities, lastPageIsHome : true, createForScreenshots : true); var testProjectDetails = SetUpTestProjectForInitialScreenshotComparison(refAppDetails, otherProjDetails[i], GetExclusionAreasForVisualEquivalencyTest(projectType, page), noClickCount); var(testSuccess, testOutput) = RunWinAppDriverTests(testProjectDetails); if (testSuccess) { UninstallAppx(otherProjDetails[i].PackageFullName); RemoveCertificate(otherProjDetails[i].CertificatePath); // Parent of images folder also contains the test project Fs.SafeDeleteDirectory(Path.Combine(testProjectDetails.imagesFolder, "..")); } else { allTestsPassed = false; if (Directory.Exists(testProjectDetails.imagesFolder) && Directory.GetFiles(testProjectDetails.imagesFolder, "*.*-Diff.png").Any()) { outputMessages += $"Failing test images in {testProjectDetails.imagesFolder}{Environment.NewLine}{Environment.NewLine}{outputMessages}"; } else { outputMessages += $"{Environment.NewLine}{string.Join(Environment.NewLine, testOutput)}"; } } } if (allTestsPassed) { UninstallAppx(refAppDetails.PackageFullName); RemoveCertificate(refAppDetails.CertificatePath); } Assert.True(allTestsPassed, outputMessages.TrimStart()); }
public async Task RunBasicAccessibilityChecksAgainstEachPageUwpAsync() { // This test does not run against all combinations as other tests ensure output is the same for each combination. // In theory this means that there should be no need to repeat tests as they would find the same things. // As this is a slow test to run we do not want to increase overall execution time unnecessarily. ExecutionEnvironment.CheckRunningAsAdmin(); WinAppDriverHelper.CheckIsInstalled(); WinAppDriverHelper.StartIfNotRunning(); var pagesFoundWithIssues = 0; var rootFolder = $"{Path.GetPathRoot(Environment.CurrentDirectory)}UIT\\ALLY\\{DateTime.Now:dd_HHmmss}\\"; var reportFolder = Path.Combine(rootFolder, "Reports"); var pageIdentities = AllTestablePages(Frameworks.CodeBehind); VisualComparisonTestDetails appDetails = null; async Task ForOpenedPage(string pageName, WindowsDriver <WindowsElement> session, string projectName) { if (pageName == "Map") { // For location permission if (await ClickYesOnPopUpAsync(session)) { await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompt } } else if (pageName == "Camera") { var cameraPermission = await ClickYesOnPopUpAsync(session); // For camera permission var microphonePermission = await ClickYesOnPopUpAsync(session); // For microphone permission if (cameraPermission && microphonePermission) { await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompts } } var processes = Process.GetProcessesByName(projectName); var config = Axe.Windows.Automation.Config.Builder.ForProcessId(processes[0].Id); config.WithOutputFileFormat(Axe.Windows.Automation.OutputFileFormat.A11yTest); config.WithOutputDirectory(reportFolder); var scanner = Axe.Windows.Automation.ScannerFactory.CreateScanner(config.Build()); try { var scanResults = scanner.Scan(); if (scanResults.ErrorCount > 0) { pagesFoundWithIssues++; } } catch (Axe.Windows.Automation.AxeWindowsAutomationException exc) { Assert.True(exc == null, exc.ToString()); } } try { appDetails = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, ProjectTypes.SplitView, Frameworks.CodeBehind, pageIdentities); using (var appSession = WinAppDriverHelper.LaunchAppx(appDetails.PackageFamilyName)) { appSession.Manage().Window.Maximize(); await Task.Delay(TimeSpan.FromSeconds(2)); var menuItems = appSession.FindElementsByClassName("Microsoft.UI.Xaml.Controls.NavigationViewItem"); foreach (var menuItem in menuItems) { menuItem.Click(); await Task.Delay(TimeSpan.FromMilliseconds(1500)); // Allow page to load and animations to complete await ForOpenedPage(menuItem.Text, appSession, appDetails.ProjectName); } } // Don't leave the app maximized in case we want to open the app again. // Some controls handle layout differently when the app is first opened maximized VirtualKeyboard.RestoreMaximizedWindow(); } finally { if (appDetails != null) { UninstallAppx(appDetails.PackageFullName); RemoveCertificate(appDetails.CertificatePath); } WinAppDriverHelper.StopIfRunning(); } Assert.True(0 == pagesFoundWithIssues, $"Accessibility issues found. For details, see reports in '{reportFolder}'."); }