public void SetUp()
{
WinAppDriverHelper.CheckIsInstalled();
WinAppDriverHelper.StartIfNotRunning();
VirtualKeyboard.MinimizeAllWindows();
}
public async Task EnsureFrameworksProduceIdenticalOutputForEachPageInNavViewAsync()
{
var genIdentities = AllPagesThatSupportSimpleTesting();
ExecutionEnvironment.CheckRunningAsAdmin();
WinAppDriverHelper.CheckIsInstalled();
var app1Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, "SplitView", "MVVMBasic", genIdentities);
var errors = new List <string>();
foreach (var framework in GetAdditionalCsFrameworks())
{
var app2Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, "SplitView", framework[0].ToString(), genIdentities);
var pageExclusions = new Dictionary <string, string>();
pageExclusions.Add("Settings", "new ImageComparer.ExclusionArea(new Rectangle(480, 360, 450, 40), 1.25f)");
var testProjectDetails = SetUpTestProjectForAllNavViewPagesComparison(app1Details, app2Details, pageExclusions);
var(testSuccess, testOutput) = RunWinAppDriverTests(testProjectDetails);
// Note that failing tests will leave the projects behind, plus the apps and test certificates installed
if (testSuccess)
{
UninstallAppx(app2Details.PackageFullName);
RemoveCertificate(app2Details.CertificatePath);
// Parent of images folder also contains the test project
Fs.SafeDeleteDirectory(Path.Combine(testProjectDetails.imagesFolder, ".."));
}
else
{
errors.AddRange(testOutput);
// A diff image is automatically created if the outputs differ
if (Directory.GetFiles(testProjectDetails.imagesFolder, "DIFF-*.png").Any())
{
errors.Add($"Failing test images in {testProjectDetails.imagesFolder}");
}
}
}
var outputMessages = string.Join(Environment.NewLine, errors);
if (outputMessages.Any())
{
Assert.True(false, $"{Environment.NewLine}{Environment.NewLine}{outputMessages}");
}
else
{
UninstallAppx(app1Details.PackageFullName);
RemoveCertificate(app1Details.CertificatePath);
}
}
// There are tests with hardcoded projectType and framework values to make rerunning/debugging only some of the tests easier
private async Task EnsureLanguageLaunchPageVisualsAreEquivalentAsync(string projectType, string framework, string page)
{
var genIdentities = new[] { page };
ExecutionEnvironment.CheckRunningAsAdmin();
WinAppDriverHelper.CheckIsInstalled();
var app1Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, projectType, framework, genIdentities, lastPageIsHome : true, createForScreenshots : true);
var app2Details = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.VisualBasic, projectType, framework, genIdentities, lastPageIsHome : true, createForScreenshots : true);
var noClickCount = 0;
if (page == "wts.Page.Map")
{
noClickCount = 1;
}
else if (page == "wts.Page.Camera")
{
noClickCount = 2;
}
var longPause = page.EndsWith(".WebView") || page.EndsWith(".MediaPlayer");
var testProjectDetails = SetUpTestProjectForInitialScreenshotComparison(app1Details, app2Details, GetExclusionAreasForVisualEquivalencyTest(projectType, page), noClickCount, longPause);
var(testSuccess, testOutput) = RunWinAppDriverTests(testProjectDetails);
// Note that failing tests will leave the projects behind, plus the apps and test certificates installed
if (testSuccess)
{
UninstallAppx(app1Details.PackageFullName);
UninstallAppx(app2Details.PackageFullName);
RemoveCertificate(app1Details.CertificatePath);
RemoveCertificate(app2Details.CertificatePath);
// Parent of images folder also contains the test project
Fs.SafeDeleteDirectory(Path.Combine(testProjectDetails.imagesFolder, ".."));
}
var outputMessages = string.Join(Environment.NewLine, testOutput);
// A diff image is automatically created if the outputs differ
if (Directory.Exists(testProjectDetails.imagesFolder) &&
Directory.GetFiles(testProjectDetails.imagesFolder, "*.*-Diff.png").Any())
{
Assert.True(
testSuccess,
$"Failing test images in {testProjectDetails.imagesFolder}{Environment.NewLine}{Environment.NewLine}{outputMessages}");
}
else
{
Assert.True(testSuccess, outputMessages);
}
}
public async Task RunBasicAccessibilityChecksAgainstEachPageWpfAsync()
{
// This test does not run against all combinations but relies on other tests to ensure output is the same for each combination.
// In theory this means that there should be no need to repeat tests as they would find the same things.
// As this is a slow test to run we do not want to increase overall execution time unnecessarily.
ExecutionEnvironment.CheckRunningAsAdmin();
WinAppDriverHelper.CheckIsInstalled();
WinAppDriverHelper.StartIfNotRunning();
var pagesFoundWithIssues = 0;
var rootFolder = $"{Path.GetPathRoot(Environment.CurrentDirectory)}UIT\\ALLY\\{DateTime.Now:dd_HHmmss}\\";
var reportFolder = Path.Combine(rootFolder, "Reports");
void ForOpenedPage(string projectName)
{
var processes = Process.GetProcessesByName(projectName);
var config = Axe.Windows.Automation.Config.Builder.ForProcessId(processes[0].Id);
config.WithOutputFileFormat(Axe.Windows.Automation.OutputFileFormat.A11yTest);
config.WithOutputDirectory(reportFolder);
var scanner = Axe.Windows.Automation.ScannerFactory.CreateScanner(config.Build());
try
{
var scanResults = scanner.Scan();
if (scanResults.ErrorCount > 0)
{
pagesFoundWithIssues++;
}
}
catch (Axe.Windows.Automation.AxeWindowsAutomationException exc)
{
Assert.True(exc == null, exc.ToString());
}
}
try
{
var wpfTestPages = new List <string>
{
"wts.Wpf.Page.Blank",
"wts.Wpf.Page.ListDetails",
"wts.Wpf.Page.Settings",
"wts.Wpf.Page.WebView",
};
var appDetails = await SetUpWpfProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, ProjectTypes.SplitView, Frameworks.MVVMLight, wpfTestPages);
var exePath = Path.Combine(appDetails.ProjectPath, appDetails.ProjectName, "bin", "Release", "netcoreapp3.1", $"{appDetails.ProjectName}.exe");
using (var appSession = WinAppDriverHelper.LaunchExe(exePath))
{
appSession.Manage().Window.Maximize();
await Task.Delay(TimeSpan.FromSeconds(2));
var menuItems = appSession.FindElementsByClassName("ListBoxItem");
foreach (var menuItem in menuItems)
{
menuItem.Click();
await Task.Delay(TimeSpan.FromMilliseconds(1500)); // Allow page to load and animations to complete
ForOpenedPage(appDetails.ProjectName);
}
}
// Don't leave the app maximized in case we want to open the app again.
// Some controls handle layout differently when the app is first opened maximized
VirtualKeyboard.RestoreMaximizedWindow();
}
finally
{
WinAppDriverHelper.StopIfRunning();
}
Assert.True(0 == pagesFoundWithIssues, $"Accessibility issues found. For details, see reports in '{reportFolder}'.");
}
private async Task EnsureCanNavigateToEveryPageWithoutErrorAsync(string framework, string language, string projectType)
{
var pageIdentities = AllTestablePages();
ExecutionEnvironment.CheckRunningAsAdmin();
WinAppDriverHelper.CheckIsInstalled();
WinAppDriverHelper.StartIfNotRunning();
VisualComparisonTestDetails appDetails = null;
int pagesOpenedSuccessfully = 0;
try
{
appDetails = await SetUpProjectForUiTestComparisonAsync(language, projectType, framework, pageIdentities);
using (var appSession = WinAppDriverHelper.LaunchAppx(appDetails.PackageFamilyName))
{
appSession.Manage().Window.Maximize();
await Task.Delay(TimeSpan.FromSeconds(2));
var menuItems = appSession.FindElementsByClassName("ListViewItem");
foreach (var menuItem in menuItems)
{
menuItem.Click();
Debug.WriteLine("Opened: " + menuItem.Text);
await Task.Delay(TimeSpan.FromMilliseconds(1500)); // Allow page to load and animations to complete
if (menuItem.Text == "Map")
{
// For location permission
if (await ClickYesOnPopUpAsync(appSession))
{
await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompt
pagesOpenedSuccessfully++;
}
else
{
Assert.True(false, "Failed to click \"Yes\" on popup for Map permission.");
}
}
else if (menuItem.Text == "Camera")
{
var cameraPermission = await ClickYesOnPopUpAsync(appSession); // For camera permission
var microphonePermission = await ClickYesOnPopUpAsync(appSession); // For microphone permission
if (cameraPermission && microphonePermission)
{
await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompts
pagesOpenedSuccessfully++;
}
else
{
Assert.True(false, "Failed to click \"Yes\" on popups for Camera page permissions.");
}
}
else
{
pagesOpenedSuccessfully++;
}
}
// Don't leave the app maximized in case we want to open the app again.
// Some controls handle layout differently when the app is first opened maximized
VirtualKeyboard.RestoreMaximizedWindow();
}
}
finally
{
if (appDetails != null)
{
UninstallAppx(appDetails.PackageFullName);
RemoveCertificate(appDetails.CertificatePath);
}
WinAppDriverHelper.StopIfRunning();
}
var expectedPageCount = pageIdentities.Length + 1; // Add 1 for"Main page" added as well by default
Assert.True(expectedPageCount == pagesOpenedSuccessfully, "Not all pages were opened successfully.");
}
public async Task EnsureFrameworkLaunchPageVisualsAreEquivalentAsync(string projectType, string page, string[] frameworks)
{
var genIdentities = new[] { page };
var noClickCount = 0;
if (page == "wts.Page.Map")
{
noClickCount = 1;
}
else if (page == "wts.Page.Camera")
{
noClickCount = 2;
}
ExecutionEnvironment.CheckRunningAsAdmin();
WinAppDriverHelper.CheckIsInstalled();
// MVVMBasic is considered the reference version. Compare generated apps with equivalent in other frameworks
var refAppDetails = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, projectType, "MVVMBasic", genIdentities, lastPageIsHome : true, createForScreenshots : true);
var otherProjDetails = new VisualComparisonTestDetails[frameworks.Length];
bool allTestsPassed = true;
var outputMessages = string.Empty;
for (int i = 0; i < frameworks.Length; i++)
{
string framework = frameworks[i];
otherProjDetails[i] = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, projectType, framework, genIdentities, lastPageIsHome : true, createForScreenshots : true);
var testProjectDetails = SetUpTestProjectForInitialScreenshotComparison(refAppDetails, otherProjDetails[i], GetExclusionAreasForVisualEquivalencyTest(projectType, page), noClickCount);
var(testSuccess, testOutput) = RunWinAppDriverTests(testProjectDetails);
if (testSuccess)
{
UninstallAppx(otherProjDetails[i].PackageFullName);
RemoveCertificate(otherProjDetails[i].CertificatePath);
// Parent of images folder also contains the test project
Fs.SafeDeleteDirectory(Path.Combine(testProjectDetails.imagesFolder, ".."));
}
else
{
allTestsPassed = false;
if (Directory.Exists(testProjectDetails.imagesFolder) &&
Directory.GetFiles(testProjectDetails.imagesFolder, "*.*-Diff.png").Any())
{
outputMessages += $"Failing test images in {testProjectDetails.imagesFolder}{Environment.NewLine}{Environment.NewLine}{outputMessages}";
}
else
{
outputMessages += $"{Environment.NewLine}{string.Join(Environment.NewLine, testOutput)}";
}
}
}
if (allTestsPassed)
{
UninstallAppx(refAppDetails.PackageFullName);
RemoveCertificate(refAppDetails.CertificatePath);
}
Assert.True(allTestsPassed, outputMessages.TrimStart());
}
public async Task RunBasicAccessibilityChecksAgainstEachPageUwpAsync()
{
// This test does not run against all combinations as other tests ensure output is the same for each combination.
// In theory this means that there should be no need to repeat tests as they would find the same things.
// As this is a slow test to run we do not want to increase overall execution time unnecessarily.
ExecutionEnvironment.CheckRunningAsAdmin();
WinAppDriverHelper.CheckIsInstalled();
WinAppDriverHelper.StartIfNotRunning();
var pagesFoundWithIssues = 0;
var rootFolder = $"{Path.GetPathRoot(Environment.CurrentDirectory)}UIT\\ALLY\\{DateTime.Now:dd_HHmmss}\\";
var reportFolder = Path.Combine(rootFolder, "Reports");
var pageIdentities = AllTestablePages(Frameworks.CodeBehind);
VisualComparisonTestDetails appDetails = null;
async Task ForOpenedPage(string pageName, WindowsDriver <WindowsElement> session, string projectName)
{
if (pageName == "Map")
{
// For location permission
if (await ClickYesOnPopUpAsync(session))
{
await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompt
}
}
else if (pageName == "Camera")
{
var cameraPermission = await ClickYesOnPopUpAsync(session); // For camera permission
var microphonePermission = await ClickYesOnPopUpAsync(session); // For microphone permission
if (cameraPermission && microphonePermission)
{
await Task.Delay(TimeSpan.FromSeconds(2)); // Allow page to load after accepting prompts
}
}
var processes = Process.GetProcessesByName(projectName);
var config = Axe.Windows.Automation.Config.Builder.ForProcessId(processes[0].Id);
config.WithOutputFileFormat(Axe.Windows.Automation.OutputFileFormat.A11yTest);
config.WithOutputDirectory(reportFolder);
var scanner = Axe.Windows.Automation.ScannerFactory.CreateScanner(config.Build());
try
{
var scanResults = scanner.Scan();
if (scanResults.ErrorCount > 0)
{
pagesFoundWithIssues++;
}
}
catch (Axe.Windows.Automation.AxeWindowsAutomationException exc)
{
Assert.True(exc == null, exc.ToString());
}
}
try
{
appDetails = await SetUpProjectForUiTestComparisonAsync(ProgrammingLanguages.CSharp, ProjectTypes.SplitView, Frameworks.CodeBehind, pageIdentities);
using (var appSession = WinAppDriverHelper.LaunchAppx(appDetails.PackageFamilyName))
{
appSession.Manage().Window.Maximize();
await Task.Delay(TimeSpan.FromSeconds(2));
var menuItems = appSession.FindElementsByClassName("Microsoft.UI.Xaml.Controls.NavigationViewItem");
foreach (var menuItem in menuItems)
{
menuItem.Click();
await Task.Delay(TimeSpan.FromMilliseconds(1500)); // Allow page to load and animations to complete
await ForOpenedPage(menuItem.Text, appSession, appDetails.ProjectName);
}
}
// Don't leave the app maximized in case we want to open the app again.
// Some controls handle layout differently when the app is first opened maximized
VirtualKeyboard.RestoreMaximizedWindow();
}
finally
{
if (appDetails != null)
{
UninstallAppx(appDetails.PackageFullName);
RemoveCertificate(appDetails.CertificatePath);
}
WinAppDriverHelper.StopIfRunning();
}
Assert.True(0 == pagesFoundWithIssues, $"Accessibility issues found. For details, see reports in '{reportFolder}'.");
}
