// ------------------------------------------------------------------------------------ // Detailed Results Title Entry // Generate a new title entry void GenerateNewTitleEntry(TestEntry input) { string title = input.suiteName + " - " + input.typeName; // If new suite add new suite name to title ResultsEntry newEntry = GenerateResultsEntry(resultsEntryTitlePrefab); // Generate a results entry newEntry.SetupTitle(title); // Setup the title instance if (titleEntriesBelow.Count == 0) // If first entry { SetHoverTitleEntry(title); // Set hover title } titleEntriesBelow.Add(newEntry); // Add to list of titles below top of scroll rect listHeight -= newEntry.GetComponent <RectTransform>().sizeDelta.y; // Track height for next entry }
// Load specified test public IEnumerator LoadSpecificTest(TestEntry inputEntry) { if (SceneManager.GetActiveScene().name != inputEntry.scenePath) // If current scene name does not match requested { SceneManager.LoadScene(inputEntry.scenePath); // Load requested scene while (!levelWasLoaded) // Wait for load { yield return(null); } levelWasLoaded = false; // Reset } Console.Instance.Write(DebugLevel.Logic, MessageLevel.Log, "Loading test " + inputEntry.testName); // Write to console StartTest(inputEntry, runnerType); // Start the test }
// Get a logic instance and set model instance on it TestLogicBase GetLogicInstance(string suiteName, Test activeTest, TestEntry activeEntry) { Console.Instance.Write(DebugLevel.Full, MessageLevel.Log, "Getting logic instance"); // Write to console TestLogicBase output; // Create logic instance var modelType = TestTypes.GetTypeFromIndex(activeEntry.typeValue); // Get the model type from its index if (!FindModelInstanceOfType(modelType, out activeModelInstance)) // Check for model instance matching criteria { activeModelInstance = CreateNewModelInstance(modelType); // Create model instance } activeModelInstance.SetLogic(); // Set the logic reference on the model output = TestTypeManager.Instance.GetLogicInstanceFromName(activeModelInstance.logic.ToString().Replace("GraphicsTestFramework.", "").Replace("Logic", "")); // Get test logic instance output.SetSuiteName(suiteName); // Set suite name on the logic output.SetModel(activeModelInstance); // Set the active test model in the logic TestTypeManager.Instance.SetActiveLogic(output); // Set as active test logic return(output); // Return }
// ------------------------------------------------------------------------------------ // Test Execution // Start an individual test (called by TestRunner) public void StartTest(TestEntry inputTest, RunnerType runnerType) { Console.Instance.Write(DebugLevel.Logic, MessageLevel.Log, "Starting test " + inputTest.testName); // Write to console TestLogicBase activeTestLogic = null; if (!isAnalytic) // If not analytic mode { activeTest = SuiteManager.GetTest(inputTest); // Get the active test from suite manager activeTestLogic = GetLogicInstance(SuiteManager.GetSuiteName(inputTest.suiteIndex), inputTest); // Get active test logic instance } else { activeTest = Common.GenerateTestFromTestEntry(inputTest); // Get the active test from structure activeTestLogic = GetLogicInstance(TestStructure.Instance.GetSuiteNameFromIndex(inputTest.suiteIndex), inputTest); // Get active test logic instance } StartCoroutine(activeTestLogic.SetupTest(inputTest, runnerType)); // Setup test }
// Convert the test structure into a runner based on current selection and runner type void GenerateTestRunner(TestStructure.Structure inputStructure) { runner = new Runner(); // Create new Runner instance for (int su = 0; su < inputStructure.suites.Count; su++) // Iterate suites { if (inputStructure.suites[su].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation { string suiteName = inputStructure.suites[su].suiteName; for (int ty = 0; ty < inputStructure.suites[su].types.Count; ty++) // Iterate types { if (inputStructure.suites[su].types[ty].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation { string typeName = inputStructure.suites[su].types[ty].typeName; int typeIndex = inputStructure.suites[su].types[ty].typeIndex; for (int gr = 0; gr < inputStructure.suites[su].types[ty].groups.Count; gr++) // Iterate scenes { if (inputStructure.suites[su].types[ty].groups[gr].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation { string groupName = inputStructure.suites[su].types[ty].groups[gr].groupName; for (int te = 0; te < inputStructure.suites[su].types[ty].groups[gr].tests.Count; te++) // Iterate tests { if (inputStructure.suites[su].types[ty].groups[gr].tests[te].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation { string testName = inputStructure.suites[su].types[ty].groups[gr].tests[te].testName; string scenePath = inputStructure.suites[su].types[ty].groups[gr].tests[te].scenePath; if (!inputStructure.suites[su].types[ty].groups[gr].tests[te].baseline || runnerType != RunnerType.Resolve) // If baseline resolution mode return all with no baselines { TestEntry newTest = new TestEntry(suiteName, groupName, scenePath, typeName, testName, typeIndex, su, gr, ty, te); // Create new TestEntry instance runner.tests.Add(newTest); // Add to runner } } } } } } } } } Console.Instance.Write(DebugLevel.Logic, MessageLevel.Log, "Generated test runner"); // Write to console if (runnerType != RunnerType.Resolve) // If in Resolve mode we start tests manually { StartTests(); // Start tests } }
// Enable the resolve menu void EnableResolveMenu() { Console.Instance.Write(DebugLevel.Full, MessageLevel.Log, "Enabling resolve menu"); // Write to console resolveWindow.parent.SetActive(true); // Enable resolve menu resolveWindow.message.text = TestRunner.Instance.runner.tests.Count + " baseline files are missing. This situation must be resolved before continuing. Manually resolve baselines for the listed tests."; // Set text float entryHeight = 0; // Reset for (int i = 0; i < TestRunner.Instance.runner.tests.Count; i++) // Iterate items to resolve { TestEntry currentTest = TestRunner.Instance.runner.tests[i]; // Get data GameObject go = Instantiate(listEntryPrefab, resolveWindow.contentRect, false); // Create instance RectTransform goRect = go.GetComponent <RectTransform>(); // Get rect goRect.anchoredPosition = new Vector2(0, entryHeight); // Set position MenuResolveListEntry newEntry = go.GetComponent <MenuResolveListEntry>(); // Get script reference newEntry.Setup(currentTest.suiteName + " - " + currentTest.typeName, currentTest.groupName + " - " + currentTest.testName); // Setup entryHeight -= goRect.sizeDelta.y; // Add to position tracker } resolveWindow.contentRect.sizeDelta = new Vector2(resolveWindow.contentRect.sizeDelta.x, -entryHeight); // Set content rect size }
// ------------------------------------------------------------------------------------ // Test Execution // Set initial information for test at beginning of test run public IEnumerator SetupTest(TestEntry inputEntry, RunnerType runType) { ProgressScreen.Instance.SetState(true, ProgressType.LocalSave, "Preparing test"); // Enable ProgressScreen testWasRan = false; // Reset activeTestEntry = inputEntry; // Store active TestEntry activeRunType = runType; // Store active RunnerType SetSettings(); // Set settings to internal yield return(new WaitForEndOfFrame()); // Wait for settings SetupResultsStructs(); // Setup the results structs to be filled CheckForBaseline(); // Check for baselines Console.Instance.Write(DebugLevel.Full, MessageLevel.Log, this.GetType().Name + " set up test " + activeTestEntry.testName); // Write to console ResultsIOData localResult; // Used for certain active run types switch (activeRunType) { case RunnerType.Automation: TestPreProcess(); // Start pre-process break; case RunnerType.Manual: localResult = ResultsIO.Instance.RetrieveResult(suiteName, GetType().ToString().Replace("GraphicsTestFramework.", "").Replace("Logic", ""), activeResultData.common); // Try get local result if (localResult == null) // If not found { TestPreProcess(); // Start pre-process } else // If found { UseLocalResult(localResult); // Use local } break; case RunnerType.Results: localResult = ResultsIO.Instance.RetrieveResult(suiteName, GetType().ToString().Replace("GraphicsTestFramework.", "").Replace("Logic", ""), activeResultData.common); // Try get local result UseLocalResult(localResult); // Use local break; case RunnerType.Resolve: TestPreProcess(); // Start pre-process break; } }
public void CallLoadSpecificTest(TestEntry inputEntry) { StartCoroutine(LoadSpecificTest(inputEntry)); }
public ResultsEntryData(TestEntry inputEntry, ResultsIOData inputData, ResultsIOData inputDataB) { testEntry = inputEntry; resultsData = inputData; resultsDataB = inputDataB; }
public void GetSlackResultsMessage() { List <Attachment> attachments = new List <Attachment>(); Attachment attachment = new Attachment("", false); string previousSuiteName = ""; int localTestCount = 0; int localFailCount = 0; int globalTestCount = 0; int globalFailCount = 0; bool resultsFound = false; for (int i = 0; i < TestRunner.Instance.runner.tests.Count; i++) { TestEntry currentEntry = TestRunner.Instance.runner.tests[i]; if (currentEntry.suiteName != previousSuiteName) { previousSuiteName = currentEntry.suiteName; if (resultsFound == true) { attachment.text += "Ran " + localTestCount.ToString() + ", Passed " + (localTestCount - localFailCount).ToString() + ", Failed " + localFailCount.ToString(); attachments.Add(attachment); globalTestCount += localTestCount; globalFailCount += localFailCount; localTestCount = 0; localFailCount = 0; } attachment = new Attachment(currentEntry.suiteName + ": ", true); } ResultsDataCommon common = BuildResultsDataCommon(currentEntry.groupName, currentEntry.testName); // Build results data common to retrieve results ResultsIOData data = ResultsIO.Instance.RetrieveResult(currentEntry.suiteName, currentEntry.typeName, common); // Retrieve results data if (data != null) { resultsFound = true; localTestCount++; //search for PassFail field to avoid hardcoding int passFailIndex = -1; for (int f = 0; f < data.fieldNames.Count; f++) { if (data.fieldNames [f] == "PassFail") { passFailIndex = f; } } if (data.resultsRow[0].resultsColumn[passFailIndex] == "False") { localFailCount++; attachment.passFail = false; } } } if (TestRunner.Instance.runner.tests.Count > 0) { attachment.text += "Ran " + localTestCount.ToString() + ", Passed " + (localTestCount - localFailCount).ToString() + ", Failed " + localFailCount.ToString(); attachments.Add(attachment); globalTestCount += localTestCount; globalFailCount += localFailCount; } SystemData sysData = Master.Instance.GetSystemData(); string message = "*UTF completed run. Ran " + globalTestCount.ToString() + ", Passed " + (globalTestCount - globalFailCount).ToString() + ", Failed " + globalFailCount.ToString() + "*" + Environment.NewLine + "_" + sysData.Device + "_" + Environment.NewLine + "_" + sysData.Platform + " - " + sysData.API + "_" + Environment.NewLine + "_" + sysData.UnityVersion + "_"; Attachment[] attachmentArray = attachments.ToArray(); currentMessage = new Message(message, attachmentArray); }
// Get a specific test public static Test GetTest(TestEntry inputEntry) { ProjectSettings projectSettings = GetProjectSettings(); // Get the suite list return(projectSettings.suiteList[inputEntry.suiteIndex].groups[inputEntry.groupIndex].tests[inputEntry.testIndex]); // Return requested }
// Get analytic data from a specific test structure entry public ResultsIOData RequestAnalyticData(int dataSet, TestEntry testEntry) { TestResults testResults = (TestResults)testStructure.suites[testEntry.suiteIndex].types[testEntry.typeIndex].groups[testEntry.groupIndex].tests[testEntry.testIndex]; // Get entry and cast return(dataSet == 0 ? testResults.dataA : testResults.dataB); // Return data set }