// ------------------------------------------------------------------------------------
// Detailed Results Title Entry
// Generate a new title entry
void GenerateNewTitleEntry(TestEntry input)
{
string title = input.suiteName + " - " + input.typeName; // If new suite add new suite name to title
ResultsEntry newEntry = GenerateResultsEntry(resultsEntryTitlePrefab); // Generate a results entry
newEntry.SetupTitle(title); // Setup the title instance
if (titleEntriesBelow.Count == 0) // If first entry
{
SetHoverTitleEntry(title); // Set hover title
}
titleEntriesBelow.Add(newEntry); // Add to list of titles below top of scroll rect
listHeight -= newEntry.GetComponent <RectTransform>().sizeDelta.y; // Track height for next entry
}
// Load specified test
public IEnumerator LoadSpecificTest(TestEntry inputEntry)
{
if (SceneManager.GetActiveScene().name != inputEntry.scenePath) // If current scene name does not match requested
{
SceneManager.LoadScene(inputEntry.scenePath); // Load requested scene
while (!levelWasLoaded) // Wait for load
{
yield return(null);
}
levelWasLoaded = false; // Reset
}
Console.Instance.Write(DebugLevel.Logic, MessageLevel.Log, "Loading test " + inputEntry.testName); // Write to console
StartTest(inputEntry, runnerType); // Start the test
}
// Get a logic instance and set model instance on it
TestLogicBase GetLogicInstance(string suiteName, Test activeTest, TestEntry activeEntry)
{
Console.Instance.Write(DebugLevel.Full, MessageLevel.Log, "Getting logic instance"); // Write to console
TestLogicBase output; // Create logic instance
var modelType = TestTypes.GetTypeFromIndex(activeEntry.typeValue); // Get the model type from its index
if (!FindModelInstanceOfType(modelType, out activeModelInstance)) // Check for model instance matching criteria
{
activeModelInstance = CreateNewModelInstance(modelType); // Create model instance
}
activeModelInstance.SetLogic(); // Set the logic reference on the model
output = TestTypeManager.Instance.GetLogicInstanceFromName(activeModelInstance.logic.ToString().Replace("GraphicsTestFramework.", "").Replace("Logic", "")); // Get test logic instance
output.SetSuiteName(suiteName); // Set suite name on the logic
output.SetModel(activeModelInstance); // Set the active test model in the logic
TestTypeManager.Instance.SetActiveLogic(output); // Set as active test logic
return(output); // Return
}
// ------------------------------------------------------------------------------------
// Test Execution
// Start an individual test (called by TestRunner)
public void StartTest(TestEntry inputTest, RunnerType runnerType)
{
Console.Instance.Write(DebugLevel.Logic, MessageLevel.Log, "Starting test " + inputTest.testName); // Write to console
TestLogicBase activeTestLogic = null;
if (!isAnalytic) // If not analytic mode
{
activeTest = SuiteManager.GetTest(inputTest); // Get the active test from suite manager
activeTestLogic = GetLogicInstance(SuiteManager.GetSuiteName(inputTest.suiteIndex), inputTest); // Get active test logic instance
}
else
{
activeTest = Common.GenerateTestFromTestEntry(inputTest); // Get the active test from structure
activeTestLogic = GetLogicInstance(TestStructure.Instance.GetSuiteNameFromIndex(inputTest.suiteIndex), inputTest); // Get active test logic instance
}
StartCoroutine(activeTestLogic.SetupTest(inputTest, runnerType)); // Setup test
}
// Convert the test structure into a runner based on current selection and runner type
void GenerateTestRunner(TestStructure.Structure inputStructure)
{
runner = new Runner(); // Create new Runner instance
for (int su = 0; su < inputStructure.suites.Count; su++) // Iterate suites
{
if (inputStructure.suites[su].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation
{
string suiteName = inputStructure.suites[su].suiteName;
for (int ty = 0; ty < inputStructure.suites[su].types.Count; ty++) // Iterate types
{
if (inputStructure.suites[su].types[ty].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation
{
string typeName = inputStructure.suites[su].types[ty].typeName;
int typeIndex = inputStructure.suites[su].types[ty].typeIndex;
for (int gr = 0; gr < inputStructure.suites[su].types[ty].groups.Count; gr++) // Iterate scenes
{
if (inputStructure.suites[su].types[ty].groups[gr].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation
{
string groupName = inputStructure.suites[su].types[ty].groups[gr].groupName;
for (int te = 0; te < inputStructure.suites[su].types[ty].groups[gr].tests.Count; te++) // Iterate tests
{
if (inputStructure.suites[su].types[ty].groups[gr].tests[te].selectionState != 0 || runnerType != RunnerType.Automation) // If selected or automation
{
string testName = inputStructure.suites[su].types[ty].groups[gr].tests[te].testName;
string scenePath = inputStructure.suites[su].types[ty].groups[gr].tests[te].scenePath;
if (!inputStructure.suites[su].types[ty].groups[gr].tests[te].baseline || runnerType != RunnerType.Resolve) // If baseline resolution mode return all with no baselines
{
TestEntry newTest = new TestEntry(suiteName, groupName, scenePath, typeName, testName, typeIndex, su, gr, ty, te); // Create new TestEntry instance
runner.tests.Add(newTest); // Add to runner
}
}
}
}
}
}
}
}
}
Console.Instance.Write(DebugLevel.Logic, MessageLevel.Log, "Generated test runner"); // Write to console
if (runnerType != RunnerType.Resolve) // If in Resolve mode we start tests manually
{
StartTests(); // Start tests
}
}
// Enable the resolve menu
void EnableResolveMenu()
{
Console.Instance.Write(DebugLevel.Full, MessageLevel.Log, "Enabling resolve menu"); // Write to console
resolveWindow.parent.SetActive(true); // Enable resolve menu
resolveWindow.message.text = TestRunner.Instance.runner.tests.Count + " baseline files are missing. This situation must be resolved before continuing. Manually resolve baselines for the listed tests."; // Set text
float entryHeight = 0; // Reset
for (int i = 0; i < TestRunner.Instance.runner.tests.Count; i++) // Iterate items to resolve
{
TestEntry currentTest = TestRunner.Instance.runner.tests[i]; // Get data
GameObject go = Instantiate(listEntryPrefab, resolveWindow.contentRect, false); // Create instance
RectTransform goRect = go.GetComponent <RectTransform>(); // Get rect
goRect.anchoredPosition = new Vector2(0, entryHeight); // Set position
MenuResolveListEntry newEntry = go.GetComponent <MenuResolveListEntry>(); // Get script reference
newEntry.Setup(currentTest.suiteName + " - " + currentTest.typeName, currentTest.groupName + " - " + currentTest.testName); // Setup
entryHeight -= goRect.sizeDelta.y; // Add to position tracker
}
resolveWindow.contentRect.sizeDelta = new Vector2(resolveWindow.contentRect.sizeDelta.x, -entryHeight); // Set content rect size
}
// ------------------------------------------------------------------------------------
// Test Execution
// Set initial information for test at beginning of test run
public IEnumerator SetupTest(TestEntry inputEntry, RunnerType runType)
{
ProgressScreen.Instance.SetState(true, ProgressType.LocalSave, "Preparing test"); // Enable ProgressScreen
testWasRan = false; // Reset
activeTestEntry = inputEntry; // Store active TestEntry
activeRunType = runType; // Store active RunnerType
SetSettings(); // Set settings to internal
yield return(new WaitForEndOfFrame()); // Wait for settings
SetupResultsStructs(); // Setup the results structs to be filled
CheckForBaseline(); // Check for baselines
Console.Instance.Write(DebugLevel.Full, MessageLevel.Log, this.GetType().Name + " set up test " + activeTestEntry.testName); // Write to console
ResultsIOData localResult; // Used for certain active run types
switch (activeRunType)
{
case RunnerType.Automation:
TestPreProcess(); // Start pre-process
break;
case RunnerType.Manual:
localResult = ResultsIO.Instance.RetrieveResult(suiteName, GetType().ToString().Replace("GraphicsTestFramework.", "").Replace("Logic", ""), activeResultData.common); // Try get local result
if (localResult == null) // If not found
{
TestPreProcess(); // Start pre-process
}
else // If found
{
UseLocalResult(localResult); // Use local
}
break;
case RunnerType.Results:
localResult = ResultsIO.Instance.RetrieveResult(suiteName, GetType().ToString().Replace("GraphicsTestFramework.", "").Replace("Logic", ""), activeResultData.common); // Try get local result
UseLocalResult(localResult); // Use local
break;
case RunnerType.Resolve:
TestPreProcess(); // Start pre-process
break;
}
}
public void CallLoadSpecificTest(TestEntry inputEntry)
{
StartCoroutine(LoadSpecificTest(inputEntry));
}
public ResultsEntryData(TestEntry inputEntry, ResultsIOData inputData, ResultsIOData inputDataB)
{
testEntry = inputEntry;
resultsData = inputData;
resultsDataB = inputDataB;
}
public void GetSlackResultsMessage()
{
List <Attachment> attachments = new List <Attachment>();
Attachment attachment = new Attachment("", false);
string previousSuiteName = "";
int localTestCount = 0;
int localFailCount = 0;
int globalTestCount = 0;
int globalFailCount = 0;
bool resultsFound = false;
for (int i = 0; i < TestRunner.Instance.runner.tests.Count; i++)
{
TestEntry currentEntry = TestRunner.Instance.runner.tests[i];
if (currentEntry.suiteName != previousSuiteName)
{
previousSuiteName = currentEntry.suiteName;
if (resultsFound == true)
{
attachment.text += "Ran " + localTestCount.ToString() + ", Passed " + (localTestCount - localFailCount).ToString() + ", Failed " + localFailCount.ToString();
attachments.Add(attachment);
globalTestCount += localTestCount;
globalFailCount += localFailCount;
localTestCount = 0;
localFailCount = 0;
}
attachment = new Attachment(currentEntry.suiteName + ": ", true);
}
ResultsDataCommon common = BuildResultsDataCommon(currentEntry.groupName, currentEntry.testName); // Build results data common to retrieve results
ResultsIOData data = ResultsIO.Instance.RetrieveResult(currentEntry.suiteName, currentEntry.typeName, common); // Retrieve results data
if (data != null)
{
resultsFound = true;
localTestCount++;
//search for PassFail field to avoid hardcoding
int passFailIndex = -1;
for (int f = 0; f < data.fieldNames.Count; f++)
{
if (data.fieldNames [f] == "PassFail")
{
passFailIndex = f;
}
}
if (data.resultsRow[0].resultsColumn[passFailIndex] == "False")
{
localFailCount++;
attachment.passFail = false;
}
}
}
if (TestRunner.Instance.runner.tests.Count > 0)
{
attachment.text += "Ran " + localTestCount.ToString() + ", Passed " + (localTestCount - localFailCount).ToString() + ", Failed " + localFailCount.ToString();
attachments.Add(attachment);
globalTestCount += localTestCount;
globalFailCount += localFailCount;
}
SystemData sysData = Master.Instance.GetSystemData();
string message = "*UTF completed run. Ran " + globalTestCount.ToString() + ", Passed " + (globalTestCount - globalFailCount).ToString() + ", Failed " + globalFailCount.ToString() + "*" + Environment.NewLine + "_" + sysData.Device + "_" + Environment.NewLine + "_" + sysData.Platform + " - " + sysData.API + "_" + Environment.NewLine + "_" + sysData.UnityVersion + "_";
Attachment[] attachmentArray = attachments.ToArray();
currentMessage = new Message(message, attachmentArray);
}
// Get a specific test
public static Test GetTest(TestEntry inputEntry)
{
ProjectSettings projectSettings = GetProjectSettings(); // Get the suite list
return(projectSettings.suiteList[inputEntry.suiteIndex].groups[inputEntry.groupIndex].tests[inputEntry.testIndex]); // Return requested
}
// Get analytic data from a specific test structure entry
public ResultsIOData RequestAnalyticData(int dataSet, TestEntry testEntry)
{
TestResults testResults = (TestResults)testStructure.suites[testEntry.suiteIndex].types[testEntry.typeIndex].groups[testEntry.groupIndex].tests[testEntry.testIndex]; // Get entry and cast
return(dataSet == 0 ? testResults.dataA : testResults.dataB); // Return data set
}
