public override void Set()
{
if (!AutomationMaster.AllMethodsInFramework.Any())
{
AutomationMaster.SetAllMethodsInFramework();
}
string rawFavoritesData = FileBroker.GetNonUnityTextResource(AutomationMaster.UnitTestMode ? FileResource.FavoritesUnit : FileResource.Favorites);
List <string> favoritesEachRaw = rawFavoritesData.Split(AutomationMaster.DELIMITER).ToList();
FavoritesList = new List <KeyValuePair <string, List <KeyValuePair <bool, string> > > >();
for (int f = 0; f < favoritesEachRaw.Count; f++)
{
//Invalid data
if (!favoritesEachRaw[f].Trim().Contains(internalDelimiter))
{
continue;
}
List <string> pieces = favoritesEachRaw[f].Split(new string[] { internalDelimiter }, StringSplitOptions.None).ToList();
pieces = pieces.RemoveNullAndEmpty(); //Remove possible empties due to line breaks at end of file.
string name = pieces.First();
List <KeyValuePair <bool, string> > tests = new List <KeyValuePair <bool, string> >();
for (int t = 1; t < pieces.Count; t++)
{
tests.Add(new KeyValuePair <bool, string>(pieces[t].EndsWith(classIndicator) ? true : false, pieces[t].EndsWith(classIndicator) ? pieces[t].Replace(classIndicator, string.Empty) : pieces[t]));
}
FavoritesList.Add(new KeyValuePair <string, List <KeyValuePair <bool, string> > >(name, tests));
}
}
//Record logs from Unity console.
public static void GetLog(string message, string stackTrace, LogType type) {
Log newLog = new Log() {
message = message,
stackTrace = stackTrace,
type = type,
};
Logs.Add(newLog);
//If we exceed the maximum storage space.
if(Logs.Count >= MAX_LOG_COUNT_HISTORY) {
Logs.RemoveAt(0); //Remove oldest log.
}
//If an unhandled exception has occurred in the Trilleon Framework during a test run, handle it!
if(type == LogType.Exception) {
#if UNITY_EDITOR
EditorApplication.UnlockReloadAssemblies();
#endif
//If this error occurred directly in the Trilleon Framework (not editor), then stop all tests and report on the exception.
if(AutomationMaster.Busy && AutomationMaster.Initialized && stackTrace.Contains("TrilleonAutomation.") && !stackTrace.Contains("/Editor/") && !ReportOnce) {
ReportOnce = true;
AutomationMaster.Arbiter.SendCommunication("An unhandled exception occurred in the Trilleon Framework, disrupting the test run execution");
string stack = string.Format("The unhandled exception was: {0} {1}", message, stackTrace);
if(stack.Length > ConnectionStrategy.MaxMessageLength) {
stack = stack.Substring(0, ConnectionStrategy.MaxMessageLength - 50);
}
AutomationMaster.Arbiter.SendCommunication(stack);
string assertionData = string.Join("**", AutomationMaster.CurrentTestContext.Assertions.ToArray());
if(assertionData.Length > ConnectionStrategy.MaxMessageLength) {
int startingIndex = assertionData.Length - ConnectionStrategy.MaxMessageLength - 50;
assertionData = stack.Substring(startingIndex, stack.Length - startingIndex - 1);
}
AutomationMaster.Arbiter.SendCommunication(string.Format("ASSERTION DATA:[{0}]", assertionData));
AutomationMaster.AutomationReport.ReportUnhandledException(message, stackTrace);
#if UNITY_EDITOR
//Reset test runner
GameObject helper = GameObject.Find(TestMonitorHelpers.NAME);
if(helper != null)
AutomationMaster.Destroy(helper);
#endif
AutomationMaster.Destroy(AutomationMaster.StaticSelf);
AutomationMaster.Initialize();
AutomationMaster.StaticSelfComponent.ResetTestRunner();
AutoConsole.PostMessage("Exception in framework killed TestRunner. Framework reset and ready for new commands.", MessageLevel.Abridged);
} else if(AutomationMaster.Busy && AutomationMaster.Initialized && !stackTrace.Contains("/Editor/")) {
AutomationMaster.TestRunContext.Exceptions.Add(newLog);
}
#if UNITY_EDITOR
if(!AutomationMaster.Initialized && stackTrace.Contains("TrilleonAutomation.") && message.ToLower().Contains("object reference")) {
//Without AutomationMaster.Initialize(), a null reference error will occur trying to use most functionality in the Trilleon framework.
string missingRefError = "Object reference error in Trilleon framework without initialization. This is most often caused when TrilleonAutomation.Initialize() is not called. This method is required in your Game's startup logic to activate Trilleon.";
Debug.LogError(missingRefError);
AutoConsole.PostMessage(missingRefError, MessageLevel.Abridged);
}
#endif
if(ConnectionStrategy.TrilleonConnectionStrategy == ConnectionStrategyType.Socket) {
AutomationMaster.StaticSelf.GetComponent<SocketConnectionStrategy>().Stop();
}
}
}
public static void GetMostRecentsResults(string status = "", string plainText = "")
{
_testsMeta = new List <KeyValuePair <string, string[]> >();
//TODO: Remove tests from file that no longer are automation tests.
bool skipCreate = string.IsNullOrEmpty(status);
List <string> resultsRaw = new List <string>();
string txt = FileBroker.GetNonUnityTextResource(FileResource.LatestTestResults);
resultsRaw = txt.Split(new string[] { "\n" }, StringSplitOptions.None).ToList();
if (resultsRaw.Count == 0 && !skipCreate)
{
fileText.AppendLine(plainText);
}
else
{
List <Type> currentClasses = AutomationMaster.GetAutomationClasses();
List <MethodInfo> currentMethods = new List <MethodInfo>();
currentClasses.ForEach(x => currentMethods.AddRange(x.GetMethods()));
//Filter out tests that no longer exist, have been renamed, or are no longer marked as automation methods.
currentMethods = currentMethods.FindAll(x => {
Automation[] aut = (Automation[])Attribute.GetCustomAttributes(x, typeof(Automation));
return(aut != null && aut.Length > 0);
});
for (int i = 0; i < resultsRaw.Count; i++)
{
if (!string.IsNullOrEmpty(resultsRaw[i]))
{
string[] row = resultsRaw[i].Split(AutomationMaster.DELIMITER);
string testName = string.Empty;
for (int r = 0; r < row.Length; r++)
{
string[] keyVal = row[r].Split(':');
if (keyVal[0] == "name")
{
testName = keyVal[1];
}
}
if (!string.IsNullOrEmpty(testName))
{
//Ignore references to methods that are no longer valid automation methods for whatever reason.
if (currentMethods.FindAll(x => x.Name == testName).Any())
{
_testsMeta.Add(new KeyValuePair <string, string[]>(testName, row));
}
}
}
}
if (!skipCreate)
{
List <KeyValuePair <string, string[]> > matches = testsMeta.FindAll(x => x.Key == AutomationMaster.CurrentTestContext.TestName);
if (!matches.Any())
{
_testsMeta.Add(new KeyValuePair <string, string[]>(AutomationMaster.CurrentTestContext.TestName, plainText.Split(AutomationMaster.DELIMITER)));
}
else
{
_testsMeta.Remove(matches.First());
string[] newValues = new string[] {
string.Format("status:{0}", status),
string.Format("name:{0}", AutomationMaster.CurrentTestContext.TestName),
string.Format("class:{0}", AutomationMaster.CurrentTestContext.ClassName),
string.Format("test_categories:{0}", string.Join(",", AutomationMaster.CurrentTestContext.Categories.ToArray())),
};
KeyValuePair <string, string[]> newInsert = new KeyValuePair <string, string[]>(AutomationMaster.CurrentTestContext.TestName, newValues);
_testsMeta.Add(newInsert);
}
for (int f = 0; f < testsMeta.Count; f++)
{
fileText.AppendLine(string.Join(AutomationMaster.DELIMITER.ToString(), _testsMeta[f].Value));
}
}
}
}
void GetDependencyClassTestStructure()
{
//Get all automation test methods.
List <KeyValuePair <Type, MethodInfo> > allTestMethods = new List <KeyValuePair <Type, MethodInfo> >();
List <Type> AutomationClasses = AutomationMaster.GetAutomationClasses();
List <Type> masterDependencyClasses = AutomationClasses.FindAll(x => x.GetCustomAttributes(typeof(DependencyClass), false).ToList().Any());
List <Type> masterlessDependencyClasses = AutomationClasses.FindAll(x => {
return(!x.GetCustomAttributes(typeof(DependencyClass), false).ToList().Any() && x.GetMethods().ToList().FindAll(y => {
DependencyTest dt = (DependencyTest)Attribute.GetCustomAttribute(y, typeof(DependencyTest));
return dt != null;
}).Any());
});
for (int i = 0; i < AutomationClasses.Count; i++)
{
List <MethodInfo> methods = AutomationClasses[i].GetMethods().ToList().FindAll(y => y.GetCustomAttributes(typeof(Automation), false).ToList().Any());
for (int x = 0; x < methods.Count; x++)
{
allTestMethods.Add(new KeyValuePair <Type, MethodInfo>(AutomationClasses[i], methods[x]));
}
}
//Add groupings of master dependencies (DependencyTests under DependencyClasses)
int classId = 0;
for (int a = 0; a < masterDependencyClasses.Count; a++)
{
List <KeyValuePair <Type, MethodInfo> > allMasterDependencyMethods = new List <KeyValuePair <Type, MethodInfo> >();
for (int all = 0; all < masterDependencyClasses.Count; all++)
{
List <MethodInfo> allMethods = masterDependencyClasses[all].GetMethods().ToList().FindAll(y => y.GetCustomAttributes(typeof(DependencyTest), false).ToList().Any());
for (int am = 0; am < allMethods.Count; am++)
{
allMasterDependencyMethods.Add(new KeyValuePair <Type, MethodInfo>(masterDependencyClasses[all], allMethods[am]));
}
}
List <MethodInfo> thisDependencyClassIdMethods = allMasterDependencyMethods.FindAll(x => {
//Return all methods under the current DependencyClass ID.
DependencyClass dc = (DependencyClass)Attribute.GetCustomAttribute(x.Key, typeof(DependencyClass));
return(dc != null && dc.order == classId);
}).ExtractListOfValuesFromKeyValList();
thisDependencyClassIdMethods = thisDependencyClassIdMethods.FindAll(x => {
//Return all methods that have a DependencyTest attribute.
DependencyTest dt = (DependencyTest)Attribute.GetCustomAttribute(x, typeof(DependencyTest));
return(dt != null);
});
//If any methods found.
if (thisDependencyClassIdMethods.Any())
{
KeyValuePair <int, string> DepClassThis = new KeyValuePair <int, string>(classId, string.Empty); //This Dep Class id.
List <KeyValuePair <Type, MethodInfo> > DepTestThese = new List <KeyValuePair <Type, MethodInfo> >();
for (int ms = 0; ms < thisDependencyClassIdMethods.Count; ms++)
{
List <MethodInfo> nextMethod = thisDependencyClassIdMethods.FindAll(x => {
DependencyTest dt = (DependencyTest)Attribute.GetCustomAttribute(x, typeof(DependencyTest));
return(dt.order == ms + 1);
});
if (nextMethod.Any())
{
//Add method to next order in list.
DepTestThese.Add(new KeyValuePair <Type, MethodInfo>(allMasterDependencyMethods.FindAll(x => x.Value.Name == nextMethod.First().Name).First().Key, nextMethod.First()));
}
else
{
int duplicateId = 0;
List <KeyValuePair <string, int> > ordersFound = new List <KeyValuePair <string, int> >();
//Check if there is a duplicate ID, rather than a missing ID.
for (int d = 0; d < thisDependencyClassIdMethods.Count; d++)
{
DependencyTest dt = (DependencyTest)Attribute.GetCustomAttribute(thisDependencyClassIdMethods[d], typeof(DependencyTest));
if (ordersFound.FindAll(x => x.Value == dt.order).Any())
{
duplicateId = dt.order;
ordersFound.Add(new KeyValuePair <string, int>(thisDependencyClassIdMethods[d].Name, dt.order));
break;
}
else
{
ordersFound.Add(new KeyValuePair <string, int>(thisDependencyClassIdMethods[d].Name, dt.order));
}
}
List <Type> allTestClassesThatShareThisMasterId = allMasterDependencyMethods.FindAll(x => {
DependencyClass dc = (DependencyClass)Attribute.GetCustomAttribute(x.Key, typeof(DependencyClass));
return(dc.order == DepClassThis.Key);
}).ExtractListOfKeysFromKeyValList().Distinct();
StringBuilder classNameList = new StringBuilder();
for (int nl = 0; nl < allTestClassesThatShareThisMasterId.Count; nl++)
{
classNameList.Append(allTestClassesThatShareThisMasterId[nl].Name);
if (nl + 1 != allTestClassesThatShareThisMasterId.Count)
{
classNameList.Append(", ");
}
}
if (duplicateId > 0)
{
StringBuilder testNameList = new StringBuilder();
for (int nl = 0; nl < ordersFound.Count; nl++)
{
testNameList.Append(ordersFound[nl].Key);
if (nl + 1 != ordersFound.Count)
{
testNameList.Append(", ");
}
}
_errorInDependencyUsage = true;
throw new UnityException(string.Format("There multiple tests with the DependencyTest ID of {0} under the DependencyClass ( Name(s): {1} - DependencyClass ID: {2} ). Tests with duplicate ID ( {3} )", duplicateId, classNameList.ToString(), DepClassThis.Key, testNameList.ToString()));
}
else
{
_errorInDependencyUsage = true;
throw new UnityException(string.Format("There should be a DependencyTest of ID {0} under the DependencyClass ( Name(s): {1} - ID: {2} )", ms + 1, classNameList.ToString(), DepClassThis.Key));
}
}
}
//Add this pairing to the building list of all Dependencies.
DependencyOrderingMaster.Add(new KeyValuePair <KeyValuePair <int, string>, List <KeyValuePair <Type, MethodInfo> > >(DepClassThis, DepTestThese));
}
else
{
break;
}
classId++;
}
//Add groupings of master dependencies (DependencyTests under DependencyClasses)
for (int b = 0; b < masterlessDependencyClasses.Count; b++)
{
List <KeyValuePair <Type, MethodInfo> > DepTestThese = new List <KeyValuePair <Type, MethodInfo> >();
KeyValuePair <int, string> DepClassThis = new KeyValuePair <int, string>(0, masterlessDependencyClasses[b].Name); //This Dep Class id.
//Add each method based on the id of the Dependency Test.
List <MethodInfo> allMasterlessDependencyMethods = masterlessDependencyClasses[b].GetMethods().ToList().FindAll(y => y.GetCustomAttributes(typeof(DependencyTest), false).ToList().Any());
for (int ms = 0; ms < allMasterlessDependencyMethods.Count; ms++)
{
MethodInfo nextMethod = allMasterlessDependencyMethods.FindAll(x => {
DependencyTest dc = (DependencyTest)Attribute.GetCustomAttribute(x, typeof(DependencyTest));
return(dc.order == ms + 1);
}).First();
//Add method to next order in list.
DepTestThese.Add(new KeyValuePair <Type, MethodInfo>(masterlessDependencyClasses[b], nextMethod));
}
//Add this pairing to the building list of all Dependencies.
DependencyOrderingMasterless.Add(new KeyValuePair <KeyValuePair <int, string>, List <KeyValuePair <Type, MethodInfo> > >(DepClassThis, DepTestThese));
}
//Filter debug classes, unless debug classes represent ALL existing dependency archituecture tests. In that case, show debug tests to demonstrate how this editor window works.
List <KeyValuePair <KeyValuePair <int, string>, List <KeyValuePair <Type, MethodInfo> > > > filteredDebugMaster = DependencyOrderingMaster.FindAll(x => !x.Value.FindAll(t => t.Key.GetCustomAttributes(typeof(DebugClass), false).Length > 0).Any());
List <KeyValuePair <KeyValuePair <int, string>, List <KeyValuePair <Type, MethodInfo> > > > filteredDebugMasterfilteredDebugMaster = DependencyOrderingMasterless.FindAll(x => !x.Value.FindAll(t => t.Key.GetCustomAttributes(typeof(DebugClass), false).Length > 0).Any());
if (filteredDebugMaster.Any() || filteredDebugMasterfilteredDebugMaster.Any())
{
DependencyOrderingMaster = filteredDebugMaster;
DependencyOrderingMasterless = filteredDebugMasterfilteredDebugMaster;
}
}
//DependencyWebs consists of a Key (the id of the web list) and a Value (a list of KeyValuePairs which consist of a DependencyWeb method and the method that this declares as a dependency.
void MapDependencies()
{
//Get all automation test methods.
List <KeyValuePair <string, MethodInfo> > allTestMethods = new List <KeyValuePair <string, MethodInfo> >();
List <Type> AutomationClasses = AutomationMaster.GetAutomationClasses().FindAll(x => lastPassDetectedNoWebTestsSoDisplayDebugAsExample || !x.GetCustomAttributes(false).OfType <DebugClass>().Any()); //Ignore debug classes.
for (int i = 0; i < AutomationClasses.Count(); i++)
{
List <MethodInfo> methods = AutomationClasses[i].GetMethods().Where(y => y.GetCustomAttributes(false).OfType <Automation>().Any()).ToList();
for (int x = 0; x < methods.Count(); x++)
{
allTestMethods.Add(new KeyValuePair <string, MethodInfo>(methods[x].Name, methods[x]));
}
}
//From all methods, get methods that declare dependencies.
List <KeyValuePair <string, MethodInfo> > allDependencyTests = allTestMethods.Where(x => {
return(x.Value.GetCustomAttributes(typeof(DependencyWeb), false).Any());
}).ToList();
testsAndTheirDependenciesList = new List <KeyValuePair <string, string[]> >();
if (allDependencyTests.Count == 0)
{
lastPassDetectedNoWebTestsSoDisplayDebugAsExample = true; //Show DependencyTests marked as Debug, as a demo for what this window shows when a user has their own DependencyTests.
return;
}
//Get list of every test name associated with its declared dependencies.
for (int i = 0; i < allDependencyTests.Count; i++)
{
DependencyWeb dw = (DependencyWeb)Attribute.GetCustomAttribute(allDependencyTests[i].Value, typeof(DependencyWeb));
List <string> dtNames = dw.Dependencies;
dtNames.AddRange(dw.OneOfDependencies);
testsAndTheirDependenciesList.Add(new KeyValuePair <string, string[]>(allDependencyTests[i].Key, dtNames.ToArray()));
}
//List of all methods.
List <string> allInvolvedTestMethods = new List <string>();
//Build dependency web connections.
for (int t = 0; t < testsAndTheirDependenciesList.Count; t++)
{
allInvolvedTestMethods.Add(testsAndTheirDependenciesList[t].Key);
allInvolvedTestMethods.AddRange(testsAndTheirDependenciesList[t].Value);
}
allInvolvedTestMethods = allInvolvedTestMethods.Distinct().ToList();
for (int all = 0; all < allInvolvedTestMethods.Count; all++)
{
string method = allInvolvedTestMethods[all];
DependencyNode node = new DependencyNode();
node.TestName = method;
List <KeyValuePair <string, string[]> > matchChild = testsAndTheirDependenciesList.Where(x => x.Key == method).ToList();
if (matchChild.Any())
{
List <string> children = matchChild.Single().Value.ToList();
for (int c = 0; c < children.Count; c++)
{
node.AddDependency(DependencyNodeConnectionType.Outgoing, children[c]);
}
}
List <string> parents = testsAndTheirDependenciesList.Where(x => x.Value.Contains(method)).Select(x => x.Key).ToList();
if (parents.Any())
{
for (int p = 0; p < parents.Count; p++)
{
node.AddDependency(DependencyNodeConnectionType.Incoming, parents[p]);
}
}
DependencyWebs.Add(node);
}
}
protected IEnumerator Unifier(bool b, bool inverse, string message, FailureContext newFailureContext, params int[] testRailsId)
{
//If test was already marked as a failure, and test has flag indicating that it should continue despite failure, ignore.
if (!AutomationMaster.CurrentTestContext.IsSuccess || ((AutomationMaster.TryContinueOnFailure || MideExecution_MarkTestToTryContinueAfterFail) && IsFailing))
{
UnitTestStepFailure = isSoft = isTry = quiet = false; //Reset the UnitTestStepFailure, Soft, Try, and Quiet flags.
yield break;
}
//Automatically label this assertion as quiet if the previous assertion is identical to this one.
quiet = quiet ? true : AutomationMaster.CurrentTestContext.Assertions.Last() == message;
_failureContext = newFailureContext;
if ((!b && inverse) || (b && !inverse))
{
if (isTry && !quiet)
{
AutomationMaster.CurrentTestContext.AddAssertion(string.Format("**TRY_SUCCESS**{0}", message));
UnitTestStepFailure = isSoft = isTry = quiet = false; //Reset the UnitTestStepFailure, Soft, Try, and Quiet flags.
yield break;
}
ConcurrentFailures = 0;
AutomationMaster.CurrentTestContext.IsSuccess = true;
if (!string.IsNullOrEmpty(message) && !isTry && !quiet)
{
AutomationMaster.CurrentTestContext.AddAssertion(message);
}
}
else
{
//TODO: UnitTestStepFailure - Determine if an assertion has failed within the context of a TestObject "steps" method. If so, set this to true. Used to disabled certain TestRunner reactive logic, such as screenshots.
if (isTry)
{
if (!quiet)
{
AutomationMaster.CurrentTestContext.AddAssertion(string.Format("**TRY_FAIL**{0}", message));
}
UnitTestStepFailure = isSoft = isTry = quiet = false; //Reset the UnitTestStepFailure, Soft, Try, and Quiet flags.
yield break;
}
IsFailing = true;
bool recordLogDetails = newFailureContext != FailureContext.Skipped;
SetReflectedTestData();
string recentLogs = AutomationReport.EncodeCharactersForJson(AutoConsole.ReturnLatestLogs(5));
if (newFailureContext == FailureContext.Skipped)
{
AutomationMaster.TestRunContext.Skipped.Add(AutomationMaster.CurrentTestContext.TestName);
}
else
{
AutomationMaster.TestRunContext.Failed.Add(AutomationMaster.CurrentTestContext.TestName, new string[] {
message,
recentLogs.ToString(),
lineNumber
});
}
AutomationMaster.CurrentTestContext.IsSuccess = false;
AutomationMaster.CurrentTestContext.AddAssertion(message);
AutomationMaster.CurrentTestContext.ErrorDetails += string.Format("Error Message [{0}] : Test Line [{1}] : Debug Logs [{2}] ", message, string.Format("Line [{0}] Call [{1}]", lineNumber, lineCall), (recordLogDetails ? recentLogs : string.Format("#SKIPPED#{0}", message)));
AutomationMaster.CurrentTestContext.ErrorDetails += string.Format(" FULL STACK: [{0}]", Environment.StackTrace.Replace(" at", string.Format(" {0} at", AutomationMaster.NEW_LINE_INDICATOR)));
if (failureContext != FailureContext.Skipped)
{
//Take screenshot if a failure is not a "Skip" failure (In which case a test does not run at all, and there is no value in taking a screenshot as the current screen has no relevance to the reason it failed).
yield return(StartCoroutine(AutomationMaster.StaticSelfComponent.TakeScreenshot()));
screenshotRequestTime = DateTime.UtcNow;
}
//Handle errors occurring outside of the context of the current test's execution. Only certain contexts require additional handling over what is offered by default.
switch (AutomationMaster.ExecutionContext)
{
case AutomationMaster.CurrentExecutionContext.SetUpClass:
AutomationMaster.AutoSkips.Add(new KeyValuePair <string[], string>(new string[] { "class", AutomationMaster.CurrentTestContext.ClassName }, string.Format("FAILURE OCCURRED IN SETUPCLASS:", message)));
break;
case AutomationMaster.CurrentExecutionContext.SetUp:
AutomationMaster.AutoSkips.Add(new KeyValuePair <string[], string>(new string[] { "test", AutomationMaster.CurrentTestContext.TestName }, string.Format("FAILURE OCCURRED IN SETUP:", message)));
break;
case AutomationMaster.CurrentExecutionContext.TearDownClass:
yield return(StartCoroutine(Q.assert.Warn(string.Format("A failure occurred in the TearDownClass logic for the test \"{0}.{1}\". This fails the last-run test, and may cause other undesirable behavior for downstream test execution.", AutomationMaster.CurrentTestContext.ClassName, AutomationMaster.CurrentTestContext.TestName))));
//Will automatically handle the failure of this test.
break;
case AutomationMaster.CurrentExecutionContext.TearDown:
//Will automatically handle the failure of this test.
case AutomationMaster.CurrentExecutionContext.Test:
//Will automatically handle the failure of this test.
default:
break;
}
if ((AutomationMaster.TryContinueOnFailure || MideExecution_MarkTestToTryContinueAfterFail) && ConcurrentFailures > 5)
{
AutomationMaster.OverrideContinueOnFailureAfterTooManyConcurrentFailures = true;
}
#if UNITY_EDITOR
AutomationMaster.PauseEditorOnFailure();
#endif
//Any FailureContext beyond TestMethod will not have an instantiated test method.
if (!AutomationMaster.TryContinueOnFailure)
{
if ((!isSoft && AutomationMaster.OverrideContinueOnFailureAfterTooManyConcurrentFailures) || (!MideExecution_MarkTestToTryContinueAfterFail && (_failureContext == FailureContext.TestMethod || _failureContext == FailureContext.Default) && failureContext != FailureContext.Skipped))
{
try {
AutomationMaster.CurrentTestMethod.Stop(); //Kill current test, only if the currently queued test has been initialized.
} catch { }
yield return(new WaitForEndOfFrame()); //Allow all Coroutines to be stopped before returning control. In reality, the coroutine calling this will be stopped, so control will never be returned anyway.
}
}
if (!isSoft && (AutomationMaster.TryContinueOnFailure || MideExecution_MarkTestToTryContinueAfterFail))
{
ConcurrentFailures++;
}
}
if (testRailsId.Length > 0)
{
AutomationReport.MarkTestRailsTestCase(AutomationMaster.CurrentTestContext.IsSuccess ? "Passed" : "Failed", testRailsId);
}
AutoConsole.PostMessage(string.Format("Assert [{0}] |{1}| {2}", AutomationMaster.CurrentTestContext.TestName, AutomationMaster.CurrentTestContext.IsSuccess ? "Success" : "Failure", message), MessageLevel.Verbose, ConsoleMessageType.TestRunnerUpdate);
UnitTestStepFailure = isSoft = isTry = quiet = false; //Reset the UnitTestStepFailure, Soft, Try, and Quiet flags.
yield return(null);
}
/// <summary>
/// Handles incoming pubsub messages. Expects JSON format.
/// </summary>
public IEnumerator HandleMessage(string result)
{
//Ignore duplicate or empty messages. Ignore messages not meant for this client.
if (lastMessageReceived == result || string.IsNullOrEmpty(result.Trim()))
{
yield break;
}
lastMessageReceived = result;
List <KeyValuePair <string, string> > parameters = DeserializeJsonString(result);
if (!LocalRunLaunch)
{
//If no context or identity is provided, then this is not a valid command. If the DeviceUdid is not valid, then ignore the command.
if (!parameters.FindAll(x => x.Key.ToLower() == "grid_source").Any() || !parameters.FindAll(x => x.Key.ToLower() == "grid_identity").Any())
{
yield break;
}
string source = parameters.Find(x => x.Key == "grid_source").Value;
string identity = parameters.Find(x => x.Key == "grid_identity").Value;
string buddy = parameters.Find(x => x.Key == "grid_identity_buddy").Value;
//If message simply contains no reference to this GridIdentity OR the identity is self, then it is chatter and can be ignored.
bool isChatter = !result.Contains(GridIdentity);
bool isInvalid = string.IsNullOrEmpty(TestRunId) ? !parameters.FindAll(x => x.Key == "set_test_run_id").Any() : parameters.FindAll(x => x.Key == "test_run_id").Any() && TestRunId != parameters.Find(x => x.Key == "test_run_id").Value;
bool isEcho = identity == GridIdentity && source == "client";
//If message is from a client source where the identity is not that of this client, but contains this client's identity, then this it is a BuddySystem message.
bool isBuddyMessage = source != "server" && parameters.FindAll(x => x.Key.StartsWith("buddy_")).Any() && buddy == GridIdentity && identity == BuddyHandler.BuddyName;
//If this message is meant for a different client, or is an echo from the current client, simply ignore the message.
if (!isBuddyMessage && (isChatter || isEcho || isInvalid))
{
yield break;
}
}
else if (!LocalRunLaunch && parameters.FindAll(x => x.Key.ToLower() == "grid_identity").Any() ? parameters.Find(x => x.Key == "grid_identity").Value != GridIdentity : false)
{
yield break;
}
LastMessage = DateTime.Now;
if (parameters.Count > 0)
{
//Process each command.
for (int c = 0; c < parameters.Count; c++)
{
string command = parameters[c].Key.ToLower();
string message = parameters[c].Value.TrimEnd(',');
bool isRecognizedCommand = true;
switch (command.ToLower())
{
case "change_connection_strategy":
AutomationMaster.ConnectionStrategy.ChangeConnectionStrategy(message);
break;
case "change_communications_identity":
AutomationMaster.ConnectionStrategy.UpdateChannelIdentity(message);
break;
case "no_interval_screenshots":
AutomationMaster.NoIntervalScreenshots = true;
break;
case "ignore_memory_tracking":
AutomationMaster.IgnoreMemoryTracking = true;
break;
case "health_check":
SendCommunication(string.Format("heartbeat_{0}", (++AutomationMaster.HeartBeatIndex).ToString(), "0"));
break;
case "buddy_ignore_all":
AutomationMaster.IgnoreAllBuddyTests = true;
AutomationMaster.LockIgnoreBuddyTestsFlag = true; //Prevents Test editor window from implicitly updating the Ignore flag.
break;
case "buddy_ready_for_tests":
//TODO: Refactor and re-add GRIDLOCK logic. Without it, BuddySystem will not informatively report that both buddies are reporting as the same role.
//if((BuddyHandler.IsPrimary && message == "primary") || (!BuddyHandler.IsPrimary && message == "secondary")) {
//Gridlock. One client must be the primary, and one must be the secondary.
//SendCommunication("buddy_gridlock_detected", "0");
//BuddyHandler.RoleGridLock = true;
//} else {
SendCommunication("buddy_ready_for_tests_acknowledged", BuddyHandler.IsPrimary ? "primary" : "secondary");
BuddyHandler.IsBuddyReadyForBuddyTests = true;
//}
break;
case "buddy_ready_for_tests_acknowledged":
BuddyHandler.HasBuddyAcknowledgedOurReadiness = true;
break;
case "buddy_switching_roles":
BuddyHandler.BuddyHasSuccessfullySwitchRoles = true;
break;
case "buddy_requesting_required_details":
//Send/Resend details required by Primary Buddy.
BuddyHandler.SendBasicBuddyDetails();
break;
case "buddy_starting_reaction":
BuddyHandler.SecondaryReactionsStarted = true;
break;
case "buddy_tearing_down":
BuddyHandler.BuddyTearingDown = true;
break;
case "buddy_data_update":
BuddyHandler.SetCurrentBuddyRequiredDetails(message);
break;
case "buddy_primary_test_complete":
BuddyHandler.CurrentPrimaryTest = message;
BuddyHandler.ReadyForReactionTests = true;
BuddyHandler.SendBuddyCommunication("buddy_xyz", string.Format("Buddy Primary Test Completion ({0}) Acknowledged ({1}) %%%%", BuddyHandler.CurrentPrimaryTest, BuddyHandler.ReadyForReactionTests));
break;
case "buddy_primary_pretest_commands":
AutomationMaster.BuddyHandler.PreTestCommandReceived(message);
break;
case "buddy_secondary_pretest_commands_complete":
BuddyHandler.BuddyProcessingCommands = false;
break;
case "buddy_secondary_pretest_commands_failure":
BuddyHandler.BuddyCommandExecutionFailure = true;
BuddyHandler.BuddyProcessingCommands = false;
BuddyHandler.BuddyCommandFailure = message;
break;
case "buddy_secondary_tests_complete":
BuddyHandler.WaitingForBuddyToCompleteReactionTests = false;
break;
case "buddy_primary_test_failed":
BuddyHandler.PrimaryFailed = true;
break;
case "buddy_primary_complete_action_tests":
BuddyHandler.IsPrimaryFinishedWithActionTests = true;
break;
case "loop_tests":
//This command should be sent before or at the same time as the run command. Sending it after may result in failing to have the desired effect.
List <KeyValuePair <string, int> > loopTests = new List <KeyValuePair <string, int> >();
List <string> RawRequests = message.Split(AutomationMaster.DELIMITER).ToList();
for (int x = 0; x < RawRequests.Count; x++)
{
string testName = RawRequests[x].Split('@').First();
string count = RawRequests[x].Split('@').Last();
if (RawRequests[x].Split('@').ToList().Count != 2 || count.ToInt() == 0)
{
AutoConsole.PostMessage("Provided loop_tests command is invalid. The value must be a string and then integer, separated by an @ symbol.");
continue;
}
loopTests.Add(new KeyValuePair <string, int>(testName, count.ToInt()));
}
AutomationMaster.LoopTests = loopTests;
break;
case "request_response":
switch (message)
{
case "screenshot":
AutomationMaster.AwaitingScreenshot = false;
break;
default:
break;
}
break;
case "request_buddy":
//AutomationMaster.BuddyRequest(message, "newbuddy");
break;
case "set_test_run_id":
TestRunId = message;
break;
case "manual_set_buddy_primary":
BuddyHandler.BuddyName = message;
BuddyHandler.IsPrimary = true;
BuddyHandler.SendBasicBuddyDetails();
BuddyIdentity = message;
break;
case "manual_set_buddy_secondary":
BuddyHandler.BuddyName = message;
BuddyHandler.IsPrimary = false;
BuddyHandler.SendBasicBuddyDetails();
BuddyIdentity = message;
break;
case "no_test_rails_reporting":
AutomationReport.IgnoreTestRailsReporting = true;
break;
case "server_heartbeat":
AutomationMaster.ServerHeartbeatReceived();
break;
case "console_command":
List <string> commands = message.Trim().Split('|').ToList();
for (int co = 0; co < commands.Count; co++)
{
string com = string.Format("{0} {1}", commands[co].Split('$').First(), commands[co].Split('$').Last());
Q.SendConsoleCommand(com);
AutoConsole.PostMessage(string.Format("Ran Command: {0}", com), MessageLevel.Abridged);
}
break;
case "server_broker_response":
Q.request.CommandResponseReceived(message);
break;
case "automation_command":
if (AutomationMaster.Busy)
{
SendCommunication("Notification", "Busy completing previous test run.");
break;
}
SendCommunication("Notification", "Beginning pre-run checks.");
if (parameters.Find(x => x.Key == "grid_source").Value == "server")
{
AutomationMaster.IsServerListening = true;
}
//Incoming server-based runs will require the carrot before "rt all", for example, to run unit tests instead of automation.
if (message.StartsWith("^"))
{
AutomationMaster.UnitTestMode = true;
}
yield return(StartCoroutine(Q.driver.WaitRealTime(1)));
AutomationMaster.Busy = true;
AutomationMaster.LockIgnoreBuddyTestsFlag = true;
//Split string and discard only command prefix. Also allows for spaces in test Category names.
message = message.TrimStart().TrimEnd().Replace(", ", ",").Split(new char[] { ' ' }, 2)[1].Trim().ToLower();
if (message == "all")
{
AutomationMaster.LaunchType = LaunchType.All;
}
else if (message.StartsWith("*") && message.Contains(","))
{
message = message.Replace("*", string.Empty);
AutomationMaster.LaunchType = LaunchType.MultipleMethodNames;
}
else if (message.StartsWith("*"))
{
message = message.Replace("*", string.Empty);
AutomationMaster.LaunchType = LaunchType.MethodName;
}
else if (message.StartsWith("&&"))
{
message = message.Replace("&&", string.Empty);
AutomationMaster.LaunchType = LaunchType.Mix;
}
else if (message.Contains(","))
{
AutomationMaster.LaunchType = LaunchType.MultipleCategoryNames;
}
else
{
AutomationMaster.LaunchType = LaunchType.CategoryName;
}
//Wait until loading of game is complete to attempt a launch of the automation suite
yield return(StartCoroutine(Q.game.WaitForGameLoadingComplete()));
StartCoroutine(AutomationMaster.StaticSelfComponent.BeginTestLaunch(message));
break;
case "buddy_secondary_test_complete":
case "buddy_requesting_value_ready":
case "buddy_setting_ready_to":
//Commands that do not require any action, but should be considered valid for logging purposes (isRecognizedCommand).
break;
default:
isRecognizedCommand = false;
break;
}
Arbiter.LocalRunLaunch = false;
if (isRecognizedCommand && !string.IsNullOrEmpty(message))
{
AutoConsole.PostMessage(string.Format("SENDER [{0}] - COMMAND [{1}] - MESSAGE [{2}]", parameters.Find(x => x.Key == "grid_identity").Value, command, message), ConsoleMessageType.Pubsub);
}
}
}
}
