private static void OnAfterCreateNode(ITestNode node) { if (node.GetType().GetInterfaces().Contains((typeof(ITestCase)))) { Console.Write("."); } }
private static void WriteTestResult(ITestNode testNode, IFixtureResult result) { ConsoleText.ColorCodes cColor = ConsoleText.ColorCodes.Red; string status = " FAIL! "; if (result.Right == (result.Right + result.Wrong + result.Exceptions)) { cColor = ConsoleText.ColorCodes.Green; status = " PASS! "; } else if ((result.Exceptions != 0) && (result.Wrong == 0)) { cColor = ConsoleText.ColorCodes.Yellow; status = " FAIL! "; } else { cColor = ConsoleText.ColorCodes.Red; status = " FAIL! "; } ConsoleText.SetColor(cColor); Console.Write(status); ConsoleText.SetColor(ConsoleText.DEFAULT_COLOR); if (testNode != null) { Console.Write(testNode.Name + "\t"); } Console.Write("Assertions:\t Pass:{0} \t Fail:{1} \t Exception:{2} \t {3}", result.Right, result.Wrong, result.Exceptions, result.GetTimeSpan().TotalSeconds + "sec"); }
private void AddRelationship(ITestNode node, ITestNode baseNode, ModelRelationshipStereotype stereotype) { var underlyingNode = ModelService.GetUnderlyingNode(node); var underlyingBaseNode = ModelService.GetUnderlyingNode(baseNode); ModelService.AddRelationship(underlyingNode.Id, underlyingBaseNode.Id, stereotype); }
private void NotifyOnAfterCreateNode(ITestNode node) { if (OnAfterCreateNode != null) { OnAfterCreateNode(node); } }
/// <summary> /// Create the list of tests specified by the context. /// </summary> /// <param name="Context"></param> /// <returns></returns> IEnumerable <ITestNode> CreateTestList(UnrealTestContext Context, Params DefaultParams, ArgumentWithParams PlatformParams = null) { List <ITestNode> NodeList = new List <ITestNode>(); IEnumerable <string> Namespaces = Context.Options.Namespaces.Split(',').Select(S => S.Trim()); List <string> BuildIssues = new List <string>(); UnrealTargetPlatform UnrealPlatform = UnrealTargetPlatform.Unknown; if (!Enum.TryParse(PlatformParams.Argument, true, out UnrealPlatform)) { throw new AutomationException("Could not convert platform {0} to a valid UnrealTargetPlatform", PlatformParams.Argument); } //List<string> Platforms = Globals.Params.ParseValue("platform") // Create an instance of each test and add it to the executor foreach (var Test in Context.Options.TestList) { // create a copy of the context for this test UnrealTestContext TestContext = (UnrealTestContext)Context.Clone(); // if test specifies platforms, filter for this context if (Test.Platforms.Count() > 0 && Test.Platforms.Where(Plat => Plat.Argument == PlatformParams.Argument).Count() == 0) { continue; } if (Blacklist.Instance.IsTestBlacklisted(Test.TestName, UnrealPlatform, TestContext.BuildInfo.Branch)) { Gauntlet.Log.Info("Test {0} is currently blacklisted on {1} in branch {2}", Test.TestName, UnrealPlatform, TestContext.BuildInfo.Branch); continue; } // combine global and test-specific params Params CombinedParams = new Params(DefaultParams.AllArguments.Concat(Test.TestParams.AllArguments).ToArray()); // parse any target constraints List <string> PerfSpecArgs = CombinedParams.ParseValues("PerfSpec", false); string PerfSpecArg = PerfSpecArgs.Count > 0 ? PerfSpecArgs.Last() : "Unspecified"; EPerfSpec PerfSpec; if (!Enum.TryParse <EPerfSpec>(PerfSpecArg, true, out PerfSpec)) { throw new AutomationException("Unable to convert perfspec '{0}' into an EPerfSpec", PerfSpec); } TestContext.Constraint = new UnrealTargetConstraint(UnrealPlatform, PerfSpec); TestContext.TestParams = CombinedParams; // This will throw if the test cannot be created ITestNode NewTest = Utils.TestConstructor.ConstructTest <ITestNode, UnrealTestContext>(Test.TestName, TestContext, Namespaces); NodeList.Add(NewTest); } return(NodeList); }
public static string GetFullPath(ITestNode node) { string parentPath = string.Empty; FindParentPath(node, ref parentPath); parentPath += node.Name; return(parentPath); }
private void CreateTextTable(int tableId, ITestNode testNode, ref StringBuilder sb) { var textDataTable = Services.Get <IFixtureTextData>(tableId + ""); textDataTable.ContentText = sb.ToString(); sb.Length = 0; NotifyOnAfterCreateNode(textDataTable); testNode.AddChildNode(textDataTable); }
public void AddNode(ITestNode node, ITestNode parentNode = null) { var stereotype = GetNodeType(node); var parentWrapperNodeId = parentNode == null ? null : (ModelNodeId?)GetWrapperNode(parentNode).Id; var wrapperNode = _modelService.AddNode(node.Name, stereotype, node, parentWrapperNodeId); AddItemToCurrentGroup(wrapperNode); }
public static void FindParentPath(ITestNode node, ref string parentPath) { if ((node.ParentNode == null)) { return; } parentPath = parentPath.Insert(0, node.ParentNode.Name + "\\"); FindParentPath(node.ParentNode, ref parentPath); }
private string GetOutputFile(ITestNode node) { if (Directory.Exists(OutputDir)) Directory.CreateDirectory(OutputDir); _runningNumber++; string runningFileName = _runningNumber.ToString(); runningFileName = runningFileName.PadLeft(6, '0'); string status = (node.TestResult.Pass) ? "Pass" : "Fail"; return OutputDir + "\\" + runningFileName + "-" + status + ".html"; }
public void LoadData(ITestNode node) { var localNode = node as IRedefineTestNodeProperties; if (localNode != null) { Services.Get <IUtilService>().ReadFromFile( localNode.FullFileName, localNode.ContentList); } Parse(localNode); }
private static void OnTestComplete(ITestNode node) { WriteTestResult(node, node.TestResult); Console.WriteLine(); if ((_exitOnFail) && (node.TestResult.Wrong + node.TestResult.Exceptions != 0)) { var addIn = node.Services.Get <ITestFrameworkAddin>(); GenerateReport(addIn.GetReportWriter()); Console.WriteLine("test run stopped on failure."); Environment.Exit(11); } }
public static void GetAllChildren(ITestNode node, ref IList <ITestNode> list) { if (node == null) { return; } foreach (ITestNode selectedNode in node) { list.Add(selectedNode); GetAllChildren(selectedNode, ref list); } }
private static void ApplyNodeLevel(ITestNode node) { if (node == null) { return; } foreach (ITestNode selectedNode in node) { selectedNode.LevelNo = selectedNode.ParentNode.LevelNo + 1; ApplyNodeLevel(selectedNode); } }
private static void OnErrorProcessingNodes(ITestNode node) { if (node is ITestLink) { ConsoleText.SetColor(ConsoleText.ColorCodes.Yellow); Console.WriteLine(); Console.Write("Warning: processing include scripts: [{0}]:[{1}]", node.ParentNode, node); } else { ConsoleText.SetColor(ConsoleText.ColorCodes.Red); Console.WriteLine(); Console.Write("Unidentified error while processing node " + node); } ConsoleText.SetColor(ConsoleText.DEFAULT_COLOR); }
private static ModelNodeStereotype GetNodeType(ITestNode node) { switch (node) { case ClassNode _: return(ModelNodeStereotypes.Class); case InterfaceNode _: return(ModelNodeStereotypes.Interface); case PropertyNode _: return(ModelNodeStereotypes.Property); default: throw new Exception($"Unexpected node type {node?.GetType().Name}"); } }
public void GetVariables(ITestNode node, ref IList <string> varList) { if (node == null) { return; } foreach (ITestNode selectedNode in node) { if ((selectedNode as IFixtureTextData) != null) { var fixtureTextdata = selectedNode as IFixtureTextData; string[] lines = fixtureTextdata.ContentText.Split('\r'); foreach (string s in lines) { string line = s; if (line.ToLower().Contains("!path")) { line = line.Replace("\n", ""); line = line.Replace("\r", "").Trim(); string valueToAdd = "path=" + line.Substring(6).Trim(); if (varList.IndexOf(valueToAdd) == -1) { varList.Add(valueToAdd); } } if (line.ToLower().Contains("!define")) { line = line.Replace("\n", ""); line = line.Replace("\r", ""); line = line.Replace("{", ""); line = line.Replace("}", ""); line = line.Substring(8).Trim(); int firstSpace = line.IndexOf(' '); string firstPart = line.Substring(0, firstSpace).Trim(); string secondPart = line.Replace(firstPart, "").Trim(); string valueToAdd = firstPart + "=" + secondPart; if (varList.IndexOf(valueToAdd) == -1) { varList.Add(valueToAdd); } } } } } GetVariables(node.ParentNode, ref varList); }
private static void ContainsRunnableItems(ITestNode node, ref bool result) { if ((node == null) || result) { return; } foreach (ITestNode selectedNode in node) { if ((selectedNode as ISuite) != null) { result |= (selectedNode as ISuite).IsRunnable; } else if ((selectedNode as ITestCase) != null) { result = true; } ContainsRunnableItems(selectedNode, ref result); } }
public ITestNode AddChildNode(ITestNode node) { PersistanceNode.AddChildNode(node.PersistanceNode); return(node); }
public TreeChildEnumerator(ITestNode node) { _node = node; }
public TestExecutionInfo(ITestNode InNode) { FirstReadyCheckTime = PreStartTime = PostStartTime = EndTime = DateTime.MinValue; TestNode = InNode; CancellationReason = ""; }
/// <summary> /// Executes the provided tests. Currently tests are executed synchronously /// </summary> /// <param name="Context"></param> public bool ExecuteTests(TestExecutorOptions InOptions, IEnumerable <ITestNode> RequiredTests) { Options = InOptions; Log.Info("Preparing to start {0} automation test(s)", RequiredTests.Count()); // install a cancel handler so we can stop parallel-for gracefully Action CancelHandler = delegate() { Log.Info("Cancelling Tests"); IsCancelled = true; }; Action PostCancelHandler = delegate() { HaveReceivedPostAbort = true; }; Globals.AbortHandlers.Add(CancelHandler); Globals.PostAbortHandlers.Add(PostCancelHandler); StartTime = DateTime.Now; CurrentTestPass = 0; IsRunning = true; List <int> FailedPassList = new List <int>(); int MaxParallelTasks = 0; int MaxStartingTasks = 0; // sort by priority if (Options.Parallel > 1) { RequiredTests = RequiredTests.OrderBy(Node => Node.Priority); } for (CurrentTestPass = 0; CurrentTestPass < Options.TestLoops; CurrentTestPass++) { // do not start a pass if cancelled if (IsCancelled) { break; } if (CurrentTestPass > 0) { // if repeating tests wait a little bit. If there was a crash CR might still be // processing things. Thread.Sleep(10000); } DateTime StartPassTime = DateTime.Now; Log.Info("Starting test pass {0} of {1}", CurrentTestPass + 1, Options.TestLoops); // Tests that we want to run List <TestExecutionInfo> PendingTests = RequiredTests.Select(N => new TestExecutionInfo(N)).ToList(); // Tests that are in the process of starting List <TestExecutionInfo> StartingTests = new List <TestExecutionInfo>(); List <Thread> StartingTestThreads = new List <Thread>(); // Tests that have been started and we're ticking/checking List <TestExecutionInfo> RunningTests = new List <TestExecutionInfo>(); // Completed tests List <TestExecutionInfo> CompletedTests = new List <TestExecutionInfo>(); DateTime LastUpdateMsg = DateTime.MinValue; DateTime LastReadyCheck = DateTime.MinValue; DateTime LastStatusUpdateTime = DateTime.MinValue; const double ReadyCheckPeriod = 30.0; const double StatusUpdatePeriod = 60.0; while (CompletedTests.Count() < RequiredTests.Count() && IsCancelled == false) { Monitor.Enter(Globals.MainLock); int SecondsRunning = (int)(DateTime.Now - StartPassTime).TotalSeconds; int InProgressCount = RunningTests.Count() + StartingTests.Count(); double TimeSinceLastReadyCheck = (DateTime.Now - LastReadyCheck).TotalSeconds; // Are any tests ready to run? if (InProgressCount < Options.Parallel && PendingTests.Count() > 0 && TimeSinceLastReadyCheck >= ReadyCheckPeriod) { TestExecutionInfo TestToStart = null; List <ITestNode> TestsFailingToStart = new List <ITestNode>(); // find a node that can run, and // find the first test that can run.... for (int i = 0; i < PendingTests.Count(); i++) { TestExecutionInfo NodeInfo = PendingTests[i]; ITestNode Node = NodeInfo.TestNode; bool IsTestReady = false; try { IsTestReady = Node.IsReadyToStart(); } catch (System.Exception ex) { Log.Error("Test {0} threw an exception during ready check. Ex: {1}", Node, ex); PendingTests[i] = null; NodeInfo.PreStartTime = NodeInfo.PostStartTime = NodeInfo.EndTime = DateTime.Now; CompletedTests.Add(NodeInfo); } if (IsTestReady) { // if ready then take it and stop looking TestToStart = NodeInfo; if (NodeInfo.FirstReadyCheckTime == DateTime.MinValue) { NodeInfo.FirstReadyCheckTime = DateTime.Now; } break; } else { // track the time that this test should have been able to run due to no other tests // consuming resources (at least locally...) // TODO - how can tests express resource requirements in a generic way? // TODO - what about the situation where no tests can run so all FirstCheck times are set, but // then a test starts and consumes all resources? if (RunningTests.Count() == 0 && StartingTests.Count() == 0) { if (NodeInfo.FirstReadyCheckTime == DateTime.MinValue) { NodeInfo.FirstReadyCheckTime = DateTime.Now; } double TimeWaiting = (DateTime.Now - NodeInfo.FirstReadyCheckTime).TotalSeconds; if (TimeWaiting >= Options.Wait) { Log.Warning("Test {0} has been waiting to run resource-free for {1:00} seconds. Removing from wait list", Node, TimeWaiting); PendingTests[i] = null; NodeInfo.PreStartTime = NodeInfo.PostStartTime = NodeInfo.EndTime = DateTime.Now; NodeInfo.Result = TestExecutionInfo.ExecutionResult.TimedOut; CompletedTests.Add(NodeInfo); } } } } // remove anything we nulled PendingTests = PendingTests.Where(T => T != null).ToList(); if (TestToStart != null) { Log.Info("Test {0} is ready to run", TestToStart); PendingTests.Remove(TestToStart); StartingTests.Add(TestToStart); // StartTest is the only thing we do on a thread because it's likely to be the most time consuming // as build are copied so will get the most benefit from happening in parallel Thread StartThread = new Thread(() => { Thread.CurrentThread.IsBackground = true; // start the test, this also fills in the pre/post start times bool Started = StartTest(TestToStart, CurrentTestPass, Options.TestLoops); lock (Globals.MainLock) { if (Started == false) { TestToStart.PostStartTime = TestToStart.EndTime = DateTime.Now; CompletedTests.Add(TestToStart); Log.Error("Test {0} failed to start", TestToStart); } else { RunningTests.Add(TestToStart); } StartingTests.Remove(TestToStart); StartingTestThreads.Remove(Thread.CurrentThread); } }); if (StartingTests.Count > MaxStartingTasks) { MaxStartingTasks = StartingTests.Count; } // track the thread and start it StartingTestThreads.Add(StartThread); StartThread.Start(); } else { // don't check for a while as querying kits for availability can be expensive LastReadyCheck = DateTime.Now; } } // Tick all running tests foreach (TestExecutionInfo TestInfo in RunningTests) { TestResult Result = TickTest(TestInfo); // invalid = no result yet if (Result == TestResult.Invalid) { TimeSpan RunningTime = DateTime.Now - TestInfo.PostStartTime; if ((SecondsRunning % 60) == 0) { Log.Verbose("Test {0} is still running. {1:00} seconds elapsed, will timeout in {2:00} seconds", TestInfo, RunningTime.TotalSeconds, TestInfo.TestNode.MaxDuration - RunningTime.TotalSeconds); LastUpdateMsg = DateTime.Now; } } else { TestInfo.EndTime = DateTime.Now; TestInfo.Result = Result == TestResult.Passed ? TestExecutionInfo.ExecutionResult.Passed : TestExecutionInfo.ExecutionResult.Failed; CompletedTests.Add(TestInfo); } } // remove any tests that were completed RunningTests = RunningTests.Where(R => CompletedTests.Contains(R) == false).ToList(); if ((DateTime.Now - LastStatusUpdateTime).TotalSeconds >= StatusUpdatePeriod) { LastStatusUpdateTime = DateTime.Now; Log.Info("Status: Completed:{0}, Running:{1}, Starting: {2}, Waiting:{3}", CompletedTests.Count(), RunningTests.Count(), StartingTests.Count(), PendingTests.Count()); } if (InProgressCount > MaxParallelTasks) { MaxParallelTasks = RunningTests.Count(); } // Release our global lock before we loop Monitor.Exit(Globals.MainLock); // sleep a while before we tick our running tasks again Thread.Sleep(500); } if (IsCancelled) { DateTime StartTime = DateTime.Now; Log.Info("Cleaning up pending and running tests."); while (HaveReceivedPostAbort == false) { Thread.Sleep(500); double Elapsed = (DateTime.Now - StartTime).TotalSeconds; if (Elapsed >= 5) { Log.Error("Giving up waiting for tests after {0:00} seconds", Elapsed); break; } } // tick anything running, this will also check IsCancelled and stop them // forcibly kill anything waiting if (StartingTestThreads.Count > 0) { foreach (Thread T in StartingTestThreads) { Log.Info("Aborting startup thread"); T.Abort(); } Thread.Sleep(1000); } foreach (TestExecutionInfo TestInfo in StartingTests) { Log.Info("Forcing pending test {0} to run CleanupTest", TestInfo.TestNode.Name); TestInfo.TestNode.CleanupTest(); CompletedTests.Add(TestInfo); } foreach (TestExecutionInfo TestInfo in RunningTests) { Log.Info("Ticking test {0} to cancel", TestInfo.TestNode.Name); TestResult Res = TickTest(TestInfo); CompletedTests.Add(TestInfo); if (Res != TestResult.Failed) { Log.Warning("Ticking of cancelled test {0} returnd {1}", TestInfo.TestNode.Name, Res); } } } else { TimeSpan PassDuration = DateTime.Now - StartPassTime; int FailedCount = 0; int TestCount = CompletedTests.Count; CompletedTests.ForEach(T => { TimeSpan TimeWaiting = T.FirstReadyCheckTime - T.PreStartTime; TimeSpan SetupTime = T.PostStartTime - T.PreStartTime; TimeSpan TestDuration = T.EndTime - T.PostStartTime; // status msg, kept uniform to avoid spam on notifiers (ie. don't include timestamps, etc) string Msg = string.Format("Test {0} {1}", T.TestNode, T.Result); if (T.Result != TestExecutionInfo.ExecutionResult.Passed) { FailedCount++; } Log.Info(Msg); // log test timing to info Log.Info(string.Format("Test Time: {0:mm\\:ss} (Waited:{1:mm\\:ss}, Setup:{2:mm\\:ss})", TestDuration, TimeWaiting, SetupTime)); }); if (Options.Parallel > 1) { Log.Info("MaxParallelTasks: {0}", MaxParallelTasks); Log.Info("MaxStartingTasks: {0}", MaxStartingTasks); } // report all tests ReportMasterSummary(CurrentTestPass + 1, Options.TestLoops, PassDuration, CompletedTests); if (FailedCount > 0) { FailedPassList.Add(CurrentTestPass); if (Options.StopOnError) { break; } } } // show details for multi passes if (Options.TestLoops > 1) { Log.Info("Completed all passes. {0} of {1} completed without error", CurrentTestPass + 1 - FailedPassList.Count(), Options.TestLoops); if (FailedPassList.Count > 0) { string FailedList = string.Join(",", FailedPassList); Log.Warning("Failed passes: " + FailedList); } } } IsRunning = false; Globals.AbortHandlers.Remove(CancelHandler); Globals.PostAbortHandlers.Remove(PostCancelHandler); return(FailedPassList.Count == 0 && !IsCancelled); }
private IModelNode GetWrapperNode([NotNull] ITestNode testNode) { return(_modelService.LatestModel.Nodes.Single(i => i.Payload.Equals(testNode))); }
public IModelNode GetUnderlyingNode(ITestNode node) => GetWrapperNode(node);
private void AddInheritance(ITestNode node, ITestNode baseNode) { AddRelationship(node, baseNode, ModelRelationshipStereotypes.Inheritance); }
public void Run(ITestNode node) { }
private void AddImplements(ITestNode node, ITestNode baseNode) { AddRelationship(node, baseNode, ModelRelationshipStereotypes.Implementation); }