internal bool KillEmulator() { bool emCrash = false; Harness.SetDeviceDoneEvent(); foreach (Process p in Process.GetProcessesByName("Microsoft.SPOT.Emulator.Sample.SampleEmulator")) { p.Kill(); emCrash = true; } return(emCrash); }
internal HarnessExecutionResult Run(string[] args) { TimedTest test; HarnessExecutionResult hResult = HarnessExecutionResult.Unavailable; bool runTestsIndividually = false; m_log = new XmlLog(); // Prelim: Set the env, parse the arguments, set the result paths and set the test list. SetEnvironment(); if (args != null && args.Length > 0) { if (!ParseArguments(args, ref runTestsIndividually)) { return HarnessExecutionResult.InvalidArguments; } } SetResultPaths(); BaseTest[] testList = BuildTestList(runTestsIndividually); // Create a new harness object and set the properties. Harness harness = new Harness(IsDevEnvironment); if (m_transport != null) { harness.Transport = m_transport; } else { // harness constructor assigns default transport m_transport = harness.Transport; } if (m_device != null) { harness.Device = m_device; } else { // harness constructor assigns default device m_device = harness.Device; } // Execute each of the solution files using Harness. for (int i = 0; i < testList.Length; i++) { if (testList[i] == null) { continue; } if (this.Transport.ToLower().Contains("tcpip") && (testList[i].Name.ToLower().Contains("netinfotests.sln"))) { continue; } hResult = HarnessExecutionResult.Unavailable; int attempts = 0; while ((hResult != HarnessExecutionResult.Success && hResult != HarnessExecutionResult.Abort) && attempts++ < 3) { test = new TimedTest(testList[i], harness, m_log); // Kill any emulators running from previous runs. TerminateRunningEmulators(test, testList[i]); try { hResult = test.Execute(); if (hResult == HarnessExecutionResult.Unavailable) { Utils.WriteToEventLog("Harness returned an unavailable result after running the test: " + testList[i].Name + ". No of tries so far = " + attempts); string deviceStatus = DeviceStatus(hResult); Utils.WriteToEventLog("Device status after unavailable from harness: " + deviceStatus); } // Test did not execute because the device was dead. // If so, reset power to the device and re-run test. if (hResult == HarnessExecutionResult.NoConnection) { Utils.WriteToEventLog("Harness returned an NoConnection result after running the test: " + testList[i].Name + ". No of tries so far = " + attempts); string deviceStatus = DeviceStatus(hResult); Utils.WriteToEventLog("Device status after noconnection from harness: " + deviceStatus); } // Test did not succeed running in three attempts. if (hResult == HarnessExecutionResult.TimeOut) { Utils.WriteToEventLog("Test: " + test.Test.Name + " failed."); harness.MFTestResult = Harness.Result.Fail; GetTestResultDetails(harness, testList[i]); Console.WriteLine("Test Result: " + harness.MFTestResult); test.SendMail(); break; } // Test did not succeed running in three attempts. if (hResult != HarnessExecutionResult.Success && attempts >= 3) { Utils.WriteToEventLog("Test: " + test.Test.Name + " failed."); harness.MFTestResult = Harness.Result.Fail; GetTestResultDetails(harness, testList[i]); Console.WriteLine("Test Result: " + harness.MFTestResult); } // Test succeeded with 3 attempts or an abort was sent by harness. if ((hResult == HarnessExecutionResult.Success && attempts < 4) || (hResult == HarnessExecutionResult.Abort)) { GetTestResultDetails(harness, testList[i]); if (!string.IsNullOrEmpty(m_device)) { string deviceStatus = DeviceStatus(hResult); Utils.WriteToEventLog("Device status after running " + testList[i].Name + ": " + deviceStatus); if (!IsProfilerRun) { m_log.AddDeviceStatusToLog("Device ping result after running " + testList[i].Name + ": " + deviceStatus); } if (string.Equals(deviceStatus.ToLower(), "noconnection")) { throw new ApplicationException("Device did not reboot correctly after " + "running the test: " + testList[i].Name); } } if (!IsProfilerRun) { Console.WriteLine("Test Result: " + harness.MFTestResult); } } } catch (Exception ex) { if (ex is FileNotFoundException) { if (!IsProfilerRun) { Console.WriteLine(ex.ToString()); try { m_log.AddCommentToLog(ex.ToString()); } catch { } Utils.WriteToEventLog( string.Format("Exception in TestSystem.cs: {0}", ex.ToString())); hResult = HarnessExecutionResult.Abort; } } } // Wait for a few seconds before starting the next test when running on devices. if (!string.Equals(m_transport.ToLower(), "emulator")) { System.Threading.Thread.Sleep(5000); } } } // Update test results and logs location. m_didAllTestsPass = ((tests.FailCount > 0) || (tests.PassCount == 0)) ? false : true; UpdateLogFolder(); return hResult; }
private void UpdateTestDetails(Harness harness, BaseTest currentTest, Harness.Result mfResult, int totalTestMethods, XmlDocument doc) { string[] seperator = { "::" }; try { currentTest.EndTime = harness.EndTime; currentTest.LogFile = harness.LogFile; currentTest.Result = mfResult.ToString(); currentTest.StartTime = harness.StartTime; if (m_isProfilerRun) { ArrayList methodList = Utils.ReadProfilerLogFile(currentTest.LogFile, currentTest.ExeLocation); for (int i = 0; i < methodList.Count; i++) { string[] vals = methodList[i].ToString().Split('\t'); int exclTime = 0; if (!string.IsNullOrEmpty(vals[3])) { exclTime = Convert.ToInt32(vals[3].Trim()); } int inclTime = 0; if (!string.IsNullOrEmpty(vals[4])) { inclTime = Convert.ToInt32(vals[4].Trim()); } ProfilerTestMethod ptm = new ProfilerTestMethod(); ptm.TestMethod = vals[2].Split(seperator, StringSplitOptions.None)[1].Trim(); ptm.InclusiveTime = inclTime; ptm.ExclusiveTime = exclTime; currentTest.TestMethods.Add(ptm); } } else { // Update specific results count. switch (mfResult) { case Harness.Result.Pass: m_passCount++; break; case Harness.Result.Fail: m_failCount++; break; case Harness.Result.Skip: m_skipCount++; break; case Harness.Result.KnownFailure: m_knownFailCount++; break; } int pc = 0, fc = 0, sc = 0, kc = 0; try { // Get total passcount, failcount, skipcount and known failure count for the test XmlNodeList passNodes = doc.GetElementsByTagName("PassCount"); foreach (XmlNode passNode in passNodes) { pc += Convert.ToInt32(passNode.ChildNodes[0].InnerText); } XmlNodeList failNodes = doc.GetElementsByTagName("FailCount"); foreach (XmlNode failNode in failNodes) { fc += Convert.ToInt32(failNode.ChildNodes[0].InnerText); } XmlNodeList skipNodes = doc.GetElementsByTagName("SkipCount"); foreach (XmlNode skipNode in skipNodes) { sc += Convert.ToInt32(skipNode.ChildNodes[0].InnerText); } XmlNodeList knownFailNodes = doc.GetElementsByTagName("KnownFailureCount"); foreach (XmlNode knownFailNode in knownFailNodes) { kc += Convert.ToInt32(knownFailNode.ChildNodes[0].InnerText); } } catch { } currentTest.TestMethodFailCount = fc; currentTest.TestMethodKnownFailureCount = kc; currentTest.TestMethodPassCount = pc; currentTest.TestMethodSkipCount = sc; currentTest.TotalTestCases = totalTestMethods; } m_mfTestList.Add(currentTest); } catch (Exception ex) { Console.WriteLine("WARNING: " + ex.ToString()); } }
private void GetTestResultDetails(Harness harness, BaseTest currentTest) { // Get the number of test methods in the log file for the test. // Open the xml file and get a count of number of <TestMethod> sections. int totalTestMethods = 0; XmlDocument doc = new XmlDocument(); Harness.Result mfResult = Harness.Result.NotKnown; if (!m_isProfilerRun) { try { doc.Load(m_logStore + harness.LogFile); XmlNodeList testLogNodes = doc.SelectNodes("/SPOT_Platform_Test/TestLog"); foreach (XmlNode tlNode in testLogNodes) { XmlNodeList testMethodNodes = tlNode.SelectNodes("TestMethod"); totalTestMethods += testMethodNodes.Count; } } catch (XmlException) { harness.TestResults = Harness.Result.Fail; } catch (Exception ex) { Console.WriteLine("WARNING: " + ex.ToString()); } try { mfResult = harness.MFTestResult; } catch { mfResult = Harness.Result.Fail; } } UpdateTestDetails(harness, currentTest, mfResult, totalTestMethods, doc); // Add to totalTestCases count. // This will give you the number of test cases executed during the entire run in all the tests. m_totalTestCases += totalTestMethods; }
/// <summary> /// Overloaded Constructor /// </summary> /// <param name="test">The name of the sln file.</param> /// <param name="harness">The Harness instance.</param> /// <param name="log">Path to log file. </param> internal TimedTest(BaseTest test, Harness harness, XmlLog log) { this.m_test = test; this.m_harness = harness; this.m_log = log; }