private void AddTest(ref BaseTest[] list, ref int index, string slnPath) { if (m_isProfilerRun) { list[index] = new ProfilerTest(); } else { list[index] = new MicroFrameworkTest(); } list[index].Name = slnPath; list[index++].Location = slnPath; }
private BaseTest[] BuildSolutionFileList(string dirProjPath) { // Scan build.dirproj to generate the test list. //List<string> staticList = new List<string>(); ArrayList test_list = new ArrayList(); if (m_isStaticList) { foreach (string file in File.ReadAllLines(m_staticList)) { //staticList.Add(file.ToLower()); test_list.Add(file.ToLower()); } } else { test_list = BuildTestList(dirProjPath, new ArrayList()); // Add the DPWS fixture tests which are built by the build system to the test list. string dpwsFixtureTestPath = ""; string flav = Environment.GetEnvironmentVariable("FLAVOR"); if (string.IsNullOrEmpty(flav)) { flav = "Release"; } dpwsFixtureTestPath = string.Format(@"{0}\BuildOutput\public\{1}\test\server\dpws", Environment.GetEnvironmentVariable("SPOCLIENT"), flav); if (!Directory.Exists(dpwsFixtureTestPath)) { if (flav.ToLower() == "debug") flav = "Release"; else flav = "Debug"; dpwsFixtureTestPath = string.Format(@"{0}\BuildOutput\public\{1}\test\server\dpws", Environment.GetEnvironmentVariable("SPOCLIENT"), flav); } if (Directory.Exists(dpwsFixtureTestPath)) { FileInfo[] dpwsSlnFiles = new DirectoryInfo(dpwsFixtureTestPath).GetFiles("*.sln"); foreach (FileInfo dpwsSlnFile in dpwsSlnFiles) { test_list.Add(dpwsSlnFile.FullName); } } } // Copy the contents of the local arraylist to the string array. BaseTest[] list = new BaseTest[test_list.Count * m_runCount]; for (int rc = 0; rc < m_runCount; rc++) { for (int i = 0; i < list.Length; i++) { //if (m_isStaticList) //{ // string test = Path.GetFileName(test_list[i].ToString()); // if (!staticList.Contains(test)) // continue; //} if (m_isProfilerRun) { list[i] = new ProfilerTest(); } else { list[i] = new MicroFrameworkTest(); } list[i].Name = test_list[i].ToString(); list[i].Location = test_list[i].ToString(); } } return list; }
private BaseTest[] BuildTestList(bool runTestsIndividually) { DirectoryInfo[] dirs; BaseTest[] list; if (runTestsIndividually) { list = new BaseTest[m_runCount]; for (int i = 0; i < m_runCount; i++) { if (m_isProfilerRun) { list[i] = new ProfilerTest(); } else { list[i] = new MicroFrameworkTest(); } list[i].Name = m_test; list[i].Location = m_test; } } else { // The machine is a test box with the sdk installation and not a dev box. if (!IsDevEnvironment) { if (m_isProfilerRun) { dirs = new DirectoryInfo(string.Format("{0}ManagedProfilerTests", InstallRoot)).GetDirectories(); } else { dirs = new DirectoryInfo(string.Format("{0}TestCases", InstallRoot)).GetDirectories(); } list = BuildSolutionFileList(dirs); } else { // The machine is a dev box. if (m_isProfilerRun) { dirs = new DirectoryInfo(string.Format(@"{0}Tests\Performance\ProfilerTests", InstallRoot)).GetDirectories(); list = BuildSolutionFileList(dirs); } else { list = BuildSolutionFileList(InstallRoot + "build.dirproj"); } } } return list; }
private BaseTest[] BuildSolutionFileList(DirectoryInfo[] dirs) { int index = 0; BaseTest[] list = new BaseTest[dirs.Length * m_runCount]; for (int i = 0; i < m_runCount; i++) { foreach (DirectoryInfo dir in dirs) { // The installed location will have only one sln file per test. FileInfo[] slnFiles = dir.GetFiles("*.sln"); if (slnFiles.Length > 0) { string slnPath = dir.Name + "\\" + slnFiles[0].Name; if (!string.IsNullOrEmpty(slnPath)) { // Add only the tests specified in the testlist if one is supplied. if (!m_isStaticList) { AddTest(ref list, ref index, slnPath); } else { List<string> staticList = new List<string>(); foreach (string file in File.ReadAllLines(m_staticList)) { if (string.Equals(slnFiles[0].Name, Path.GetFileName(file), StringComparison.InvariantCultureIgnoreCase)) { AddTest(ref list, ref index, slnPath); } } } } } } } return list; }
private void GetTestResultDetails(Harness harness, BaseTest currentTest) { // Get the number of test methods in the log file for the test. // Open the xml file and get a count of number of <TestMethod> sections. int totalTestMethods = 0; XmlDocument doc = new XmlDocument(); Harness.Result mfResult = Harness.Result.NotKnown; if (!m_isProfilerRun) { try { doc.Load(m_logStore + harness.LogFile); XmlNodeList testLogNodes = doc.SelectNodes("/SPOT_Platform_Test/TestLog"); foreach (XmlNode tlNode in testLogNodes) { XmlNodeList testMethodNodes = tlNode.SelectNodes("TestMethod"); totalTestMethods += testMethodNodes.Count; } } catch (XmlException) { harness.TestResults = Harness.Result.Fail; } catch (Exception ex) { Console.WriteLine("WARNING: " + ex.ToString()); } try { mfResult = harness.MFTestResult; } catch { mfResult = Harness.Result.Fail; } } UpdateTestDetails(harness, currentTest, mfResult, totalTestMethods, doc); // Add to totalTestCases count. // This will give you the number of test cases executed during the entire run in all the tests. m_totalTestCases += totalTestMethods; }
private void UpdateTestDetails(Harness harness, BaseTest currentTest, Harness.Result mfResult, int totalTestMethods, XmlDocument doc) { string[] seperator = { "::" }; try { currentTest.EndTime = harness.EndTime; currentTest.LogFile = harness.LogFile; currentTest.Result = mfResult.ToString(); currentTest.StartTime = harness.StartTime; if (m_isProfilerRun) { ArrayList methodList = Utils.ReadProfilerLogFile(currentTest.LogFile, currentTest.ExeLocation); for (int i = 0; i < methodList.Count; i++) { string[] vals = methodList[i].ToString().Split('\t'); int exclTime = 0; if (!string.IsNullOrEmpty(vals[3])) { exclTime = Convert.ToInt32(vals[3].Trim()); } int inclTime = 0; if (!string.IsNullOrEmpty(vals[4])) { inclTime = Convert.ToInt32(vals[4].Trim()); } ProfilerTestMethod ptm = new ProfilerTestMethod(); ptm.TestMethod = vals[2].Split(seperator, StringSplitOptions.None)[1].Trim(); ptm.InclusiveTime = inclTime; ptm.ExclusiveTime = exclTime; currentTest.TestMethods.Add(ptm); } } else { // Update specific results count. switch (mfResult) { case Harness.Result.Pass: m_passCount++; break; case Harness.Result.Fail: m_failCount++; break; case Harness.Result.Skip: m_skipCount++; break; case Harness.Result.KnownFailure: m_knownFailCount++; break; } int pc = 0, fc = 0, sc = 0, kc = 0; try { // Get total passcount, failcount, skipcount and known failure count for the test XmlNodeList passNodes = doc.GetElementsByTagName("PassCount"); foreach (XmlNode passNode in passNodes) { pc += Convert.ToInt32(passNode.ChildNodes[0].InnerText); } XmlNodeList failNodes = doc.GetElementsByTagName("FailCount"); foreach (XmlNode failNode in failNodes) { fc += Convert.ToInt32(failNode.ChildNodes[0].InnerText); } XmlNodeList skipNodes = doc.GetElementsByTagName("SkipCount"); foreach (XmlNode skipNode in skipNodes) { sc += Convert.ToInt32(skipNode.ChildNodes[0].InnerText); } XmlNodeList knownFailNodes = doc.GetElementsByTagName("KnownFailureCount"); foreach (XmlNode knownFailNode in knownFailNodes) { kc += Convert.ToInt32(knownFailNode.ChildNodes[0].InnerText); } } catch { } currentTest.TestMethodFailCount = fc; currentTest.TestMethodKnownFailureCount = kc; currentTest.TestMethodPassCount = pc; currentTest.TestMethodSkipCount = sc; currentTest.TotalTestCases = totalTestMethods; } m_mfTestList.Add(currentTest); } catch (Exception ex) { Console.WriteLine("WARNING: " + ex.ToString()); } }
/// <summary> /// Overloaded Constructor /// </summary> /// <param name="test">The name of the sln file.</param> /// <param name="harness">The Harness instance.</param> /// <param name="log">Path to log file. </param> internal TimedTest(BaseTest test, Harness harness, XmlLog log) { this.m_test = test; this.m_harness = harness; this.m_log = log; }
private void TerminateRunningEmulators(TimedTest test, BaseTest bTest) { if (string.Equals(m_transport.ToLower(), "emulator")) { try { test.KillEmulator(); } catch (Exception ex) { Utils.WriteToEventLog("An exception was thrown when killing the emulator " + "before executing " + bTest.Name + " : " + ex.ToString()); } } string[] onboardFlashes = new string[] { Path.Combine(Path.GetDirectoryName(bTest.Location) , "OnBoardFlash.dat"), Path.Combine(Directory.GetCurrentDirectory() , "OnBoardFlash.dat") }; foreach (string onboardFlash in onboardFlashes) { if (File.Exists(onboardFlash)) { try { File.Delete(onboardFlash); } catch { } } } }
private string BuildFilePath(BaseTest test, ref string xslPath) { string toolSrcPath = @"\test\Platform\"; string srcPath = @"\test\Platform\Tests\"; string enlistmentPath = GetEnlistmentPath; bool isTestMsiInstalled = IsTestMsiInstalled; string file = string.Empty; xslPath = "Results.xsl"; if (!String.IsNullOrEmpty(test.Name)) { // If dev environment, use the test files from the enlistment. // Else If the test msi is installed, use the test files from the installed location. // Else, neither enlistment exists nor test msi is installed - throw a file not found exception. if (m_isDevEnvironment) { if (!Path.IsPathRooted(test.Name)) { test.Name = Path.Combine(Environment.GetEnvironmentVariable("SPOCLIENT"), test.Name); } if (File.Exists(test.Name)) { file = Path.GetFullPath(test.Name); } else { if (test is ProfilerTest) { file = enlistmentPath + srcPath + @"\Performance\ProfilerTests\" + test.Name; } else { file = enlistmentPath + srcPath + test.Name; } } if (!File.Exists(file)) { throw new FileNotFoundException(); } // Set the log folder path. m_mfLogDirectory = enlistmentPath + toolSrcPath + @"Tools\MFTestSystem\Results"; } else if (isTestMsiInstalled) { m_mfLogDirectory = m_mfLogDirectory = string.Format("{0}{1}\\{2}\\", m_installPath, "Results", TestSystem.TestResultId); if (test is ProfilerTest) { srcPath = string.Format("{0}{1}", m_installPath, @"ManagedProfilerTests\"); } else { // If the file doesn't exist assume that the path is not an absolute path. srcPath = string.Format("{0}{1}", m_installPath, @"TestCases\"); toolSrcPath = string.Format("{0}{1}", m_installPath, @"Tools\"); } if (!File.Exists(test.Location)) { file = srcPath + test.Location; } else { file = test.Location; } } else { throw new System.Exception("Could not find the tests. Please install the test msi"); } if (!File.Exists(file)) { Console.WriteLine("\tERROR: File not found: " + file); Utils.WriteToEventLog("File Not Found: " + file); throw new System.ArgumentException( "Specified file " + file + " could not be found."); } } test.Location = file; return(file); }
internal HarnessExecutionResult Run(BaseTest test, XmlLog log) { m_initialTime = new DateTime(); string file = string.Empty; string csprojFilePath = string.Empty; m_logFileSaved = false; m_debugText = new StringBuilder(); Thread desktopThread = null; NamedPipeServerStream pipeStream = null; try { // Set the file paths. file = BuildFilePath(test, ref m_xslPath); Console.WriteLine("\nTest: " + test.Name); string pathForFile = file.Substring(0, file.LastIndexOf(@"\")); if (!pathForFile.EndsWith(@"\")) { pathForFile = string.Format(@"{0}\", pathForFile); } string slnFileName = test.Name.Substring(test.Name.LastIndexOf(@"\") + 1); try { // Get reference list and build the test ArrayList referenceList = GetProjectReferences(file, ref csprojFilePath); if (m_currentAppType != TestType.DeviceDesktop) { BuildTest(csprojFilePath, m_currentAppType); } else { pipeStream = new NamedPipeServerStream("MFHarnessPipe", PipeDirection.InOut, 1, PipeTransmissionMode.Message, PipeOptions.WriteThrough); } // Set log file name. SetLogFileName(test.Name); test.LogFile = m_outputXmlFileName; if (!(test is ProfilerTest)) { m_log = log; m_log.StartLog(m_outputXmlFileName, m_xslPath); } // Get build and exe path. string buildPath = GetBuildPath(file); test.ExeLocation = buildPath + m_assemblyName + ".exe"; // If this is a profiler test, run the test and return. if (test is ProfilerTest) { return(RunProfilerTest(file, test.ExeLocation, buildPath, referenceList)); } string testName = m_logFileName.Replace(".xml", string.Empty); StartCodeCoverage(testName); m_startTime = DateTime.Now; switch (m_currentAppType) { case TestType.Device: try { RunDeviceTest(buildPath, test.ExeLocation, referenceList); } catch (Exception ex) { Close(); return(HarnessResult(ex)); } break; case TestType.DeviceDesktop: // Desktop apps under // DevBox: %spoclient%\Test\Platform\Tests\Desktop\Applications\ // TestBox: %programfiles%\Microsoft .NET Micro Framework\<version>\Tests\Desktop\ desktopThread = StartDesktopApplication(pathForFile, ref csprojFilePath); pipeStream.WaitForConnection(); // Get references and build the device test. GetProjectReferences(file, ref csprojFilePath); BuildTest(csprojFilePath, TestType.Device); RunDeviceTest(buildPath, test.ExeLocation, referenceList); break; case TestType.Desktop: RunDesktopTest(test.ExeLocation); break; } m_endTime = DateTime.Now; StopCodeCoverage(testName); } catch (Exception ex) { m_endTime = DateTime.Now; if (null != m_log) { m_log.WriteElementString("Test_Exception", ex.Message + ex.StackTrace); } Utils.WriteToEventLog("Exception: " + ex.ToString()); Close(); if (ex is ApplicationException && ex.Message.ToLower().Contains("build failure")) { return(HarnessExecutionResult.Abort); } else { return(HarnessExecutionResult.Unavailable); } } finally { if (m_currentAppType == TestType.DeviceDesktop) { StopDesktopApplication(pipeStream, desktopThread); } Close(); if (null != m_log) { switch (m_currentAppType) { case TestType.DeviceDesktop: SaveLogFile(test.Location, TestType.DeviceDesktop); break; default: SaveLogFile(test.Location); break; } } // Change test results back to unknown for the next test. TestResults = Result.NotKnown; } if (!string.IsNullOrEmpty(file)) { if (!string.Equals(m_transport.ToLower(), "emulator")) { m_log.SynchronizeLogTime(file, m_initialTime); } } else { return(HarnessExecutionResult.Unavailable); } } catch (Exception ex) { if (ex is FileNotFoundException) { throw ex; } Utils.WriteToEventLog(string.Format("Exception in Harness: {0}", ex.ToString())); return(HarnessExecutionResult.Abort); } return(HarnessExecutionResult.Success); }
private string BuildFilePath(BaseTest test, ref string xslPath) { string toolSrcPath = @"\test\Platform\"; string srcPath = @"\test\Platform\Tests\"; string enlistmentPath = GetEnlistmentPath; bool isTestMsiInstalled = IsTestMsiInstalled; string file = string.Empty; xslPath = "Results.xsl"; if (!String.IsNullOrEmpty(test.Name)) { // If dev environment, use the test files from the enlistment. // Else If the test msi is installed, use the test files from the installed location. // Else, neither enlistment exists nor test msi is installed - throw a file not found exception. if (m_isDevEnvironment) { if (!Path.IsPathRooted(test.Name)) { test.Name = Path.Combine(Environment.GetEnvironmentVariable("SPOCLIENT"), test.Name); } if (File.Exists(test.Name)) { file = Path.GetFullPath(test.Name); } else { if (test is ProfilerTest) { file = enlistmentPath + srcPath + @"\Performance\ProfilerTests\" + test.Name; } else { file = enlistmentPath + srcPath + test.Name; } } if (!File.Exists(file)) { throw new FileNotFoundException(); } // Set the log folder path. m_mfLogDirectory = enlistmentPath + toolSrcPath + @"Tools\MFTestSystem\Results"; } else if (isTestMsiInstalled) { m_mfLogDirectory = m_mfLogDirectory = string.Format("{0}{1}\\{2}\\", m_installPath, "Results", TestSystem.TestResultId); if (test is ProfilerTest) { srcPath = string.Format("{0}{1}", m_installPath, @"ManagedProfilerTests\"); } else { // If the file doesn't exist assume that the path is not an absolute path. srcPath = string.Format("{0}{1}", m_installPath, @"TestCases\"); toolSrcPath = string.Format("{0}{1}", m_installPath, @"Tools\"); } if (!File.Exists(test.Location)) { file = srcPath + test.Location; } else { file = test.Location; } } else { throw new System.Exception("Could not find the tests. Please install the test msi"); } if (!File.Exists(file)) { Console.WriteLine("\tERROR: File not found: " + file); Utils.WriteToEventLog("File Not Found: " + file); throw new System.ArgumentException( "Specified file " + file + " could not be found."); } } test.Location = file; return file; }
internal HarnessExecutionResult Run(BaseTest test, XmlLog log) { m_initialTime = new DateTime(); string file = string.Empty; string csprojFilePath = string.Empty; m_logFileSaved = false; m_debugText = new StringBuilder(); Thread desktopThread = null; NamedPipeServerStream pipeStream = null; try { // Set the file paths. file = BuildFilePath(test, ref m_xslPath); Console.WriteLine("\nTest: " + test.Name); string pathForFile = file.Substring(0, file.LastIndexOf(@"\")); if (!pathForFile.EndsWith(@"\")) { pathForFile = string.Format(@"{0}\", pathForFile); } string slnFileName = test.Name.Substring(test.Name.LastIndexOf(@"\") + 1); try { // Get reference list and build the test ArrayList referenceList = GetProjectReferences(file, ref csprojFilePath); if (m_currentAppType != TestType.DeviceDesktop) { BuildTest(csprojFilePath, m_currentAppType); } else { pipeStream = new NamedPipeServerStream("MFHarnessPipe", PipeDirection.InOut, 1, PipeTransmissionMode.Message, PipeOptions.WriteThrough); } // Set log file name. SetLogFileName(test.Name); test.LogFile = m_outputXmlFileName; if (!(test is ProfilerTest)) { m_log = log; m_log.StartLog(m_outputXmlFileName, m_xslPath); } // Get build and exe path. string buildPath = GetBuildPath(file); test.ExeLocation = buildPath + m_assemblyName + ".exe"; // If this is a profiler test, run the test and return. if (test is ProfilerTest) { return RunProfilerTest(file, test.ExeLocation, buildPath, referenceList); } string testName = m_logFileName.Replace(".xml", string.Empty); StartCodeCoverage(testName); m_startTime = DateTime.Now; switch(m_currentAppType) { case TestType.Device: try { RunDeviceTest(buildPath, test.ExeLocation, referenceList); } catch (Exception ex) { Close(); return HarnessResult(ex); } break; case TestType.DeviceDesktop: // Desktop apps under // DevBox: %spoclient%\Test\Platform\Tests\Desktop\Applications\ // TestBox: %programfiles%\Microsoft .NET Micro Framework\<version>\Tests\Desktop\ desktopThread = StartDesktopApplication(pathForFile, ref csprojFilePath); pipeStream.WaitForConnection(); // Get references and build the device test. GetProjectReferences(file, ref csprojFilePath); BuildTest(csprojFilePath, TestType.Device); RunDeviceTest(buildPath, test.ExeLocation, referenceList); break; case TestType.Desktop: RunDesktopTest(test.ExeLocation); break; } m_endTime = DateTime.Now; StopCodeCoverage(testName); } catch (Exception ex) { m_endTime = DateTime.Now; if (null != m_log) { m_log.WriteElementString("Test_Exception", ex.Message + ex.StackTrace); } Utils.WriteToEventLog("Exception: " + ex.ToString()); Close(); if (ex is ApplicationException && ex.Message.ToLower().Contains("build failure")) { return HarnessExecutionResult.Abort; } else { return HarnessExecutionResult.Unavailable; } } finally { if (m_currentAppType == TestType.DeviceDesktop) { StopDesktopApplication(pipeStream, desktopThread); } Close(); if (null != m_log) { switch(m_currentAppType) { case TestType.DeviceDesktop: SaveLogFile(test.Location, TestType.DeviceDesktop); break; default: SaveLogFile(test.Location); break; } } // Change test results back to unknown for the next test. TestResults = Result.NotKnown; } if (!string.IsNullOrEmpty(file)) { if (!string.Equals(m_transport.ToLower(), "emulator")) { m_log.SynchronizeLogTime(file, m_initialTime); } } else { return HarnessExecutionResult.Unavailable; } } catch (Exception ex) { if (ex is FileNotFoundException) { throw ex; } Utils.WriteToEventLog(string.Format("Exception in Harness: {0}", ex.ToString())); return HarnessExecutionResult.Abort; } return HarnessExecutionResult.Success; }