/// <summary> /// retrieves the run logs for the test when the steps are not reported to Qc (like in ST) /// </summary> /// <param name="currentTest"></param> /// <returns>the test run log</returns> private string GetTestRunLog(ITSTest currentTest) { string TestLog = "log\\vtd_user.log"; IRun lastrun = currentTest.LastRun as IRun; string retVal = ""; if (lastrun != null) { try { IExtendedStorage storage = lastrun.ExtendedStorage as IExtendedStorage; List list; bool wasFatalError; var path = storage.LoadEx(TestLog, true, out list, out wasFatalError); string logPath = Path.Combine(path, TestLog); if (File.Exists(logPath)) { retVal = File.ReadAllText(logPath).TrimEnd(); } } catch { retVal = ""; } } retVal = ConsoleWriter.FilterXmlProblematicChars(retVal); return(retVal); }
/// <summary> /// writes a summary of the test run after it's over /// </summary> /// <param name="prevTest"></param> private void WriteTestRunSummary(ITSTest prevTest) { int prevRunId = ConsoleWriter.ActiveTestRun.PrevRunId; int runid = GetTestRunId(prevTest); if (runid > prevRunId) { string stepsString = GetTestStepsDescFromQc(prevTest); if (string.IsNullOrWhiteSpace(stepsString) && ConsoleWriter.ActiveTestRun.TestState != TestState.Error) { stepsString = GetTestRunLog(prevTest); } if (!string.IsNullOrWhiteSpace(stepsString)) { ConsoleWriter.WriteLine(stepsString); } string linkStr = GetTestRunLink(prevTest, runid); ConsoleWriter.WriteLine("\n" + string.Format(Resources.AlmRunnerDisplayLink, linkStr)); } ConsoleWriter.WriteLine(DateTime.Now.ToString(Launcher.DateFormat) + " " + Resources.AlmRunnerTestCompleteCaption + " " + prevTest.Name + ((runid > prevRunId) ? ", " + Resources.AlmRunnerRunIdCaption + " " + runid : "") + "\n-------------------------------------------------------------------------------------------------------"); }
private void SetTestResults(ITSTest currentTest, IExecutionStatus executionStatus, ITestSet targetTestSet, TestRunResults activeTestDesc, TestSuiteRunResults runDesc, string testPath, string abortFilename) { // write the status for each test for (int k = 1; k <= executionStatus.Count; ++k) { if (System.IO.File.Exists(abortFilename)) { break; } TestExecStatus testExecStatusObj = executionStatus[k]; currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId]; if (currentTest == null) { ConsoleWriter.WriteLine(string.Format("currentTest is null for test.{0} after whole execution", k)); continue; } activeTestDesc = UpdateTestStatus(runDesc, targetTestSet, testExecStatusObj, false); UpdateCounters(activeTestDesc, runDesc); activeTestDesc.TestPath = testPath; } }
/// <summary> /// gets a link string for the test run in Qc /// </summary> /// <param name="prevTest"></param> /// <param name="runid"></param> /// <returns></returns> private string GetTestRunLink(ITSTest prevTest, int runid) { bool oldQc = CheckIsOldQc(); ITestSet set = prevTest.TestSet; string testRunLink = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/TestLabModule-000000003649890581?EntityType=IRun&EntityID=" + runid; string testRunLinkQc10 = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/Test%20Lab?Action=FindRun&TestSetID=" + set.ID + "&TestInstanceID=" + prevTest.ID + "&RunID=" + runid; string linkStr = (oldQc ? testRunLinkQc10 : testRunLink); return(linkStr); }
/// <summary> /// gets the runId for the given test /// </summary> /// <param name="currentTest">a test instance</param> /// <returns>the run id</returns> private static int GetTestRunId(ITSTest currentTest) { int runid = -1; IRun lastrun = currentTest.LastRun as IRun; if (lastrun != null) { runid = lastrun.ID; } return(runid); }
/// <summary> /// summerizes test steps after test has run /// </summary> /// <param name="test"></param> /// <returns>a string containing descriptions of step states and messags</returns> string GetTestStepsDescFromQc(ITSTest test) { StringBuilder sb = new StringBuilder(); try { //get runs for the test RunFactory rfactory = test.RunFactory; List runs = rfactory.NewList(""); if (runs.Count == 0) { return(""); } //get steps from run StepFactory stepFact = runs[runs.Count].StepFactory; List steps = stepFact.NewList(""); if (steps.Count == 0) { return(""); } //go over steps and format a string foreach (IStep step in steps) { sb.Append("Step: " + step.Name); if (!string.IsNullOrWhiteSpace(step.Status)) { sb.Append(", Status: " + step.Status); } string desc = step["ST_DESCRIPTION"] as string; if (!string.IsNullOrEmpty(desc)) { desc = "\n\t" + desc.Trim().Replace("\n", "\t").Replace("\r", ""); if (!string.IsNullOrWhiteSpace(desc)) { sb.AppendLine(desc); } } } } catch (Exception ex) { sb.AppendLine("Exception while reading step data: " + ex.Message); } return(sb.ToString().TrimEnd()); }
/// <summary> /// updates the test status in our list of tests /// </summary> /// <param name="targetTestSet"></param> /// <param name="testExecStatusObj"></param> private TestRunResults UpdateTestStatus(TestSuiteRunResults runResults, ITestSet targetTestSet, TestExecStatus testExecStatusObj, bool onlyUpdateState) { TestRunResults qTest = null; ITSTest currentTest = null; try { //find the test for the given status object currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId]; //find the test in our list int testIndex = GetIdxByTestName(currentTest.Name, runResults); qTest = runResults.TestRuns[testIndex]; if (qTest.TestType == null) { qTest.TestType = GetTestType(currentTest); } //update the state qTest.PrevTestState = qTest.TestState; qTest.TestState = GetTsStateFromQcState(testExecStatusObj.Status); if (!onlyUpdateState) { try { //duration and status are updated according to the run qTest.Runtime = TimeSpan.FromSeconds(currentTest.LastRun.Field("RN_DURATION")); } catch { //a problem getting duration, maybe the test isn't done yet - don't stop the flow.. } switch (qTest.TestState) { case TestState.Failed: qTest.FailureDesc = GenerateFailedLog(currentTest.LastRun); if (string.IsNullOrWhiteSpace(qTest.FailureDesc)) { qTest.FailureDesc = testExecStatusObj.Status + " : " + testExecStatusObj.Message; } break; case TestState.Error: qTest.ErrorDesc = testExecStatusObj.Status + " : " + testExecStatusObj.Message; break; default: break; } //check qc version for link type bool oldQc = CheckIsOldQc(); //string testLink = "<a href=\"testdirector:mydtqc01.isr.hp.com:8080/qcbin," + m_qcProject + "," + m_qcDomain + "," + targetTestSet.Name+ ";test-instance:" + testExecStatusObj.TestInstance + "\"> Alm link</a>"; string serverURl = m_qcServer.TrimEnd("/".ToCharArray()); if (serverURl.ToLower().StartsWith("http://")) { serverURl = serverURl.Substring(7); } //string testLinkInLabQc10 = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/Test%20Lab?Action=FindTestInstance&TestSetID=" + targetTestSet.ID + "&TestInstanceID=" + testExecStatusObj.TSTestId; //string testLinkInLab = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/TestLabModule-000000003649890581?EntityType=ITestInstance&EntityID=" + testExecStatusObj.TSTestId; int runid = GetTestRunId(currentTest); string linkStr = GetTestRunLink(currentTest, runid); string statusString = GetTsStateFromQcState(testExecStatusObj.Status as string).ToString(); ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerTestStat, currentTest.Name, statusString, testExecStatusObj.Message, linkStr)); runResults.TestRuns[testIndex] = qTest; } } catch (Exception ex) { ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerErrorGettingStat, currentTest.Name, ex.Message)); } return(qTest); }
/// <summary> /// runs a test set with given parameters (and a valid connection to the QC server) /// </summary> /// <param name="tsFolderName">testSet folder name</param> /// <param name="tsName">testSet name</param> /// <param name="timeout">-1 for unlimited, or number of miliseconds</param> /// <param name="runMode">run on LocalMachine or remote</param> /// <param name="runHost">if run on remote machine - remote machine name</param> /// <returns></returns> public TestSuiteRunResults RunTestSet(string tsFolderName, string tsName, double timeout, QcRunMode runMode, string runHost) { string currentTestSetInstances = ""; TestSuiteRunResults runDesc = new TestSuiteRunResults(); TestRunResults activeTestDesc = null; var tsFactory = tdConnection.TestSetFactory; var tsTreeManager = (ITestSetTreeManager)tdConnection.TestSetTreeManager; List tsList = null; string tsPath = "Root\\" + tsFolderName; ITestSetFolder tsFolder = null; try { tsFolder = (ITestSetFolder)tsTreeManager.get_NodeByPath(tsPath); } catch (COMException ex) { //not found tsFolder = null; } if (tsFolder == null) { //node wasn't found, folder = null ConsoleWriter.WriteErrLine(string.Format(Resources.AlmRunnerNoSuchFolder, tsFolder)); //this will make sure run will fail at the end. (since there was an error) Launcher.ExitCode = Launcher.ExitCodeEnum.Failed; return(null); } else { tsList = tsFolder.FindTestSets(tsName); } if (tsList == null) { ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerCantFindTest, tsName)); //this will make sure run will fail at the end. (since there was an error) Launcher.ExitCode = Launcher.ExitCodeEnum.Failed; return(null); } ITestSet targetTestSet = null; foreach (ITestSet ts in tsList) { if (ts.Name.Equals(tsName, StringComparison.InvariantCultureIgnoreCase)) { targetTestSet = ts; break; } } if (targetTestSet == null) { ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerCantFindTest, tsName)); //this will make sure run will fail at the end. (since there was an error) Launcher.ExitCode = Launcher.ExitCodeEnum.Failed; return(null); } ConsoleWriter.WriteLine(Resources.GeneralDoubleSeperator); ConsoleWriter.WriteLine(Resources.AlmRunnerStartingExecution); ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerDisplayTest, tsName, targetTestSet.ID)); ITSScheduler Scheduler = null; try { //need to run this to install everyhting needed http://AlmServer:8080/qcbin/start_a.jsp?common=true //start the scheduler Scheduler = targetTestSet.StartExecution(""); } catch (Exception ex) { Scheduler = null; } try { currentTestSetInstances = GetTestInstancesString(targetTestSet); } catch (Exception ex) { } if (Scheduler == null) { Console.WriteLine(GetAlmNotInstalledError()); //proceeding with program execution is tasteless, since nothing will run without a properly installed QC. Environment.Exit((int)Launcher.ExitCodeEnum.Failed); } TSTestFactory tsTestFactory = targetTestSet.TSTestFactory; ITDFilter2 tdFilter = tsTestFactory.Filter; tdFilter["TC_CYCLE_ID"] = targetTestSet.ID.ToString(); IList tList = tsTestFactory.NewList(tdFilter.Text); try { //set up for the run depending on where the test instances are to execute switch (runMode) { case QcRunMode.RUN_LOCAL: // run all tests on the local machine Scheduler.RunAllLocally = true; break; case QcRunMode.RUN_REMOTE: // run tests on a specified remote machine Scheduler.TdHostName = runHost; break; // RunAllLocally must not be set for remote invocation of tests. As such, do not do this: Scheduler.RunAllLocally = False case QcRunMode.RUN_PLANNED_HOST: // run on the hosts as planned in the test set Scheduler.RunAllLocally = false; break; } } catch (Exception ex) { ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerProblemWithHost, ex.Message)); } ConsoleWriter.WriteLine(Resources.AlmRunnerNumTests + tList.Count); int i = 1; foreach (ITSTest3 test in tList) { string runOnHost = runHost; if (runMode == QcRunMode.RUN_PLANNED_HOST) { runOnHost = test.HostName; } //if host isn't taken from QC (PLANNED) and not from the test definition (REMOTE), take it from LOCAL (machineName) string hostName = runOnHost; if (runMode == QcRunMode.RUN_LOCAL) { hostName = Environment.MachineName; } ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerDisplayTestRunOnHost, i, test.Name, hostName)); Scheduler.RunOnHost[test.ID] = runOnHost; var testResults = new TestRunResults(); testResults.TestName = test.Name; runDesc.TestRuns.Add(testResults); i = i + 1; } Stopwatch sw = Stopwatch.StartNew(); Stopwatch testSw = null; try { //tests are actually run Scheduler.Run(); } catch (Exception ex) { ConsoleWriter.WriteLine(Resources.AlmRunnerRunError + ex.Message); } ConsoleWriter.WriteLine(Resources.AlmRunnerSchedStarted + DateTime.Now.ToString(Launcher.DateFormat)); ConsoleWriter.WriteLine(Resources.SingleSeperator); IExecutionStatus executionStatus = Scheduler.ExecutionStatus; bool tsExecutionFinished = false; ITSTest prevTest = null; ITSTest currentTest = null; string abortFilename = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\stop" + Launcher.UniqueTimeStamp + ".txt"; //wait for the tests to end ("normally" or because of the timeout) while ((tsExecutionFinished == false) && (timeout == -1 || sw.Elapsed.TotalSeconds < timeout)) { executionStatus.RefreshExecStatusInfo("all", true); tsExecutionFinished = executionStatus.Finished; if (System.IO.File.Exists(abortFilename)) { break; } for (int j = 1; j <= executionStatus.Count; ++j) { TestExecStatus testExecStatusObj = executionStatus[j]; activeTestDesc = UpdateTestStatus(runDesc, targetTestSet, testExecStatusObj, true); if (activeTestDesc.PrevTestState != activeTestDesc.TestState) { TestState tstate = activeTestDesc.TestState; if (tstate == TestState.Running) { currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId]; int testIndex = GetIdxByTestName(currentTest.Name, runDesc); int prevRunId = GetTestRunId(currentTest); runDesc.TestRuns[testIndex].PrevRunId = prevRunId; //closing previous test if (prevTest != null) { WriteTestRunSummary(prevTest); } //starting new test prevTest = currentTest; //assign the new test the consol writer so it will gather the output ConsoleWriter.ActiveTestRun = runDesc.TestRuns[testIndex]; ConsoleWriter.WriteLine(DateTime.Now.ToString(Launcher.DateFormat) + " Running: " + currentTest.Name); //tell user that the test is running ConsoleWriter.WriteLine(DateTime.Now.ToString(Launcher.DateFormat) + " Running test: " + activeTestDesc.TestName + ", Test id: " + testExecStatusObj.TestId + ", Test instance id: " + testExecStatusObj.TSTestId); //start timing the new test run string foldername = ""; ITestSetFolder folder = targetTestSet.TestSetFolder as ITestSetFolder; if (folder != null) { foldername = folder.Name.Replace(".", "_"); } //the test group is it's test set. (dots are problematic since jenkins parses them as seperators between packadge and class) activeTestDesc.TestGroup = foldername + "\\" + targetTestSet.Name; activeTestDesc.TestGroup = activeTestDesc.TestGroup.Replace(".", "_"); } TestState enmState = GetTsStateFromQcState(testExecStatusObj.Status as string); string statusString = enmState.ToString(); if (enmState == TestState.Running) { ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerStat, activeTestDesc.TestName, testExecStatusObj.TSTestId, statusString)); } else if (enmState != TestState.Waiting) { ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerStatWithMessage, activeTestDesc.TestName, testExecStatusObj.TSTestId, statusString, testExecStatusObj.Message)); } if (System.IO.File.Exists(abortFilename)) { break; } } } //wait 0.2 seconds Thread.Sleep(200); //check for abortion if (System.IO.File.Exists(abortFilename)) { _blnRunCancelled = true; ConsoleWriter.WriteLine(Resources.GeneralStopAborted); //stop all test instances in this testSet. Scheduler.Stop(currentTestSetInstances); ConsoleWriter.WriteLine(Resources.GeneralAbortedByUser); //stop working Environment.Exit((int)Launcher.ExitCodeEnum.Aborted); } } //check status for each test if (timeout == -1 || sw.Elapsed.TotalSeconds < timeout) { //close last test if (prevTest != null) { WriteTestRunSummary(prevTest); } //done with all tests, stop collecting output in the testRun object. ConsoleWriter.ActiveTestRun = null; for (int k = 1; k <= executionStatus.Count; ++k) { if (System.IO.File.Exists(abortFilename)) { break; } TestExecStatus testExecStatusObj = executionStatus[k]; activeTestDesc = UpdateTestStatus(runDesc, targetTestSet, testExecStatusObj, false); UpdateCounters(activeTestDesc, runDesc); currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId]; string testPath = "Root\\" + tsFolderName + "\\" + tsName + "\\" + activeTestDesc.TestName; activeTestDesc.TestPath = testPath; } //update the total runtime runDesc.TotalRunTime = sw.Elapsed; ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerTestsetDone, tsName, DateTime.Now.ToString(Launcher.DateFormat))); } else { _blnRunCancelled = true; ConsoleWriter.WriteLine(Resources.GeneralTimedOut); Launcher.ExitCode = Launcher.ExitCodeEnum.Aborted; } return(runDesc); }
/// <summary> /// Initializes a new instance of the <see cref="TestCaseInstance"/> class. /// </summary> /// <param name="test">The test<see cref="ITSTest"/>.</param> /// <param name="buildNumber"> The build number for this test run. </param> /// <param name="environment"> The environmnet the test is run on.</param> public TestCaseInstance(ITSTest test, string environment, string buildNumber) { this.Test = test; this.Name = test.Name.Substring(test.Name.LastIndexOf("]") + 1); // remove the test case instance number this.StartTestRun(environment, buildNumber); }
/// <summary> /// summerizes test steps after test has run /// </summary> /// <param name="test"></param> /// <returns>a string containing descriptions of step states and messags</returns> string GetTestStepsDescFromQc(ITSTest test) { StringBuilder sb = new StringBuilder(); try { //get runs for the test RunFactory rfactory = test.RunFactory; List runs = rfactory.NewList(""); if (runs.Count == 0) return ""; //get steps from run StepFactory stepFact = runs[runs.Count].StepFactory; List steps = stepFact.NewList(""); if (steps.Count == 0) return ""; //go over steps and format a string foreach (IStep step in steps) { sb.Append("Step: " + step.Name); if (!string.IsNullOrWhiteSpace(step.Status)) sb.Append(", Status: " + step.Status); string desc = step["ST_DESCRIPTION"] as string; if (!string.IsNullOrEmpty(desc)) { desc = "\n\t" + desc.Trim().Replace("\n", "\t").Replace("\r", ""); if (!string.IsNullOrWhiteSpace(desc)) sb.AppendLine(desc); } } } catch (Exception ex) { sb.AppendLine("Exception while reading step data: " + ex.Message); } return sb.ToString().TrimEnd(); }
/// <summary> /// retrieves the run logs for the test when the steps are not reported to Qc (like in ST) /// </summary> /// <param name="currentTest"></param> /// <returns>the test run log</returns> private string GetTestRunLog(ITSTest currentTest) { string TestLog = "log\\vtd_user.log"; IRun lastrun = currentTest.LastRun as IRun; string retVal = ""; if (lastrun != null) { try { IExtendedStorage storage = lastrun.ExtendedStorage as IExtendedStorage; List list; bool wasFatalError; var path = storage.LoadEx(TestLog, true, out list, out wasFatalError); string logPath = Path.Combine(path, TestLog); if (File.Exists(logPath)) { retVal = File.ReadAllText(logPath).TrimEnd(); } } catch { retVal = ""; } } retVal = ConsoleWriter.FilterXmlProblematicChars(retVal); return retVal; }
/// <summary> /// gets a link string for the test run in Qc /// </summary> /// <param name="prevTest"></param> /// <param name="runid"></param> /// <returns></returns> private string GetTestRunLink(ITSTest prevTest, int runid) { bool oldQc = CheckIsOldQc(); bool useSSL = (m_qcServer.Contains("https://")); ITestSet set = prevTest.TestSet; string testRunLink = useSSL ? ("tds://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("https://", "") + "/TestLabModule-000000003649890581?EntityType=IRun&EntityID=" + runid) : ("td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/TestLabModule-000000003649890581?EntityType=IRun&EntityID=" + runid); string testRunLinkQc10 = useSSL ? ("tds://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("https://", "") + "/Test%20Lab?Action=FindRun&TestSetID=" + set.ID + "&TestInstanceID=" + prevTest.ID + "&RunID=" + runid) : ("td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/Test%20Lab?Action=FindRun&TestSetID=" + set.ID + "&TestInstanceID=" + prevTest.ID + "&RunID=" + runid); string linkStr = (oldQc ? testRunLinkQc10 : testRunLink); return linkStr; }
/// <summary> /// gets the runId for the given test /// </summary> /// <param name="currentTest">a test instance</param> /// <returns>the run id</returns> private static int GetTestRunId(ITSTest currentTest) { int runid = -1; IRun lastrun = currentTest.LastRun as IRun; if (lastrun != null) runid = lastrun.ID; return runid; }
/// <summary> /// writes a summary of the test run after it's over /// </summary> /// <param name="prevTest"></param> private void WriteTestRunSummary(ITSTest prevTest) { int prevRunId = ConsoleWriter.ActiveTestRun.PrevRunId; int runid = GetTestRunId(prevTest); if (runid > prevRunId) { string stepsString = GetTestStepsDescFromQc(prevTest); if (string.IsNullOrWhiteSpace(stepsString) && ConsoleWriter.ActiveTestRun.TestState != TestState.Error) stepsString = GetTestRunLog(prevTest); if (!string.IsNullOrWhiteSpace(stepsString)) ConsoleWriter.WriteLine(stepsString); string linkStr = GetTestRunLink(prevTest, runid); ConsoleWriter.WriteLine("\n" + string.Format(Resources.AlmRunnerDisplayLink, linkStr)); } ConsoleWriter.WriteLine(DateTime.Now.ToString(Launcher.DateFormat) + " " + Resources.AlmRunnerTestCompleteCaption + " " + prevTest.Name + ((runid > prevRunId) ? ", " + Resources.AlmRunnerRunIdCaption + " " + runid : "") + "\n-------------------------------------------------------------------------------------------------------"); }