/// <summary> /// Retrieves the automated test script from the server for the provided automated test run and populates /// the test run info as well as the test script attachment info /// </summary> /// <param name="remoteAutomatedTestRun">The api data object from the server</param> /// <param name="projectId">The id of the current project</param> /// <returns>The automated test run object that will be passed to the plugins</returns> public AutomatedTestRun PopulateAutomatedTestScript(RemoteAutomatedTestRun remoteAutomatedTestRun, int projectId) { try { //Create a new automated test run object AutomatedTestRun automatedTestRun = new AutomatedTestRun(); if (this._client.Connection_ConnectToProject(projectId)) { if (remoteAutomatedTestRun.AutomationAttachmentId.HasValue) { RemoteDocument remoteDocument = this._client.Document_RetrieveById(remoteAutomatedTestRun.AutomationAttachmentId.Value); if (remoteDocument == null) { string strMessage = "Could not retrieve attachment for TC:" + remoteAutomatedTestRun.TestCaseId + "."; Logger.LogMessage(strMessage, System.Diagnostics.EventLogEntryType.Warning); return(null); } else { //Populate the automated test run object from api test run object automatedTestRun.TestRunId = new Guid(); //Generates new random GUID //Standard Fields automatedTestRun.Name = remoteAutomatedTestRun.Name; automatedTestRun.AutomationEngineId = remoteAutomatedTestRun.AutomationEngineId; automatedTestRun.AutomationHostId = remoteAutomatedTestRun.AutomationHostId; automatedTestRun.AutomationEngineToken = remoteAutomatedTestRun.AutomationEngineToken; automatedTestRun.ReleaseId = remoteAutomatedTestRun.ReleaseId; automatedTestRun.TestCaseId = remoteAutomatedTestRun.TestCaseId; automatedTestRun.TestSetId = remoteAutomatedTestRun.TestSetId; automatedTestRun.TestSetTestCaseId = remoteAutomatedTestRun.TestSetTestCaseId; automatedTestRun.TestRunTypeId = remoteAutomatedTestRun.TestRunTypeId; automatedTestRun.TestSetTestCaseId = remoteAutomatedTestRun.TestSetTestCaseId; //Parameters if (remoteAutomatedTestRun.Parameters != null) { automatedTestRun.Parameters = new List <TestRunParameter>(); foreach (RemoteTestSetTestCaseParameter parameter in remoteAutomatedTestRun.Parameters) { TestRunParameter testRunParameter = new TestRunParameter(); testRunParameter.Name = parameter.Name; testRunParameter.Value = parameter.Value; automatedTestRun.Parameters.Add(testRunParameter); } } //Custom Properties automatedTestRun.Text01 = remoteAutomatedTestRun.Text01; automatedTestRun.Text02 = remoteAutomatedTestRun.Text02; automatedTestRun.Text03 = remoteAutomatedTestRun.Text03; automatedTestRun.Text04 = remoteAutomatedTestRun.Text04; automatedTestRun.Text05 = remoteAutomatedTestRun.Text05; automatedTestRun.Text06 = remoteAutomatedTestRun.Text06; automatedTestRun.Text07 = remoteAutomatedTestRun.Text07; automatedTestRun.Text08 = remoteAutomatedTestRun.Text08; automatedTestRun.Text09 = remoteAutomatedTestRun.Text09; automatedTestRun.Text10 = remoteAutomatedTestRun.Text10; automatedTestRun.List01 = remoteAutomatedTestRun.List01; automatedTestRun.List02 = remoteAutomatedTestRun.List02; automatedTestRun.List03 = remoteAutomatedTestRun.List03; automatedTestRun.List04 = remoteAutomatedTestRun.List04; automatedTestRun.List05 = remoteAutomatedTestRun.List05; automatedTestRun.List06 = remoteAutomatedTestRun.List06; automatedTestRun.List07 = remoteAutomatedTestRun.List07; automatedTestRun.List08 = remoteAutomatedTestRun.List08; automatedTestRun.List09 = remoteAutomatedTestRun.List09; automatedTestRun.List10 = remoteAutomatedTestRun.List10; //Populate the script info automatedTestRun.FilenameOrUrl = remoteDocument.FilenameOrUrl; automatedTestRun.Type = (AutomatedTestRun.AttachmentType)remoteDocument.AttachmentTypeId; automatedTestRun.Size = remoteDocument.Size; if (automatedTestRun.Type == AutomatedTestRun.AttachmentType.File) { //Need to actually download the test script automatedTestRun.TestScript = this._client.Document_OpenFile(remoteDocument.AttachmentId.Value); } //Return the populated test run return(automatedTestRun); } } else { string strMessage = "Could not find attachment for TC:" + remoteAutomatedTestRun.TestCaseId + ", ID #" + remoteAutomatedTestRun.AutomationAttachmentId + "."; Logger.LogMessage(strMessage, System.Diagnostics.EventLogEntryType.Warning); return(null); } } else { string strMessage = "Could not log on to system."; Logger.LogMessage(strMessage, System.Diagnostics.EventLogEntryType.Warning); return(null); } } catch (Exception ex) { Logger.LogMessage(ex); return(null); } }
/// <summary>Takes an automated test run and creates a test run in the application.</summary> /// <param name="automatedTestRun">An automated test run.</param> /// <param name="projectId">The projectId of the test run.</param> /// <returns>Status on updating.</returns> public bool UpdateAutomatedTestRun(AutomatedTestRun automatedTestRun, int projectId) { try { if (this._client.Connection_ConnectToProject(projectId)) { //Create the RemoteTestRun. RemoteAutomatedTestRun updTestRun = new RemoteAutomatedTestRun(); updTestRun.AutomationEngineId = updTestRun.AutomationEngineId; updTestRun.AutomationHostId = automatedTestRun.AutomationHostId; updTestRun.AutomationEngineToken = automatedTestRun.AutomationEngineToken; updTestRun.EndDate = automatedTestRun.EndDate; updTestRun.ExecutionStatusId = (int)automatedTestRun.ExecutionStatus; updTestRun.List01 = automatedTestRun.List01; updTestRun.List02 = automatedTestRun.List02; updTestRun.List03 = automatedTestRun.List03; updTestRun.List04 = automatedTestRun.List04; updTestRun.List05 = automatedTestRun.List05; updTestRun.List06 = automatedTestRun.List06; updTestRun.List07 = automatedTestRun.List07; updTestRun.List08 = automatedTestRun.List08; updTestRun.List09 = automatedTestRun.List09; updTestRun.List10 = automatedTestRun.List10; updTestRun.Name = automatedTestRun.Name; //Copy parameters.. if (automatedTestRun.Parameters != null) { List <RemoteTestSetTestCaseParameter> trParms = new List <RemoteTestSetTestCaseParameter>(); foreach (TestRunParameter parameter in automatedTestRun.Parameters) { RemoteTestSetTestCaseParameter testRunParam = new RemoteTestSetTestCaseParameter(); testRunParam.Name = parameter.Name; testRunParam.Value = parameter.Value; trParms.Add(testRunParam); } updTestRun.Parameters = trParms.ToArray(); } updTestRun.ProjectId = projectId; updTestRun.ReleaseId = automatedTestRun.ReleaseId; updTestRun.RunnerAssertCount = automatedTestRun.RunnerAssertCount; updTestRun.RunnerMessage = automatedTestRun.RunnerMessage; updTestRun.RunnerName = automatedTestRun.RunnerName; updTestRun.RunnerStackTrace = automatedTestRun.RunnerStackTrace; updTestRun.RunnerTestName = automatedTestRun.RunnerTestName; updTestRun.StartDate = automatedTestRun.StartDate; updTestRun.TestCaseId = automatedTestRun.TestCaseId; updTestRun.TesterId = automatedTestRun.TesterId; //updTestRun.TestRunId = automatedTestRun.TestRunId; updTestRun.TestRunTypeId = automatedTestRun.TestRunTypeId; updTestRun.TestSetId = automatedTestRun.TestSetId; updTestRun.TestSetTestCaseId = automatedTestRun.TestSetTestCaseId; updTestRun.Text01 = automatedTestRun.Text01; updTestRun.Text02 = automatedTestRun.Text02; updTestRun.Text03 = automatedTestRun.Text03; updTestRun.Text04 = automatedTestRun.Text04; updTestRun.Text05 = automatedTestRun.Text05; updTestRun.Text06 = automatedTestRun.Text06; updTestRun.Text07 = automatedTestRun.Text07; updTestRun.Text08 = automatedTestRun.Text08; updTestRun.Text09 = automatedTestRun.Text09; updTestRun.Text10 = automatedTestRun.Text10; //That's it, go create the test run.. this._client.TestRun_RecordAutomated1(updTestRun); //Return true. return(true); } else { Logger.LogMessage("Could not connect to ProjectId " + projectId + ".", System.Diagnostics.EventLogEntryType.Warning); return(false); } } catch (Exception ex) { Logger.LogMessage(ex); return(false); } }
/// <summary> /// This is the main method that is used to start automated test execution /// </summary> /// <param name="automatedTestRun">The automated test run object</param> /// <returns>Either the populated test run or an exception</returns> public override AutomatedTestRun StartExecution(AutomatedTestRun automatedTestRun) { //Set status to OK base.status = EngineStatus.OK; string path = ""; try { if (Properties.Settings.Default.TraceLogging) { LogEvent("Starting test execution", EventLogEntryType.Information); } DateTime startDate = DateTime.Now; //See if we have any parameters we need to pass to the automation engine Dictionary <string, string> parameters = new Dictionary <string, string>(); if (automatedTestRun.Parameters == null) { if (Properties.Settings.Default.TraceLogging) { LogEvent("Test Run has no parameters", EventLogEntryType.Information); } } else { if (Properties.Settings.Default.TraceLogging) { LogEvent("Test Run has parameters", EventLogEntryType.Information); } foreach (TestRunParameter testRunParameter in automatedTestRun.Parameters) { string parameterName = testRunParameter.Name.ToLowerInvariant(); if (!parameters.ContainsKey(parameterName)) { //Make sure the parameters are lower case if (Properties.Settings.Default.TraceLogging) { LogEvent("Adding test run parameter " + parameterName + " = " + testRunParameter.Value, EventLogEntryType.Information); } parameters.Add(parameterName, testRunParameter.Value); } } } //We store the script output in a temp file in the remote launch folder string outputFolder = System.Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData) + @"\RemoteLaunch"; if (!Directory.Exists(outputFolder)) { Directory.CreateDirectory(outputFolder); } string outputFile = Path.Combine(outputFolder, "JMeterEngine_Output.log"); //Delete the file if it already exists if (File.Exists(outputFile)) { File.Delete(outputFile); } //See if we have an attached or linked test script if (automatedTestRun.Type == AutomatedTestRun.AttachmentType.URL) { //The "URL" of the test is actually the full file path of the file that contains the test script //Some automation engines need additional parameters which can be provided by allowing the test script filename //to consist of multiple elements separated by a specific character. //Conventionally, most engines use the pipe (|) character to delimit the different elements //See if we have any pipes in the 'filename' that contains arguments or parameters string[] filenameElements = automatedTestRun.FilenameOrUrl.Split('|'); //To make it easier, we have certain shortcuts that can be used in the path // string path = automatedTestRun.FilenameOrUrl; path = filenameElements[0]; string arguments = ""; //To make it easier, we have certain shortcuts that can be used in the path //This allows the same test to be run on different machines with different physical folder layouts path = path.Replace("[MyDocuments]", Environment.GetFolderPath(System.Environment.SpecialFolder.MyDocuments)); path = path.Replace("[CommonDocuments]", Environment.GetFolderPath(System.Environment.SpecialFolder.CommonDocuments)); path = path.Replace("[DesktopDirectory]", Environment.GetFolderPath(System.Environment.SpecialFolder.DesktopDirectory)); path = path.Replace("[ProgramFiles]", Environment.GetFolderPath(System.Environment.SpecialFolder.ProgramFiles)); path = path.Replace("[ProgramFilesX86]", Environment.GetFolderPath(System.Environment.SpecialFolder.ProgramFilesX86)); //See if we have any arguments (not parameters) if (filenameElements.Length > 1) { //Replace any special folders in the arguments as well arguments = filenameElements[1]; arguments = arguments.Replace("[MyDocuments]", Environment.GetFolderPath(System.Environment.SpecialFolder.MyDocuments)); arguments = arguments.Replace("[CommonDocuments]", Environment.GetFolderPath(System.Environment.SpecialFolder.CommonDocuments)); arguments = arguments.Replace("[DesktopDirectory]", Environment.GetFolderPath(System.Environment.SpecialFolder.DesktopDirectory)); arguments = arguments.Replace("[ProgramFiles]", Environment.GetFolderPath(System.Environment.SpecialFolder.ProgramFiles)); arguments = arguments.Replace("[ProgramFilesX86]", Environment.GetFolderPath(System.Environment.SpecialFolder.ProgramFilesX86)); //Replace the special test case and test run id tokens arguments = arguments.Replace("[TestCaseId]", automatedTestRun.TestCaseId.ToString()); arguments = arguments.Replace("[TestRunId]", automatedTestRun.TestRunId.ToString()); if (automatedTestRun.TestSetId.HasValue) { arguments = arguments.Replace("[TestSetId]", automatedTestRun.TestSetId.Value.ToString()); } if (automatedTestRun.ReleaseId.HasValue) { arguments = arguments.Replace("[ReleaseId]", automatedTestRun.ReleaseId.Value.ToString()); } } //First make sure that the file exists if (File.Exists(path)) { if (Properties.Settings.Default.TraceLogging) { LogEvent("Executing " + Constants.EXTERNAL_SYSTEM_NAME + " test located at " + path, EventLogEntryType.Information); } //Construct the command line arguments and working folder string commandDirectory = Properties.Settings.Default.Location; string command = "jmeter.bat"; string commandArgs = "-n "; //Run in non-GUI mode //Add on the test file commandArgs += "-t \"" + path + "\" "; //Add on the output file commandArgs += "-l \"" + outputFile + "\" "; //Add other arguments commandArgs += " " + arguments + " "; //Add on any parameters if (parameters != null && parameters.Count > 0) { foreach (KeyValuePair <string, string> parameter in parameters) { //Escape double-quotes (i.e. " --> "") string paramName = parameter.Key; if (paramName.Contains('"')) { paramName = paramName.Replace("\"", "\"\""); } string paramValue = parameter.Value; if (paramValue.Contains('"')) { paramValue = paramValue.Replace("\"", "\"\"\""); } commandArgs += "-J" + paramName + "=\"" + paramValue + "\" "; } } //Make sure the exe exists string jmeterPath = Path.Combine(commandDirectory, command); if (!File.Exists(jmeterPath)) { throw new FileNotFoundException("Unable to find " + Constants.EXTERNAL_SYSTEM_NAME + " at " + jmeterPath); } //Execute the JMeter command line: //jmeter.exe -n -t [test path] -l [log file path] -J [paramName=paramValue] [arguments] ProcessStartInfo startInfo = new ProcessStartInfo(); startInfo.FileName = jmeterPath; startInfo.Arguments = commandArgs; startInfo.WorkingDirectory = commandDirectory; startInfo.UseShellExecute = false; startInfo.ErrorDialog = false; //Start the process and wait until completed Process process = new Process(); process.StartInfo = startInfo; process.Start(); process.WaitForExit(); process.Close(); } else { throw new FileNotFoundException("Unable to find a " + Constants.EXTERNAL_SYSTEM_NAME + " test at " + path); } } else { //The JMeter automation engine doesn't support embedded/attached scripts, so we throw the following exception: throw new InvalidOperationException("The " + Constants.EXTERNAL_SYSTEM_NAME + " automation engine only supports linked test scripts"); } //Capture the time that it took to run the test DateTime endDate = DateTime.Now; //Now extract the test results //Make sure that the output file exists if (!File.Exists(outputFile)) { throw new FileNotFoundException("Unable to find an output log for " + Constants.EXTERNAL_SYSTEM_NAME + " at " + outputFile); } XmlDocument xmlDoc = new XmlDocument(); try { xmlDoc.Load(outputFile); } catch (XmlException) { //Happens if the file is not XML or the XML is not well formed string outputFileText = System.IO.File.ReadAllText(outputFile); throw new ApplicationException("Unable to parse the " + Constants.EXTERNAL_SYSTEM_NAME + " XML Output file '" + outputFile + "' - " + outputFileText); } //Extract the results from the XML log-file //Default to N/A in case there are no assertions. TestRun.TestStatusEnum executionStatus = TestRun.TestStatusEnum.NotApplicable; string externalTestDetailedResults = ""; XmlNodeList xmlAssertionResults = xmlDoc.SelectNodes("//assertionResult"); //Loop through each result node int failureCount = 0; int errorCount = 0; if (xmlAssertionResults.Count > 0) { //As long as we have one result, mark as passed executionStatus = TestRun.TestStatusEnum.Passed; } foreach (XmlNode xmlAssertionResult in xmlAssertionResults) { //Get the URL being accessed string label = ""; label = xmlAssertionResult.ParentNode.Attributes["lb"].Value; if (xmlAssertionResult.ParentNode.Attributes["lb"] != null) { label = xmlAssertionResult.ParentNode.Attributes["lb"].Value; } if (xmlAssertionResult != null) { string name = label; if (xmlAssertionResult.SelectSingleNode("name") != null) { if (String.IsNullOrEmpty(label)) { name = xmlAssertionResult.SelectSingleNode("name").InnerText; } else { name = xmlAssertionResult.SelectSingleNode("name").InnerText + " (" + label + ")"; } } string failure = "false"; if (xmlAssertionResult.SelectSingleNode("failure") != null) { failure = xmlAssertionResult.SelectSingleNode("failure").InnerText; } string error = "false"; if (xmlAssertionResult.SelectSingleNode("error") != null) { error = xmlAssertionResult.SelectSingleNode("error").InnerText; } string failureMessage = ""; if (xmlAssertionResult.SelectSingleNode("failureMessage") != null) { failureMessage = xmlAssertionResult.SelectSingleNode("failureMessage").InnerText; } if (!String.IsNullOrEmpty(failure) && failure.ToLowerInvariant() == "true") { failureCount++; executionStatus = TestRun.TestStatusEnum.Failed; } if (!String.IsNullOrEmpty(error) && error.ToLowerInvariant() == "true") { errorCount++; executionStatus = TestRun.TestStatusEnum.Failed; } string message = String.Format("{0}: failure={1}, error={2}, message='{3}'", name, failure, error, failureMessage); externalTestDetailedResults += message + "\n"; } } string externalTestSummary = String.Format("Ran with {0} failures and {1} errors", failureCount, errorCount); //Populate the Test Run object with the results if (String.IsNullOrEmpty(automatedTestRun.RunnerName)) { automatedTestRun.RunnerName = this.ExtensionName.Substring(0, 13); } automatedTestRun.RunnerTestName = Path.GetFileNameWithoutExtension(path); //Specify the start/end dates automatedTestRun.StartDate = startDate; automatedTestRun.EndDate = endDate; //The result log automatedTestRun.ExecutionStatus = executionStatus; automatedTestRun.RunnerMessage = externalTestSummary; automatedTestRun.RunnerStackTrace = externalTestDetailedResults; //Report as complete base.status = EngineStatus.OK; return(automatedTestRun); } catch (Exception exception) { //Log the error and denote failure LogEvent(exception.Message + " (" + exception.StackTrace + ")", EventLogEntryType.Error); //Report as completed with error base.status = EngineStatus.Error; throw exception; } }
/* Leave this function un-implemented */ public override AutomatedTestRun StartExecution(AutomatedTestRun automatedTestRun) { //Not used since we implement the V4 API instead throw new NotImplementedException(); }
/// <summary> /// Actually runs the SmarteStudio/Rapise test run and extracts the results /// </summary> /// <param name="automatedTestRun">The automated test run</param> /// <returns>The test run with populated results</returns> /// <param name="documents">Any documents to the test case</param> /// <param name="projectId">The id of the Spira project</param> public AutomatedTestRun StartExecution(AutomatedTestRun automatedTestRun, int projectId, List <AttachedDocument> documents) { //TODO: SmarteSoft needs to implement this method throw new NotImplementedException(); }
public void Execute() { const string METHOD = "Execute()"; Logger.LogTrace(CLASS + METHOD + " Enter"); //Loop through each test, execute it and then finally raise the event. bool isBlocked = false; for (int i = 0; i < this._testsToRun.TestRuns.Count() && !isBlocked; i++) { //DEBUG: Is it contained? RemoteAutomatedTestRun automatedTestRun = this._testsToRun.TestRuns.ElementAt(i); int testCaseId = automatedTestRun.TestCaseId; string strLog2 = " Checking TC:" + testCaseId + ", RunnerName: '" + automatedTestRun.AutomationEngineToken; Logger.LogTrace(CLASS + METHOD + strLog2); //Get the test details. Copy to the return set in case there's an error. int projectId = this._testsToRun.TestRuns.ElementAt(i).ProjectId.Value; AutomatedTestRun testToExecute = this._client.PopulateAutomatedTestScript(automatedTestRun, projectId); AutomatedTestRun testComplete = testToExecute; //Execute the test. try { if (testToExecute != null) { try { Logger.LogTrace(CLASS + METHOD + " Executing test TC:" + testCaseId); //Create an instance of the automation engine AutomationEngine engine = new AutomationEngine(); engine.ApplicationLog = Logger.EventLog; //Download any documents attached to the test case //Since they need to be passed to the test engine List <AttachedDocument> attachedDocs = this._client.GetAttachedDocuments(projectId, testCaseId); //Execute test. testComplete.StartDate = DateTime.Now; testComplete = engine.StartExecution(testToExecute, projectId, attachedDocs); } catch (Exception ex) { //Log it first. Logger.LogMessage(ex, CLASS + METHOD + " Error running test."); //Mark test as blocked, insert data. testComplete.ExecutionStatus = TestRun.TestStatusEnum.Blocked; testComplete.RunnerName = automatedTestRun.AutomationEngineToken; testComplete.EndDate = testComplete.StartDate; //Get exception message. string strStack = "Stack Trace:" + Environment.NewLine + ex.StackTrace; string strMsg = ex.Message; while (ex.InnerException != null) { strMsg += Environment.NewLine + ex.InnerException.Message; ex = ex.InnerException; } testComplete.RunnerMessage = strMsg; testComplete.RunnerStackTrace = "Exception:" + Environment.NewLine + strMsg + Environment.NewLine + Environment.NewLine + strStack; Logger.LogTrace(CLASS + METHOD + " Sending result to server."); try { this._client.UpdateAutomatedTestRun(testComplete, this._testsToRun.ProjectId); } catch (Exception ex2) { Logger.LogMessage(ex2, CLASS + METHOD + " Could not send result to server."); } } } else { Logger.LogMessage(CLASS + METHOD + " Retrieved test was null. Cannot execute." + Environment.NewLine + "Skipping TC:" + testCaseId + " in TX:" + automatedTestRun.TestSetId, System.Diagnostics.EventLogEntryType.Error); } //Record test. if (testComplete != null) { Logger.LogMessage(CLASS + METHOD + " Test finished. Recording results."); try { this._client.UpdateAutomatedTestRun(testComplete, this._testsToRun.ProjectId); } catch (Exception ex) { Logger.LogMessage(ex, CLASS + METHOD + " Error saving results."); } } else { Logger.LogMessage(CLASS + METHOD + " Returned tstComplete variable null! Not recording.", EventLogEntryType.Error); } } catch (Exception ex) { Logger.LogMessage(ex, CLASS + METHOD + " Uncaught error occurred when running test! Cancelling rest of pending tests."); this.WorkCompleted(this, new WorkCompletedArgs(this._testsToRun, ex, WorkCompletedArgs.WorkCompletedStatusEnum.Error)); isBlocked = true; } //Now, report on specific test. this.WorkProgress(this, new WorkProgressArgs()); } if (!isBlocked) { this.WorkCompleted(this, new WorkCompletedArgs(this._testsToRun, null, WorkCompletedArgs.WorkCompletedStatusEnum.OK)); } }