public void LoadSavedJobs() { lock (this) { this.Configured.Set(); if (Server.IsRemote) { // retreiving saved jobs and adding to the job queue and jobmap foreach (SavedJob saved in Entities.SavedJobs) { if (string.IsNullOrEmpty(saved.VFUrl) == false && Server.JenkinsUrl != saved.VFUrl) { continue; } Job job = new JobImpl(Server) { Title = saved.Title, Status = Job.StatusEnum.StartedOnServer, // TODO: wrong, but not a big deal, will be re-read from the server RunCommand = saved.RunCommand, WorkingDirectory = saved.WorkingDirectory, }; try { Directory.CreateDirectory(saved.WorkingDirectory); } catch (Exception e) { Trace.TraceError(e.ToString()); } var jenkinsJobInfo = JenkinsInstance.GetJobInfo(saved.JobName); if (jenkinsJobInfo == null) { Trace.TraceInformation("Saved job {0} ({1}) no longer exists on server. Marking failed.", job.Title, jenkinsJobInfo); job.Status = Job.StatusEnum.FailedAbortOnServer; Entities.SavedJobs.DeleteObject(saved); } else { Trace.TraceInformation("Adding saved job {0} ({1})", job.Title, jenkinsJobInfo); ((JobImpl)job).remoteInfo = JobMap[job] = new RemoteJobInfo() { JenkinsJobName = jenkinsJobInfo.name }; Server.AddJob(job); Entities.SavedJobs.Detach(saved); } } Entities.SaveChanges(); // start timer for remote execution monitoring this.RestartMonitorTimer(); } } }
private void MonitorJenkinsJob(Job job, RemoteJobInfo remoteJobInfo) { string jenkinsJobName = remoteJobInfo.JenkinsJobName; if (job.Status == Job.StatusEnum.RedownloadQueued) { // n.b. job may not exist on server var success = SaveWorkspace(job, remoteJobInfo); job.Status = success; return; } Jenkins.Job.Job jenkinsJob; try { jenkinsJob = JenkinsInstance.GetJobInfo(jenkinsJobName, rethrow: true); } catch (WebException e) { if (e is Jenkins.WebExceptionWithStatusCode && ((Jenkins.WebExceptionWithStatusCode)e).StatusCode == 404) { Trace.TraceError("Job {0} no longer exists (404 received from server). Marking job as Failed", jenkinsJobName); job.Status = Job.StatusEnum.Failed; return; } throw; } if (jenkinsJob == null) { // should not be reachable Trace.TraceError("Could not retrieve status for job {0}. Marking job as Failed", jenkinsJobName); job.Status = Job.StatusEnum.Failed; return; } if (jenkinsJob.lastSuccessfulBuild != null) { JenkinsInstance.SaveLastConsole(jenkinsJob.name, Path.Combine(job.WorkingDirectory, "consoleText.txt")); // delete workspace.zip here if it exists, or SaveWorkspace will try to resume // n.b. "If the file to be deleted does not exist, no exception is thrown." File.Delete(Path.Combine(job.WorkingDirectory, "workspace.zip")); var success = SaveWorkspace(job, remoteJobInfo); if (Server.WipeWorkspaceOnSuccess) { // wipe workspace before we delete the job JenkinsInstance.WipeWorkSpace(jenkinsJob.name); } if (false && Server.DeleteJobOnSuccess) // META-2493 META-2492 { // delete job from server JenkinsInstance.DeleteJob(jenkinsJob.name); } job.Status = success; } else if (jenkinsJob.lastFailedBuild != null) { JenkinsInstance.SaveLastConsole(jenkinsJob.name, Path.Combine(job.WorkingDirectory, "consoleText.txt")); File.Copy(Path.Combine(job.WorkingDirectory, "consoleText.txt"), Path.Combine(job.WorkingDirectory, LocalPool.Failed), true); try { // delete workspace.zip here if it exists, or SaveWorkspace will try to resume // n.b. "If the file to be deleted does not exist, no exception is thrown." File.Delete(Path.Combine(job.WorkingDirectory, "workspace.zip")); job.Status = SaveWorkspace(job, remoteJobInfo); } catch (Exception e) { Trace.TraceError("Error retrieving results for {0} ({1}): {2}", job.Title, jenkinsJob.name, e.ToString()); job.Status = Job.StatusEnum.FailedToDownload; } } else if (jenkinsJob.lastCompletedBuild != null) { // build got cancelled JenkinsInstance.SaveLastConsole(jenkinsJob.name, Path.Combine(job.WorkingDirectory, "consoleText.txt")); File.Copy(Path.Combine(job.WorkingDirectory, "consoleText.txt"), Path.Combine(job.WorkingDirectory, LocalPool.Failed), true); job.Status = Job.StatusEnum.FailedAbortOnServer; } else if (jenkinsJob.inQueue) { if (job.Status == Job.StatusEnum.AbortOnServerRequested) { job.Status = Job.StatusEnum.FailedAbortOnServer; JenkinsInstance.StopJob(remoteJobInfo.JenkinsJobName); } else { job.Status = Job.StatusEnum.QueuedOnServer; if (jenkinsJob.queueItem != null) { if (jenkinsJob.queueItem.stuck) { //Trace.TraceError(Newtonsoft.Json.JsonConvert.SerializeObject(jenkinsJob.queueItem)); Trace.TraceError("In queue reason: {0}", jenkinsJob.queueItem.why); } else { Trace.TraceWarning("In queue reason: {0}", jenkinsJob.queueItem.why); } } } } else if (jenkinsJob.nextBuildNumber > 1) { if (jenkinsJob.lastBuild != null && jenkinsJob.lastBuild.number < jenkinsJob.nextBuildNumber) { if (job.Status == Job.StatusEnum.AbortOnServerRequested) { // n.b. job may not exist on server JenkinsInstance.StopJob(remoteJobInfo.JenkinsJobName); } else { job.Status = Job.StatusEnum.RunningOnServer; } } } else if (job.Status == Job.StatusEnum.AbortOnServerRequested) { job.Status = Job.StatusEnum.FailedAbortOnServer; JenkinsInstance.StopJob(remoteJobInfo.JenkinsJobName); } // TODO: how to detect if it got cancelled in the queue, before it runs? }
private void Run(JobImpl job) { string failedLog = Path.Combine(job.WorkingDirectory, LocalPool.Failed); File.Delete(failedLog); if (string.IsNullOrEmpty(job.RunCommand) // || (job.RunCommand.Split(' ').Count() > 1 && //File.Exists(Path.Combine(job.WorkingDirectory, job.RunCommand.Split(' ').FirstOrDefault())) == false) ) { Trace.TraceError("Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? "")); using (StreamWriter writer = new StreamWriter(failedLog)) { writer.WriteLine("ERROR: Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? "")); } job.Status = Job.StatusEnum.FailedExecution; return; } if (Server.IsRemote) { Trace.TraceInformation("Prepare Job for remote execution: {0} {1}", job.Id, job.Title); try { // zip working directory string zipFile = Path.Combine(job.WorkingDirectory, "source_data.zip"); string zipPy = Path.Combine(job.WorkingDirectory, "zip.py"); // if zip.py does not exist create if (File.Exists(zipPy) == false) { using (StreamWriter writer = new StreamWriter(zipPy)) { writer.WriteLine(@"#!/usr/bin/python import zipfile import sys import os import os.path path_join = os.path.join if sys.platform == 'win32': def path_join(*args): return '\\\\?\\' + os.path.join(os.getcwd(), os.path.join(*args)) output_filename = 'source_data.zip' if os.path.exists(output_filename): os.remove(output_filename) # LS_Dyna workers have RHEL6. RHEL6 has Python2.6, which doesnt have zipfile.ZipFile.__exit__ http://bugs.python.org/issue5511 . So we dont use 'with' z = zipfile.ZipFile(output_filename, 'w', allowZip64=True) try: parent_dir_name = os.path.basename(os.getcwd()) os.chdir('..') for dirpath, dirs, files in os.walk(parent_dir_name): # Fix META-1850: make sure all dirs are copied. for d in dirs: dn = path_join(dirpath, d) z.write(dn, arcname=os.path.join(dirpath, d), compress_type=zipfile.ZIP_DEFLATED) for f in files: if output_filename == f: continue fn = path_join(dirpath, f) #print fn z.write(fn, arcname=os.path.join(dirpath, f), compress_type=zipfile.ZIP_DEFLATED) finally: z.close() "); } } if (File.Exists(zipFile) == false) { // call zip.py to zip the package if it does not exist job.Status = Job.StatusEnum.ZippingPackage; ProcessStartInfo psi = new ProcessStartInfo(META.VersionInfo.PythonVEnvExe) { Arguments = "-E \"" + zipPy + "\"", WorkingDirectory = job.WorkingDirectory, WindowStyle = ProcessWindowStyle.Hidden, UseShellExecute = false, CreateNoWindow = true, RedirectStandardError = true, }; Process proc = new Process() { StartInfo = psi, }; proc.Start(); string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0) { job.Status = Job.StatusEnum.Failed; Trace.TraceError("zip.py failed with exit code {0}. stderr was {1}", proc.ExitCode, stderr); return; } } if (File.Exists(zipFile) == false) { job.Status = Job.StatusEnum.Failed; Trace.TraceError("zip.py did not produce {0}", zipFile); return; } if (this.IsServiceOnline() == false) { // put all job into a waiting status if the server is disconnected job.Status = Job.StatusEnum.WaitingForStart; // hack: maybe the jenkins job server is down. don't hammer the server Thread.Sleep(Jenkins.Jenkins.VF_JOB_POLL_FREQUENCY); // put the job back to the list and try to start it again later JobsToBeStarted.Add(() => { job.Start(); }); return; } // create a job on the remote machine // KMS: add random hex number because SoT steps have the same WorkingDirectory // ZsL: the 8 digit random hex numbers were not enough. If users had many jobs 500+ got exception the // jobname is not unique: problems in database due jobname is the primary key and on jenkins too. var randomid = Path.GetFileName(job.WorkingDirectory); //Guid.NewGuid().ToString("N"); var anotherRandomId = Path.GetFileNameWithoutExtension(Path.GetRandomFileName()); string jobname = string.Format("{0}_{1}{2}", Server.UserName, randomid, anotherRandomId); string description = string.Format("{0}_{1}_{2:00000}_{3}", Server.UserName, job.Title, job.Id, randomid); jobname = jobname.Replace(' ', '_'); string cmd = job.RunCommand; string labels = job.Labels; job.Status = Job.StatusEnum.UploadPackage; string getUrl; if (String.IsNullOrWhiteSpace(job.GetURLOverride) == false) { getUrl = "r'" + job.GetURLOverride + "'"; } else { getUrl = "os.environ['sourceGetUrl']"; // parameter provided to Jenkins by VF string zipFileUrl = JenkinsInstance.UploadFileToVF(zipFile, jobname, delegate(int percent) { //TODO: UI Callback for Jenkins upload progress }); if (zipFileUrl == null) { job.Status = Job.StatusEnum.FailedToUploadServer; return; } } // 1. {0} description // 2. {1} working directory name // 3. {2} runcommand that needs to be executed in the working directory // 4. {3} required slave node (Label expression) // 5. {4} source.zip GET URL // 6. {5} zip.py server side hook // 7. {6} name of the zipped results file string resultZipName = "results.zip"; string resultZipPy = string.Format(job.ResultsZip, resultZipName, Path.GetFileName(job.WorkingDirectory)); var config_xml = String.Format( Properties.Resources.job_config, SecurityElement.Escape(description), Path.GetFileName(job.WorkingDirectory), SecurityElement.Escape(cmd), SecurityElement.Escape(labels), SecurityElement.Escape(getUrl), SecurityElement.Escape(resultZipPy).Replace("\\", "\\\\"), SecurityElement.Escape(resultZipName)); var jobinfonew = JenkinsInstance.CreateJob(jobname, config_xml); if (jobinfonew == null) { // job creation failed job.Status = Job.StatusEnum.FailedToUploadServer; return; } job.Status = Job.StatusEnum.PostedToServer; string returned_config_xml = JenkinsInstance.GetJobConfig(jobname); if (returned_config_xml != null) { if (returned_config_xml.IndexOf("org.jvnet.hudson.plugins.Jython") == -1) { string logFilename = System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed); File.WriteAllText(logFilename, "Jenkins does not have the Jython plugin installed"); Trace.TraceError("Jenkins does not have the Jython plugin installed"); job.Status = Job.StatusEnum.FailedToUploadServer; return; } } else { } // FIXME throw? // send zip and start job if (JenkinsInstance.BuildJob(jobname, jobname, job.BuildQuery) == null) { job.Status = Job.StatusEnum.FailedToUploadServer; return; } // on rerun, job is already in the map job.remoteInfo = JobMap[job] = new RemoteJobInfo() { JenkinsJobName = jobname }; job.Status = Job.StatusEnum.StartedOnServer; } catch (Exception ex) { JenkinsInstance.DebugLog.WriteLine(ex.ToString().Replace("\n", " ")); job.Status = Job.StatusEnum.FailedToUploadServer; //TODO: Jenkins upload error callback //MessageBox.Show(ex.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error); Trace.TraceError(ex.ToString()); } } else { // if local pool.EnqueueJob(job); } }
/// <summary> /// /// </summary> /// <param name="jenkinsJob"></param> /// <param name="job"></param> /// <returns>Indicates if job failed or not.</returns> private Job.StatusEnum SaveWorkspace( Job job, RemoteJobInfo remoteJobInfo) { job.Status = Job.StatusEnum.DownloadResults; if (remoteJobInfo.ResultsGetURL == null) { if (JenkinsInstance.DownloadFileFromVF(remoteJobInfo.JenkinsJobName, ref remoteJobInfo.ResultsGetURL) == false) { Trace.TraceError("DownloadFileFromVF failed {0}", job.Title); job.Status = Job.StatusEnum.FailedToDownload; return(Job.StatusEnum.FailedToDownload); } } bool result = JenkinsInstance.DownloadFileFromSwift(Path.Combine(job.WorkingDirectory, "workspace.zip"), remoteJobInfo.ResultsGetURL, delegate(int percent) { //TODO: Download progress notification for client /*lvJobQueue.BeginInvoke((MethodInvoker)delegate () * { * ListViewItem item = lvJobQueue.Items.Cast<ListViewItem>(). * FirstOrDefault(x => x.SubItems[(int)Headers.Id].Text == job.Id.ToString()); * if (item != null) * { * item.SubItems[(int)Headers.Progress].Text = percent + "%"; * } * });*/ }); if (result == false) { job.Status = Job.StatusEnum.FailedToDownload; return(Job.StatusEnum.FailedToDownload); } // unzip package string unzipPy = Path.Combine(job.WorkingDirectory, "unzip.py"); if (File.Exists(unzipPy) == false) { using (StreamWriter writer = new StreamWriter(unzipPy)) { writer.WriteLine(@"#!/usr/bin/py import os import os.path import sys import shutil import zipfile path_join = os.path.join if sys.platform == 'win32': def path_join(*args): return '\\\\?\\' + os.path.join(os.getcwd(), os.path.join(*args)) try: parent_dir_name = os.path.basename(os.getcwd()) zip = zipfile.ZipFile('workspace.zip') # OLD version zip.namelist()[0] is unpredictable #root_src_dir = zip.namelist()[0] + parent_dir_name # ASSUMPTION workspace.zip has always the parent_dir_name as a zipped directory root_src_dir = parent_dir_name print root_src_dir for entry in zip.infolist(): if entry.filename.startswith(root_src_dir): dest = entry.filename[len(root_src_dir)+1:] if dest == '': continue if dest.endswith('/'): if not os.path.isdir(dest): os.mkdir(dest) else: if os.path.basename(dest) != 'workspace.zip': entry.filename = dest zip.extract(entry, path=path_join(os.getcwd())) except Exception as msg: import traceback sys.stderr.write(traceback.format_exc()) with open('_FAILED.txt', 'wb') as f_out: f_out.write(str(msg)) f_out.write('\nMost likely due to a too long file-path for Windows (max 260).') if os.name == 'nt': os._exit(3) elif os.name == 'posix': os._exit(os.EX_OSFILE) "); } } // call unzip.py to unzip the package ProcessStartInfo psi = new ProcessStartInfo(META.VersionInfo.PythonVEnvExe) { Arguments = "-E \"" + unzipPy + "\"", WorkingDirectory = job.WorkingDirectory, WindowStyle = ProcessWindowStyle.Hidden, UseShellExecute = false, CreateNoWindow = true, RedirectStandardError = true, }; Process proc = new Process() { StartInfo = psi, }; proc.Start(); string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); if (proc.ExitCode != 0) { string logFilename = System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed); File.WriteAllText(logFilename, "unzip.py failed:\n" + stderr); Trace.TraceError("unzip.py failed {0}", job.Title); return(Job.StatusEnum.Failed); } else { if (File.Exists(System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed))) { return(Job.StatusEnum.FailedExecution); } return(Job.StatusEnum.Succeeded); } }