public void LoadSavedJobs()
{
lock (this)
{
this.Configured.Set();
if (Server.IsRemote)
{
// retreiving saved jobs and adding to the job queue and jobmap
foreach (SavedJob saved in Entities.SavedJobs)
{
if (string.IsNullOrEmpty(saved.VFUrl) == false && Server.JenkinsUrl != saved.VFUrl)
{
continue;
}
Job job = new JobImpl(Server)
{
Title = saved.Title,
Status = Job.StatusEnum.StartedOnServer, // TODO: wrong, but not a big deal, will be re-read from the server
RunCommand = saved.RunCommand,
WorkingDirectory = saved.WorkingDirectory,
};
try
{
Directory.CreateDirectory(saved.WorkingDirectory);
}
catch (Exception e)
{
Trace.TraceError(e.ToString());
}
var jenkinsJobInfo = JenkinsInstance.GetJobInfo(saved.JobName);
if (jenkinsJobInfo == null)
{
Trace.TraceInformation("Saved job {0} ({1}) no longer exists on server. Marking failed.", job.Title, jenkinsJobInfo);
job.Status = Job.StatusEnum.FailedAbortOnServer;
Entities.SavedJobs.DeleteObject(saved);
}
else
{
Trace.TraceInformation("Adding saved job {0} ({1})", job.Title, jenkinsJobInfo);
((JobImpl)job).remoteInfo = JobMap[job] = new RemoteJobInfo()
{
JenkinsJobName = jenkinsJobInfo.name
};
Server.AddJob(job);
Entities.SavedJobs.Detach(saved);
}
}
Entities.SaveChanges();
// start timer for remote execution monitoring
this.RestartMonitorTimer();
}
}
}
private void MonitorJenkinsJob(Job job, RemoteJobInfo remoteJobInfo)
{
string jenkinsJobName = remoteJobInfo.JenkinsJobName;
if (job.Status == Job.StatusEnum.RedownloadQueued)
{
// n.b. job may not exist on server
var success = SaveWorkspace(job, remoteJobInfo);
job.Status = success;
return;
}
Jenkins.Job.Job jenkinsJob;
try
{
jenkinsJob = JenkinsInstance.GetJobInfo(jenkinsJobName, rethrow: true);
}
catch (WebException e)
{
if (e is Jenkins.WebExceptionWithStatusCode && ((Jenkins.WebExceptionWithStatusCode)e).StatusCode == 404)
{
Trace.TraceError("Job {0} no longer exists (404 received from server). Marking job as Failed", jenkinsJobName);
job.Status = Job.StatusEnum.Failed;
return;
}
throw;
}
if (jenkinsJob == null)
{
// should not be reachable
Trace.TraceError("Could not retrieve status for job {0}. Marking job as Failed", jenkinsJobName);
job.Status = Job.StatusEnum.Failed;
return;
}
if (jenkinsJob.lastSuccessfulBuild != null)
{
JenkinsInstance.SaveLastConsole(jenkinsJob.name, Path.Combine(job.WorkingDirectory, "consoleText.txt"));
// delete workspace.zip here if it exists, or SaveWorkspace will try to resume
// n.b. "If the file to be deleted does not exist, no exception is thrown."
File.Delete(Path.Combine(job.WorkingDirectory, "workspace.zip"));
var success = SaveWorkspace(job, remoteJobInfo);
if (Server.WipeWorkspaceOnSuccess)
{
// wipe workspace before we delete the job
JenkinsInstance.WipeWorkSpace(jenkinsJob.name);
}
if (false && Server.DeleteJobOnSuccess) // META-2493 META-2492
{
// delete job from server
JenkinsInstance.DeleteJob(jenkinsJob.name);
}
job.Status = success;
}
else if (jenkinsJob.lastFailedBuild != null)
{
JenkinsInstance.SaveLastConsole(jenkinsJob.name, Path.Combine(job.WorkingDirectory, "consoleText.txt"));
File.Copy(Path.Combine(job.WorkingDirectory, "consoleText.txt"), Path.Combine(job.WorkingDirectory, LocalPool.Failed), true);
try
{
// delete workspace.zip here if it exists, or SaveWorkspace will try to resume
// n.b. "If the file to be deleted does not exist, no exception is thrown."
File.Delete(Path.Combine(job.WorkingDirectory, "workspace.zip"));
job.Status = SaveWorkspace(job, remoteJobInfo);
}
catch (Exception e)
{
Trace.TraceError("Error retrieving results for {0} ({1}): {2}", job.Title, jenkinsJob.name, e.ToString());
job.Status = Job.StatusEnum.FailedToDownload;
}
}
else if (jenkinsJob.lastCompletedBuild != null)
{
// build got cancelled
JenkinsInstance.SaveLastConsole(jenkinsJob.name, Path.Combine(job.WorkingDirectory, "consoleText.txt"));
File.Copy(Path.Combine(job.WorkingDirectory, "consoleText.txt"), Path.Combine(job.WorkingDirectory, LocalPool.Failed), true);
job.Status = Job.StatusEnum.FailedAbortOnServer;
}
else if (jenkinsJob.inQueue)
{
if (job.Status == Job.StatusEnum.AbortOnServerRequested)
{
job.Status = Job.StatusEnum.FailedAbortOnServer;
JenkinsInstance.StopJob(remoteJobInfo.JenkinsJobName);
}
else
{
job.Status = Job.StatusEnum.QueuedOnServer;
if (jenkinsJob.queueItem != null)
{
if (jenkinsJob.queueItem.stuck)
{
//Trace.TraceError(Newtonsoft.Json.JsonConvert.SerializeObject(jenkinsJob.queueItem));
Trace.TraceError("In queue reason: {0}", jenkinsJob.queueItem.why);
}
else
{
Trace.TraceWarning("In queue reason: {0}", jenkinsJob.queueItem.why);
}
}
}
}
else if (jenkinsJob.nextBuildNumber > 1)
{
if (jenkinsJob.lastBuild != null &&
jenkinsJob.lastBuild.number < jenkinsJob.nextBuildNumber)
{
if (job.Status == Job.StatusEnum.AbortOnServerRequested)
{
// n.b. job may not exist on server
JenkinsInstance.StopJob(remoteJobInfo.JenkinsJobName);
}
else
{
job.Status = Job.StatusEnum.RunningOnServer;
}
}
}
else if (job.Status == Job.StatusEnum.AbortOnServerRequested)
{
job.Status = Job.StatusEnum.FailedAbortOnServer;
JenkinsInstance.StopJob(remoteJobInfo.JenkinsJobName);
}
// TODO: how to detect if it got cancelled in the queue, before it runs?
}
private void Run(JobImpl job)
{
string failedLog = Path.Combine(job.WorkingDirectory, LocalPool.Failed);
File.Delete(failedLog);
if (string.IsNullOrEmpty(job.RunCommand)
// || (job.RunCommand.Split(' ').Count() > 1 &&
//File.Exists(Path.Combine(job.WorkingDirectory, job.RunCommand.Split(' ').FirstOrDefault())) == false)
)
{
Trace.TraceError("Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? ""));
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine("ERROR: Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? ""));
}
job.Status = Job.StatusEnum.FailedExecution;
return;
}
if (Server.IsRemote)
{
Trace.TraceInformation("Prepare Job for remote execution: {0} {1}", job.Id, job.Title);
try
{
// zip working directory
string zipFile = Path.Combine(job.WorkingDirectory, "source_data.zip");
string zipPy = Path.Combine(job.WorkingDirectory, "zip.py");
// if zip.py does not exist create
if (File.Exists(zipPy) == false)
{
using (StreamWriter writer = new StreamWriter(zipPy))
{
writer.WriteLine(@"#!/usr/bin/python
import zipfile
import sys
import os
import os.path
path_join = os.path.join
if sys.platform == 'win32':
def path_join(*args):
return '\\\\?\\' + os.path.join(os.getcwd(), os.path.join(*args))
output_filename = 'source_data.zip'
if os.path.exists(output_filename):
os.remove(output_filename)
# LS_Dyna workers have RHEL6. RHEL6 has Python2.6, which doesnt have zipfile.ZipFile.__exit__ http://bugs.python.org/issue5511 . So we dont use 'with'
z = zipfile.ZipFile(output_filename, 'w', allowZip64=True)
try:
parent_dir_name = os.path.basename(os.getcwd())
os.chdir('..')
for dirpath, dirs, files in os.walk(parent_dir_name):
# Fix META-1850: make sure all dirs are copied.
for d in dirs:
dn = path_join(dirpath, d)
z.write(dn, arcname=os.path.join(dirpath, d), compress_type=zipfile.ZIP_DEFLATED)
for f in files:
if output_filename == f:
continue
fn = path_join(dirpath, f)
#print fn
z.write(fn, arcname=os.path.join(dirpath, f), compress_type=zipfile.ZIP_DEFLATED)
finally:
z.close()
");
}
}
if (File.Exists(zipFile) == false)
{
// call zip.py to zip the package if it does not exist
job.Status = Job.StatusEnum.ZippingPackage;
ProcessStartInfo psi = new ProcessStartInfo(META.VersionInfo.PythonVEnvExe)
{
Arguments = "-E \"" + zipPy + "\"",
WorkingDirectory = job.WorkingDirectory,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardError = true,
};
Process proc = new Process()
{
StartInfo = psi,
};
proc.Start();
string stderr = proc.StandardError.ReadToEnd();
proc.WaitForExit();
if (proc.ExitCode != 0)
{
job.Status = Job.StatusEnum.Failed;
Trace.TraceError("zip.py failed with exit code {0}. stderr was {1}", proc.ExitCode, stderr);
return;
}
}
if (File.Exists(zipFile) == false)
{
job.Status = Job.StatusEnum.Failed;
Trace.TraceError("zip.py did not produce {0}", zipFile);
return;
}
if (this.IsServiceOnline() == false)
{
// put all job into a waiting status if the server is disconnected
job.Status = Job.StatusEnum.WaitingForStart;
// hack: maybe the jenkins job server is down. don't hammer the server
Thread.Sleep(Jenkins.Jenkins.VF_JOB_POLL_FREQUENCY);
// put the job back to the list and try to start it again later
JobsToBeStarted.Add(() => { job.Start(); });
return;
}
// create a job on the remote machine
// KMS: add random hex number because SoT steps have the same WorkingDirectory
// ZsL: the 8 digit random hex numbers were not enough. If users had many jobs 500+ got exception the
// jobname is not unique: problems in database due jobname is the primary key and on jenkins too.
var randomid = Path.GetFileName(job.WorkingDirectory); //Guid.NewGuid().ToString("N");
var anotherRandomId = Path.GetFileNameWithoutExtension(Path.GetRandomFileName());
string jobname = string.Format("{0}_{1}{2}", Server.UserName, randomid, anotherRandomId);
string description = string.Format("{0}_{1}_{2:00000}_{3}", Server.UserName, job.Title,
job.Id, randomid);
jobname = jobname.Replace(' ', '_');
string cmd = job.RunCommand;
string labels = job.Labels;
job.Status = Job.StatusEnum.UploadPackage;
string getUrl;
if (String.IsNullOrWhiteSpace(job.GetURLOverride) == false)
{
getUrl = "r'" + job.GetURLOverride + "'";
}
else
{
getUrl = "os.environ['sourceGetUrl']"; // parameter provided to Jenkins by VF
string zipFileUrl = JenkinsInstance.UploadFileToVF(zipFile, jobname, delegate(int percent)
{
//TODO: UI Callback for Jenkins upload progress
});
if (zipFileUrl == null)
{
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
}
// 1. {0} description
// 2. {1} working directory name
// 3. {2} runcommand that needs to be executed in the working directory
// 4. {3} required slave node (Label expression)
// 5. {4} source.zip GET URL
// 6. {5} zip.py server side hook
// 7. {6} name of the zipped results file
string resultZipName = "results.zip";
string resultZipPy = string.Format(job.ResultsZip, resultZipName, Path.GetFileName(job.WorkingDirectory));
var config_xml = String.Format(
Properties.Resources.job_config,
SecurityElement.Escape(description),
Path.GetFileName(job.WorkingDirectory),
SecurityElement.Escape(cmd),
SecurityElement.Escape(labels),
SecurityElement.Escape(getUrl),
SecurityElement.Escape(resultZipPy).Replace("\\", "\\\\"),
SecurityElement.Escape(resultZipName));
var jobinfonew = JenkinsInstance.CreateJob(jobname, config_xml);
if (jobinfonew == null)
{
// job creation failed
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
job.Status = Job.StatusEnum.PostedToServer;
string returned_config_xml = JenkinsInstance.GetJobConfig(jobname);
if (returned_config_xml != null)
{
if (returned_config_xml.IndexOf("org.jvnet.hudson.plugins.Jython") == -1)
{
string logFilename = System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed);
File.WriteAllText(logFilename, "Jenkins does not have the Jython plugin installed");
Trace.TraceError("Jenkins does not have the Jython plugin installed");
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
}
else
{
} // FIXME throw?
// send zip and start job
if (JenkinsInstance.BuildJob(jobname, jobname, job.BuildQuery) == null)
{
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
// on rerun, job is already in the map
job.remoteInfo = JobMap[job] = new RemoteJobInfo()
{
JenkinsJobName = jobname
};
job.Status = Job.StatusEnum.StartedOnServer;
}
catch (Exception ex)
{
JenkinsInstance.DebugLog.WriteLine(ex.ToString().Replace("\n", " "));
job.Status = Job.StatusEnum.FailedToUploadServer;
//TODO: Jenkins upload error callback
//MessageBox.Show(ex.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
Trace.TraceError(ex.ToString());
}
}
else
{
// if local
pool.EnqueueJob(job);
}
}
/// <summary>
///
/// </summary>
/// <param name="jenkinsJob"></param>
/// <param name="job"></param>
/// <returns>Indicates if job failed or not.</returns>
private Job.StatusEnum SaveWorkspace(
Job job,
RemoteJobInfo remoteJobInfo)
{
job.Status = Job.StatusEnum.DownloadResults;
if (remoteJobInfo.ResultsGetURL == null)
{
if (JenkinsInstance.DownloadFileFromVF(remoteJobInfo.JenkinsJobName, ref remoteJobInfo.ResultsGetURL) == false)
{
Trace.TraceError("DownloadFileFromVF failed {0}", job.Title);
job.Status = Job.StatusEnum.FailedToDownload;
return(Job.StatusEnum.FailedToDownload);
}
}
bool result = JenkinsInstance.DownloadFileFromSwift(Path.Combine(job.WorkingDirectory, "workspace.zip"), remoteJobInfo.ResultsGetURL, delegate(int percent)
{
//TODO: Download progress notification for client
/*lvJobQueue.BeginInvoke((MethodInvoker)delegate ()
* {
* ListViewItem item = lvJobQueue.Items.Cast<ListViewItem>().
* FirstOrDefault(x => x.SubItems[(int)Headers.Id].Text == job.Id.ToString());
* if (item != null)
* {
* item.SubItems[(int)Headers.Progress].Text = percent + "%";
* }
* });*/
});
if (result == false)
{
job.Status = Job.StatusEnum.FailedToDownload;
return(Job.StatusEnum.FailedToDownload);
}
// unzip package
string unzipPy = Path.Combine(job.WorkingDirectory, "unzip.py");
if (File.Exists(unzipPy) == false)
{
using (StreamWriter writer = new StreamWriter(unzipPy))
{
writer.WriteLine(@"#!/usr/bin/py
import os
import os.path
import sys
import shutil
import zipfile
path_join = os.path.join
if sys.platform == 'win32':
def path_join(*args):
return '\\\\?\\' + os.path.join(os.getcwd(), os.path.join(*args))
try:
parent_dir_name = os.path.basename(os.getcwd())
zip = zipfile.ZipFile('workspace.zip')
# OLD version zip.namelist()[0] is unpredictable
#root_src_dir = zip.namelist()[0] + parent_dir_name
# ASSUMPTION workspace.zip has always the parent_dir_name as a zipped directory
root_src_dir = parent_dir_name
print root_src_dir
for entry in zip.infolist():
if entry.filename.startswith(root_src_dir):
dest = entry.filename[len(root_src_dir)+1:]
if dest == '':
continue
if dest.endswith('/'):
if not os.path.isdir(dest):
os.mkdir(dest)
else:
if os.path.basename(dest) != 'workspace.zip':
entry.filename = dest
zip.extract(entry, path=path_join(os.getcwd()))
except Exception as msg:
import traceback
sys.stderr.write(traceback.format_exc())
with open('_FAILED.txt', 'wb') as f_out:
f_out.write(str(msg))
f_out.write('\nMost likely due to a too long file-path for Windows (max 260).')
if os.name == 'nt':
os._exit(3)
elif os.name == 'posix':
os._exit(os.EX_OSFILE)
");
}
}
// call unzip.py to unzip the package
ProcessStartInfo psi = new ProcessStartInfo(META.VersionInfo.PythonVEnvExe)
{
Arguments = "-E \"" + unzipPy + "\"",
WorkingDirectory = job.WorkingDirectory,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardError = true,
};
Process proc = new Process()
{
StartInfo = psi,
};
proc.Start();
string stderr = proc.StandardError.ReadToEnd();
proc.WaitForExit();
if (proc.ExitCode != 0)
{
string logFilename = System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed);
File.WriteAllText(logFilename, "unzip.py failed:\n" + stderr);
Trace.TraceError("unzip.py failed {0}", job.Title);
return(Job.StatusEnum.Failed);
}
else
{
if (File.Exists(System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed)))
{
return(Job.StatusEnum.FailedExecution);
}
return(Job.StatusEnum.Succeeded);
}
}
