private void Run(JobImpl job)
{
string failedLog = Path.Combine(job.WorkingDirectory, LocalPool.Failed);
File.Delete(failedLog);
if (string.IsNullOrEmpty(job.RunCommand)
// || (job.RunCommand.Split(' ').Count() > 1 &&
//File.Exists(Path.Combine(job.WorkingDirectory, job.RunCommand.Split(' ').FirstOrDefault())) == false)
)
{
Trace.TraceError("Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? ""));
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine("ERROR: Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? ""));
}
job.Status = Job.StatusEnum.FailedExecution;
return;
}
if (Server.IsRemote)
{
Trace.TraceInformation("Prepare Job for remote execution: {0} {1}", job.Id, job.Title);
try
{
// zip working directory
string zipFile = Path.Combine(job.WorkingDirectory, "source_data.zip");
string zipPy = Path.Combine(job.WorkingDirectory, "zip.py");
// if zip.py does not exist create
if (File.Exists(zipPy) == false)
{
using (StreamWriter writer = new StreamWriter(zipPy))
{
writer.WriteLine(@"#!/usr/bin/python
import zipfile
import sys
import os
import os.path
path_join = os.path.join
if sys.platform == 'win32':
def path_join(*args):
return '\\\\?\\' + os.path.join(os.getcwd(), os.path.join(*args))
output_filename = 'source_data.zip'
if os.path.exists(output_filename):
os.remove(output_filename)
# LS_Dyna workers have RHEL6. RHEL6 has Python2.6, which doesnt have zipfile.ZipFile.__exit__ http://bugs.python.org/issue5511 . So we dont use 'with'
z = zipfile.ZipFile(output_filename, 'w', allowZip64=True)
try:
parent_dir_name = os.path.basename(os.getcwd())
os.chdir('..')
for dirpath, dirs, files in os.walk(parent_dir_name):
# Fix META-1850: make sure all dirs are copied.
for d in dirs:
dn = path_join(dirpath, d)
z.write(dn, arcname=os.path.join(dirpath, d), compress_type=zipfile.ZIP_DEFLATED)
for f in files:
if output_filename == f:
continue
fn = path_join(dirpath, f)
#print fn
z.write(fn, arcname=os.path.join(dirpath, f), compress_type=zipfile.ZIP_DEFLATED)
finally:
z.close()
");
}
}
if (File.Exists(zipFile) == false)
{
// call zip.py to zip the package if it does not exist
job.Status = Job.StatusEnum.ZippingPackage;
ProcessStartInfo psi = new ProcessStartInfo(META.VersionInfo.PythonVEnvExe)
{
Arguments = "-E \"" + zipPy + "\"",
WorkingDirectory = job.WorkingDirectory,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardError = true,
};
Process proc = new Process()
{
StartInfo = psi,
};
proc.Start();
string stderr = proc.StandardError.ReadToEnd();
proc.WaitForExit();
if (proc.ExitCode != 0)
{
job.Status = Job.StatusEnum.Failed;
Trace.TraceError("zip.py failed with exit code {0}. stderr was {1}", proc.ExitCode, stderr);
return;
}
}
if (File.Exists(zipFile) == false)
{
job.Status = Job.StatusEnum.Failed;
Trace.TraceError("zip.py did not produce {0}", zipFile);
return;
}
if (this.IsServiceOnline() == false)
{
// put all job into a waiting status if the server is disconnected
job.Status = Job.StatusEnum.WaitingForStart;
// hack: maybe the jenkins job server is down. don't hammer the server
Thread.Sleep(Jenkins.Jenkins.VF_JOB_POLL_FREQUENCY);
// put the job back to the list and try to start it again later
JobsToBeStarted.Add(() => { job.Start(); });
return;
}
// create a job on the remote machine
// KMS: add random hex number because SoT steps have the same WorkingDirectory
// ZsL: the 8 digit random hex numbers were not enough. If users had many jobs 500+ got exception the
// jobname is not unique: problems in database due jobname is the primary key and on jenkins too.
var randomid = Path.GetFileName(job.WorkingDirectory); //Guid.NewGuid().ToString("N");
var anotherRandomId = Path.GetFileNameWithoutExtension(Path.GetRandomFileName());
string jobname = string.Format("{0}_{1}{2}", Server.UserName, randomid, anotherRandomId);
string description = string.Format("{0}_{1}_{2:00000}_{3}", Server.UserName, job.Title,
job.Id, randomid);
jobname = jobname.Replace(' ', '_');
string cmd = job.RunCommand;
string labels = job.Labels;
job.Status = Job.StatusEnum.UploadPackage;
string getUrl;
if (String.IsNullOrWhiteSpace(job.GetURLOverride) == false)
{
getUrl = "r'" + job.GetURLOverride + "'";
}
else
{
getUrl = "os.environ['sourceGetUrl']"; // parameter provided to Jenkins by VF
string zipFileUrl = JenkinsInstance.UploadFileToVF(zipFile, jobname, delegate(int percent)
{
//TODO: UI Callback for Jenkins upload progress
});
if (zipFileUrl == null)
{
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
}
// 1. {0} description
// 2. {1} working directory name
// 3. {2} runcommand that needs to be executed in the working directory
// 4. {3} required slave node (Label expression)
// 5. {4} source.zip GET URL
// 6. {5} zip.py server side hook
// 7. {6} name of the zipped results file
string resultZipName = "results.zip";
string resultZipPy = string.Format(job.ResultsZip, resultZipName, Path.GetFileName(job.WorkingDirectory));
var config_xml = String.Format(
Properties.Resources.job_config,
SecurityElement.Escape(description),
Path.GetFileName(job.WorkingDirectory),
SecurityElement.Escape(cmd),
SecurityElement.Escape(labels),
SecurityElement.Escape(getUrl),
SecurityElement.Escape(resultZipPy).Replace("\\", "\\\\"),
SecurityElement.Escape(resultZipName));
var jobinfonew = JenkinsInstance.CreateJob(jobname, config_xml);
if (jobinfonew == null)
{
// job creation failed
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
job.Status = Job.StatusEnum.PostedToServer;
string returned_config_xml = JenkinsInstance.GetJobConfig(jobname);
if (returned_config_xml != null)
{
if (returned_config_xml.IndexOf("org.jvnet.hudson.plugins.Jython") == -1)
{
string logFilename = System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed);
File.WriteAllText(logFilename, "Jenkins does not have the Jython plugin installed");
Trace.TraceError("Jenkins does not have the Jython plugin installed");
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
}
else
{
} // FIXME throw?
// send zip and start job
if (JenkinsInstance.BuildJob(jobname, jobname, job.BuildQuery) == null)
{
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
// on rerun, job is already in the map
job.remoteInfo = JobMap[job] = new RemoteJobInfo()
{
JenkinsJobName = jobname
};
job.Status = Job.StatusEnum.StartedOnServer;
}
catch (Exception ex)
{
JenkinsInstance.DebugLog.WriteLine(ex.ToString().Replace("\n", " "));
job.Status = Job.StatusEnum.FailedToUploadServer;
//TODO: Jenkins upload error callback
//MessageBox.Show(ex.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
Trace.TraceError(ex.ToString());
}
}
else
{
// if local
pool.EnqueueJob(job);
}
}
private void RunTestBenchAndStart(CyPhySoT.TestBench testbench)
{
//this.OpenProject();
// Updating with the active objects
testbench.Project = this.Project;
testbench.OriginalProjectFileName = sotConfig.OriginalProjectFileName;
Exception interpreterError = null;
try
{
// run interpreters
testbench.Run();
}
catch (Exception e)
{
interpreterError = e;
}
//this.SaveAndCloseProject();
// TODO: catch exceptions from interpreter
string title = string.Format("{0}__{1}", currentObjectName, testbench.Name);
Trace.TraceInformation("Job needs to be posted {0}", title);
// Set up job properties
JobImpl job = this.TestBenchJobMap[testbench];
job.RunCommand = testbench.RunCommand;
job.WorkingDirectory = testbench.OutputDirectory;
job.Title = sotName;
job.TestBenchName = testbench.Name;
// artifacts/labels
job.Labels = testbench.Labels;
if (string.IsNullOrWhiteSpace(job.Labels))
{
job.Labels = Job.DefaultLabels;
}
job.BuildQuery = testbench.BuildQuery;
if (string.IsNullOrWhiteSpace(job.BuildQuery))
{
job.BuildQuery = Job.DefaultBuildQuery;
}
if (string.IsNullOrWhiteSpace(testbench.ResultsZip) == false)
{
job.ResultsZip = testbench.ResultsZip;
}
if (sotConfig.MultiJobRun)
{
CyPhySoT.TestBench tb = testbench.UpstreamTestBenches.FirstOrDefault();
JobImpl upstreamJob;
if (tb != null && TestBenchJobMap.TryGetValue(tb, out upstreamJob) && upstreamJob.remoteInfo != null)
{
job.GetURLOverride = upstreamJob.remoteInfo.ResultsGetURL;
}
}
job.JobStatusChanged += JobStatusChanged;
job.Status = Job.StatusEnum.WaitingForStart;
this.Server.AddJob(job);
if (interpreterError != null)
{
string failed_txt = Path.Combine(testbench.OutputDirectory, "_FAILED.txt");
if (File.Exists(failed_txt) == false)
{
File.WriteAllText(failed_txt, String.Format("Interpreter {0} failed: {1}", testbench.ProgId, interpreterError.ToString()));
}
job.Status = Job.StatusEnum.Failed;
}
else
{
if (this.JobAction != null)
{
this.JobAction(job);
}
job.Start();
}
}