private void JobStatusChanged(Job job_, Job.StatusEnum status)
{
try
{
JobImpl job = (JobImpl)job_;
Trace.TraceInformation("Form JobStatusChanged was called for: {0}, which changed status to {1}", job.Title, status.ToString());
if (Server.IsRemote)
{
if (status == Job.StatusEnum.StartedOnServer)
{
RemoteJobInfo remoteJobInfo;
if (JobMap.TryGetValue(job, out remoteJobInfo) == false)
{
Trace.TraceError("Could not find jenkins job for {0}. Persistence will not work", job.Title);
}
else
{
SavedJob saved = SavedJob.CreateSavedJob(
job.Title, job.WorkingDirectory, job.RunCommand, job.Status.ToString(), remoteJobInfo.JenkinsJobName);
saved.VFUrl = Server.JenkinsUrl;
Entities.SavedJobs.AddObject(saved);
Entities.SaveChanges();
Entities.SavedJobs.Detach(saved);
}
}
else if (Job.IsFailedStatus(status) || status == Job.StatusEnum.Succeeded)
{
RemoteJobInfo remoteJobInfo;
if (JobMap.TryGetValue(job, out remoteJobInfo) == false)
{
Trace.TraceError("Could not find jenkins job for {0}. It may remain in the database", job.Title);
}
else
{
foreach (var entity in Entities.SavedJobs.Where(x => x.JobName == remoteJobInfo.JenkinsJobName))
{
Entities.SavedJobs.DeleteObject(entity);
}
Entities.SaveChanges();
}
}
}
if (status == Job.StatusEnum.Ready)
{
JobsToBeStarted.Add(delegate() { StartJob(job); });
}
}
catch (Exception ex)
{
//TODO: Propagate error back to UI?
Trace.TraceError(ex.ToString());
}
}
public void LoadSavedJobs()
{
lock (this)
{
this.Configured.Set();
if (Server.IsRemote)
{
// retreiving saved jobs and adding to the job queue and jobmap
foreach (SavedJob saved in Entities.SavedJobs)
{
if (string.IsNullOrEmpty(saved.VFUrl) == false && Server.JenkinsUrl != saved.VFUrl)
{
continue;
}
Job job = new JobImpl(Server)
{
Title = saved.Title,
Status = Job.StatusEnum.StartedOnServer, // TODO: wrong, but not a big deal, will be re-read from the server
RunCommand = saved.RunCommand,
WorkingDirectory = saved.WorkingDirectory,
};
try
{
Directory.CreateDirectory(saved.WorkingDirectory);
}
catch (Exception e)
{
Trace.TraceError(e.ToString());
}
var jenkinsJobInfo = JenkinsInstance.GetJobInfo(saved.JobName);
if (jenkinsJobInfo == null)
{
Trace.TraceInformation("Saved job {0} ({1}) no longer exists on server. Marking failed.", job.Title, jenkinsJobInfo);
job.Status = Job.StatusEnum.FailedAbortOnServer;
Entities.SavedJobs.DeleteObject(saved);
}
else
{
Trace.TraceInformation("Adding saved job {0} ({1})", job.Title, jenkinsJobInfo);
((JobImpl)job).remoteInfo = JobMap[job] = new RemoteJobInfo()
{
JenkinsJobName = jenkinsJobInfo.name
};
Server.AddJob(job);
Entities.SavedJobs.Detach(saved);
}
}
Entities.SaveChanges();
// start timer for remote execution monitoring
this.RestartMonitorTimer();
}
}
}
private void JobAddedHandler(JobImpl job, Job.StatusEnum status)
{
lock (this)
{
job.JobStatusChanged += JobStatusChanged;
if (job.Status == Job.StatusEnum.Ready)
{
JobsToBeStarted.Add(() => { StartJob(job); });
}
}
if (JobAdded != null)
{
JobAdded(this, new JobAddedEventArgs
{
Job = job,
Status = status
});
}
}
private void Run(JobImpl job)
{
string failedLog = Path.Combine(job.WorkingDirectory, LocalPool.Failed);
File.Delete(failedLog);
if (string.IsNullOrEmpty(job.RunCommand)
// || (job.RunCommand.Split(' ').Count() > 1 &&
//File.Exists(Path.Combine(job.WorkingDirectory, job.RunCommand.Split(' ').FirstOrDefault())) == false)
)
{
Trace.TraceError("Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? ""));
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine("ERROR: Job will not be executed because the runCommand is empty or does not exist: {0}", Path.Combine(job.WorkingDirectory, job.RunCommand ?? ""));
}
job.Status = Job.StatusEnum.FailedExecution;
return;
}
if (Server.IsRemote)
{
Trace.TraceInformation("Prepare Job for remote execution: {0} {1}", job.Id, job.Title);
try
{
// zip working directory
string zipFile = Path.Combine(job.WorkingDirectory, "source_data.zip");
string zipPy = Path.Combine(job.WorkingDirectory, "zip.py");
// if zip.py does not exist create
if (File.Exists(zipPy) == false)
{
using (StreamWriter writer = new StreamWriter(zipPy))
{
writer.WriteLine(@"#!/usr/bin/python
import zipfile
import sys
import os
import os.path
path_join = os.path.join
if sys.platform == 'win32':
def path_join(*args):
return '\\\\?\\' + os.path.join(os.getcwd(), os.path.join(*args))
output_filename = 'source_data.zip'
if os.path.exists(output_filename):
os.remove(output_filename)
# LS_Dyna workers have RHEL6. RHEL6 has Python2.6, which doesnt have zipfile.ZipFile.__exit__ http://bugs.python.org/issue5511 . So we dont use 'with'
z = zipfile.ZipFile(output_filename, 'w', allowZip64=True)
try:
parent_dir_name = os.path.basename(os.getcwd())
os.chdir('..')
for dirpath, dirs, files in os.walk(parent_dir_name):
# Fix META-1850: make sure all dirs are copied.
for d in dirs:
dn = path_join(dirpath, d)
z.write(dn, arcname=os.path.join(dirpath, d), compress_type=zipfile.ZIP_DEFLATED)
for f in files:
if output_filename == f:
continue
fn = path_join(dirpath, f)
#print fn
z.write(fn, arcname=os.path.join(dirpath, f), compress_type=zipfile.ZIP_DEFLATED)
finally:
z.close()
");
}
}
if (File.Exists(zipFile) == false)
{
// call zip.py to zip the package if it does not exist
job.Status = Job.StatusEnum.ZippingPackage;
ProcessStartInfo psi = new ProcessStartInfo(META.VersionInfo.PythonVEnvExe)
{
Arguments = "-E \"" + zipPy + "\"",
WorkingDirectory = job.WorkingDirectory,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardError = true,
};
Process proc = new Process()
{
StartInfo = psi,
};
proc.Start();
string stderr = proc.StandardError.ReadToEnd();
proc.WaitForExit();
if (proc.ExitCode != 0)
{
job.Status = Job.StatusEnum.Failed;
Trace.TraceError("zip.py failed with exit code {0}. stderr was {1}", proc.ExitCode, stderr);
return;
}
}
if (File.Exists(zipFile) == false)
{
job.Status = Job.StatusEnum.Failed;
Trace.TraceError("zip.py did not produce {0}", zipFile);
return;
}
if (this.IsServiceOnline() == false)
{
// put all job into a waiting status if the server is disconnected
job.Status = Job.StatusEnum.WaitingForStart;
// hack: maybe the jenkins job server is down. don't hammer the server
Thread.Sleep(Jenkins.Jenkins.VF_JOB_POLL_FREQUENCY);
// put the job back to the list and try to start it again later
JobsToBeStarted.Add(() => { job.Start(); });
return;
}
// create a job on the remote machine
// KMS: add random hex number because SoT steps have the same WorkingDirectory
// ZsL: the 8 digit random hex numbers were not enough. If users had many jobs 500+ got exception the
// jobname is not unique: problems in database due jobname is the primary key and on jenkins too.
var randomid = Path.GetFileName(job.WorkingDirectory); //Guid.NewGuid().ToString("N");
var anotherRandomId = Path.GetFileNameWithoutExtension(Path.GetRandomFileName());
string jobname = string.Format("{0}_{1}{2}", Server.UserName, randomid, anotherRandomId);
string description = string.Format("{0}_{1}_{2:00000}_{3}", Server.UserName, job.Title,
job.Id, randomid);
jobname = jobname.Replace(' ', '_');
string cmd = job.RunCommand;
string labels = job.Labels;
job.Status = Job.StatusEnum.UploadPackage;
string getUrl;
if (String.IsNullOrWhiteSpace(job.GetURLOverride) == false)
{
getUrl = "r'" + job.GetURLOverride + "'";
}
else
{
getUrl = "os.environ['sourceGetUrl']"; // parameter provided to Jenkins by VF
string zipFileUrl = JenkinsInstance.UploadFileToVF(zipFile, jobname, delegate(int percent)
{
//TODO: UI Callback for Jenkins upload progress
});
if (zipFileUrl == null)
{
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
}
// 1. {0} description
// 2. {1} working directory name
// 3. {2} runcommand that needs to be executed in the working directory
// 4. {3} required slave node (Label expression)
// 5. {4} source.zip GET URL
// 6. {5} zip.py server side hook
// 7. {6} name of the zipped results file
string resultZipName = "results.zip";
string resultZipPy = string.Format(job.ResultsZip, resultZipName, Path.GetFileName(job.WorkingDirectory));
var config_xml = String.Format(
Properties.Resources.job_config,
SecurityElement.Escape(description),
Path.GetFileName(job.WorkingDirectory),
SecurityElement.Escape(cmd),
SecurityElement.Escape(labels),
SecurityElement.Escape(getUrl),
SecurityElement.Escape(resultZipPy).Replace("\\", "\\\\"),
SecurityElement.Escape(resultZipName));
var jobinfonew = JenkinsInstance.CreateJob(jobname, config_xml);
if (jobinfonew == null)
{
// job creation failed
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
job.Status = Job.StatusEnum.PostedToServer;
string returned_config_xml = JenkinsInstance.GetJobConfig(jobname);
if (returned_config_xml != null)
{
if (returned_config_xml.IndexOf("org.jvnet.hudson.plugins.Jython") == -1)
{
string logFilename = System.IO.Path.Combine(job.WorkingDirectory, LocalPool.Failed);
File.WriteAllText(logFilename, "Jenkins does not have the Jython plugin installed");
Trace.TraceError("Jenkins does not have the Jython plugin installed");
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
}
else
{
} // FIXME throw?
// send zip and start job
if (JenkinsInstance.BuildJob(jobname, jobname, job.BuildQuery) == null)
{
job.Status = Job.StatusEnum.FailedToUploadServer;
return;
}
// on rerun, job is already in the map
job.remoteInfo = JobMap[job] = new RemoteJobInfo()
{
JenkinsJobName = jobname
};
job.Status = Job.StatusEnum.StartedOnServer;
}
catch (Exception ex)
{
JenkinsInstance.DebugLog.WriteLine(ex.ToString().Replace("\n", " "));
job.Status = Job.StatusEnum.FailedToUploadServer;
//TODO: Jenkins upload error callback
//MessageBox.Show(ex.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
Trace.TraceError(ex.ToString());
}
}
else
{
// if local
pool.EnqueueJob(job);
}
}
private void StartJob(JobImpl job)
{
//TODO: Update service status here
Run(job);
}
public override Job CreateJob()
{
Job job = new JobImpl(this);
return(job);
}
private void RunTestBenchAndStart(CyPhySoT.TestBench testbench)
{
//this.OpenProject();
// Updating with the active objects
testbench.Project = this.Project;
testbench.OriginalProjectFileName = sotConfig.OriginalProjectFileName;
Exception interpreterError = null;
try
{
// run interpreters
testbench.Run();
}
catch (Exception e)
{
interpreterError = e;
}
//this.SaveAndCloseProject();
// TODO: catch exceptions from interpreter
string title = string.Format("{0}__{1}", currentObjectName, testbench.Name);
Trace.TraceInformation("Job needs to be posted {0}", title);
// Set up job properties
JobImpl job = this.TestBenchJobMap[testbench];
job.RunCommand = testbench.RunCommand;
job.WorkingDirectory = testbench.OutputDirectory;
job.Title = sotName;
job.TestBenchName = testbench.Name;
// artifacts/labels
job.Labels = testbench.Labels;
if (string.IsNullOrWhiteSpace(job.Labels))
{
job.Labels = Job.DefaultLabels;
}
job.BuildQuery = testbench.BuildQuery;
if (string.IsNullOrWhiteSpace(job.BuildQuery))
{
job.BuildQuery = Job.DefaultBuildQuery;
}
if (string.IsNullOrWhiteSpace(testbench.ResultsZip) == false)
{
job.ResultsZip = testbench.ResultsZip;
}
if (sotConfig.MultiJobRun)
{
CyPhySoT.TestBench tb = testbench.UpstreamTestBenches.FirstOrDefault();
JobImpl upstreamJob;
if (tb != null && TestBenchJobMap.TryGetValue(tb, out upstreamJob) && upstreamJob.remoteInfo != null)
{
job.GetURLOverride = upstreamJob.remoteInfo.ResultsGetURL;
}
}
job.JobStatusChanged += JobStatusChanged;
job.Status = Job.StatusEnum.WaitingForStart;
this.Server.AddJob(job);
if (interpreterError != null)
{
string failed_txt = Path.Combine(testbench.OutputDirectory, "_FAILED.txt");
if (File.Exists(failed_txt) == false)
{
File.WriteAllText(failed_txt, String.Format("Interpreter {0} failed: {1}", testbench.ProgId, interpreterError.ToString()));
}
job.Status = Job.StatusEnum.Failed;
}
else
{
if (this.JobAction != null)
{
this.JobAction(job);
}
job.Start();
}
}
public void Run()
{
if (string.IsNullOrEmpty(this.WorkingDirectory) ||
Directory.Exists(this.WorkingDirectory) == false)
{
// nothing needs to be done.
return;
}
// TODO: exception handling...
// get manifest file
string manifestFile = Path.Combine(this.WorkingDirectory, "manifest.sot.json");
if (File.Exists(manifestFile) == false)
{
Trace.TraceError("{0} file does not exist", manifestFile);
return;
}
// this could fail... exception handling!
sotConfig = Newtonsoft.Json.JsonConvert.DeserializeObject <CyPhySoT.SotConfig>(File.ReadAllText(manifestFile));
// get mga filename
string mgaFileName = Path.Combine(this.WorkingDirectory, sotConfig.ProjectFileName);
if (File.Exists(mgaFileName) == false)
{
Trace.TraceError("{0} file does not exist", mgaFileName);
return;
}
this.ProjectConnStr = "MGA=" + mgaFileName;
Semaphore sem = null;
if (criticalSection.TryGetValue(this.ProjectConnStr, out sem) == false)
{
criticalSection[this.ProjectConnStr] = new Semaphore(1, 1);
}
// Load GME model
this.OpenProject();
if (sotConfig.MultiJobRun)
{
Project.BeginTransactionInNewTerr(transactiontype_enum.TRANSACTION_READ_ONLY);
try
{
this.TestBenches = GetMultiJobRunTestBenches(CurrentObj, Path.GetDirectoryName(sotConfig.ProjectFileName));
}
finally
{
Project.AbortTransaction();
}
}
else
{
CyPhySoT.CyPhySoTInterpreter sotInterpreter = new CyPhySoT.CyPhySoTInterpreter();
this.TestBenches = sotInterpreter.GetTestbenchesFromModel(this.Project, (MgaModel)this.CurrentObj, Path.GetDirectoryName(this.WorkingDirectory));
Project.BeginTransactionInNewTerr(transactiontype_enum.TRANSACTION_NON_NESTED);
try
{
// replace tb in TestBenches that corresponds to the MultiJobRun with the tbs from GetMultiJobRunTestBenches
foreach (var tb in TestBenches.ToList <CyPhySoT.TestBench>())
{
var modelProcessor = AnalysisModelProcessor.GetAnalysisModelProcessor((IMgaModel)tb.CurrentObj);
if (modelProcessor is MultiJobRunProcessor)
{
List <CyPhySoT.TestBench> multiTbs = GetMultiJobRunTestBenches(tb.CurrentObj, tb.OutputDirectory);
foreach (var upstreamTb in tb.UpstreamTestBenches)
{
upstreamTb.DownstreamTestBenches.Remove(tb);
upstreamTb.DownstreamTestBenches.Add(multiTbs[0]);
multiTbs[0].UpstreamTestBenches.Add(upstreamTb);
}
foreach (var downstreamTb in tb.DownstreamTestBenches)
{
downstreamTb.UpstreamTestBenches.Remove(tb);
downstreamTb.UpstreamTestBenches.Add(multiTbs[multiTbs.Count - 1]);
multiTbs[multiTbs.Count - 1].DownstreamTestBenches.Add(downstreamTb);
}
this.TestBenches.AddRange(multiTbs);
this.TestBenches.Remove(tb);
}
}
}
finally
{
Project.AbortTransaction();
}
}
// TODO: update value flow!
// Test bench/job map
foreach (var testbench in this.TestBenches)
{
JobImpl job = new JobImpl(this.Server);
this.TestBenchJobMap.Add(testbench, job);
}
try
{
// propagate the values between test benches
PropagateValueFlow();
}
catch (JobFailure ex)
{
// FIXME: add some GUI notification
string failed_txt = Path.Combine(this.WorkingDirectory, "_FAILED.txt");
File.WriteAllText(failed_txt, String.Format("SoT {0} failed: {1}", this.sotName, ex.ToString()));
this.CloseProject();
return;
}
// get all testbenches from sot that can be run without dependency
foreach (var testbench in this.TestBenches.Where(x => x.UpstreamTestBenches.Count == 0))
{
// call interpreters for this test bench and post it to the job manager
RunTestBenchAndStart(testbench);
}
this.CloseProject();
}
