protected internal override void Kill (IEnumerable<int> pids, SynchronizedStreamWriter log)
{
foreach (var pid in pids)
RenderStackTraceWithGdb (pid, log);
KillImpl (pids, log);
}
public override void Terminate (SynchronizedStreamWriter log)
{
bool success = TerminateJobObject (job_handle, 1);
Logger.Log ("JobWindows: terminated job object with status: {0}", success);
CloseHandle (job_handle);
job_handle = IntPtr.Zero;
}
internal static void RenderStackTraceWithGdb (int pid, SynchronizedStreamWriter log)
{
log.WriteLine (string.Format ("\n * Fetching stack trace for process {0} * \n", pid));
var template = Path.GetTempFileName ();
try {
bool using_lldb = false;
string debugger;
if (File.Exists ("/usr/bin/gdb")) {
using_lldb = false;
debugger = "/usr/bin/gdb";
} else if (File.Exists ("/usr/bin/lldb")) {
using_lldb = true;
debugger = "/usr/bin/lldb";
} else {
using_lldb = false; // lets hope "gdb" is somewhere.
debugger = "gdb";
}
var commands = new StringBuilder ();
using (var dbg = new Job ()) {
dbg.StartInfo.UseShellExecute = false;
dbg.StartInfo.FileName = debugger;
if (using_lldb) {
commands.AppendFormat ("process attach --pid {0}\n", pid);
commands.Append ("thread list\n");
commands.Append ("thread backtrace all\n");
commands.Append ("detach\n");
commands.Append ("quit\n");
dbg.StartInfo.Arguments = "--source \"" + template + "\"";
} else {
commands.AppendFormat ("attach {0}\n", pid);
commands.Append ("info target\n");
commands.Append ("info threads\n");
commands.Append ("thread apply all bt\n");
dbg.StartInfo.Arguments = "-batch -x \"" + template + "\"";
}
File.WriteAllText (template, commands.ToString ());
var reader = new ProcessReader (log);
reader.Setup (dbg);
dbg.Start ();
reader.Start ();
try {
if (!dbg.WaitForExit (1000 * 30 /* 30 seconds */))
throw new ApplicationException (string.Format ("The 'gdb' process didn't exit in 30 seconds.", dbg.StartInfo.FileName));
} finally {
reader.Join ();
}
}
} finally {
try {
File.Delete (template);
} catch {
}
}
}
/// <summary>
/// Implementation using kill
/// </summary>
/// <param name="pids"></param>
internal static void KillImpl (IEnumerable<int> pids, SynchronizedStreamWriter log)
{
using (Process kill = new Process ()) {
kill.StartInfo.FileName = "kill";
kill.StartInfo.Arguments = "-9 ";
foreach (int pid in pids) {
kill.StartInfo.Arguments += pid.ToString () + " ";
}
log.WriteLine (string.Format ("\n * Killing the processes {0} * ", kill.StartInfo.Arguments.Substring (3)));
kill.StartInfo.UseShellExecute = false;
kill.Start ();
if (!kill.WaitForExit (1000 * 15 /* 15 seconds */))
throw new ApplicationException (string.Format ("The 'kill' process didn't exit in 15 seconds."));
}
}
internal static void PrintProcessesImplPS (SynchronizedStreamWriter log)
{
using (var ps = new Job ()) {
ps.StartInfo.FileName = "ps";
ps.StartInfo.Arguments = "aux";
var reader = new ProcessReader (log);
reader.Setup (ps);
ps.Start ();
reader.Start ();
try {
if (!ps.WaitForExit (1000 * 30 /* 30 seconds */))
throw new ApplicationException (string.Format ("The 'ps' process didn't exit in 30 seconds."));
} finally {
reader.Join ();
}
}
}
internal static void RenderStackTraceWithGdb (int pid, SynchronizedStreamWriter log)
{
log.WriteLine (string.Format ("\n * Fetching stack trace for process {0} (name '{1}') * \n", pid, GetProcessName (pid)));
using (var gdb = new Job ()) {
gdb.StartInfo.FileName = "gdb";
gdb.StartInfo.Arguments = string.Format ("-ex attach {0} --ex \"info target\" --ex \"info threads\" --ex \"thread apply all bt\" --batch", pid);
var reader = new ProcessReader (log);
reader.Setup (gdb);
gdb.Start ();
reader.Start ();
try {
if (!gdb.WaitForExit (1000 * 30 /* 30 seconds */))
throw new ApplicationException (string.Format ("The 'gdb' process didn't exit in 30 seconds."));
} finally {
reader.Join ();
}
}
}
public override void PrintProcesses (SynchronizedStreamWriter log)
{
PrintProcessesImplPS (log);
}
protected internal override void Kill (int pid, SynchronizedStreamWriter log)
{
this.Kill (new int [] { pid }, log);
}
public abstract void PrintProcesses (SynchronizedStreamWriter log);
public ProcessReader (SynchronizedStreamWriter output)
{
this.log = output;
}
/// <summary>
/// Default KillTree implementation.
/// </summary>
/// <param name="p"></param>
public virtual void KillTree (Process p, SynchronizedStreamWriter log)
{
List<int> processes = new List<int> ();
FindChildren (p.Id, processes);
Kill (processes, log);
}
public override void PrintProcesses (SynchronizedStreamWriter log)
{
log.WriteLine ("IProcessHelper.PrintProcesses is unimplemented on windows");
}
private static void Build(BuildInfo info)
{
try {
ProcessReader process_reader;
string log_file = Path.Combine(info.BUILDER_DATA_LOG_DIR, info.command.command + ".log");
DateTime local_starttime = DateTime.Now;
DBState result;
DBCommand command = info.command;
int exitcode = 0;
ReportBuildStateResponse response;
Logger.Log("{0} Builder started new thread for sequence {1} step {2}", info.number, info.command.sequence, info.command.command);
/* Check if step has been aborted already */
info.work.State = WebService.GetWorkStateSafe(info.work);
if (info.work.State != DBState.NotDone && info.work.State != DBState.Executing)
{
/* If we're in an executing state, we're restarting a command for whatever reason (crash, reboot, etc) */
Logger.Log("{0} Builder found that step {1} is not ready to start, it's in a '{2}' state", info.number, info.command.command, info.work.State);
return;
}
result = DBState.Executing;
info.work.starttime = DBRecord.DatabaseNow;
info.work.State = result;
info.work.host_id = info.host.id;
info.work = WebService.ReportBuildStateSafe(info.work).Work;
using (Job p = ProcessHelper.CreateJob()) {
using (FileStream fs = new FileStream(log_file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) {
using (var log = new SynchronizedStreamWriter(new StreamWriter(fs))) {
p.StartInfo.FileName = info.command.filename;
p.StartInfo.Arguments = string.Format(info.command.arguments, Path.Combine(info.temp_dir, info.command.command), info.temp_dir);
if (!string.IsNullOrEmpty(info.command.working_directory))
{
p.StartInfo.WorkingDirectory = Path.Combine(info.BUILDER_DATA_SOURCE_DIR, info.command.working_directory);
}
else
{
p.StartInfo.WorkingDirectory = info.BUILDER_DATA_SOURCE_DIR;
}
// set environment variables
p.StartInfo.EnvironmentVariables ["BUILD_LANE"] = info.lane.lane;
p.StartInfo.EnvironmentVariables ["BUILD_COMMAND"] = info.command.command;
p.StartInfo.EnvironmentVariables ["BUILD_REVISION"] = info.revision.revision;
p.StartInfo.EnvironmentVariables ["BUILD_INSTALL"] = Configuration.CygwinizePath(info.BUILDER_DATA_INSTALL_DIR);
p.StartInfo.EnvironmentVariables ["BUILD_DATA_LANE"] = Configuration.GetDataLane(info.lane.id);
p.StartInfo.EnvironmentVariables ["BUILD_DATA_SOURCE"] = info.BUILDER_DATA_SOURCE_DIR;
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY"] = info.lane.repository;
p.StartInfo.EnvironmentVariables ["BUILD_HOST"] = Configuration.Host;
p.StartInfo.EnvironmentVariables ["BUILD_WORK_HOST"] = info.host_being_worked_for;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_MAX_REVISION"] = info.lane.max_revision;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_MIN_REVISION"] = info.lane.min_revision;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_COMMIT_FILTER"] = info.lane.commit_filter;
int r = 0;
foreach (string repo in info.lane.repository.Split(','))
{
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_" + r.ToString()] = repo;
r++;
}
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_SPACE"] = info.lane.repository.Replace(',', ' ');
p.StartInfo.EnvironmentVariables ["BUILD_SEQUENCE"] = "0";
p.StartInfo.EnvironmentVariables ["BUILD_SCRIPT_DIR"] = info.temp_dir;
p.StartInfo.EnvironmentVariables ["LD_LIBRARY_PATH"] = Configuration.CygwinizePath(Configuration.GetLdLibraryPath(info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["PKG_CONFIG_PATH"] = Configuration.CygwinizePath(Configuration.GetPkgConfigPath(info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["PATH"] = Configuration.CygwinizePath(Configuration.GetPath(info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["C_INCLUDE_PATH"] = Configuration.CygwinizePath(Configuration.GetCIncludePath(info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["CPLUS_INCLUDE_PATH"] = Configuration.CygwinizePath(Configuration.GetCPlusIncludePath(info.lane.id, info.revision.revision));
// We need to remove all paths from environment variables that were
// set for this executable to work so that they don't mess with
// whatever we're trying to execute
string [] bot_dependencies = new string [] { "PATH", "LD_LIBRARY_PATH", "PKG_CONFIG_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "AC_LOCAL_PATH", "MONO_PATH" };
foreach (string bot_dependency in bot_dependencies)
{
if (!p.StartInfo.EnvironmentVariables.ContainsKey(bot_dependency))
{
continue;
}
List <string> paths = new List <string> (p.StartInfo.EnvironmentVariables [bot_dependency].Split(new char [] { ':' /* XXX: does windows use ';' here? */ }, StringSplitOptions.None));
for (int i = paths.Count - 1; i >= 0; i--)
{
if (paths [i].Contains("bot-dependencies"))
{
paths.RemoveAt(i);
}
}
p.StartInfo.EnvironmentVariables [bot_dependency] = string.Join(":", paths.ToArray());
}
if (info.environment_variables != null)
{
// order is important here, we need to loop over the array in the same order get got the variables.
for (int e = 0; e < info.environment_variables.Count; e++)
{
info.environment_variables [e].Evaluate(p.StartInfo.EnvironmentVariables);
}
}
process_reader = new ProcessReader(log);
process_reader.Setup(p);
p.Start();
process_reader.Start();
while (!p.WaitForExit(60000 /* 1 minute */))
{
if (p.HasExited)
{
break;
}
// Check if step has been aborted.
info.work.State = WebService.GetWorkStateSafe(info.work);
if (info.work.State == DBState.Aborted)
{
result = DBState.Aborted;
try {
exitcode = 255;
Logger.Log("{1} The build step '{0}' has been aborted, terminating it.", info.command.command, info.number);
p.Terminate(log);
log.WriteLine(string.Format("{1} The build step '{0}' was aborted, terminated it.", info.command.command, info.number));
} catch (Exception ex) {
Logger.Log("{1} Exception while killing build step: {0}", ex.ToString(), info.number);
}
break;
}
// Check if step has timedout
bool timedout = false;
string timeoutReason = null;
int timeout = 30;
if ((DateTime.Now > local_starttime.AddMinutes(info.command.timeout)))
{
timedout = true;
timeoutReason = string.Format("The build step '{0}' didn't finish in {1} minute(s).", info.command.command, info.command.timeout);
}
else if (log.LastStamp.AddMinutes(timeout) <= DateTime.Now)
{
timedout = true;
timeoutReason = string.Format("The build step '{0}' has had no output for {1} minute(s).", info.command.command, timeout);
}
if (!timedout)
{
continue;
}
try {
result = DBState.Timeout;
exitcode = 255;
Logger.Log("{0} {1}", info.number, timeoutReason);
log.WriteLine("\n*** Timed out. Proceeding to get stack traces for all child processes. ***");
p.Terminate(log);
log.WriteLine("\n * " + timeoutReason + " * \n");
} catch (Exception ex) {
Logger.Log("{1} Exception while terminating build step: {0}", ex.ToString(), info.number);
}
break;
}
// Sleep a bit so that the process has enough time to finish
System.Threading.Thread.Sleep(1000);
process_reader.Join();
if (p.HasExited)
{
exitcode = p.ExitCode;
}
else
{
Logger.Log("{1} Step: {0}: the process didn't exit in time.", command.command, info.number);
exitcode = 1;
}
if (result == DBState.Executing)
{
if (exitcode == 0)
{
result = DBState.Success;
}
else
{
result = DBState.Failed;
}
}
else if (result == DBState.Aborted)
{
result = DBState.Failed;
}
}
}
}
info.work.State = result;
using (TextReader reader = new StreamReader(new FileStream(log_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
info.work.CalculateSummary(reader);
info.work.endtime = DBRecord.DatabaseNow;
response = WebService.ReportBuildStateSafe(info.work);
info.work = response.Work;
WebService.UploadFilesSafe(info.work, new string [] { log_file }, new bool [] { false });
// Gather files from logged commands and from the upload_files glob
CheckLog(log_file, info);
if (!string.IsNullOrEmpty(info.command.upload_files))
{
foreach (string glob in info.command.upload_files.Split(','))
{
// TODO: handle globs in directory parts
// TODO: allow absolute paths?
Logger.Log("Uploading files from glob {0}", glob);
// TODO: allow hidden files also
WebService.UploadFilesSafe(info.work, Directory.GetFiles(Path.Combine(info.BUILDER_DATA_SOURCE_DIR, Path.GetDirectoryName(glob)), Path.GetFileName(glob)), null);
}
}
if (response.RevisionWorkCompleted)
{
// Cleanup after us.
string base_dir = Configuration.GetDataRevisionDir(info.lane.id, info.revision.revision);
FileUtilities.TryDeleteDirectoryRecursive(base_dir);
}
Logger.Log("{4} Revision {0}, executed step '{1}' in {2} s, ExitCode: {3}, State: {4}", info.revision.revision, info.command.command, info.work.duration, exitcode, info.number, info.work.State);
} catch (Exception ex) {
info.work.State = DBState.Failed;
info.work.summary = ex.Message;
info.work = WebService.ReportBuildStateSafe(info.work).Work;
Logger.Log("{3} Revision {0}, got exception '{1}': \n{2}", info.revision.revision, ex.Message, ex.StackTrace, info.number);
throw;
} finally {
Logger.Log("{0} Builder finished thread for sequence {0}", info.number);
if ((info.work.State == DBState.Failed || info.work.State == DBState.Aborted || info.work.State == DBState.Timeout) && !info.command.nonfatal)
{
failed_hostlanes.Add(info.hostlane);
}
}
}
private static void Build (BuildInfo info)
{
try {
ProcessReader process_reader;
string log_file = Path.Combine (info.BUILDER_DATA_LOG_DIR, info.command.command + ".log");
DateTime local_starttime = DateTime.Now;
DBState result;
DBCommand command = info.command;
int exitcode = 0;
ReportBuildStateResponse response;
Builder.log.DebugFormat ("{0} Builder started new thread for sequence {1} step {2}", info.number, info.command.sequence, info.command.command);
/* Check if step has been aborted already */
info.work.State = WebService.GetWorkStateSafe (info.work);
if (info.work.State != DBState.NotDone && info.work.State != DBState.Executing) {
/* If we're in an executing state, we're restarting a command for whatever reason (crash, reboot, etc) */
log.WarnFormat ("{0} Builder found that step {1} is not ready to start, it's in a '{2}' state", info.number, info.command.command, info.work.State);
return;
}
result = DBState.Executing;
info.work.starttime = DBRecord.DatabaseNow;
info.work.State = result;
info.work.host_id = info.host.id;
info.work = WebService.ReportBuildStateSafe (info.work).Work;
using (Job p = ProcessHelper.CreateJob ()) {
using (FileStream fs = new FileStream (log_file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) {
using (var log = new SynchronizedStreamWriter (new StreamWriter (fs))) {
p.StartInfo.FileName = info.command.filename;
p.StartInfo.Arguments = string.Format (info.command.arguments, Path.Combine (info.temp_dir, info.command.command), info.temp_dir);
if (!string.IsNullOrEmpty (info.command.working_directory))
p.StartInfo.WorkingDirectory = Path.Combine (info.BUILDER_DATA_SOURCE_DIR, info.command.working_directory);
else
p.StartInfo.WorkingDirectory = info.BUILDER_DATA_SOURCE_DIR;
// set environment variables
p.StartInfo.EnvironmentVariables ["BUILD_LANE"] = info.lane.lane;
p.StartInfo.EnvironmentVariables ["BUILD_COMMAND"] = info.command.command;
p.StartInfo.EnvironmentVariables ["BUILD_REVISION"] = info.revision.revision;
p.StartInfo.EnvironmentVariables ["BUILD_INSTALL"] = Configuration.CygwinizePath (info.BUILDER_DATA_INSTALL_DIR);
p.StartInfo.EnvironmentVariables ["BUILD_DATA_LANE"] = Configuration.GetDataLane (info.lane.id);
p.StartInfo.EnvironmentVariables ["BUILD_DATA_SOURCE"] = info.BUILDER_DATA_SOURCE_DIR;
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY"] = info.lane.repository;
p.StartInfo.EnvironmentVariables ["BUILD_HOST"] = Configuration.Host;
p.StartInfo.EnvironmentVariables ["BUILD_WORK_HOST"] = info.host_being_worked_for;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_MAX_REVISION"] = info.lane.max_revision;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_MIN_REVISION"] = info.lane.min_revision;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_COMMIT_FILTER"] = info.lane.commit_filter;
int r = 0;
foreach (string repo in info.lane.repository.Split (',')) {
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_" + r.ToString ()] = repo;
r++;
}
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_SPACE"] = info.lane.repository.Replace (',', ' ');
p.StartInfo.EnvironmentVariables ["BUILD_SEQUENCE"] = "0";
p.StartInfo.EnvironmentVariables ["BUILD_SCRIPT_DIR"] = info.temp_dir;
p.StartInfo.EnvironmentVariables ["LD_LIBRARY_PATH"] = Configuration.CygwinizePath (Configuration.GetLdLibraryPath (info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["PKG_CONFIG_PATH"] = Configuration.CygwinizePath (Configuration.GetPkgConfigPath (info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["PATH"] = Configuration.CygwinizePath (Configuration.GetPath (info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["C_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCIncludePath (info.lane.id, info.revision.revision));
p.StartInfo.EnvironmentVariables ["CPLUS_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCPlusIncludePath (info.lane.id, info.revision.revision));
// We need to remove all paths from environment variables that were
// set for this executable to work so that they don't mess with
// whatever we're trying to execute
string [] bot_dependencies = new string [] { "PATH", "LD_LIBRARY_PATH", "PKG_CONFIG_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "AC_LOCAL_PATH", "MONO_PATH" };
foreach (string bot_dependency in bot_dependencies) {
if (!p.StartInfo.EnvironmentVariables.ContainsKey (bot_dependency))
continue;
List<string> paths = new List<string> (p.StartInfo.EnvironmentVariables [bot_dependency].Split (new char [] { ':' /* XXX: does windows use ';' here? */}, StringSplitOptions.None));
for (int i = paths.Count - 1; i >= 0; i--) {
if (paths [i].Contains ("bot-dependencies"))
paths.RemoveAt (i);
}
p.StartInfo.EnvironmentVariables [bot_dependency] = string.Join (":", paths.ToArray ());
}
if (info.environment_variables != null) {
// order is important here, we need to loop over the array in the same order get got the variables.
for (int e = 0; e < info.environment_variables.Count; e++) {
info.environment_variables [e].Evaluate (p.StartInfo.EnvironmentVariables);
}
}
process_reader = new ProcessReader (log);
process_reader.Setup (p, command.timestamp);
p.Start ();
process_reader.Start ();
while (!p.WaitForExit (60000 /* 1 minute */)) {
info.progress_stamp = DateTime.Now;
if (p.HasExited)
break;
// Check if step has been aborted.
info.work.State = WebService.GetWorkStateSafe (info.work);
if (info.work.State == DBState.Aborted) {
result = DBState.Aborted;
try {
exitcode = 255;
Builder.log.InfoFormat ("{1} The build step '{0}' has been aborted, terminating it.", info.command.command, info.number);
p.Terminate (log);
log.WriteLine (string.Format ("{1} The build step '{0}' was aborted, terminated it.", info.command.command, info.number));
} catch (Exception ex) {
Builder.log.ErrorFormat ("{1} Exception while killing build step: {0}", ex, info.number);
}
break;
}
// Check if step has timedout
bool timedout = false;
string timeoutReason = null;
int deadlock_timeout;
if (!info.command.deadlock_timeout.HasValue) {
deadlock_timeout = Configuration.NoOutputTimeout;
} else {
deadlock_timeout = info.command.deadlock_timeout.Value;
}
if ((DateTime.Now > local_starttime.AddMinutes (info.command.timeout))) {
timedout = true;
timeoutReason = string.Format ("The build step '{0}' didn't finish in {1} minute(s).", info.command.command, info.command.timeout);
} else if (deadlock_timeout > 0 && log.LastStamp.AddMinutes (deadlock_timeout) <= DateTime.Now) {
timedout = true;
timeoutReason = string.Format ("The build step '{0}' has had no output for {1} minute(s).", info.command.command, deadlock_timeout);
}
if (!timedout)
continue;
try {
result = DBState.Timeout;
exitcode = 255;
Builder.log.ErrorFormat ("{0} {1}", info.number, timeoutReason);
log.WriteLine ("\n*** Timed out. Proceeding to get stack traces for all child processes. ***");
log.WriteLine ("\n*** " + timeoutReason + " *** \n");
p.Terminate (log);
log.WriteLine ("\n * " + timeoutReason + " * \n");
} catch (Exception ex) {
Builder.log.ErrorFormat ("{1} Exception while terminating build step: {0}", ex, info.number);
}
break;
}
// Sleep a bit so that the process has enough time to finish
System.Threading.Thread.Sleep (1000);
process_reader.Join ();
if (p.HasExited) {
exitcode = p.ExitCode;
} else {
Builder.log.ErrorFormat ("{1} Step: {0}: the process didn't exit in time.", command.command, info.number);
exitcode = 1;
}
if (result == DBState.Executing) {
if (exitcode == 0) {
result = DBState.Success;
} else {
result = DBState.Failed;
}
} else if (result == DBState.Aborted) {
result = DBState.Failed;
}
}
}
}
info.work.State = result;
using (TextReader reader = new StreamReader (new FileStream (log_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
info.work.CalculateSummary (reader);
info.work.endtime = DBRecord.DatabaseNow;
response = WebService.ReportBuildStateSafe (info.work);
info.work = response.Work;
info.progress_stamp = DateTime.Now;
try {
WebService.UploadFilesSafe (info.work, new string [] { log_file }, new bool [] { false });
// Gather files from logged commands and from the upload_files glob
CheckLog (log_file, info);
if (!string.IsNullOrEmpty (info.command.upload_files)) {
foreach (string glob in info.command.upload_files.Split (',')) {
// TODO: handle globs in directory parts
// TODO: allow absolute paths?
log.InfoFormat ("Uploading files from glob {0}", glob);
info.progress_stamp = DateTime.Now;
// TODO: allow hidden files also
WebService.UploadFilesSafe (info.work, Directory.GetFiles (Path.Combine (info.BUILDER_DATA_SOURCE_DIR, Path.GetDirectoryName (glob)), Path.GetFileName (glob)), null);
}
}
} finally {
if (response.RevisionWorkCompleted) {
// Cleanup after us.
string base_dir = Configuration.GetDataRevisionDir (info.lane.id, info.revision.revision);
FileUtilities.TryDeleteDirectoryRecursive (base_dir);
}
}
Builder.log.InfoFormat ("{4} Revision {0}, executed step '{1}' in {2} s, ExitCode: {3}, State: {4}", info.revision.revision, info.command.command, info.work.duration, exitcode, info.number, info.work.State);
} catch (Exception ex) {
info.work.State = DBState.Failed;
info.work.summary = ex.Message;
info.work = WebService.ReportBuildStateSafe (info.work).Work;
log.ErrorFormat ("{3} Revision {0}, got exception '{1}': \n{2}", info.revision.revision, ex.Message, ex.StackTrace, info.number);
throw;
} finally {
log.DebugFormat ("{0} Builder finished thread for sequence {0}", info.number);
if ((info.work.State == DBState.Failed || info.work.State == DBState.Aborted || info.work.State == DBState.Timeout) && !info.command.nonfatal) {
failed_hostlanes.Add (info.hostlane);
}
}
}
/// <summary>
/// Terminate this job and all the processes within.
/// </summary>
public virtual void Terminate (SynchronizedStreamWriter logstream)
{
p.KillTree (logstream);
}
/// <summary>
/// Kills this process and all its child processes (recursively, grand children are killed too).
/// Also waits until the process has actually exited.
/// </summary>
/// <param name="process"></param>
public static void KillTree (this Process process, SynchronizedStreamWriter log)
{
GetHelper ().KillTree (process, log);
}
protected virtual internal void Kill (IEnumerable<int> pids, SynchronizedStreamWriter log)
{
foreach (int pid in pids) {
Kill (pid, log);
}
}
/// <summary>
/// Default Kill implementation.
/// </summary>
/// <param name="pid"></param>
protected virtual internal void Kill (int pid, SynchronizedStreamWriter log)
{
using (Process p = Process.GetProcessById (pid)) {
p.Kill ();
}
}
