internal static void RenderStackTraceWithGdb (int pid, SynchronizedStreamWriter log) { log.WriteLine (string.Format ("\n * Fetching stack trace for process {0} * \n", pid)); var template = Path.GetTempFileName (); try { bool using_lldb = false; string debugger; if (File.Exists ("/usr/bin/gdb")) { using_lldb = false; debugger = "/usr/bin/gdb"; } else if (File.Exists ("/usr/bin/lldb")) { using_lldb = true; debugger = "/usr/bin/lldb"; } else { using_lldb = false; // lets hope "gdb" is somewhere. debugger = "gdb"; } var commands = new StringBuilder (); using (var dbg = new Job ()) { dbg.StartInfo.UseShellExecute = false; dbg.StartInfo.FileName = debugger; if (using_lldb) { commands.AppendFormat ("process attach --pid {0}\n", pid); commands.Append ("thread list\n"); commands.Append ("thread backtrace all\n"); commands.Append ("detach\n"); commands.Append ("quit\n"); dbg.StartInfo.Arguments = "--source \"" + template + "\""; } else { commands.AppendFormat ("attach {0}\n", pid); commands.Append ("info target\n"); commands.Append ("info threads\n"); commands.Append ("thread apply all bt\n"); dbg.StartInfo.Arguments = "-batch -x \"" + template + "\""; } File.WriteAllText (template, commands.ToString ()); var reader = new ProcessReader (log); reader.Setup (dbg); dbg.Start (); reader.Start (); try { if (!dbg.WaitForExit (1000 * 30 /* 30 seconds */)) throw new ApplicationException (string.Format ("The 'gdb' process didn't exit in 30 seconds.", dbg.StartInfo.FileName)); } finally { reader.Join (); } } } finally { try { File.Delete (template); } catch { } } }
internal static void PrintProcessesImplPS (SynchronizedStreamWriter log) { using (var ps = new Job ()) { ps.StartInfo.FileName = "ps"; ps.StartInfo.Arguments = "aux"; var reader = new ProcessReader (log); reader.Setup (ps); ps.Start (); reader.Start (); try { if (!ps.WaitForExit (1000 * 30 /* 30 seconds */)) throw new ApplicationException (string.Format ("The 'ps' process didn't exit in 30 seconds.")); } finally { reader.Join (); } } }
internal static void RenderStackTraceWithGdb (int pid, SynchronizedStreamWriter log) { log.WriteLine (string.Format ("\n * Fetching stack trace for process {0} (name '{1}') * \n", pid, GetProcessName (pid))); using (var gdb = new Job ()) { gdb.StartInfo.FileName = "gdb"; gdb.StartInfo.Arguments = string.Format ("-ex attach {0} --ex \"info target\" --ex \"info threads\" --ex \"thread apply all bt\" --batch", pid); var reader = new ProcessReader (log); reader.Setup (gdb); gdb.Start (); reader.Start (); try { if (!gdb.WaitForExit (1000 * 30 /* 30 seconds */)) throw new ApplicationException (string.Format ("The 'gdb' process didn't exit in 30 seconds.")); } finally { reader.Join (); } } }
internal static void PrintProcessesImplPS(SynchronizedStreamWriter log) { using (var ps = new Job()) { ps.StartInfo.FileName = "ps"; ps.StartInfo.Arguments = "aux"; var reader = new ProcessReader(log); reader.Setup(ps); ps.Start(); reader.Start(); try { if (!ps.WaitForExit(1000 * 30 /* 30 seconds */)) { throw new ApplicationException(string.Format("The 'ps' process didn't exit in 30 seconds.")); } } finally { reader.Join(); } } }
internal static void RenderStackTraceWithGdb(int pid, SynchronizedStreamWriter log) { log.WriteLine(string.Format("\n * Fetching stack trace for process {0} * \n", pid)); using (var gdb = new Job()) { gdb.StartInfo.FileName = "gdb"; gdb.StartInfo.Arguments = string.Format("-ex attach {0} --ex \"info target\" --ex \"info threads\" --ex \"thread apply all bt\" --batch", pid); var reader = new ProcessReader(log); reader.Setup(gdb); gdb.Start(); reader.Start(); try { if (!gdb.WaitForExit(1000 * 30 /* 30 seconds */)) { throw new ApplicationException(string.Format("The 'gdb' process didn't exit in 30 seconds.")); } } finally { reader.Join(); } } }
internal static void RenderStackTraceWithGdb(int pid, SynchronizedStreamWriter log) { log.WriteLine(string.Format("\n * Fetching stack trace for process {0} * \n", pid)); var template = Path.GetTempFileName(); try { bool using_lldb = false; string debugger; if (File.Exists("/usr/bin/gdb")) { using_lldb = false; debugger = "/usr/bin/gdb"; } else if (File.Exists("/usr/bin/lldb")) { using_lldb = true; debugger = "/usr/bin/lldb"; } else { using_lldb = false; // lets hope "gdb" is somewhere. debugger = "gdb"; } var commands = new StringBuilder(); using (var dbg = new Job()) { dbg.StartInfo.UseShellExecute = false; dbg.StartInfo.FileName = debugger; if (using_lldb) { commands.AppendFormat("process attach --pid {0}\n", pid); commands.Append("script lldb.debugger.HandleCommand (\"thread list\")\n"); commands.Append("script lldb.debugger.HandleCommand (\"thread backtrace all\")\n"); commands.Append("detach\n"); commands.Append("quit\n"); dbg.StartInfo.Arguments = "--source \"" + template + "\""; } else { commands.AppendFormat("attach {0}\n", pid); commands.Append("info target\n"); commands.Append("info threads\n"); commands.Append("thread apply all bt\n"); dbg.StartInfo.Arguments = "-batch -x \"" + template + "\""; } File.WriteAllText(template, commands.ToString()); var reader = new ProcessReader(log); reader.Setup(dbg); dbg.Start(); reader.Start(); try { if (!dbg.WaitForExit(1000 * 30 /* 30 seconds */)) { throw new ApplicationException(string.Format("The 'gdb' process didn't exit in 30 seconds.", dbg.StartInfo.FileName)); } } finally { reader.Join(); } } } finally { try { File.Delete(template); } catch { } } }
private static void Build (BuildInfo info) { try { ProcessReader process_reader; string log_file = Path.Combine (info.BUILDER_DATA_LOG_DIR, info.command.command + ".log"); DateTime local_starttime = DateTime.Now; DBState result; DBCommand command = info.command; int exitcode = 0; ReportBuildStateResponse response; Builder.log.DebugFormat ("{0} Builder started new thread for sequence {1} step {2}", info.number, info.command.sequence, info.command.command); /* Check if step has been aborted already */ info.work.State = WebService.GetWorkStateSafe (info.work); if (info.work.State != DBState.NotDone && info.work.State != DBState.Executing) { /* If we're in an executing state, we're restarting a command for whatever reason (crash, reboot, etc) */ log.WarnFormat ("{0} Builder found that step {1} is not ready to start, it's in a '{2}' state", info.number, info.command.command, info.work.State); return; } result = DBState.Executing; info.work.starttime = DBRecord.DatabaseNow; info.work.State = result; info.work.host_id = info.host.id; info.work = WebService.ReportBuildStateSafe (info.work).Work; using (Job p = ProcessHelper.CreateJob ()) { using (FileStream fs = new FileStream (log_file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) { using (var log = new SynchronizedStreamWriter (new StreamWriter (fs))) { p.StartInfo.FileName = info.command.filename; p.StartInfo.Arguments = string.Format (info.command.arguments, Path.Combine (info.temp_dir, info.command.command), info.temp_dir); if (!string.IsNullOrEmpty (info.command.working_directory)) p.StartInfo.WorkingDirectory = Path.Combine (info.BUILDER_DATA_SOURCE_DIR, info.command.working_directory); else p.StartInfo.WorkingDirectory = info.BUILDER_DATA_SOURCE_DIR; // set environment variables p.StartInfo.EnvironmentVariables ["BUILD_LANE"] = info.lane.lane; p.StartInfo.EnvironmentVariables ["BUILD_COMMAND"] = info.command.command; p.StartInfo.EnvironmentVariables ["BUILD_REVISION"] = info.revision.revision; p.StartInfo.EnvironmentVariables ["BUILD_INSTALL"] = Configuration.CygwinizePath (info.BUILDER_DATA_INSTALL_DIR); p.StartInfo.EnvironmentVariables ["BUILD_DATA_LANE"] = Configuration.GetDataLane (info.lane.id); p.StartInfo.EnvironmentVariables ["BUILD_DATA_SOURCE"] = info.BUILDER_DATA_SOURCE_DIR; p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY"] = info.lane.repository; p.StartInfo.EnvironmentVariables ["BUILD_HOST"] = Configuration.Host; p.StartInfo.EnvironmentVariables ["BUILD_WORK_HOST"] = info.host_being_worked_for; p.StartInfo.EnvironmentVariables ["BUILD_LANE_MAX_REVISION"] = info.lane.max_revision; p.StartInfo.EnvironmentVariables ["BUILD_LANE_MIN_REVISION"] = info.lane.min_revision; p.StartInfo.EnvironmentVariables ["BUILD_LANE_COMMIT_FILTER"] = info.lane.commit_filter; int r = 0; foreach (string repo in info.lane.repository.Split (',')) { p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_" + r.ToString ()] = repo; r++; } p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_SPACE"] = info.lane.repository.Replace (',', ' '); p.StartInfo.EnvironmentVariables ["BUILD_SEQUENCE"] = "0"; p.StartInfo.EnvironmentVariables ["BUILD_SCRIPT_DIR"] = info.temp_dir; p.StartInfo.EnvironmentVariables ["LD_LIBRARY_PATH"] = Configuration.CygwinizePath (Configuration.GetLdLibraryPath (info.lane.id, info.revision.revision)); p.StartInfo.EnvironmentVariables ["PKG_CONFIG_PATH"] = Configuration.CygwinizePath (Configuration.GetPkgConfigPath (info.lane.id, info.revision.revision)); p.StartInfo.EnvironmentVariables ["PATH"] = Configuration.CygwinizePath (Configuration.GetPath (info.lane.id, info.revision.revision)); p.StartInfo.EnvironmentVariables ["C_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCIncludePath (info.lane.id, info.revision.revision)); p.StartInfo.EnvironmentVariables ["CPLUS_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCPlusIncludePath (info.lane.id, info.revision.revision)); // We need to remove all paths from environment variables that were // set for this executable to work so that they don't mess with // whatever we're trying to execute string [] bot_dependencies = new string [] { "PATH", "LD_LIBRARY_PATH", "PKG_CONFIG_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "AC_LOCAL_PATH", "MONO_PATH" }; foreach (string bot_dependency in bot_dependencies) { if (!p.StartInfo.EnvironmentVariables.ContainsKey (bot_dependency)) continue; List<string> paths = new List<string> (p.StartInfo.EnvironmentVariables [bot_dependency].Split (new char [] { ':' /* XXX: does windows use ';' here? */}, StringSplitOptions.None)); for (int i = paths.Count - 1; i >= 0; i--) { if (paths [i].Contains ("bot-dependencies")) paths.RemoveAt (i); } p.StartInfo.EnvironmentVariables [bot_dependency] = string.Join (":", paths.ToArray ()); } if (info.environment_variables != null) { // order is important here, we need to loop over the array in the same order get got the variables. for (int e = 0; e < info.environment_variables.Count; e++) { info.environment_variables [e].Evaluate (p.StartInfo.EnvironmentVariables); } } process_reader = new ProcessReader (log); process_reader.Setup (p, command.timestamp); p.Start (); process_reader.Start (); while (!p.WaitForExit (60000 /* 1 minute */)) { info.progress_stamp = DateTime.Now; if (p.HasExited) break; // Check if step has been aborted. info.work.State = WebService.GetWorkStateSafe (info.work); if (info.work.State == DBState.Aborted) { result = DBState.Aborted; try { exitcode = 255; Builder.log.InfoFormat ("{1} The build step '{0}' has been aborted, terminating it.", info.command.command, info.number); p.Terminate (log); log.WriteLine (string.Format ("{1} The build step '{0}' was aborted, terminated it.", info.command.command, info.number)); } catch (Exception ex) { Builder.log.ErrorFormat ("{1} Exception while killing build step: {0}", ex, info.number); } break; } // Check if step has timedout bool timedout = false; string timeoutReason = null; int deadlock_timeout; if (!info.command.deadlock_timeout.HasValue) { deadlock_timeout = Configuration.NoOutputTimeout; } else { deadlock_timeout = info.command.deadlock_timeout.Value; } if ((DateTime.Now > local_starttime.AddMinutes (info.command.timeout))) { timedout = true; timeoutReason = string.Format ("The build step '{0}' didn't finish in {1} minute(s).", info.command.command, info.command.timeout); } else if (deadlock_timeout > 0 && log.LastStamp.AddMinutes (deadlock_timeout) <= DateTime.Now) { timedout = true; timeoutReason = string.Format ("The build step '{0}' has had no output for {1} minute(s).", info.command.command, deadlock_timeout); } if (!timedout) continue; try { result = DBState.Timeout; exitcode = 255; Builder.log.ErrorFormat ("{0} {1}", info.number, timeoutReason); log.WriteLine ("\n*** Timed out. Proceeding to get stack traces for all child processes. ***"); log.WriteLine ("\n*** " + timeoutReason + " *** \n"); p.Terminate (log); log.WriteLine ("\n * " + timeoutReason + " * \n"); } catch (Exception ex) { Builder.log.ErrorFormat ("{1} Exception while terminating build step: {0}", ex, info.number); } break; } // Sleep a bit so that the process has enough time to finish System.Threading.Thread.Sleep (1000); process_reader.Join (); if (p.HasExited) { exitcode = p.ExitCode; } else { Builder.log.ErrorFormat ("{1} Step: {0}: the process didn't exit in time.", command.command, info.number); exitcode = 1; } if (result == DBState.Executing) { if (exitcode == 0) { result = DBState.Success; } else { result = DBState.Failed; } } else if (result == DBState.Aborted) { result = DBState.Failed; } } } } info.work.State = result; using (TextReader reader = new StreamReader (new FileStream (log_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) info.work.CalculateSummary (reader); info.work.endtime = DBRecord.DatabaseNow; response = WebService.ReportBuildStateSafe (info.work); info.work = response.Work; info.progress_stamp = DateTime.Now; try { WebService.UploadFilesSafe (info.work, new string [] { log_file }, new bool [] { false }); // Gather files from logged commands and from the upload_files glob CheckLog (log_file, info); if (!string.IsNullOrEmpty (info.command.upload_files)) { foreach (string glob in info.command.upload_files.Split (',')) { // TODO: handle globs in directory parts // TODO: allow absolute paths? log.InfoFormat ("Uploading files from glob {0}", glob); info.progress_stamp = DateTime.Now; // TODO: allow hidden files also WebService.UploadFilesSafe (info.work, Directory.GetFiles (Path.Combine (info.BUILDER_DATA_SOURCE_DIR, Path.GetDirectoryName (glob)), Path.GetFileName (glob)), null); } } } finally { if (response.RevisionWorkCompleted) { // Cleanup after us. string base_dir = Configuration.GetDataRevisionDir (info.lane.id, info.revision.revision); FileUtilities.TryDeleteDirectoryRecursive (base_dir); } } Builder.log.InfoFormat ("{4} Revision {0}, executed step '{1}' in {2} s, ExitCode: {3}, State: {4}", info.revision.revision, info.command.command, info.work.duration, exitcode, info.number, info.work.State); } catch (Exception ex) { info.work.State = DBState.Failed; info.work.summary = ex.Message; info.work = WebService.ReportBuildStateSafe (info.work).Work; log.ErrorFormat ("{3} Revision {0}, got exception '{1}': \n{2}", info.revision.revision, ex.Message, ex.StackTrace, info.number); throw; } finally { log.DebugFormat ("{0} Builder finished thread for sequence {0}", info.number); if ((info.work.State == DBState.Failed || info.work.State == DBState.Aborted || info.work.State == DBState.Timeout) && !info.command.nonfatal) { failed_hostlanes.Add (info.hostlane); } } }