internal static void RenderStackTraceWithGdb (int pid, SynchronizedStreamWriter log)
		{
			log.WriteLine (string.Format ("\n * Fetching stack trace for process {0} * \n", pid));

			var template = Path.GetTempFileName ();
			try {
				bool using_lldb = false;
				string debugger;

				if (File.Exists ("/usr/bin/gdb")) {
					using_lldb = false;
					debugger = "/usr/bin/gdb";
				} else if (File.Exists ("/usr/bin/lldb")) {
					using_lldb = true;
					debugger = "/usr/bin/lldb";
				} else {
					using_lldb = false; // lets hope "gdb" is somewhere.
					debugger = "gdb";
				}

				var commands = new StringBuilder ();
				using (var dbg = new Job ()) {
					dbg.StartInfo.UseShellExecute = false;
					dbg.StartInfo.FileName = debugger;
					if (using_lldb) {
						commands.AppendFormat ("process attach --pid {0}\n", pid);
						commands.Append ("thread list\n");
						commands.Append ("thread backtrace all\n");
						commands.Append ("detach\n");
						commands.Append ("quit\n");
						dbg.StartInfo.Arguments = "--source \"" + template + "\"";
					} else {
						commands.AppendFormat ("attach {0}\n", pid);
						commands.Append ("info target\n");
						commands.Append ("info threads\n");
						commands.Append ("thread apply all bt\n");
						dbg.StartInfo.Arguments = "-batch -x \"" + template + "\"";
					}
					File.WriteAllText (template, commands.ToString ());

					var reader = new ProcessReader (log);
					reader.Setup (dbg);
					dbg.Start ();
					reader.Start ();

					try {
						if (!dbg.WaitForExit (1000 * 30 /* 30 seconds */))
							throw new ApplicationException (string.Format ("The 'gdb' process didn't exit in 30 seconds.", dbg.StartInfo.FileName));
					} finally {
						reader.Join ();
					}
				}
			} finally {
				try {
					File.Delete (template);
				} catch {
				}
			}
		}
		internal static void PrintProcessesImplPS (SynchronizedStreamWriter log)
		{
			using (var ps = new Job ()) {
				ps.StartInfo.FileName = "ps";
				ps.StartInfo.Arguments = "aux";

				var reader = new ProcessReader (log);
				reader.Setup (ps);
				ps.Start ();
				reader.Start ();

				try {
					if (!ps.WaitForExit (1000 * 30 /* 30 seconds */))
						throw new ApplicationException (string.Format ("The 'ps' process didn't exit in 30 seconds."));
				} finally {
					reader.Join ();
				}
			}
		}
		internal static void RenderStackTraceWithGdb (int pid, SynchronizedStreamWriter log)
		{
			log.WriteLine (string.Format ("\n * Fetching stack trace for process {0} (name '{1}') * \n", pid, GetProcessName (pid)));

			using (var gdb = new Job ()) {
				gdb.StartInfo.FileName = "gdb";
				gdb.StartInfo.Arguments = string.Format ("-ex attach {0} --ex \"info target\" --ex \"info threads\" --ex \"thread apply all bt\" --batch", pid);

				var reader = new ProcessReader (log);
				reader.Setup (gdb);
				gdb.Start ();
				reader.Start ();

				try {
					if (!gdb.WaitForExit (1000 * 30 /* 30 seconds */))
						throw new ApplicationException (string.Format ("The 'gdb' process didn't exit in 30 seconds."));
				} finally {
					reader.Join ();
				}
			}
		}
        internal static void PrintProcessesImplPS(SynchronizedStreamWriter log)
        {
            using (var ps = new Job()) {
                ps.StartInfo.FileName  = "ps";
                ps.StartInfo.Arguments = "aux";

                var reader = new ProcessReader(log);
                reader.Setup(ps);
                ps.Start();
                reader.Start();

                try {
                    if (!ps.WaitForExit(1000 * 30 /* 30 seconds */))
                    {
                        throw new ApplicationException(string.Format("The 'ps' process didn't exit in 30 seconds."));
                    }
                } finally {
                    reader.Join();
                }
            }
        }
        internal static void RenderStackTraceWithGdb(int pid, SynchronizedStreamWriter log)
        {
            log.WriteLine(string.Format("\n * Fetching stack trace for process {0} * \n", pid));

            using (var gdb = new Job()) {
                gdb.StartInfo.FileName  = "gdb";
                gdb.StartInfo.Arguments = string.Format("-ex attach {0} --ex \"info target\" --ex \"info threads\" --ex \"thread apply all bt\" --batch", pid);

                var reader = new ProcessReader(log);
                reader.Setup(gdb);
                gdb.Start();
                reader.Start();

                try {
                    if (!gdb.WaitForExit(1000 * 30 /* 30 seconds */))
                    {
                        throw new ApplicationException(string.Format("The 'gdb' process didn't exit in 30 seconds."));
                    }
                } finally {
                    reader.Join();
                }
            }
        }
        internal static void RenderStackTraceWithGdb(int pid, SynchronizedStreamWriter log)
        {
            log.WriteLine(string.Format("\n * Fetching stack trace for process {0} * \n", pid));

            var template = Path.GetTempFileName();

            try {
                bool   using_lldb = false;
                string debugger;

                if (File.Exists("/usr/bin/gdb"))
                {
                    using_lldb = false;
                    debugger   = "/usr/bin/gdb";
                }
                else if (File.Exists("/usr/bin/lldb"))
                {
                    using_lldb = true;
                    debugger   = "/usr/bin/lldb";
                }
                else
                {
                    using_lldb = false;                     // lets hope "gdb" is somewhere.
                    debugger   = "gdb";
                }

                var commands = new StringBuilder();
                using (var dbg = new Job()) {
                    dbg.StartInfo.UseShellExecute = false;
                    dbg.StartInfo.FileName        = debugger;
                    if (using_lldb)
                    {
                        commands.AppendFormat("process attach --pid {0}\n", pid);
                        commands.Append("script lldb.debugger.HandleCommand (\"thread list\")\n");
                        commands.Append("script lldb.debugger.HandleCommand (\"thread backtrace all\")\n");
                        commands.Append("detach\n");
                        commands.Append("quit\n");
                        dbg.StartInfo.Arguments = "--source \"" + template + "\"";
                    }
                    else
                    {
                        commands.AppendFormat("attach {0}\n", pid);
                        commands.Append("info target\n");
                        commands.Append("info threads\n");
                        commands.Append("thread apply all bt\n");
                        dbg.StartInfo.Arguments = "-batch -x \"" + template + "\"";
                    }
                    File.WriteAllText(template, commands.ToString());

                    var reader = new ProcessReader(log);
                    reader.Setup(dbg);
                    dbg.Start();
                    reader.Start();

                    try {
                        if (!dbg.WaitForExit(1000 * 30 /* 30 seconds */))
                        {
                            throw new ApplicationException(string.Format("The 'gdb' process didn't exit in 30 seconds.", dbg.StartInfo.FileName));
                        }
                    } finally {
                        reader.Join();
                    }
                }
            } finally {
                try {
                    File.Delete(template);
                } catch {
                }
            }
        }
Beispiel #7
0
		private static void Build (BuildInfo info)
		{
			try {
				ProcessReader process_reader;
				string log_file = Path.Combine (info.BUILDER_DATA_LOG_DIR, info.command.command + ".log");
				DateTime local_starttime = DateTime.Now;
				DBState result;
				DBCommand command = info.command;
				int exitcode = 0;
				ReportBuildStateResponse response;

				Builder.log.DebugFormat ("{0} Builder started new thread for sequence {1} step {2}", info.number, info.command.sequence, info.command.command);

				/* Check if step has been aborted already */
				info.work.State = WebService.GetWorkStateSafe (info.work);
				if (info.work.State != DBState.NotDone && info.work.State != DBState.Executing) {
					/* If we're in an executing state, we're restarting a command for whatever reason (crash, reboot, etc) */
					log.WarnFormat ("{0} Builder found that step {1} is not ready to start, it's in a '{2}' state", info.number, info.command.command, info.work.State);
					return;
				}
				result = DBState.Executing;

				info.work.starttime = DBRecord.DatabaseNow;
				info.work.State = result;
				info.work.host_id = info.host.id;
				info.work = WebService.ReportBuildStateSafe (info.work).Work;

				using (Job p = ProcessHelper.CreateJob ()) {
					using (FileStream fs = new FileStream (log_file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) {
						using (var log = new SynchronizedStreamWriter (new StreamWriter (fs))) {
							p.StartInfo.FileName = info.command.filename;
							p.StartInfo.Arguments = string.Format (info.command.arguments, Path.Combine (info.temp_dir, info.command.command), info.temp_dir);
							if (!string.IsNullOrEmpty (info.command.working_directory))
								p.StartInfo.WorkingDirectory = Path.Combine (info.BUILDER_DATA_SOURCE_DIR, info.command.working_directory);
							else
								p.StartInfo.WorkingDirectory = info.BUILDER_DATA_SOURCE_DIR;

							// set environment variables
							p.StartInfo.EnvironmentVariables ["BUILD_LANE"] = info.lane.lane;
							p.StartInfo.EnvironmentVariables ["BUILD_COMMAND"] = info.command.command;
							p.StartInfo.EnvironmentVariables ["BUILD_REVISION"] = info.revision.revision;
							p.StartInfo.EnvironmentVariables ["BUILD_INSTALL"] = Configuration.CygwinizePath (info.BUILDER_DATA_INSTALL_DIR);
							p.StartInfo.EnvironmentVariables ["BUILD_DATA_LANE"] = Configuration.GetDataLane (info.lane.id);
							p.StartInfo.EnvironmentVariables ["BUILD_DATA_SOURCE"] = info.BUILDER_DATA_SOURCE_DIR;
							p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY"] = info.lane.repository;
							p.StartInfo.EnvironmentVariables ["BUILD_HOST"] = Configuration.Host;
							p.StartInfo.EnvironmentVariables ["BUILD_WORK_HOST"] = info.host_being_worked_for;
							p.StartInfo.EnvironmentVariables ["BUILD_LANE_MAX_REVISION"] = info.lane.max_revision;
							p.StartInfo.EnvironmentVariables ["BUILD_LANE_MIN_REVISION"] = info.lane.min_revision;
							p.StartInfo.EnvironmentVariables ["BUILD_LANE_COMMIT_FILTER"] = info.lane.commit_filter;

							int r = 0;
							foreach (string repo in info.lane.repository.Split (',')) {
								p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_" + r.ToString ()] = repo;
								r++;
							}
							p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_SPACE"] = info.lane.repository.Replace (',', ' ');
							p.StartInfo.EnvironmentVariables ["BUILD_SEQUENCE"] = "0";
							p.StartInfo.EnvironmentVariables ["BUILD_SCRIPT_DIR"] = info.temp_dir;
							p.StartInfo.EnvironmentVariables ["LD_LIBRARY_PATH"] = Configuration.CygwinizePath (Configuration.GetLdLibraryPath (info.lane.id, info.revision.revision));
							p.StartInfo.EnvironmentVariables ["PKG_CONFIG_PATH"] = Configuration.CygwinizePath (Configuration.GetPkgConfigPath (info.lane.id, info.revision.revision));
							p.StartInfo.EnvironmentVariables ["PATH"] = Configuration.CygwinizePath (Configuration.GetPath (info.lane.id, info.revision.revision));
							p.StartInfo.EnvironmentVariables ["C_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCIncludePath (info.lane.id, info.revision.revision));
							p.StartInfo.EnvironmentVariables ["CPLUS_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCPlusIncludePath (info.lane.id, info.revision.revision));

							// We need to remove all paths from environment variables that were
							// set for this executable to work so that they don't mess with 
							// whatever we're trying to execute
							string [] bot_dependencies = new string [] { "PATH", "LD_LIBRARY_PATH", "PKG_CONFIG_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "AC_LOCAL_PATH", "MONO_PATH" };
							foreach (string bot_dependency in bot_dependencies) {
								if (!p.StartInfo.EnvironmentVariables.ContainsKey (bot_dependency))
									continue;
								List<string> paths = new List<string> (p.StartInfo.EnvironmentVariables [bot_dependency].Split (new char [] { ':' /* XXX: does windows use ';' here? */}, StringSplitOptions.None));
								for (int i = paths.Count - 1; i >= 0; i--) {
									if (paths [i].Contains ("bot-dependencies"))
										paths.RemoveAt (i);
								}
								p.StartInfo.EnvironmentVariables [bot_dependency] = string.Join (":", paths.ToArray ());
							}

							if (info.environment_variables != null) {
								// order is important here, we need to loop over the array in the same order get got the variables.
								for (int e = 0; e < info.environment_variables.Count; e++) {
									info.environment_variables [e].Evaluate (p.StartInfo.EnvironmentVariables);
								}
							}

							process_reader = new ProcessReader (log);
							process_reader.Setup (p, command.timestamp);

							p.Start ();

							process_reader.Start ();

							while (!p.WaitForExit (60000 /* 1 minute */)) {
								info.progress_stamp = DateTime.Now;

								if (p.HasExited)
									break;

								// Check if step has been aborted.
								info.work.State = WebService.GetWorkStateSafe (info.work);
								if (info.work.State == DBState.Aborted) {
									result = DBState.Aborted;
									try {
										exitcode = 255;
										Builder.log.InfoFormat ("{1} The build step '{0}' has been aborted, terminating it.", info.command.command, info.number);
										p.Terminate (log);
										log.WriteLine (string.Format ("{1} The build step '{0}' was aborted, terminated it.", info.command.command, info.number));
									} catch (Exception ex) {
										Builder.log.ErrorFormat ("{1} Exception while killing build step: {0}", ex, info.number);
									}
									break;
								}
								
								// Check if step has timedout
								bool timedout = false;
								string timeoutReason = null;
								int deadlock_timeout;

								if (!info.command.deadlock_timeout.HasValue) {
									deadlock_timeout = Configuration.NoOutputTimeout;
								} else {
									deadlock_timeout = info.command.deadlock_timeout.Value;
								}

								if ((DateTime.Now > local_starttime.AddMinutes (info.command.timeout))) {
									timedout = true;
									timeoutReason = string.Format ("The build step '{0}' didn't finish in {1} minute(s).", info.command.command, info.command.timeout);
								} else if (deadlock_timeout > 0 && log.LastStamp.AddMinutes (deadlock_timeout) <= DateTime.Now) {
									timedout = true;
									timeoutReason = string.Format ("The build step '{0}' has had no output for {1} minute(s).", info.command.command, deadlock_timeout);
								}

								if (!timedout)
									continue;

								try {
									result = DBState.Timeout;
									exitcode = 255;
									Builder.log.ErrorFormat ("{0} {1}", info.number, timeoutReason);
									log.WriteLine ("\n*** Timed out. Proceeding to get stack traces for all child processes. ***");
									log.WriteLine ("\n*** " + timeoutReason + " *** \n");
									p.Terminate (log);
									log.WriteLine ("\n * " + timeoutReason + " * \n");
								} catch (Exception ex) {
									Builder.log.ErrorFormat ("{1} Exception while terminating build step: {0}", ex, info.number);
								}
								break;
							}

							// Sleep a bit so that the process has enough time to finish
							System.Threading.Thread.Sleep (1000);

							process_reader.Join ();

							if (p.HasExited) {
								exitcode = p.ExitCode;
							} else {
								Builder.log.ErrorFormat ("{1} Step: {0}: the process didn't exit in time.", command.command, info.number);
								exitcode = 1;
							}
							if (result == DBState.Executing) {
								if (exitcode == 0) {
									result = DBState.Success;
								} else {
									result = DBState.Failed;
								}
							} else if (result == DBState.Aborted) {
								result = DBState.Failed;
							}
						}
					}
				}

				info.work.State = result;
				using (TextReader reader = new StreamReader (new FileStream (log_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
					info.work.CalculateSummary (reader);
				info.work.endtime = DBRecord.DatabaseNow;
				response = WebService.ReportBuildStateSafe (info.work);
				info.work = response.Work;

				info.progress_stamp = DateTime.Now;
				try {
					WebService.UploadFilesSafe (info.work, new string [] { log_file }, new bool [] { false });

					// Gather files from logged commands and from the upload_files glob
					CheckLog (log_file, info);
					if (!string.IsNullOrEmpty (info.command.upload_files)) {
						foreach (string glob in info.command.upload_files.Split (',')) {
							// TODO: handle globs in directory parts
							// TODO: allow absolute paths?
							log.InfoFormat ("Uploading files from glob {0}", glob);
							info.progress_stamp = DateTime.Now;
							// TODO: allow hidden files also
							WebService.UploadFilesSafe (info.work, Directory.GetFiles (Path.Combine (info.BUILDER_DATA_SOURCE_DIR, Path.GetDirectoryName (glob)), Path.GetFileName (glob)), null);
						}
					}
				} finally {
					if (response.RevisionWorkCompleted) {
						// Cleanup after us.
						string base_dir = Configuration.GetDataRevisionDir (info.lane.id, info.revision.revision);
						FileUtilities.TryDeleteDirectoryRecursive (base_dir);
					}
				}

				Builder.log.InfoFormat ("{4} Revision {0}, executed step '{1}' in {2} s, ExitCode: {3}, State: {4}", info.revision.revision, info.command.command, info.work.duration, exitcode, info.number, info.work.State);
			} catch (Exception ex) {
				info.work.State = DBState.Failed;
				info.work.summary = ex.Message;
				info.work = WebService.ReportBuildStateSafe (info.work).Work;
				log.ErrorFormat ("{3} Revision {0}, got exception '{1}': \n{2}", info.revision.revision, ex.Message, ex.StackTrace, info.number);
				throw;
			} finally {
				log.DebugFormat ("{0} Builder finished thread for sequence {0}", info.number);

				if ((info.work.State == DBState.Failed || info.work.State == DBState.Aborted || info.work.State == DBState.Timeout) && !info.command.nonfatal) {
					failed_hostlanes.Add (info.hostlane);
				}
			}
		}