private static int Main2 (string [] arguments) { Lock process_lock; try { if (!Configuration.LoadConfiguration (arguments)) return 1; if (!Configuration.VerifyBuildBotConfiguration ()) return 1; process_lock = Lock.Create ("MonkeyWrench.Builder"); if (process_lock == null) { Logger.Log ("Builder could not acquire lock. Exiting"); return 1; } Logger.Log ("Builder lock aquired successfully."); } catch (Exception ex) { Logger.Log ("Could not aquire lock: {0}", ex.Message); return 1; } try { WebService = WebServices.Create (); WebService.CreateLogin (Configuration.Host, Configuration.WebServicePassword); response = WebService.GetBuildInfoMultiple (WebService.WebServiceLogin, Configuration.Host, true); if (!response.Host.enabled) { Logger.Log ("This host is disabled. Exiting."); return 0; } Logger.Log ("Builder will now build {0} lists of work items.", response.Work.Count); for (int i = 0; i < response.Work.Count; i++) { //foreach (var item in response.Work) Logger.Log ("Building list #{0}/{1}", i+ 1, response.Work.Count); Build (response.Work [i]);//item); } Logger.Log ("Builder finished successfully."); return 0; } catch (Exception ex) { Logger.Log ("An exception occurred: {0}", ex.ToString ()); return 1; } finally { process_lock.Unlock (); } }
public GetBuildInfoResponse GetBuildInfoMultiple (WebServiceLogin login, string host, bool multiple_work) { List<DBHost> hosts = new List<DBHost> (); // list of hosts to find work for List<DBHostLane> hostlanes = new List<DBHostLane> (); List<DBLane> lanes = new List<DBLane> (); GetBuildInfoResponse response = new GetBuildInfoResponse (); response.Work = new List<List<BuildInfoEntry>> (); using (DB db = new DB ()) { VerifyUserInRole (db, login, Roles.BuildBot, true); response.Host = FindHost (db, null, host); if (!response.Host.enabled) return response; // find the master hosts for this host (if any) response.MasterHosts = FindMasterHosts (db, response.Host); // get the hosts to find work for if (response.MasterHosts != null && response.MasterHosts.Count > 0) { foreach (DBMasterHost mh in response.MasterHosts) hosts.Add (DBHost_Extensions.Create (db, mh.master_host_id)); } else { hosts.Add (response.Host); } // find the enabled hostlane combinations for these hosts using (IDbCommand cmd = db.CreateCommand ()) { cmd.CommandText = "SELECT HostLane.* FROM HostLane INNER JOIN Lane ON Lane.id = HostLane.lane_id WHERE Lane.enabled = TRUE AND HostLane.enabled = TRUE AND ("; for (int i = 0; i < hosts.Count; i++) { if (i > 0) cmd.CommandText += " OR "; cmd.CommandText += " HostLane.host_id = " + hosts [i].id; } cmd.CommandText += ")"; using (IDataReader reader = cmd.ExecuteReader ()) { while (reader.Read ()) hostlanes.Add (new DBHostLane (reader)); } } if (hostlanes.Count == 0) return response; // nothing to do here lanes = db.GetAllLanes (); switch (response.Host.QueueManagement) { case DBQueueManagement.OneRevisionWorkAtATime: if (hostlanes.Count > 1) { int latest = -1; DateTime latest_date = DateTime.MaxValue; // we need to find the latest revisionwork each hostlane has completed. // we want to work on the hostlane which has waited the longest amount // of time without getting work done (but which has pending work to do). for (int i = 0; i < hostlanes.Count; i++) { DBHostLane hl = hostlanes [i]; // check if this hostlane has pending work. // this would ideally be included in the query below, but I'm not sure // how to do that while still distinguising the case where nothing has // been done ever for a hostlane. using (IDbCommand cmd = db.CreateCommand ()) { cmd.CommandText = @" SELECT RevisionWork.id FROM RevisionWork WHERE RevisionWork.host_id = @host_id AND (RevisionWork.workhost_id = @workhost_id OR RevisionWork.workhost_id IS NULL) AND RevisionWork.completed = false AND RevisionWork.state <> 9 AND RevisionWork.state <> 10 AND RevisionWork.state <> 11 AND lane_id = @lane_id LIMIT 1; "; DB.CreateParameter (cmd, "lane_id", hl.lane_id); DB.CreateParameter (cmd, "host_id", hl.host_id); DB.CreateParameter (cmd, "workhost_id", response.Host.id); object obj = cmd.ExecuteScalar (); if (obj == DBNull.Value || obj == null) { // there is nothing to do for this hostlane continue; } } // find the latest completed (this may not be correct, maybe find the latest unstarted?) // revisionwork for this hostlane. using (IDbCommand cmd = db.CreateCommand ()) { cmd.CommandText = @" SELECT RevisionWork.endtime FROM RevisionWork WHERE RevisionWork.host_id = @host_id AND (RevisionWork.workhost_id = @workhost_id OR RevisionWork.workhost_id IS NULL) AND RevisionWork.completed = true AND lane_id = @lane_id ORDER BY RevisionWork.endtime DESC LIMIT 1; "; DB.CreateParameter (cmd, "lane_id", hl.lane_id); DB.CreateParameter (cmd, "host_id", hl.host_id); DB.CreateParameter (cmd, "workhost_id", response.Host.id); object obj = cmd.ExecuteScalar (); if (obj is DateTime) { DateTime dt = (DateTime) obj; if (dt < latest_date) { latest_date = dt; latest = i; } } else { // nothing has ever been done for this hostlane. latest_date = DateTime.MinValue; latest = i; } } } if (latest >= 0) { DBHostLane tmp = hostlanes [latest]; hostlanes.Clear (); hostlanes.Add (tmp); } else { hostlanes.Clear (); // there is nothing to do at all } } break; } foreach (DBHostLane hl in hostlanes) { int counter = 10; DBRevisionWork revisionwork; DBLane lane = null; DBHost masterhost = null; foreach (DBLane l in lanes) { if (l.id == hl.lane_id) { lane = l; break; } } foreach (DBHost hh in hosts) { if (hh.id == hl.host_id) { masterhost = hh; break; } } do { revisionwork = db.GetRevisionWork (lane, masterhost, response.Host); if (revisionwork == null) break; } while (!revisionwork.SetWorkHost (db, response.Host) && counter-- > 0); if (revisionwork == null) continue; if (!revisionwork.workhost_id.HasValue || revisionwork.workhost_id != response.Host.id) continue; // couldn't lock this revisionwork. log.DebugFormat ("Found work for host {0} {4}: {1} (lane: {2} {3})", response.Host.id, revisionwork.id, revisionwork.lane_id, lane.lane, response.Host.host); DBRevision revision = DBRevision_Extensions.Create (db, revisionwork.revision_id); List<DBWorkFile> files_to_download = null; List<DBLane> dependent_lanes = null; // get dependent files List<DBLaneDependency> dependencies = lane.GetDependencies (db); if (dependencies != null && dependencies.Count > 0) { foreach (DBLaneDependency dep in dependencies) { DBLane dependent_lane; DBHost dependent_host; DBRevisionWork dep_revwork; List<DBWorkFile> work_files; if (string.IsNullOrEmpty (dep.download_files)) continue; dependent_lane = DBLane_Extensions.Create (db, dep.dependent_lane_id); dependent_host = dep.dependent_host_id.HasValue ? DBHost_Extensions.Create (db, dep.dependent_host_id.Value) : null; DBRevision dep_lane_rev = dependent_lane.FindRevision (db, revision.revision); if (dep_lane_rev == null) continue; /* Something bad happened: the lane we're dependent on does not have the same revisions we have */ dep_revwork = DBRevisionWork_Extensions.Find (db, dependent_lane, dependent_host, dep_lane_rev); work_files = dep_revwork.GetFiles (db); foreach (DBWorkFile file in work_files) { bool download = true; foreach (string exp in dep.download_files.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries)) { if (!System.Text.RegularExpressions.Regex.IsMatch (file.filename, FileUtilities.GlobToRegExp (exp))) { download = false; break; } } if (!download) continue; if (files_to_download == null) { files_to_download = new List<DBWorkFile> (); dependent_lanes = new List<DBLane> (); } files_to_download.Add (file); dependent_lanes.Add (dependent_lane); } } } List<DBWorkView2> pending_work = revisionwork.GetNextWork (db, lane, masterhost, revision, multiple_work); if (pending_work == null || pending_work.Count == 0) continue; List<DBEnvironmentVariable> environment_variables = null; using (IDbCommand cmd = db.CreateCommand ()) { foreach (int li in db.GetLaneHierarchy (lane.id)) { cmd.CommandText += string.Format (@" SELECT * FROM EnvironmentVariable WHERE (host_id = {0} OR host_id = {1} OR host_id IS NULL) AND (lane_id = {2} OR lane_id IS NULL) ORDER BY id; ;", revisionwork.workhost_id, revisionwork.host_id, li); } using (IDataReader reader = cmd.ExecuteReader ()) { var set = new HashSet<string> (); do { while (reader.Read ()) { if (environment_variables == null) environment_variables = new List<DBEnvironmentVariable> (); var ev = new DBEnvironmentVariable (reader); if (!set.Contains (ev.name)) { environment_variables.Add (ev); set.Add (ev.name); } } } while (reader.NextResult ()); } } DBHost host_being_worked_for = hosts.Find (h => h.id == revisionwork.host_id); foreach (DBWorkView2 work in pending_work) { BuildInfoEntry entry = new BuildInfoEntry (); entry.Lane = lane; entry.HostLane = hl; entry.Revision = revision; entry.Command = DBCommand_Extensions.Create (db, work.command_id); entry.FilesToDownload = files_to_download; entry.DependentLaneOfFiles = dependent_lanes; entry.Work = DBWork_Extensions.Create (db, work.id); entry.LaneFiles = lane.GetFiles (db, lanes); entry.EnvironmentVariables = environment_variables; entry.Host = host_being_worked_for; // TODO: put work with the same sequence number into one list of entries. List<BuildInfoEntry> entries = new List<BuildInfoEntry> (); entries.Add (entry); response.Work.Add (entries); } // Notify that the revision is assigned var notifyInfo = new GenericNotificationInfo (); notifyInfo.laneID = revisionwork.lane_id; notifyInfo.hostID = revisionwork.host_id; notifyInfo.revisionID = revisionwork.revision_id; notifyInfo.message = String.Format("Assigned to host '{0}' ({1})", response.Host.host, response.Host.id); notifyInfo.state = DBState.Executing; Notifications.NotifyGeneric (notifyInfo); } } return response; }
private static int Main2 (string [] arguments) { Lock process_lock; ReportBuildBotStatusResponse status_response; BuildBotStatus status; try { if (!Configuration.LoadConfiguration (arguments)) return 1; if (!Configuration.VerifyBuildBotConfiguration ()) return 1; process_lock = Lock.Create ("MonkeyWrench.Builder"); if (process_lock == null) { Logger.Log ("Builder could not acquire lock. Exiting"); return 1; } Logger.Log ("Builder lock aquired successfully."); } catch (Exception ex) { Logger.Log ("Could not aquire lock: {0}", ex.Message); return 1; } try { WebService = WebServices.Create (); WebService.CreateLogin (Configuration.Host, Configuration.WebServicePassword); status = new BuildBotStatus (); status.Host = Configuration.Host; status.FillInAssemblyAttributes (); status_response = WebService.ReportBuildBotStatus (WebService.WebServiceLogin, status); if (status_response.Exception != null) { Logger.Log ("Failed to report status: {0}", status_response.Exception.Message); return 1; } if (!string.IsNullOrEmpty (status_response.ConfiguredVersion) && status_response.ConfiguredVersion != status.AssemblyVersion) { if (!Update (status, status_response)) { Console.Error.WriteLine ("Automatic update to: {0} / {1} failed (see log for details). Please update manually.", status_response.ConfiguredVersion, status_response.ConfiguredRevision); return 2; /* Magic return code that means "automatic update failed" */ } else { Console.WriteLine ("The builder has been updated. Please run 'make build' again."); return 1; } } response = WebService.GetBuildInfoMultiple (WebService.WebServiceLogin, Configuration.Host, true); if (!response.Host.enabled) { Logger.Log ("This host is disabled. Exiting."); return 0; } Logger.Log ("Builder will now build {0} lists of work items.", response.Work.Count); for (int i = 0; i < response.Work.Count; i++) { //foreach (var item in response.Work) Logger.Log ("Building list #{0}/{1}", i+ 1, response.Work.Count); Build (response.Work [i]);//item); } Logger.Log ("Builder finished successfully."); return 0; } catch (Exception ex) { Logger.Log ("An exception occurred: {0}", ex.ToString ()); return 1; } finally { process_lock.Unlock (); } }
private static void Build (BuildInfo info) { try { object sync_object = new object (); string log_file = Path.Combine (info.BUILDER_DATA_LOG_DIR, info.command.command + ".log"); DateTime last_stamp = DateTime.Now; DateTime local_starttime = DateTime.Now; DBState result; DBCommand command = info.command; int exitcode = 0; ReportBuildStateResponse response; Logger.Log ("{0} Builder started new thread for sequence {1} step {2}", info.number, info.command.sequence, info.command.command); /* Check if step has been aborted already */ info.work.State = WebService.GetWorkStateSafe (info.work); if (info.work.State != DBState.NotDone && info.work.State != DBState.Executing) { /* If we're in an executing state, we're restarting a command for whatever reason (crash, reboot, etc) */ Logger.Log ("{0} Builder found that step {1} is not ready to start, it's in a '{2}' state", info.number, info.command.command, info.work.State); return; } result = DBState.Executing; info.work.starttime = DBRecord.DatabaseNow; info.work.State = result; info.work.host_id = info.host.id; info.work = WebService.ReportBuildStateSafe (info.work).Work; using (Job p = ProcessHelper.CreateJob ()) { using (FileStream fs = new FileStream (log_file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) { using (StreamWriter log = new StreamWriter (fs)) { p.StartInfo.FileName = info.command.filename; p.StartInfo.Arguments = string.Format (info.command.arguments, Path.Combine (info.temp_dir, info.command.command), info.temp_dir); if (!string.IsNullOrEmpty (info.command.working_directory)) p.StartInfo.WorkingDirectory = Path.Combine (info.BUILDER_DATA_SOURCE_DIR, info.command.working_directory); else p.StartInfo.WorkingDirectory = info.BUILDER_DATA_SOURCE_DIR; p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardError = true; p.StartInfo.RedirectStandardOutput = true; // set environment variables p.StartInfo.EnvironmentVariables ["BUILD_LANE"] = info.lane.lane; p.StartInfo.EnvironmentVariables ["BUILD_COMMAND"] = info.command.command; p.StartInfo.EnvironmentVariables ["BUILD_REVISION"] = info.revision.revision; p.StartInfo.EnvironmentVariables ["BUILD_INSTALL"] = Configuration.CygwinizePath (info.BUILDER_DATA_INSTALL_DIR); p.StartInfo.EnvironmentVariables ["BUILD_DATA_LANE"] = Configuration.GetDataLane (info.lane.lane); p.StartInfo.EnvironmentVariables ["BUILD_DATA_SOURCE"] = info.BUILDER_DATA_SOURCE_DIR; p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY"] = info.lane.repository; p.StartInfo.EnvironmentVariables ["BUILD_HOST"] = Configuration.Host; p.StartInfo.EnvironmentVariables ["BUILD_WORK_HOST"] = info.host_being_worked_for; p.StartInfo.EnvironmentVariables ["BUILD_LANE_MAX_REVISION"] = info.lane.max_revision; p.StartInfo.EnvironmentVariables ["BUILD_LANE_MIN_REVISION"] = info.lane.min_revision; p.StartInfo.EnvironmentVariables ["BUILD_LANE_COMMIT_FILTER"] = info.lane.commit_filter; int r = 0; foreach (string repo in info.lane.repository.Split (',')) { p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_" + r.ToString ()] = repo; r++; } p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_SPACE"] = info.lane.repository.Replace (',', ' '); p.StartInfo.EnvironmentVariables ["BUILD_SEQUENCE"] = "0"; p.StartInfo.EnvironmentVariables ["BUILD_SCRIPT_DIR"] = info.temp_dir; p.StartInfo.EnvironmentVariables ["LD_LIBRARY_PATH"] = Configuration.CygwinizePath (Configuration.GetLdLibraryPath (info.lane.lane, info.revision.revision)); p.StartInfo.EnvironmentVariables ["PKG_CONFIG_PATH"] = Configuration.CygwinizePath (Configuration.GetPkgConfigPath (info.lane.lane, info.revision.revision)); p.StartInfo.EnvironmentVariables ["PATH"] = Configuration.CygwinizePath (Configuration.GetPath (info.lane.lane, info.revision.revision)); p.StartInfo.EnvironmentVariables ["C_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCIncludePath (info.lane.lane, info.revision.revision)); p.StartInfo.EnvironmentVariables ["CPLUS_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCPlusIncludePath (info.lane.lane, info.revision.revision)); // We need to remove all paths from environment variables that were // set for this executable to work so that they don't mess with // whatever we're trying to execute string [] bot_dependencies = new string [] { "PATH", "LD_LIBRARY_PATH", "PKG_CONFIG_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "AC_LOCAL_PATH", "MONO_PATH" }; foreach (string bot_dependency in bot_dependencies) { if (!p.StartInfo.EnvironmentVariables.ContainsKey (bot_dependency)) continue; List<string> paths = new List<string> (p.StartInfo.EnvironmentVariables [bot_dependency].Split (new char [] { ':' /* XXX: does windows use ';' here? */}, StringSplitOptions.None)); for (int i = paths.Count - 1; i >= 0; i--) { if (paths [i].Contains ("bot-dependencies")) paths.RemoveAt (i); } p.StartInfo.EnvironmentVariables [bot_dependency] = string.Join (":", paths.ToArray ()); } if (info.environment_variables != null) { // order is important here, we need to loop over the array in the same order get got the variables. for (int e = 0; e < info.environment_variables.Count; e++) { info.environment_variables [e].Evaluate (p.StartInfo.EnvironmentVariables); } } Thread stdout_thread = new Thread (delegate () { try { string line; while (null != (line = p.StandardOutput.ReadLine ())) { lock (sync_object) { log.WriteLine (line); log.Flush (); } last_stamp = DateTime.Now; } } catch (Exception ex) { Logger.Log ("{1} Stdin reader thread got exception: {0}", ex.Message, info.number); } }); Thread stderr_thread = new Thread (delegate () { try { string line; while (null != (line = p.StandardError.ReadLine ())) { lock (sync_object) { log.WriteLine (line); log.Flush (); } last_stamp = DateTime.Now; } } catch (Exception ex) { Logger.Log ("{1} Stderr reader thread got exception: {0}", ex.Message, info.number); } }); p.Start (); stderr_thread.Start (); stdout_thread.Start (); while (!p.WaitForExit (60000 /* 1 minute */)) { if (p.HasExited) break; // Check if step has been aborted. info.work.State = WebService.GetWorkStateSafe (info.work); if (info.work.State == DBState.Aborted) { result = DBState.Aborted; try { exitcode = 255; Logger.Log ("{1} The build step '{0}' has been aborted, terminating it.", info.command.command, info.number); p.Terminate (); log.WriteLine ("{1} The build step '{0}' was aborted, terminated it.", info.command.command, info.number); } catch (Exception ex) { Logger.Log ("{1} Exception while killing build step: {0}", ex.ToString (), info.number); } break; } // Check if step has timedout bool timedout = false; string timeoutReason = null; int timeout = 15; if ((DateTime.Now > local_starttime.AddMinutes (info.command.timeout))) { timedout = true; timeoutReason = string.Format ("The build step '{0}' didn't finish in {1} minute(s).", info.command.command, info.command.timeout); } else if (last_stamp.AddMinutes (timeout) <= DateTime.Now) { timedout = true; timeoutReason = string.Format ("The build step '{0}' has had no output for {1} minute(s).", info.command.command, timeout); } if (!timedout) continue; try { result = DBState.Timeout; exitcode = 255; Logger.Log ("{0} {1}", info.number, timeoutReason); p.Terminate (); log.WriteLine (timeoutReason); } catch (Exception ex) { Logger.Log ("{1} Exception while terminating build step: {0}", ex.ToString (), info.number); } break; } // Sleep a bit so that the process has enough time to finish System.Threading.Thread.Sleep (1000); if (!stdout_thread.Join (TimeSpan.FromSeconds (15))) { Logger.Log ("Waited 15s for stdout thread to finish, but it didn't"); } if (!stderr_thread.Join (TimeSpan.FromSeconds (15))) { Logger.Log ("Waited 15s for stderr thread to finish, but it didn't"); } if (p.HasExited) { exitcode = p.ExitCode; } else { Logger.Log ("{1} Step: {0}: the process didn't exit in time.", command.command, info.number); exitcode = 1; } if (result == DBState.Executing) { if (exitcode == 0) { result = DBState.Success; } else { result = DBState.Failed; } } else if (result == DBState.Aborted) { result = DBState.Failed; } } } } info.work.State = result; using (TextReader reader = new StreamReader (new FileStream (log_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) info.work.CalculateSummary (reader); info.work.endtime = DBRecord.DatabaseNow; response = WebService.ReportBuildStateSafe (info.work); info.work = response.Work; WebService.UploadFilesSafe (info.work, new string [] { log_file }, new bool [] { false }); // Gather files from logged commands and from the upload_files glob CheckLog (log_file, info); if (!string.IsNullOrEmpty (info.command.upload_files)) { foreach (string glob in info.command.upload_files.Split (',')) { // TODO: handle globs in directory parts // TODO: allow absolute paths? Logger.Log ("Uploading files from glob {0}", glob); // TODO: allow hidden files also WebService.UploadFilesSafe (info.work, Directory.GetFiles (Path.Combine (info.BUILDER_DATA_SOURCE_DIR, Path.GetDirectoryName (glob)), Path.GetFileName (glob)), null); } } if (response.RevisionWorkCompleted) { // Cleanup after us. string base_dir = Configuration.GetDataRevisionDir (info.lane.lane, info.revision.revision); FileUtilities.TryDeleteDirectoryRecursive (base_dir); } Logger.Log ("{4} Revision {0}, executed step '{1}' in {2} s, ExitCode: {3}, State: {4}", info.revision.revision, info.command.command, info.work.duration, exitcode, info.number, info.work.State); } catch (Exception ex) { info.work.State = DBState.Failed; info.work.summary = ex.Message; info.work = WebService.ReportBuildStateSafe (info.work).Work; Logger.Log ("{3} Revision {0}, got exception '{1}': \n{2}", info.revision.revision, ex.Message, ex.StackTrace, info.number); throw; } finally { Logger.Log ("{0} Builder finished thread for sequence {0}", info.number); if ((info.work.State == DBState.Failed || info.work.State == DBState.Aborted || info.work.State == DBState.Timeout) && !info.command.nonfatal) { failed_hostlanes.Add (info.hostlane); } } }