private static int Main2 (string [] arguments)
{
Lock process_lock;
try {
if (!Configuration.LoadConfiguration (arguments))
return 1;
if (!Configuration.VerifyBuildBotConfiguration ())
return 1;
process_lock = Lock.Create ("MonkeyWrench.Builder");
if (process_lock == null) {
Logger.Log ("Builder could not acquire lock. Exiting");
return 1;
}
Logger.Log ("Builder lock aquired successfully.");
} catch (Exception ex) {
Logger.Log ("Could not aquire lock: {0}", ex.Message);
return 1;
}
try {
WebService = WebServices.Create ();
WebService.CreateLogin (Configuration.Host, Configuration.WebServicePassword);
response = WebService.GetBuildInfoMultiple (WebService.WebServiceLogin, Configuration.Host, true);
if (!response.Host.enabled) {
Logger.Log ("This host is disabled. Exiting.");
return 0;
}
Logger.Log ("Builder will now build {0} lists of work items.", response.Work.Count);
for (int i = 0; i < response.Work.Count; i++) {
//foreach (var item in response.Work)
Logger.Log ("Building list #{0}/{1}", i+ 1, response.Work.Count);
Build (response.Work [i]);//item);
}
Logger.Log ("Builder finished successfully.");
return 0;
} catch (Exception ex) {
Logger.Log ("An exception occurred: {0}", ex.ToString ());
return 1;
} finally {
process_lock.Unlock ();
}
}
public GetBuildInfoResponse GetBuildInfoMultiple (WebServiceLogin login, string host, bool multiple_work)
{
List<DBHost> hosts = new List<DBHost> (); // list of hosts to find work for
List<DBHostLane> hostlanes = new List<DBHostLane> ();
List<DBLane> lanes = new List<DBLane> ();
GetBuildInfoResponse response = new GetBuildInfoResponse ();
response.Work = new List<List<BuildInfoEntry>> ();
using (DB db = new DB ()) {
VerifyUserInRole (db, login, Roles.BuildBot, true);
response.Host = FindHost (db, null, host);
if (!response.Host.enabled)
return response;
// find the master hosts for this host (if any)
response.MasterHosts = FindMasterHosts (db, response.Host);
// get the hosts to find work for
if (response.MasterHosts != null && response.MasterHosts.Count > 0) {
foreach (DBMasterHost mh in response.MasterHosts)
hosts.Add (DBHost_Extensions.Create (db, mh.master_host_id));
} else {
hosts.Add (response.Host);
}
// find the enabled hostlane combinations for these hosts
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = "SELECT HostLane.* FROM HostLane INNER JOIN Lane ON Lane.id = HostLane.lane_id WHERE Lane.enabled = TRUE AND HostLane.enabled = TRUE AND (";
for (int i = 0; i < hosts.Count; i++) {
if (i > 0)
cmd.CommandText += " OR ";
cmd.CommandText += " HostLane.host_id = " + hosts [i].id;
}
cmd.CommandText += ")";
using (IDataReader reader = cmd.ExecuteReader ()) {
while (reader.Read ())
hostlanes.Add (new DBHostLane (reader));
}
}
if (hostlanes.Count == 0)
return response; // nothing to do here
lanes = db.GetAllLanes ();
switch (response.Host.QueueManagement) {
case DBQueueManagement.OneRevisionWorkAtATime:
if (hostlanes.Count > 1) {
int latest = -1;
DateTime latest_date = DateTime.MaxValue;
// we need to find the latest revisionwork each hostlane has completed.
// we want to work on the hostlane which has waited the longest amount
// of time without getting work done (but which has pending work to do).
for (int i = 0; i < hostlanes.Count; i++) {
DBHostLane hl = hostlanes [i];
// check if this hostlane has pending work.
// this would ideally be included in the query below, but I'm not sure
// how to do that while still distinguising the case where nothing has
// been done ever for a hostlane.
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = @"
SELECT RevisionWork.id
FROM RevisionWork
WHERE
RevisionWork.host_id = @host_id
AND (RevisionWork.workhost_id = @workhost_id OR RevisionWork.workhost_id IS NULL)
AND RevisionWork.completed = false
AND RevisionWork.state <> 9 AND RevisionWork.state <> 10 AND RevisionWork.state <> 11
AND lane_id = @lane_id
LIMIT 1;
";
DB.CreateParameter (cmd, "lane_id", hl.lane_id);
DB.CreateParameter (cmd, "host_id", hl.host_id);
DB.CreateParameter (cmd, "workhost_id", response.Host.id);
object obj = cmd.ExecuteScalar ();
if (obj == DBNull.Value || obj == null) {
// there is nothing to do for this hostlane
continue;
}
}
// find the latest completed (this may not be correct, maybe find the latest unstarted?)
// revisionwork for this hostlane.
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = @"
SELECT RevisionWork.endtime
FROM RevisionWork
WHERE
RevisionWork.host_id = @host_id
AND (RevisionWork.workhost_id = @workhost_id OR RevisionWork.workhost_id IS NULL)
AND RevisionWork.completed = true
AND lane_id = @lane_id
ORDER BY RevisionWork.endtime DESC
LIMIT 1;
";
DB.CreateParameter (cmd, "lane_id", hl.lane_id);
DB.CreateParameter (cmd, "host_id", hl.host_id);
DB.CreateParameter (cmd, "workhost_id", response.Host.id);
object obj = cmd.ExecuteScalar ();
if (obj is DateTime) {
DateTime dt = (DateTime) obj;
if (dt < latest_date) {
latest_date = dt;
latest = i;
}
} else {
// nothing has ever been done for this hostlane.
latest_date = DateTime.MinValue;
latest = i;
}
}
}
if (latest >= 0) {
DBHostLane tmp = hostlanes [latest];
hostlanes.Clear ();
hostlanes.Add (tmp);
} else {
hostlanes.Clear (); // there is nothing to do at all
}
}
break;
}
foreach (DBHostLane hl in hostlanes) {
int counter = 10;
DBRevisionWork revisionwork;
DBLane lane = null;
DBHost masterhost = null;
foreach (DBLane l in lanes) {
if (l.id == hl.lane_id) {
lane = l;
break;
}
}
foreach (DBHost hh in hosts) {
if (hh.id == hl.host_id) {
masterhost = hh;
break;
}
}
do {
revisionwork = db.GetRevisionWork (lane, masterhost, response.Host);
if (revisionwork == null)
break;
} while (!revisionwork.SetWorkHost (db, response.Host) && counter-- > 0);
if (revisionwork == null)
continue;
if (!revisionwork.workhost_id.HasValue || revisionwork.workhost_id != response.Host.id)
continue; // couldn't lock this revisionwork.
log.DebugFormat ("Found work for host {0} {4}: {1} (lane: {2} {3})", response.Host.id, revisionwork.id, revisionwork.lane_id, lane.lane, response.Host.host);
DBRevision revision = DBRevision_Extensions.Create (db, revisionwork.revision_id);
List<DBWorkFile> files_to_download = null;
List<DBLane> dependent_lanes = null;
// get dependent files
List<DBLaneDependency> dependencies = lane.GetDependencies (db);
if (dependencies != null && dependencies.Count > 0) {
foreach (DBLaneDependency dep in dependencies) {
DBLane dependent_lane;
DBHost dependent_host;
DBRevisionWork dep_revwork;
List<DBWorkFile> work_files;
if (string.IsNullOrEmpty (dep.download_files))
continue;
dependent_lane = DBLane_Extensions.Create (db, dep.dependent_lane_id);
dependent_host = dep.dependent_host_id.HasValue ? DBHost_Extensions.Create (db, dep.dependent_host_id.Value) : null;
DBRevision dep_lane_rev = dependent_lane.FindRevision (db, revision.revision);
if (dep_lane_rev == null)
continue; /* Something bad happened: the lane we're dependent on does not have the same revisions we have */
dep_revwork = DBRevisionWork_Extensions.Find (db, dependent_lane, dependent_host, dep_lane_rev);
work_files = dep_revwork.GetFiles (db);
foreach (DBWorkFile file in work_files) {
bool download = true;
foreach (string exp in dep.download_files.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries)) {
if (!System.Text.RegularExpressions.Regex.IsMatch (file.filename, FileUtilities.GlobToRegExp (exp))) {
download = false;
break;
}
}
if (!download)
continue;
if (files_to_download == null) {
files_to_download = new List<DBWorkFile> ();
dependent_lanes = new List<DBLane> ();
}
files_to_download.Add (file);
dependent_lanes.Add (dependent_lane);
}
}
}
List<DBWorkView2> pending_work = revisionwork.GetNextWork (db, lane, masterhost, revision, multiple_work);
if (pending_work == null || pending_work.Count == 0)
continue;
List<DBEnvironmentVariable> environment_variables = null;
using (IDbCommand cmd = db.CreateCommand ()) {
foreach (int li in db.GetLaneHierarchy (lane.id)) {
cmd.CommandText += string.Format (@"
SELECT *
FROM EnvironmentVariable
WHERE
(host_id = {0} OR host_id = {1} OR host_id IS NULL) AND (lane_id = {2} OR lane_id IS NULL)
ORDER BY id;
;", revisionwork.workhost_id, revisionwork.host_id, li);
}
using (IDataReader reader = cmd.ExecuteReader ()) {
var set = new HashSet<string> ();
do {
while (reader.Read ()) {
if (environment_variables == null)
environment_variables = new List<DBEnvironmentVariable> ();
var ev = new DBEnvironmentVariable (reader);
if (!set.Contains (ev.name)) {
environment_variables.Add (ev);
set.Add (ev.name);
}
}
} while (reader.NextResult ());
}
}
DBHost host_being_worked_for = hosts.Find (h => h.id == revisionwork.host_id);
foreach (DBWorkView2 work in pending_work) {
BuildInfoEntry entry = new BuildInfoEntry ();
entry.Lane = lane;
entry.HostLane = hl;
entry.Revision = revision;
entry.Command = DBCommand_Extensions.Create (db, work.command_id);
entry.FilesToDownload = files_to_download;
entry.DependentLaneOfFiles = dependent_lanes;
entry.Work = DBWork_Extensions.Create (db, work.id);
entry.LaneFiles = lane.GetFiles (db, lanes);
entry.EnvironmentVariables = environment_variables;
entry.Host = host_being_worked_for;
// TODO: put work with the same sequence number into one list of entries.
List<BuildInfoEntry> entries = new List<BuildInfoEntry> ();
entries.Add (entry);
response.Work.Add (entries);
}
// Notify that the revision is assigned
var notifyInfo = new GenericNotificationInfo ();
notifyInfo.laneID = revisionwork.lane_id;
notifyInfo.hostID = revisionwork.host_id;
notifyInfo.revisionID = revisionwork.revision_id;
notifyInfo.message = String.Format("Assigned to host '{0}' ({1})", response.Host.host, response.Host.id);
notifyInfo.state = DBState.Executing;
Notifications.NotifyGeneric (notifyInfo);
}
}
return response;
}
private static int Main2 (string [] arguments)
{
Lock process_lock;
ReportBuildBotStatusResponse status_response;
BuildBotStatus status;
try {
if (!Configuration.LoadConfiguration (arguments))
return 1;
if (!Configuration.VerifyBuildBotConfiguration ())
return 1;
process_lock = Lock.Create ("MonkeyWrench.Builder");
if (process_lock == null) {
Logger.Log ("Builder could not acquire lock. Exiting");
return 1;
}
Logger.Log ("Builder lock aquired successfully.");
} catch (Exception ex) {
Logger.Log ("Could not aquire lock: {0}", ex.Message);
return 1;
}
try {
WebService = WebServices.Create ();
WebService.CreateLogin (Configuration.Host, Configuration.WebServicePassword);
status = new BuildBotStatus ();
status.Host = Configuration.Host;
status.FillInAssemblyAttributes ();
status_response = WebService.ReportBuildBotStatus (WebService.WebServiceLogin, status);
if (status_response.Exception != null) {
Logger.Log ("Failed to report status: {0}", status_response.Exception.Message);
return 1;
}
if (!string.IsNullOrEmpty (status_response.ConfiguredVersion) && status_response.ConfiguredVersion != status.AssemblyVersion) {
if (!Update (status, status_response)) {
Console.Error.WriteLine ("Automatic update to: {0} / {1} failed (see log for details). Please update manually.", status_response.ConfiguredVersion, status_response.ConfiguredRevision);
return 2; /* Magic return code that means "automatic update failed" */
} else {
Console.WriteLine ("The builder has been updated. Please run 'make build' again.");
return 1;
}
}
response = WebService.GetBuildInfoMultiple (WebService.WebServiceLogin, Configuration.Host, true);
if (!response.Host.enabled) {
Logger.Log ("This host is disabled. Exiting.");
return 0;
}
Logger.Log ("Builder will now build {0} lists of work items.", response.Work.Count);
for (int i = 0; i < response.Work.Count; i++) {
//foreach (var item in response.Work)
Logger.Log ("Building list #{0}/{1}", i+ 1, response.Work.Count);
Build (response.Work [i]);//item);
}
Logger.Log ("Builder finished successfully.");
return 0;
} catch (Exception ex) {
Logger.Log ("An exception occurred: {0}", ex.ToString ());
return 1;
} finally {
process_lock.Unlock ();
}
}
private static void Build (BuildInfo info)
{
try {
object sync_object = new object ();
string log_file = Path.Combine (info.BUILDER_DATA_LOG_DIR, info.command.command + ".log");
DateTime last_stamp = DateTime.Now;
DateTime local_starttime = DateTime.Now;
DBState result;
DBCommand command = info.command;
int exitcode = 0;
ReportBuildStateResponse response;
Logger.Log ("{0} Builder started new thread for sequence {1} step {2}", info.number, info.command.sequence, info.command.command);
/* Check if step has been aborted already */
info.work.State = WebService.GetWorkStateSafe (info.work);
if (info.work.State != DBState.NotDone && info.work.State != DBState.Executing) {
/* If we're in an executing state, we're restarting a command for whatever reason (crash, reboot, etc) */
Logger.Log ("{0} Builder found that step {1} is not ready to start, it's in a '{2}' state", info.number, info.command.command, info.work.State);
return;
}
result = DBState.Executing;
info.work.starttime = DBRecord.DatabaseNow;
info.work.State = result;
info.work.host_id = info.host.id;
info.work = WebService.ReportBuildStateSafe (info.work).Work;
using (Job p = ProcessHelper.CreateJob ()) {
using (FileStream fs = new FileStream (log_file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) {
using (StreamWriter log = new StreamWriter (fs)) {
p.StartInfo.FileName = info.command.filename;
p.StartInfo.Arguments = string.Format (info.command.arguments, Path.Combine (info.temp_dir, info.command.command), info.temp_dir);
if (!string.IsNullOrEmpty (info.command.working_directory))
p.StartInfo.WorkingDirectory = Path.Combine (info.BUILDER_DATA_SOURCE_DIR, info.command.working_directory);
else
p.StartInfo.WorkingDirectory = info.BUILDER_DATA_SOURCE_DIR;
p.StartInfo.UseShellExecute = false;
p.StartInfo.RedirectStandardError = true;
p.StartInfo.RedirectStandardOutput = true;
// set environment variables
p.StartInfo.EnvironmentVariables ["BUILD_LANE"] = info.lane.lane;
p.StartInfo.EnvironmentVariables ["BUILD_COMMAND"] = info.command.command;
p.StartInfo.EnvironmentVariables ["BUILD_REVISION"] = info.revision.revision;
p.StartInfo.EnvironmentVariables ["BUILD_INSTALL"] = Configuration.CygwinizePath (info.BUILDER_DATA_INSTALL_DIR);
p.StartInfo.EnvironmentVariables ["BUILD_DATA_LANE"] = Configuration.GetDataLane (info.lane.lane);
p.StartInfo.EnvironmentVariables ["BUILD_DATA_SOURCE"] = info.BUILDER_DATA_SOURCE_DIR;
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY"] = info.lane.repository;
p.StartInfo.EnvironmentVariables ["BUILD_HOST"] = Configuration.Host;
p.StartInfo.EnvironmentVariables ["BUILD_WORK_HOST"] = info.host_being_worked_for;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_MAX_REVISION"] = info.lane.max_revision;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_MIN_REVISION"] = info.lane.min_revision;
p.StartInfo.EnvironmentVariables ["BUILD_LANE_COMMIT_FILTER"] = info.lane.commit_filter;
int r = 0;
foreach (string repo in info.lane.repository.Split (',')) {
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_" + r.ToString ()] = repo;
r++;
}
p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_SPACE"] = info.lane.repository.Replace (',', ' ');
p.StartInfo.EnvironmentVariables ["BUILD_SEQUENCE"] = "0";
p.StartInfo.EnvironmentVariables ["BUILD_SCRIPT_DIR"] = info.temp_dir;
p.StartInfo.EnvironmentVariables ["LD_LIBRARY_PATH"] = Configuration.CygwinizePath (Configuration.GetLdLibraryPath (info.lane.lane, info.revision.revision));
p.StartInfo.EnvironmentVariables ["PKG_CONFIG_PATH"] = Configuration.CygwinizePath (Configuration.GetPkgConfigPath (info.lane.lane, info.revision.revision));
p.StartInfo.EnvironmentVariables ["PATH"] = Configuration.CygwinizePath (Configuration.GetPath (info.lane.lane, info.revision.revision));
p.StartInfo.EnvironmentVariables ["C_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCIncludePath (info.lane.lane, info.revision.revision));
p.StartInfo.EnvironmentVariables ["CPLUS_INCLUDE_PATH"] = Configuration.CygwinizePath (Configuration.GetCPlusIncludePath (info.lane.lane, info.revision.revision));
// We need to remove all paths from environment variables that were
// set for this executable to work so that they don't mess with
// whatever we're trying to execute
string [] bot_dependencies = new string [] { "PATH", "LD_LIBRARY_PATH", "PKG_CONFIG_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "AC_LOCAL_PATH", "MONO_PATH" };
foreach (string bot_dependency in bot_dependencies) {
if (!p.StartInfo.EnvironmentVariables.ContainsKey (bot_dependency))
continue;
List<string> paths = new List<string> (p.StartInfo.EnvironmentVariables [bot_dependency].Split (new char [] { ':' /* XXX: does windows use ';' here? */}, StringSplitOptions.None));
for (int i = paths.Count - 1; i >= 0; i--) {
if (paths [i].Contains ("bot-dependencies"))
paths.RemoveAt (i);
}
p.StartInfo.EnvironmentVariables [bot_dependency] = string.Join (":", paths.ToArray ());
}
if (info.environment_variables != null) {
// order is important here, we need to loop over the array in the same order get got the variables.
for (int e = 0; e < info.environment_variables.Count; e++) {
info.environment_variables [e].Evaluate (p.StartInfo.EnvironmentVariables);
}
}
Thread stdout_thread = new Thread (delegate ()
{
try {
string line;
while (null != (line = p.StandardOutput.ReadLine ())) {
lock (sync_object) {
log.WriteLine (line);
log.Flush ();
}
last_stamp = DateTime.Now;
}
} catch (Exception ex) {
Logger.Log ("{1} Stdin reader thread got exception: {0}", ex.Message, info.number);
}
});
Thread stderr_thread = new Thread (delegate ()
{
try {
string line;
while (null != (line = p.StandardError.ReadLine ())) {
lock (sync_object) {
log.WriteLine (line);
log.Flush ();
}
last_stamp = DateTime.Now;
}
} catch (Exception ex) {
Logger.Log ("{1} Stderr reader thread got exception: {0}", ex.Message, info.number);
}
});
p.Start ();
stderr_thread.Start ();
stdout_thread.Start ();
while (!p.WaitForExit (60000 /* 1 minute */)) {
if (p.HasExited)
break;
// Check if step has been aborted.
info.work.State = WebService.GetWorkStateSafe (info.work);
if (info.work.State == DBState.Aborted) {
result = DBState.Aborted;
try {
exitcode = 255;
Logger.Log ("{1} The build step '{0}' has been aborted, terminating it.", info.command.command, info.number);
p.Terminate ();
log.WriteLine ("{1} The build step '{0}' was aborted, terminated it.", info.command.command, info.number);
} catch (Exception ex) {
Logger.Log ("{1} Exception while killing build step: {0}", ex.ToString (), info.number);
}
break;
}
// Check if step has timedout
bool timedout = false;
string timeoutReason = null;
int timeout = 15;
if ((DateTime.Now > local_starttime.AddMinutes (info.command.timeout))) {
timedout = true;
timeoutReason = string.Format ("The build step '{0}' didn't finish in {1} minute(s).", info.command.command, info.command.timeout);
} else if (last_stamp.AddMinutes (timeout) <= DateTime.Now) {
timedout = true;
timeoutReason = string.Format ("The build step '{0}' has had no output for {1} minute(s).", info.command.command, timeout);
}
if (!timedout)
continue;
try {
result = DBState.Timeout;
exitcode = 255;
Logger.Log ("{0} {1}", info.number, timeoutReason);
p.Terminate ();
log.WriteLine (timeoutReason);
} catch (Exception ex) {
Logger.Log ("{1} Exception while terminating build step: {0}", ex.ToString (), info.number);
}
break;
}
// Sleep a bit so that the process has enough time to finish
System.Threading.Thread.Sleep (1000);
if (!stdout_thread.Join (TimeSpan.FromSeconds (15))) {
Logger.Log ("Waited 15s for stdout thread to finish, but it didn't");
}
if (!stderr_thread.Join (TimeSpan.FromSeconds (15))) {
Logger.Log ("Waited 15s for stderr thread to finish, but it didn't");
}
if (p.HasExited) {
exitcode = p.ExitCode;
} else {
Logger.Log ("{1} Step: {0}: the process didn't exit in time.", command.command, info.number);
exitcode = 1;
}
if (result == DBState.Executing) {
if (exitcode == 0) {
result = DBState.Success;
} else {
result = DBState.Failed;
}
} else if (result == DBState.Aborted) {
result = DBState.Failed;
}
}
}
}
info.work.State = result;
using (TextReader reader = new StreamReader (new FileStream (log_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
info.work.CalculateSummary (reader);
info.work.endtime = DBRecord.DatabaseNow;
response = WebService.ReportBuildStateSafe (info.work);
info.work = response.Work;
WebService.UploadFilesSafe (info.work, new string [] { log_file }, new bool [] { false });
// Gather files from logged commands and from the upload_files glob
CheckLog (log_file, info);
if (!string.IsNullOrEmpty (info.command.upload_files)) {
foreach (string glob in info.command.upload_files.Split (',')) {
// TODO: handle globs in directory parts
// TODO: allow absolute paths?
Logger.Log ("Uploading files from glob {0}", glob);
// TODO: allow hidden files also
WebService.UploadFilesSafe (info.work, Directory.GetFiles (Path.Combine (info.BUILDER_DATA_SOURCE_DIR, Path.GetDirectoryName (glob)), Path.GetFileName (glob)), null);
}
}
if (response.RevisionWorkCompleted) {
// Cleanup after us.
string base_dir = Configuration.GetDataRevisionDir (info.lane.lane, info.revision.revision);
FileUtilities.TryDeleteDirectoryRecursive (base_dir);
}
Logger.Log ("{4} Revision {0}, executed step '{1}' in {2} s, ExitCode: {3}, State: {4}", info.revision.revision, info.command.command, info.work.duration, exitcode, info.number, info.work.State);
} catch (Exception ex) {
info.work.State = DBState.Failed;
info.work.summary = ex.Message;
info.work = WebService.ReportBuildStateSafe (info.work).Work;
Logger.Log ("{3} Revision {0}, got exception '{1}': \n{2}", info.revision.revision, ex.Message, ex.StackTrace, info.number);
throw;
} finally {
Logger.Log ("{0} Builder finished thread for sequence {0}", info.number);
if ((info.work.State == DBState.Failed || info.work.State == DBState.Aborted || info.work.State == DBState.Timeout) && !info.command.nonfatal) {
failed_hostlanes.Add (info.hostlane);
}
}
}