コード例 #1
0
        private static void Build(BuildInfo info)
        {
            try {
                ProcessReader            process_reader;
                string                   log_file        = Path.Combine(info.BUILDER_DATA_LOG_DIR, info.command.command + ".log");
                DateTime                 local_starttime = DateTime.Now;
                DBState                  result;
                DBCommand                command  = info.command;
                int                      exitcode = 0;
                ReportBuildStateResponse response;

                Logger.Log("{0} Builder started new thread for sequence {1} step {2}", info.number, info.command.sequence, info.command.command);

                /* Check if step has been aborted already */
                info.work.State = WebService.GetWorkStateSafe(info.work);
                if (info.work.State != DBState.NotDone && info.work.State != DBState.Executing)
                {
                    /* If we're in an executing state, we're restarting a command for whatever reason (crash, reboot, etc) */
                    Logger.Log("{0} Builder found that step {1} is not ready to start, it's in a '{2}' state", info.number, info.command.command, info.work.State);
                    return;
                }
                result = DBState.Executing;

                info.work.starttime = DBRecord.DatabaseNow;
                info.work.State     = result;
                info.work.host_id   = info.host.id;
                info.work           = WebService.ReportBuildStateSafe(info.work).Work;

                using (Job p = ProcessHelper.CreateJob()) {
                    using (FileStream fs = new FileStream(log_file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) {
                        using (var log = new SynchronizedStreamWriter(new StreamWriter(fs))) {
                            p.StartInfo.FileName  = info.command.filename;
                            p.StartInfo.Arguments = string.Format(info.command.arguments, Path.Combine(info.temp_dir, info.command.command), info.temp_dir);
                            if (!string.IsNullOrEmpty(info.command.working_directory))
                            {
                                p.StartInfo.WorkingDirectory = Path.Combine(info.BUILDER_DATA_SOURCE_DIR, info.command.working_directory);
                            }
                            else
                            {
                                p.StartInfo.WorkingDirectory = info.BUILDER_DATA_SOURCE_DIR;
                            }

                            // set environment variables
                            p.StartInfo.EnvironmentVariables ["BUILD_LANE"]               = info.lane.lane;
                            p.StartInfo.EnvironmentVariables ["BUILD_COMMAND"]            = info.command.command;
                            p.StartInfo.EnvironmentVariables ["BUILD_REVISION"]           = info.revision.revision;
                            p.StartInfo.EnvironmentVariables ["BUILD_INSTALL"]            = Configuration.CygwinizePath(info.BUILDER_DATA_INSTALL_DIR);
                            p.StartInfo.EnvironmentVariables ["BUILD_DATA_LANE"]          = Configuration.GetDataLane(info.lane.id);
                            p.StartInfo.EnvironmentVariables ["BUILD_DATA_SOURCE"]        = info.BUILDER_DATA_SOURCE_DIR;
                            p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY"]         = info.lane.repository;
                            p.StartInfo.EnvironmentVariables ["BUILD_HOST"]               = Configuration.Host;
                            p.StartInfo.EnvironmentVariables ["BUILD_WORK_HOST"]          = info.host_being_worked_for;
                            p.StartInfo.EnvironmentVariables ["BUILD_LANE_MAX_REVISION"]  = info.lane.max_revision;
                            p.StartInfo.EnvironmentVariables ["BUILD_LANE_MIN_REVISION"]  = info.lane.min_revision;
                            p.StartInfo.EnvironmentVariables ["BUILD_LANE_COMMIT_FILTER"] = info.lane.commit_filter;

                            int r = 0;
                            foreach (string repo in info.lane.repository.Split(','))
                            {
                                p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_" + r.ToString()] = repo;
                                r++;
                            }
                            p.StartInfo.EnvironmentVariables ["BUILD_REPOSITORY_SPACE"] = info.lane.repository.Replace(',', ' ');
                            p.StartInfo.EnvironmentVariables ["BUILD_SEQUENCE"]         = "0";
                            p.StartInfo.EnvironmentVariables ["BUILD_SCRIPT_DIR"]       = info.temp_dir;
                            p.StartInfo.EnvironmentVariables ["LD_LIBRARY_PATH"]        = Configuration.CygwinizePath(Configuration.GetLdLibraryPath(info.lane.id, info.revision.revision));
                            p.StartInfo.EnvironmentVariables ["PKG_CONFIG_PATH"]        = Configuration.CygwinizePath(Configuration.GetPkgConfigPath(info.lane.id, info.revision.revision));
                            p.StartInfo.EnvironmentVariables ["PATH"]               = Configuration.CygwinizePath(Configuration.GetPath(info.lane.id, info.revision.revision));
                            p.StartInfo.EnvironmentVariables ["C_INCLUDE_PATH"]     = Configuration.CygwinizePath(Configuration.GetCIncludePath(info.lane.id, info.revision.revision));
                            p.StartInfo.EnvironmentVariables ["CPLUS_INCLUDE_PATH"] = Configuration.CygwinizePath(Configuration.GetCPlusIncludePath(info.lane.id, info.revision.revision));

                            // We need to remove all paths from environment variables that were
                            // set for this executable to work so that they don't mess with
                            // whatever we're trying to execute
                            string [] bot_dependencies = new string [] { "PATH", "LD_LIBRARY_PATH", "PKG_CONFIG_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "AC_LOCAL_PATH", "MONO_PATH" };
                            foreach (string bot_dependency in bot_dependencies)
                            {
                                if (!p.StartInfo.EnvironmentVariables.ContainsKey(bot_dependency))
                                {
                                    continue;
                                }
                                List <string> paths = new List <string> (p.StartInfo.EnvironmentVariables [bot_dependency].Split(new char [] { ':' /* XXX: does windows use ';' here? */ }, StringSplitOptions.None));
                                for (int i = paths.Count - 1; i >= 0; i--)
                                {
                                    if (paths [i].Contains("bot-dependencies"))
                                    {
                                        paths.RemoveAt(i);
                                    }
                                }
                                p.StartInfo.EnvironmentVariables [bot_dependency] = string.Join(":", paths.ToArray());
                            }

                            if (info.environment_variables != null)
                            {
                                // order is important here, we need to loop over the array in the same order get got the variables.
                                for (int e = 0; e < info.environment_variables.Count; e++)
                                {
                                    info.environment_variables [e].Evaluate(p.StartInfo.EnvironmentVariables);
                                }
                            }

                            process_reader = new ProcessReader(log);
                            process_reader.Setup(p);

                            p.Start();

                            process_reader.Start();

                            while (!p.WaitForExit(60000 /* 1 minute */))
                            {
                                if (p.HasExited)
                                {
                                    break;
                                }

                                // Check if step has been aborted.
                                info.work.State = WebService.GetWorkStateSafe(info.work);
                                if (info.work.State == DBState.Aborted)
                                {
                                    result = DBState.Aborted;
                                    try {
                                        exitcode = 255;
                                        Logger.Log("{1} The build step '{0}' has been aborted, terminating it.", info.command.command, info.number);
                                        p.Terminate(log);
                                        log.WriteLine(string.Format("{1} The build step '{0}' was aborted, terminated it.", info.command.command, info.number));
                                    } catch (Exception ex) {
                                        Logger.Log("{1} Exception while killing build step: {0}", ex.ToString(), info.number);
                                    }
                                    break;
                                }

                                // Check if step has timedout
                                bool   timedout      = false;
                                string timeoutReason = null;
                                int    timeout       = 30;

                                if ((DateTime.Now > local_starttime.AddMinutes(info.command.timeout)))
                                {
                                    timedout      = true;
                                    timeoutReason = string.Format("The build step '{0}' didn't finish in {1} minute(s).", info.command.command, info.command.timeout);
                                }
                                else if (log.LastStamp.AddMinutes(timeout) <= DateTime.Now)
                                {
                                    timedout      = true;
                                    timeoutReason = string.Format("The build step '{0}' has had no output for {1} minute(s).", info.command.command, timeout);
                                }

                                if (!timedout)
                                {
                                    continue;
                                }

                                try {
                                    result   = DBState.Timeout;
                                    exitcode = 255;
                                    Logger.Log("{0} {1}", info.number, timeoutReason);
                                    log.WriteLine("\n*** Timed out. Proceeding to get stack traces for all child processes. ***");
                                    p.Terminate(log);
                                    log.WriteLine("\n * " + timeoutReason + " * \n");
                                } catch (Exception ex) {
                                    Logger.Log("{1} Exception while terminating build step: {0}", ex.ToString(), info.number);
                                }
                                break;
                            }

                            // Sleep a bit so that the process has enough time to finish
                            System.Threading.Thread.Sleep(1000);

                            process_reader.Join();

                            if (p.HasExited)
                            {
                                exitcode = p.ExitCode;
                            }
                            else
                            {
                                Logger.Log("{1} Step: {0}: the process didn't exit in time.", command.command, info.number);
                                exitcode = 1;
                            }
                            if (result == DBState.Executing)
                            {
                                if (exitcode == 0)
                                {
                                    result = DBState.Success;
                                }
                                else
                                {
                                    result = DBState.Failed;
                                }
                            }
                            else if (result == DBState.Aborted)
                            {
                                result = DBState.Failed;
                            }
                        }
                    }
                }

                info.work.State = result;
                using (TextReader reader = new StreamReader(new FileStream(log_file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
                    info.work.CalculateSummary(reader);
                info.work.endtime = DBRecord.DatabaseNow;
                response          = WebService.ReportBuildStateSafe(info.work);
                info.work         = response.Work;

                WebService.UploadFilesSafe(info.work, new string [] { log_file }, new bool [] { false });

                // Gather files from logged commands and from the upload_files glob
                CheckLog(log_file, info);
                if (!string.IsNullOrEmpty(info.command.upload_files))
                {
                    foreach (string glob in info.command.upload_files.Split(','))
                    {
                        // TODO: handle globs in directory parts
                        // TODO: allow absolute paths?
                        Logger.Log("Uploading files from glob {0}", glob);
                        // TODO: allow hidden files also
                        WebService.UploadFilesSafe(info.work, Directory.GetFiles(Path.Combine(info.BUILDER_DATA_SOURCE_DIR, Path.GetDirectoryName(glob)), Path.GetFileName(glob)), null);
                    }
                }

                if (response.RevisionWorkCompleted)
                {
                    // Cleanup after us.
                    string base_dir = Configuration.GetDataRevisionDir(info.lane.id, info.revision.revision);
                    FileUtilities.TryDeleteDirectoryRecursive(base_dir);
                }

                Logger.Log("{4} Revision {0}, executed step '{1}' in {2} s, ExitCode: {3}, State: {4}", info.revision.revision, info.command.command, info.work.duration, exitcode, info.number, info.work.State);
            } catch (Exception ex) {
                info.work.State   = DBState.Failed;
                info.work.summary = ex.Message;
                info.work         = WebService.ReportBuildStateSafe(info.work).Work;
                Logger.Log("{3} Revision {0}, got exception '{1}': \n{2}", info.revision.revision, ex.Message, ex.StackTrace, info.number);
                throw;
            } finally {
                Logger.Log("{0} Builder finished thread for sequence {0}", info.number);

                if ((info.work.State == DBState.Failed || info.work.State == DBState.Aborted || info.work.State == DBState.Timeout) && !info.command.nonfatal)
                {
                    failed_hostlanes.Add(info.hostlane);
                }
            }
        }