private static void ProcessNotify(GenericNotificationInfo info)
{
log.DebugFormat("Running notifiers for lane_id: {1}, revision_id: {2}, host_id: {3}, message: {0}", info.message, info.laneID, info.revisionID, info.hostID);
try {
lock (lock_obj) {
List <NotificationBase> notifiers;
if (!notifications_per_lane.TryGetValue(info.laneID, out notifiers))
{
return;
}
foreach (var notification in notifiers)
{
notification.NotifyGeneric(info);
}
}
} catch (Exception ex) {
log.ErrorFormat("Exception while processing notification: {0}", ex);
}
}
public override void NotifyGeneric(GenericNotificationInfo info)
{
string laneName, hostName, hash, repoURL;
// Fetch info from database
using (var db = new DB())
using (var cmd = db.CreateCommand(@"
SELECT lane.lane, host.host, revision.revision, lane.repository
FROM revisionwork
INNER JOIN lane ON lane.id = revisionwork.lane_id
INNER JOIN host ON host.id = revisionwork.host_id
INNER JOIN revision ON revision.id = revisionwork.revision_id
WHERE revisionwork.lane_id = @laneid AND revisionwork.host_id = @hostid AND revisionwork.revision_id = @revisionid
" )) {
DB.CreateParameter(cmd, "laneid", info.laneID);
DB.CreateParameter(cmd, "hostid", info.hostID);
DB.CreateParameter(cmd, "revisionid", info.revisionID);
using (var reader = cmd.ExecuteReader()) {
if (!reader.Read())
{
log.WarnFormat("GitHub Notifier: RevisionWork for lane {0} host {1} revision {2} no longer exists.", info.laneID, info.hostID, info.revisionID);
return;
}
laneName = reader.GetString(0);
hostName = reader.GetString(1);
hash = reader.GetString(2);
repoURL = reader.GetString(3);
}
}
string message = String.Format("Lane: {0}, Host: {1}: {2}", laneName, hostName, info.message);
var statusObj = createPayload(info.laneID, info.hostID, info.revisionID, message, STATE_TO_GITHUB[info.state]);
sendNotification(repoURL, hash, statusObj);
}
public GetBuildInfoResponse GetBuildInfoMultiple (WebServiceLogin login, string host, bool multiple_work)
{
List<DBHost> hosts = new List<DBHost> (); // list of hosts to find work for
List<DBHostLane> hostlanes = new List<DBHostLane> ();
List<DBLane> lanes = new List<DBLane> ();
GetBuildInfoResponse response = new GetBuildInfoResponse ();
response.Work = new List<List<BuildInfoEntry>> ();
using (DB db = new DB ()) {
VerifyUserInRole (db, login, Roles.BuildBot, true);
response.Host = FindHost (db, null, host);
if (!response.Host.enabled)
return response;
// find the master hosts for this host (if any)
response.MasterHosts = FindMasterHosts (db, response.Host);
// get the hosts to find work for
if (response.MasterHosts != null && response.MasterHosts.Count > 0) {
foreach (DBMasterHost mh in response.MasterHosts)
hosts.Add (DBHost_Extensions.Create (db, mh.master_host_id));
} else {
hosts.Add (response.Host);
}
// find the enabled hostlane combinations for these hosts
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = "SELECT HostLane.* FROM HostLane INNER JOIN Lane ON Lane.id = HostLane.lane_id WHERE Lane.enabled = TRUE AND HostLane.enabled = TRUE AND (";
for (int i = 0; i < hosts.Count; i++) {
if (i > 0)
cmd.CommandText += " OR ";
cmd.CommandText += " HostLane.host_id = " + hosts [i].id;
}
cmd.CommandText += ")";
using (IDataReader reader = cmd.ExecuteReader ()) {
while (reader.Read ())
hostlanes.Add (new DBHostLane (reader));
}
}
if (hostlanes.Count == 0)
return response; // nothing to do here
lanes = db.GetAllLanes ();
switch (response.Host.QueueManagement) {
case DBQueueManagement.OneRevisionWorkAtATime:
if (hostlanes.Count > 1) {
int latest = -1;
DateTime latest_date = DateTime.MaxValue;
// we need to find the latest revisionwork each hostlane has completed.
// we want to work on the hostlane which has waited the longest amount
// of time without getting work done (but which has pending work to do).
for (int i = 0; i < hostlanes.Count; i++) {
DBHostLane hl = hostlanes [i];
// check if this hostlane has pending work.
// this would ideally be included in the query below, but I'm not sure
// how to do that while still distinguising the case where nothing has
// been done ever for a hostlane.
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = @"
SELECT RevisionWork.id
FROM RevisionWork
WHERE
RevisionWork.host_id = @host_id
AND (RevisionWork.workhost_id = @workhost_id OR RevisionWork.workhost_id IS NULL)
AND RevisionWork.completed = false
AND RevisionWork.state <> 9 AND RevisionWork.state <> 10 AND RevisionWork.state <> 11
AND lane_id = @lane_id
LIMIT 1;
";
DB.CreateParameter (cmd, "lane_id", hl.lane_id);
DB.CreateParameter (cmd, "host_id", hl.host_id);
DB.CreateParameter (cmd, "workhost_id", response.Host.id);
object obj = cmd.ExecuteScalar ();
if (obj == DBNull.Value || obj == null) {
// there is nothing to do for this hostlane
continue;
}
}
// find the latest completed (this may not be correct, maybe find the latest unstarted?)
// revisionwork for this hostlane.
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = @"
SELECT RevisionWork.endtime
FROM RevisionWork
WHERE
RevisionWork.host_id = @host_id
AND (RevisionWork.workhost_id = @workhost_id OR RevisionWork.workhost_id IS NULL)
AND RevisionWork.completed = true
AND lane_id = @lane_id
ORDER BY RevisionWork.endtime DESC
LIMIT 1;
";
DB.CreateParameter (cmd, "lane_id", hl.lane_id);
DB.CreateParameter (cmd, "host_id", hl.host_id);
DB.CreateParameter (cmd, "workhost_id", response.Host.id);
object obj = cmd.ExecuteScalar ();
if (obj is DateTime) {
DateTime dt = (DateTime) obj;
if (dt < latest_date) {
latest_date = dt;
latest = i;
}
} else {
// nothing has ever been done for this hostlane.
latest_date = DateTime.MinValue;
latest = i;
}
}
}
if (latest >= 0) {
DBHostLane tmp = hostlanes [latest];
hostlanes.Clear ();
hostlanes.Add (tmp);
} else {
hostlanes.Clear (); // there is nothing to do at all
}
}
break;
}
foreach (DBHostLane hl in hostlanes) {
int counter = 10;
DBRevisionWork revisionwork;
DBLane lane = null;
DBHost masterhost = null;
foreach (DBLane l in lanes) {
if (l.id == hl.lane_id) {
lane = l;
break;
}
}
foreach (DBHost hh in hosts) {
if (hh.id == hl.host_id) {
masterhost = hh;
break;
}
}
do {
revisionwork = db.GetRevisionWork (lane, masterhost, response.Host);
if (revisionwork == null)
break;
} while (!revisionwork.SetWorkHost (db, response.Host) && counter-- > 0);
if (revisionwork == null)
continue;
if (!revisionwork.workhost_id.HasValue || revisionwork.workhost_id != response.Host.id)
continue; // couldn't lock this revisionwork.
log.DebugFormat ("Found work for host {0} {4}: {1} (lane: {2} {3})", response.Host.id, revisionwork.id, revisionwork.lane_id, lane.lane, response.Host.host);
DBRevision revision = DBRevision_Extensions.Create (db, revisionwork.revision_id);
List<DBWorkFile> files_to_download = null;
List<DBLane> dependent_lanes = null;
// get dependent files
List<DBLaneDependency> dependencies = lane.GetDependencies (db);
if (dependencies != null && dependencies.Count > 0) {
foreach (DBLaneDependency dep in dependencies) {
DBLane dependent_lane;
DBHost dependent_host;
DBRevisionWork dep_revwork;
List<DBWorkFile> work_files;
if (string.IsNullOrEmpty (dep.download_files))
continue;
dependent_lane = DBLane_Extensions.Create (db, dep.dependent_lane_id);
dependent_host = dep.dependent_host_id.HasValue ? DBHost_Extensions.Create (db, dep.dependent_host_id.Value) : null;
DBRevision dep_lane_rev = dependent_lane.FindRevision (db, revision.revision);
if (dep_lane_rev == null)
continue; /* Something bad happened: the lane we're dependent on does not have the same revisions we have */
dep_revwork = DBRevisionWork_Extensions.Find (db, dependent_lane, dependent_host, dep_lane_rev);
work_files = dep_revwork.GetFiles (db);
foreach (DBWorkFile file in work_files) {
bool download = true;
foreach (string exp in dep.download_files.Split (new char [] { ' ' }, StringSplitOptions.RemoveEmptyEntries)) {
if (!System.Text.RegularExpressions.Regex.IsMatch (file.filename, FileUtilities.GlobToRegExp (exp))) {
download = false;
break;
}
}
if (!download)
continue;
if (files_to_download == null) {
files_to_download = new List<DBWorkFile> ();
dependent_lanes = new List<DBLane> ();
}
files_to_download.Add (file);
dependent_lanes.Add (dependent_lane);
}
}
}
List<DBWorkView2> pending_work = revisionwork.GetNextWork (db, lane, masterhost, revision, multiple_work);
if (pending_work == null || pending_work.Count == 0)
continue;
List<DBEnvironmentVariable> environment_variables = null;
using (IDbCommand cmd = db.CreateCommand ()) {
foreach (int li in db.GetLaneHierarchy (lane.id)) {
cmd.CommandText += string.Format (@"
SELECT *
FROM EnvironmentVariable
WHERE
(host_id = {0} OR host_id = {1} OR host_id IS NULL) AND (lane_id = {2} OR lane_id IS NULL)
ORDER BY id;
;", revisionwork.workhost_id, revisionwork.host_id, li);
}
using (IDataReader reader = cmd.ExecuteReader ()) {
var set = new HashSet<string> ();
do {
while (reader.Read ()) {
if (environment_variables == null)
environment_variables = new List<DBEnvironmentVariable> ();
var ev = new DBEnvironmentVariable (reader);
if (!set.Contains (ev.name)) {
environment_variables.Add (ev);
set.Add (ev.name);
}
}
} while (reader.NextResult ());
}
}
DBHost host_being_worked_for = hosts.Find (h => h.id == revisionwork.host_id);
foreach (DBWorkView2 work in pending_work) {
BuildInfoEntry entry = new BuildInfoEntry ();
entry.Lane = lane;
entry.HostLane = hl;
entry.Revision = revision;
entry.Command = DBCommand_Extensions.Create (db, work.command_id);
entry.FilesToDownload = files_to_download;
entry.DependentLaneOfFiles = dependent_lanes;
entry.Work = DBWork_Extensions.Create (db, work.id);
entry.LaneFiles = lane.GetFiles (db, lanes);
entry.EnvironmentVariables = environment_variables;
entry.Host = host_being_worked_for;
// TODO: put work with the same sequence number into one list of entries.
List<BuildInfoEntry> entries = new List<BuildInfoEntry> ();
entries.Add (entry);
response.Work.Add (entries);
}
// Notify that the revision is assigned
var notifyInfo = new GenericNotificationInfo ();
notifyInfo.laneID = revisionwork.lane_id;
notifyInfo.hostID = revisionwork.host_id;
notifyInfo.revisionID = revisionwork.revision_id;
notifyInfo.message = String.Format("Assigned to host '{0}' ({1})", response.Host.host, response.Host.id);
notifyInfo.state = DBState.Executing;
Notifications.NotifyGeneric (notifyInfo);
}
}
return response;
}
public ReportBuildStateResponse ReportBuildState (WebServiceLogin login, DBWork work)
{
ReportBuildStateResponse response = new ReportBuildStateResponse ();
using (DB db = new DB ())
using (var transaction = db.BeginTransaction ()) {
VerifyUserInRole (db, login, Roles.BuildBot, true);
log.DebugFormat ("ReportBuildState, state: {0}, start time: {1}, end time: {2}", work.State, work.starttime, work.endtime);
if (work.starttime > new DateTime (2000, 1, 1) && work.endtime < work.starttime) {
// the issue here is that the server interprets the datetime as local time, while it's always as utc.
try {
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = "SELECT starttime FROM Work WHERE id = " + work.id;
var value = cmd.ExecuteScalar ();
if (value is DateTime)
work.starttime = (DateTime) value;
}
} catch (Exception ex) {
log.ErrorFormat ("ReportBuildState: Exception while fixing timezone data: {0}", ex);
}
}
work.Save (db);
work.Reload (db);
response.Work = work;
DBRevisionWork rw = DBRevisionWork_Extensions.Create (db, work.revisionwork_id);
bool was_completed = rw.completed;
rw.UpdateState (db);
if (rw.startedtime == null) {
rw.startedtime = work.starttime;
rw.Save (db);
}
if (!was_completed && rw.completed) {
rw.endtime = DBRecord.DatabaseNow;
rw.Save (db);
}
Notifications.Notify (work, rw);
if (!was_completed && rw.completed) {
var notifyInfo = new GenericNotificationInfo ();
notifyInfo.laneID = rw.lane_id;
notifyInfo.hostID = rw.host_id;
notifyInfo.revisionID = rw.revision_id;
notifyInfo.message = "Completed";
notifyInfo.state = rw.State;
Notifications.NotifyGeneric (notifyInfo);
}
response.RevisionWorkCompleted = rw.completed;
using (var cmd = db.CreateCommand ()) {
cmd.CommandText = "UPDATE Lane SET changed_date = @date WHERE id = @id;";
DB.CreateParameter (cmd, "date", DateTime.UtcNow);
DB.CreateParameter (cmd, "id", rw.lane_id);
cmd.ExecuteNonQuery ();
}
// Check if any other lane depends on this one
if (response.RevisionWorkCompleted) {
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = "SELECT 1 FROM LaneDependency WHERE dependent_lane_id = @lane_id LIMIT 1;";
DB.CreateParameter (cmd, "lane_id", rw.lane_id);
object value = cmd.ExecuteScalar ();
if (value is int) {
// If so, run the scheduler
MonkeyWrench.Scheduler.Scheduler.ExecuteSchedulerAsync (false);
}
}
}
transaction.Commit ();
return response;
}
}
public override void NotifyGeneric (GenericNotificationInfo info)
{
string laneName, hostName, hash, repoURL;
// Fetch info from database
using (var db = new DB ())
using (var cmd = db.CreateCommand (@"
SELECT lane.lane, host.host, revision.revision, lane.repository
FROM revisionwork
INNER JOIN lane ON lane.id = revisionwork.lane_id
INNER JOIN host ON host.id = revisionwork.host_id
INNER JOIN revision ON revision.id = revisionwork.revision_id
WHERE revisionwork.lane_id = @laneid AND revisionwork.host_id = @hostid AND revisionwork.revision_id = @revisionid
")) {
DB.CreateParameter (cmd, "laneid", info.laneID);
DB.CreateParameter (cmd, "hostid", info.hostID);
DB.CreateParameter (cmd, "revisionid", info.revisionID);
using (var reader = cmd.ExecuteReader ()) {
if (!reader.Read ()) {
log.WarnFormat ("GitHub Notifier: RevisionWork for lane {0} host {1} revision {2} no longer exists.", info.laneID, info.hostID, info.revisionID);
return;
}
laneName = reader.GetString (0);
hostName = reader.GetString (1);
hash = reader.GetString (2);
repoURL = reader.GetString (3);
}
}
string message = String.Format ("Lane: {0}, Host: {1}: {2}", laneName, hostName, info.message);
var statusObj = createPayload (info.laneID, info.hostID, info.revisionID, message, STATE_TO_GITHUB[info.state]);
sendNotification (repoURL, hash, statusObj);
}
/// <summary>
/// Returns true if something was added to the database.
/// </summary>
/// <param name="db"></param>
/// <param name="lane"></param>
/// <param name="host"></param>
/// <returns></returns>
public static bool AddRevisionWork (DB db)
{
var stopwatch = new Stopwatch ();
stopwatch.Start ();
int line_count = 0;
try {
using (var cmd = db.CreateCommand (@"
INSERT INTO RevisionWork (lane_id, host_id, revision_id, state)
SELECT Lane.id, Host.id, Revision.id, 10
FROM HostLane
INNER JOIN Host ON HostLane.host_id = Host.id
INNER JOIN Lane ON HostLane.lane_id = Lane.id
INNER JOIN Revision ON Revision.lane_id = lane.id
WHERE HostLane.enabled = true AND
NOT EXISTS (
SELECT 1
FROM RevisionWork
WHERE RevisionWork.lane_id = Lane.id AND RevisionWork.host_id = Host.id AND RevisionWork.revision_id = Revision.id
)
RETURNING lane_id, host_id, revision_id
"))
using (IDataReader reader = cmd.ExecuteReader ()) {
while (reader.Read ()) {
int lane_id = reader.GetInt32 (0);
int host_id = reader.GetInt32 (1);
int revision_id = reader.GetInt32 (2);
var info = new GenericNotificationInfo();
info.laneID = lane_id;
info.hostID = host_id;
info.revisionID = revision_id;
info.message = "Commit received.";
info.state = DBState.Executing;
Notifications.NotifyGeneric (info);
line_count++;
}
}
log.DebugFormat ("AddRevisionWork: Added {0} records.", line_count);
return line_count > 0;
} catch (Exception ex) {
log.ErrorFormat ("AddRevisionWork got an exception: {0}", ex);
return false;
} finally {
stopwatch.Stop ();
log.InfoFormat ("AddRevisionWork [Done in {0} seconds]", stopwatch.Elapsed.TotalSeconds);
}
}
/**
* Used for notifications not tied to a specific work. (ex. revision downloaded or assigned)
*/
public virtual void NotifyGeneric(GenericNotificationInfo info)
{
}
public static void NotifyGeneric(GenericNotificationInfo info)
{
queued_notifications.Add(new QueuedNotification {
IsInfo = true, info = info
});
}
/// <summary>
/// Returns true if something was added to the database.
/// </summary>
/// <param name="db"></param>
/// <param name="lanes"></param>
/// <param name="hostlanes"></param>
/// <returns></returns>
public static bool AddRevisionWorkSlow (DB db, List<DBLane> lanes, List<DBHostLane> hostlanes)
{
var stopwatch = new Stopwatch ();
stopwatch.Start ();
int line_count = 0;
try {
var selected_lanes = new Dictionary<int, DBLane> ();
foreach (var hl in hostlanes) {
if (!hl.enabled)
continue;
if (!selected_lanes.ContainsKey (hl.lane_id))
selected_lanes [hl.lane_id] = lanes.Find ((v) => v.id == hl.lane_id);
}
foreach (var l in lanes) {
if (l.enabled)
continue;
if (selected_lanes.ContainsKey (l.id))
selected_lanes.Remove (l.id);
}
foreach (var id in selected_lanes.Keys) {
using (var cmd = db.CreateCommand (string.Format (@"
INSERT INTO RevisionWork (lane_id, host_id, revision_id, state)
SELECT Lane.id, Host.id, Revision.id, 10
FROM HostLane
INNER JOIN Host ON HostLane.host_id = Host.id
INNER JOIN Lane ON HostLane.lane_id = Lane.id
INNER JOIN Revision ON Revision.lane_id = lane.id
WHERE HostLane.enabled = true AND Lane.id = {0} AND
NOT EXISTS (
SELECT 1
FROM RevisionWork
WHERE RevisionWork.lane_id = Lane.id AND RevisionWork.host_id = Host.id AND RevisionWork.revision_id = Revision.id
)
RETURNING lane_id, host_id, revision_id
", id)))
using (IDataReader reader = cmd.ExecuteReader ()) {
while (reader.Read ()) {
int lane_id = reader.GetInt32 (0);
int host_id = reader.GetInt32 (1);
int revision_id = reader.GetInt32 (2);
var info = new GenericNotificationInfo();
info.laneID = lane_id;
info.hostID = host_id;
info.revisionID = revision_id;
info.message = "Commit received.";
info.state = DBState.Executing;
Notifications.NotifyGeneric (info);
line_count++;
}
}
log.DebugFormat ("AddRevisionWorkSlow: Added {0} records for lane {1}.", line_count, selected_lanes [id].lane);
}
return line_count > 0;
} catch (Exception ex) {
log.ErrorFormat ("AddRevisionWorkSlow got an exception: {0}", ex);
return false;
} finally {
stopwatch.Stop ();
log.InfoFormat ("AddRevisionWorkSlow [Done in {0} seconds]", stopwatch.Elapsed.TotalSeconds);
}
}
/**
* Used for notifications not tied to a specific work. (ex. revision downloaded or assigned)
*/
public virtual void NotifyGeneric(GenericNotificationInfo info) {}
private static void ProcessNotify(GenericNotificationInfo info)
{
log.DebugFormat ("Running notifiers for lane_id: {1}, revision_id: {2}, host_id: {3}, message: {0}", info.message, info.laneID, info.revisionID, info.hostID);
try {
lock (lock_obj) {
List<NotificationBase> notifiers;
if (!notifications_per_lane.TryGetValue (info.laneID, out notifiers)) {
return;
}
foreach (var notification in notifiers) {
notification.NotifyGeneric (info);
}
}
} catch (Exception ex) {
log.ErrorFormat ("Exception while processing notification: {0}", ex);
}
}
public static void NotifyGeneric (GenericNotificationInfo info)
{
queued_notifications.Add (new QueuedNotification { IsInfo = true, info = info });
}