protected override bool UpdateRevisionsInDBInternal (DB db, DBLane lane, string repository, Dictionary<string, DBRevision> revisions, List<DBHost> hosts, List<DBHostLane> hostlanes, string min_revision, string max_revision) { string revision; bool update_steps = false; List<DateTime> used_dates; DBRevision r; List<GitEntry> log; if (string.IsNullOrEmpty (max_revision)) max_revision = "remotes/origin/master"; GITUpdater.log.InfoFormat ("Updating lane: '{0}', repository: '{1}' min revision: '{2}' max revision: '{3}'", lane.lane, repository, min_revision, max_revision); log = GetGITLog (lane, repository, min_revision, max_revision); if (log == null || log.Count == 0) { GITUpdater.log.WarnFormat ("Didn't get a git log for '{0}'", repository); return false; } GITUpdater.log.InfoFormat ("Got {0} log records", log.Count); used_dates = new List<DateTime> (); foreach (GitEntry entry in log) { string hash = entry.revision; string unix_timestamp_str = entry.timestamp; long unix_timestamp; string author = entry.author; DateTime date; if (!long.TryParse (unix_timestamp_str, out unix_timestamp)) { /* here something is wrong, this way the commit shows up as the first one so that it's easy to spot and start investigating */ date = DateTime.Now.AddYears (20); GITUpdater.log.WarnFormat ("Could not parse timestamp '{0}' for revision '{1}' in lane '{2}' in repository {3}", unix_timestamp_str, entry.revision, lane.lane, repository); } else { const long EPOCH_DIFF = 0x019DB1DED53E8000; /* 116444736000000000 nsecs */ const long RATE_DIFF = 10000000; /* 100 nsecs */ date = DateTime.FromFileTimeUtc ((unix_timestamp * RATE_DIFF) + EPOCH_DIFF); } /* * The timestamp resolution on my machine seems to be 1 second, * which means that if you commit fast enough you'll get * commits with the same date. This is a very bad thing since * the commits are order by the commit date, and if two commits * have the same date the order they're build / shown is random * (the db decides whatever it feels like). Work around this by * keeping a list of used dates and if the date has already * used, add a millisecond to it (and try again). Note that * there is still a possibility of duplicate dates: if there * already is a revision in the database with this date (from * a previous run of the scheduler). * * It may seem like there is a very small possibility of having * two commits within a second, but this happens all the time * for our test suite. */ while (used_dates.Contains (date)) { date = date.AddMilliseconds (1); } used_dates.Add (date); revision = hash; if (revision == null) continue; if (revisions.ContainsKey (revision)) { /* Check if we've saved the wrong date earlier and fix it */ if (revisions [revision].date > new DateTime (2030, 1, 1)) { /* Hopefully this code will not stay here for 20 years */ revisions [revision].date = date; revisions [revision].Save (db); GITUpdater.log.WarnFormat ("Detected wrong date in revision '{0}' in lane '{1}' in repository {2}, fixing it", revision, lane.lane, repository); } // Log (2, "Already got {0}", revision); continue; } if (!string.IsNullOrEmpty (lane.commit_filter)) { FetchFiles (entry, repository); if (DoesFilterExclude (entry, lane.commit_filter)) continue; } r = new DBRevision (); r.revision = revision; r.lane_id = lane.id; r.author = author; if (string.IsNullOrEmpty (r.author)) { GITUpdater.log.WarnFormat ("No author specified in r{0} in {1}", r.revision, repository); r.author = "?"; } r.date = date; r.log_file_id = null; r.Save (db); update_steps = true; GITUpdater.log.DebugFormat ("Saved revision '{0}' for lane '{1}' author: {2}, date: {3:yyyy/MM/dd HH:mm:ss.ffffff} {4} {5}", r.revision, lane.lane, r.author, r.date, unix_timestamp, unix_timestamp_str); } return update_steps; }
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people) { DBPerson person; try { foreach (string repository in lane.repository.Split (new char [] { ',' }, StringSplitOptions.RemoveEmptyEntries)) { string cache_dir = Configuration.GetSchedulerRepositoryCacheDirectory (repository); if (!Directory.Exists (cache_dir)) continue; using (Process git = new Process ()) { DateTime git_start = DateTime.Now; git.StartInfo.FileName = "git"; git.StartInfo.Arguments = "log -1 --pretty=format:'%aE%n%aN%n%cE%n%cN' " + revision.revision; git.StartInfo.WorkingDirectory = cache_dir; git.StartInfo.UseShellExecute = false; git.StartInfo.RedirectStandardOutput = true; git.Start (); string author_email = git.StandardOutput.ReadLine (); string author_name = git.StandardOutput.ReadLine (); string committer_email = git.StandardOutput.ReadLine (); string committer_name = git.StandardOutput.ReadLine (); // Wait 10 minutes for git to finish, otherwise abort. if (!git.WaitForExit (1000 * 60 * 10)) { GITUpdater.log.Error ("Getting commit info took more than 10 minutes, aborting."); try { git.Kill (); git.WaitForExit (10000); // Give the process 10 more seconds to completely exit. } catch (Exception ex) { GITUpdater.log.ErrorFormat ("Aborting commit info retrieval failed: {0}", ex.ToString ()); } } if (git.HasExited && git.ExitCode == 0) { GITUpdater.log.InfoFormat ("Got commit info successfully in {0} seconds", (DateTime.Now - git_start).TotalSeconds); person = new DBPerson (); person.fullname = author_name; person.Emails = new string [] { author_email }; people.Add (person); if (author_name != committer_name && !string.IsNullOrEmpty (committer_name)) { person = new DBPerson (); person.fullname = committer_name; person.Emails = new string [] {committer_email}; people.Add (person); } GITUpdater.log.DebugFormat ("Git commit info for {0}: author_name = {1} author_email: {2} committer_name: {3} committer_email: {4}", revision.revision, author_name, author_email, committer_name, committer_email); } else { GITUpdater.log.ErrorFormat ("Didn't get commit info, HasExited: {0}, ExitCode: {1}", git.HasExited, git.HasExited ? git.ExitCode.ToString () : "N/A"); } } } } catch (Exception ex) { GITUpdater.log.ErrorFormat ("Exception while trying to get commit info: {0}", ex.ToString ()); } }
public static DBRevisionWork Find (DB db, DBLane lane, DBHost host, DBRevision revision) { DBRevisionWork result; using (IDbCommand cmd = db.CreateCommand ()) { cmd.CommandText = "SELECT * FROM RevisionWork WHERE lane_id = @lane_id AND revision_id = @revision_id "; if (host != null) { cmd.CommandText += " AND host_id = @host_id;"; DB.CreateParameter (cmd, "host_id", host.id); } DB.CreateParameter (cmd, "lane_id", lane.id); DB.CreateParameter (cmd, "revision_id", revision.id); using (IDataReader reader = cmd.ExecuteReader ()) { if (!reader.Read ()) return null; result = new DBRevisionWork (reader); if (reader.Read ()) throw new ApplicationException (string.Format ("Found more than one revision work for the specified lane/host/revision ({0}/{1}/{2})", lane.lane, host == null ? "null" : host.host, revision.revision)); return result; } } }
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people) { if (lane.source_control == "git") { GITUpdater.FindPeopleForCommit (lane, revision, people); /* } else if (lane.source_control == "svn") { SVNUpdater.FindPeopleForCommit (lane, revision, people); * */ } else { log.ErrorFormat ("Unknown source control for lane {0}: {1}", lane.lane, lane.source_control); } }
public DBRevision GetRevision (string lane, int revision) { using (IDbCommand cmd = CreateCommand ()) { cmd.CommandText = "SELECT * from revisions where lane = @lane AND revision = @revision"; DB.CreateParameter (cmd, "lane", lane); DB.CreateParameter (cmd, "revision", revision.ToString ()); using (IDataReader reader = cmd.ExecuteReader ()) { if (!reader.Read ()) return null; if (reader.IsDBNull (0)) return null; DBRevision rev = new DBRevision (); rev.Load (reader); return rev; } } }
/// <summary> /// Finds pending steps for the current revision /// </summary> /// <returns></returns> public static List<DBWorkView2> GetNextWork (this DBRevisionWork rw, DB db, DBLane lane, DBHost host, DBRevision revision, bool multiple_work) { List<DBWorkView2> result = new List<DBWorkView2> (); ; if (revision == null) throw new ArgumentNullException ("revision"); if (lane == null) throw new ArgumentNullException ("lane"); if (host == null) throw new ArgumentNullException ("host"); rw.FilterPendingWork (db, db.GetWork (rw), result, multiple_work); if (result.Count == 0 && !rw.completed) { rw.completed = true; rw.UpdateState (db); } return result; }
private static void UpdateSVNDiff (object dummy) { try { Logger.Log ("SVNDiff: Thread started."); using (DB db = new DB (true)) { using (DB db_save = new DB (true)) { using (IDbCommand cmd = db.CreateCommand ()) { cmd.CommandText = @" SELECT Revision.*, Lane.repository, Lane.lane FROM Revision INNER JOIN Lane ON Lane.id = Revision.lane_id WHERE (Revision.diff IS NULL OR Revision.diff = '') AND Revision.diff_file_id IS NULL;"; using (IDataReader reader = cmd.ExecuteReader ()) { while (!quit_svn_diff && reader.Read ()) { DBRevision revision = new DBRevision (reader); string repositories = reader.GetString (reader.GetOrdinal ("repository")); string lane = reader.GetString (reader.GetOrdinal ("lane")); string diff = null; foreach (string repository in repositories.Split (',')) { diff = GetSVNDiff (lane, repository, revision.revision); if (!string.IsNullOrEmpty (diff)) break; } if (string.IsNullOrEmpty (diff)) diff = "No diff"; revision.diff_file_id = db_save.UploadString (diff, ".log", false).id; revision.Save (db_save); Logger.Log ("SVNDiff: Got diff for lane '{0}', revision '{1}'", lane, revision.revision); } } } } } Logger.Log ("SVNDiff: Thread stopping. Done: {0}", !quit_svn_diff); } catch (Exception ex) { Logger.Log ("SVNDiff: Exception: {0} \n{1}", ex.Message, ex.StackTrace); } }
//public List<string> GetRevisions(DBLane lane) //{ // List<string> result = new List<string>(); // using (IDbCommand cmd = Connection.CreateCommand()) // { // cmd.CommandText = "SELECT DISTINCT revision FROM revisions WHERE lane = @lane ORDER BY revision DESC"; // DB.CreateParameter(cmd, "lane", lane); // using (IDataReader reader = cmd.ExecuteReader()) // { // while (reader.Read()) // result.Add(reader.GetString(0)); // } // } // return result; //} public Dictionary<string, DBRevision> GetDBRevisions (int lane_id) { Dictionary<string, DBRevision> result = new Dictionary<string, DBRevision> (); DBRevision rev; using (IDbCommand cmd = CreateCommand ()) { cmd.CommandText = "SELECT * FROM Revision WHERE lane_id = @lane_id"; DB.CreateParameter (cmd, "lane_id", lane_id); using (IDataReader reader = cmd.ExecuteReader ()) { while (reader.Read ()) { rev = new DBRevision (); rev.Load (reader); result.Add (rev.revision, rev); } } } return result; }
void GenerateActionLink(StringBuilder header, DBLane lane, DBHost host, DBRevision dbr, string cmd, string short_label, string long_label, bool hidden = false) { var href = confirmViewLaneAction(lane, host, dbr, cmd, short_label); if (hidden) header.AppendFormat("- <a style='{0}' href='{1}'>{2}</a>", "display:none", href, long_label); else header.AppendFormat("- <a href='{0}'>{1}</a>", href, long_label); }
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people) { if (lane.source_control == "git") { GITUpdater.FindPeopleForCommit (lane, revision, people); /* } else if (lane.source_control == "svn") { SVNUpdater.FindPeopleForCommit (lane, revision, people); * */ } else { Logger.Log ("FindPeopleForCommit (): unknown source control: '{0}'", lane.source_control); } }
string confirmViewLaneAction(DBLane lane, DBHost host, DBRevision dbr, string action, string command) { return String.Format("javascript:confirmViewLaneAction (\"ViewLane.aspx?lane_id={0}&host_id={2}&revision_id={1}&action={3}\", \"{4}\");", lane.id, dbr.id, host.id, action, command); }
public List<DBRevision> GetDBRevisions (int lane_id, int limit) { List<DBRevision> result = new List<DBRevision> (); DBRevision rev; using (IDbCommand cmd = CreateCommand ()) { cmd.CommandText = "SELECT Revision.*, CAST (revision as int) AS r FROM Revision WHERE lane_id = @lane_id ORDER BY r DESC"; if (limit > 0) cmd.CommandText += " LIMIT " + limit.ToString (); DB.CreateParameter (cmd, "lane_id", lane_id); using (IDataReader reader = cmd.ExecuteReader ()) { while (reader.Read ()) { rev = new DBRevision (); rev.Load (reader); result.Add (rev); } } } return result; }
public List<DBRevision> GetDBRevisionsWithoutWork (int lane_id, int host_id) { List<DBRevision> result = new List<DBRevision> (); DBRevision rev; using (IDbCommand cmd = CreateCommand ()) { //cmd.CommandText = "SELECT * FROM Revision WHERE lane_id = @lane_id AND NOT EXISTS (SELECT 1 FROM Work WHERE lane_id = @lane_id AND host_id = @host_id AND revision_id = revision.id) ORDER BY date DESC"; cmd.CommandText = @" SELECT Revision.*, C FROM (SELECT RevisionWork.id, RevisionWork.revision_id, Count(Work.revisionwork_id) AS C FROM RevisionWork LEFT JOIN work ON Work.revisionwork_id = RevisionWork.id WHERE RevisionWork.lane_id = @lane_id AND RevisionWork.host_id = @host_id GROUP BY RevisionWork.id, RevisionWork.revision_id) AS T INNER JOIN Revision ON Revision.id = T.revision_id INNER JOIN RevisionWork ON T.id = RevisionWork.id WHERE C = 0 OR RevisionWork.state = 9 ORDER BY Revision.date DESC; "; DB.CreateParameter (cmd, "lane_id", lane_id); DB.CreateParameter (cmd, "host_id", host_id); using (IDataReader reader = cmd.ExecuteReader ()) { while (reader.Read ()) { rev = new DBRevision (); rev.Load (reader); result.Add (rev); } } } return result; }
public static string GenerateHeader (GetViewLaneDataResponse response, DBLane lane, DBHost host, DBRevision revision, string description) { if (!Authentication.IsInRole (response, MonkeyWrench.DataClasses.Logic.Roles.Administrator)) { return string.Format (@" <h2>{4} revision <a href='ViewLane.aspx?lane_id={0}&host_id={1}&revision_id={6}'>{5}</a> on lane '{2}' on '<a href='ViewHostHistory.aspx?host_id={1}'>{3}</a>' (<a href='ViewTable.aspx?lane_id={0}&host_id={1}'>table</a>)</h2><br/>", lane.id, host.id, lane.lane, host.host, description, revision.revision, revision.id); } else { return string.Format (@" <h2>{4} revision <a href='ViewLane.aspx?lane_id={0}&host_id={1}&revision_id={6}'>{5}</a> on lane '<a href='EditLane.aspx?lane_id={0}'>{2}</a>' on '<a href='ViewHostHistory.aspx?host_id={1}'>{3}</a>' (<a href='ViewTable.aspx?lane_id={0}&host_id={1}'>table</a>)</h2><br/>", lane.id, host.id, lane.lane, host.host, description, revision.revision, revision.id); } }
protected override bool UpdateRevisionsInDBInternal (DB db, DBLane lane, string repository,Dictionary<string, DBRevision> revisions, List<DBHost> hosts, List<DBHostLane> hostlanes, string min_revision) { string revision; XmlDocument svn_log; bool update_steps = false; DBRevision r; int min_revision_int = string.IsNullOrEmpty (min_revision) ? 0 : int.Parse (min_revision); int max_revision_int = int.MaxValue; int current_revision; string log; XmlNode n; XmlAttribute attrib; Log ("Updating '{0}'", lane.lane); if (min_revision_int == 0 && !string.IsNullOrEmpty (lane.min_revision)) min_revision_int = int.Parse (lane.min_revision); if (!string.IsNullOrEmpty (lane.max_revision)) max_revision_int = int.Parse (lane.max_revision); log = GetSVNLog (lane, repository, min_revision_int, max_revision_int); if (string.IsNullOrEmpty (log)) { Log ("Didn't get a svn log for '{0}'", repository); return false; } svn_log = new XmlDocument (); svn_log.PreserveWhitespace = true; svn_log.Load (new StringReader (log)); foreach (XmlNode node in svn_log.SelectNodes ("/log/logentry")) { revision = node.Attributes ["revision"].Value; if (revisions.ContainsKey (revision)) continue; try { current_revision = int.Parse (revision); if (current_revision < min_revision_int || current_revision > max_revision_int) continue; } catch { continue; } r = new DBRevision (); attrib = node.Attributes ["revision"]; if (attrib == null || string.IsNullOrEmpty (attrib.Value)) { Log ("An entry without revision in {0}, skipping entry", repository); continue; } r.revision = attrib.Value; r.lane_id = lane.id; n = node.SelectSingleNode ("author"); if (n != null) { r.author = n.InnerText; } else { Log ("No author specified in r{0} in {1}", r.revision, repository); r.author = "?"; } n = node.SelectSingleNode ("date"); if (n != null) { DateTime dt; if (DateTime.TryParse (n.InnerText, out dt)) { r.date = dt; } else { Log ("Could not parse the date '{0}' in r{1} in {2}", n.InnerText, r.revision, repository); r.date = DateTime.MinValue; } } else { Log ("No date specified in r{0} in {1}", r.revision, repository); r.date = DateTime.MinValue; } n = node.SelectSingleNode ("msg"); if (n != null) { r.log_file_id = db.UploadString (n.InnerText, ".log", false).id; } else { Log ("No msg specified in r{0} in {1}", r.revision, repository); r.log_file_id = null; } r.Save (db); update_steps = true; Log ("Saved revision '{0}' for lane '{1}'", r.revision, lane.lane); } return update_steps; }
public static int MoveFilesToFileSystem () { long moved_bytes = 0; Manager.log.Info ("MoveFilesToFileSystem: [START]"); using (DB db = new DB ()) { using (DB download_db = new DB ()) { while (true) { using (IDbCommand cmd = db.CreateCommand ()) { // execute this in chunks to avoid huge data transfers and slowdowns. cmd.CommandText = "SELECT * FROM File WHERE NOT file_id IS NULL LIMIT 100"; using (IDataReader reader = cmd.ExecuteReader ()) { if (!reader.Read ()) break; do { DBFile file = new DBFile (reader); byte [] buffer = new byte [1024]; int oid = file.file_id.Value; int read; string fn = FileUtilities.CreateFilename (file.md5, file.compressed_mime == MimeTypes.GZ, true); using (FileStream writer = new FileStream (fn, FileMode.Create, FileAccess.Write, FileShare.Read)) { using (Stream str = download_db.Download (file)) { while ((read = str.Read (buffer, 0, buffer.Length)) != 0) writer.Write (buffer, 0, read); } } IDbTransaction transaction = download_db.BeginTransaction (); download_db.Manager.Delete (oid); file.file_id = null; file.Save (download_db); transaction.Commit (); moved_bytes += file.size; log.InfoFormat ("MoveFilesToFileSystem: Moved oid {0} to {1} ({2} bytes, {3} total bytes moved)", oid, fn, file.size, moved_bytes); } while (reader.Read ()); } } } while (true) { using (IDbCommand cmd = db.CreateCommand ()) { // execute this in chunks to avoid huge data transfers and slowdowns. cmd.CommandText = "SELECT * FROM Revision WHERE (diff_file_id IS NULL AND NOT diff = '') OR (log_file_id IS NULL AND NOT log = '') LIMIT 100"; using (IDataReader reader = cmd.ExecuteReader ()) { if (!reader.Read ()) break; do { DBRevision revision = new DBRevision (reader); string tmpfile = null; if (!string.IsNullOrEmpty (revision.diff)) { int length = 0; if (revision.diff_file_id == null) { try { length = revision.diff.Length; tmpfile = Path.GetTempFileName (); File.WriteAllText (tmpfile, revision.diff); DBFile diff = download_db.Upload (tmpfile, ".log", false, null); revision.diff_file_id = diff.id; revision.diff = null; } finally { try { if (File.Exists (tmpfile)) File.Delete (tmpfile); } catch (Exception ex) { log.ErrorFormat ("error deleting temp file: {0}", ex); } } moved_bytes += length; log.InfoFormat ("MoveFilesToFileSystem: Moved revision {0}'s diff to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes); } } if (!string.IsNullOrEmpty (revision.log)) { int length = 0; if (revision.log_file_id == null) { try { length = revision.log.Length; tmpfile = Path.GetTempFileName (); File.WriteAllText (tmpfile, revision.log); DBFile log = download_db.Upload (tmpfile, ".log", false, null); revision.log_file_id = log.id; revision.log = null; } finally { try { if (File.Exists (tmpfile)) File.Delete (tmpfile); } catch (Exception ex) { log.ErrorFormat ("error deleting temp file: {0}", ex); } } moved_bytes += length; Manager.log.InfoFormat ("MoveFilesToFileSystem: Moved revision {0}'s log to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes); } revision.log = null; } revision.Save (download_db); } while (reader.Read ()); } } } } } Manager.log.Info ("MoveFilesToFileSystem: [Done]"); return 0; }
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people) { DBPerson person = new DBPerson (); person.fullname = revision.author; people.Add (person); }