protected override bool UpdateRevisionsInDBInternal (DB db, DBLane lane, string repository, Dictionary<string, DBRevision> revisions, List<DBHost> hosts, List<DBHostLane> hostlanes, string min_revision, string max_revision)
{
string revision;
bool update_steps = false;
List<DateTime> used_dates;
DBRevision r;
List<GitEntry> log;
if (string.IsNullOrEmpty (max_revision))
max_revision = "remotes/origin/master";
GITUpdater.log.InfoFormat ("Updating lane: '{0}', repository: '{1}' min revision: '{2}' max revision: '{3}'", lane.lane, repository, min_revision, max_revision);
log = GetGITLog (lane, repository, min_revision, max_revision);
if (log == null || log.Count == 0) {
GITUpdater.log.WarnFormat ("Didn't get a git log for '{0}'", repository);
return false;
}
GITUpdater.log.InfoFormat ("Got {0} log records", log.Count);
used_dates = new List<DateTime> ();
foreach (GitEntry entry in log) {
string hash = entry.revision;
string unix_timestamp_str = entry.timestamp;
long unix_timestamp;
string author = entry.author;
DateTime date;
if (!long.TryParse (unix_timestamp_str, out unix_timestamp)) {
/* here something is wrong, this way the commit shows up as the first one so that it's easy to spot and start investigating */
date = DateTime.Now.AddYears (20);
GITUpdater.log.WarnFormat ("Could not parse timestamp '{0}' for revision '{1}' in lane '{2}' in repository {3}", unix_timestamp_str, entry.revision, lane.lane, repository);
} else {
const long EPOCH_DIFF = 0x019DB1DED53E8000; /* 116444736000000000 nsecs */
const long RATE_DIFF = 10000000; /* 100 nsecs */
date = DateTime.FromFileTimeUtc ((unix_timestamp * RATE_DIFF) + EPOCH_DIFF);
}
/*
* The timestamp resolution on my machine seems to be 1 second,
* which means that if you commit fast enough you'll get
* commits with the same date. This is a very bad thing since
* the commits are order by the commit date, and if two commits
* have the same date the order they're build / shown is random
* (the db decides whatever it feels like). Work around this by
* keeping a list of used dates and if the date has already
* used, add a millisecond to it (and try again). Note that
* there is still a possibility of duplicate dates: if there
* already is a revision in the database with this date (from
* a previous run of the scheduler).
*
* It may seem like there is a very small possibility of having
* two commits within a second, but this happens all the time
* for our test suite.
*/
while (used_dates.Contains (date)) {
date = date.AddMilliseconds (1);
}
used_dates.Add (date);
revision = hash;
if (revision == null)
continue;
if (revisions.ContainsKey (revision)) {
/* Check if we've saved the wrong date earlier and fix it */
if (revisions [revision].date > new DateTime (2030, 1, 1)) {
/* Hopefully this code will not stay here for 20 years */
revisions [revision].date = date;
revisions [revision].Save (db);
GITUpdater.log.WarnFormat ("Detected wrong date in revision '{0}' in lane '{1}' in repository {2}, fixing it", revision, lane.lane, repository);
}
// Log (2, "Already got {0}", revision);
continue;
}
if (!string.IsNullOrEmpty (lane.commit_filter)) {
FetchFiles (entry, repository);
if (DoesFilterExclude (entry, lane.commit_filter))
continue;
}
r = new DBRevision ();
r.revision = revision;
r.lane_id = lane.id;
r.author = author;
if (string.IsNullOrEmpty (r.author)) {
GITUpdater.log.WarnFormat ("No author specified in r{0} in {1}", r.revision, repository);
r.author = "?";
}
r.date = date;
r.log_file_id = null;
r.Save (db);
update_steps = true;
GITUpdater.log.DebugFormat ("Saved revision '{0}' for lane '{1}' author: {2}, date: {3:yyyy/MM/dd HH:mm:ss.ffffff} {4} {5}", r.revision, lane.lane, r.author, r.date, unix_timestamp, unix_timestamp_str);
}
return update_steps;
}
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people)
{
DBPerson person;
try {
foreach (string repository in lane.repository.Split (new char [] { ',' }, StringSplitOptions.RemoveEmptyEntries)) {
string cache_dir = Configuration.GetSchedulerRepositoryCacheDirectory (repository);
if (!Directory.Exists (cache_dir))
continue;
using (Process git = new Process ()) {
DateTime git_start = DateTime.Now;
git.StartInfo.FileName = "git";
git.StartInfo.Arguments = "log -1 --pretty=format:'%aE%n%aN%n%cE%n%cN' " + revision.revision;
git.StartInfo.WorkingDirectory = cache_dir;
git.StartInfo.UseShellExecute = false;
git.StartInfo.RedirectStandardOutput = true;
git.Start ();
string author_email = git.StandardOutput.ReadLine ();
string author_name = git.StandardOutput.ReadLine ();
string committer_email = git.StandardOutput.ReadLine ();
string committer_name = git.StandardOutput.ReadLine ();
// Wait 10 minutes for git to finish, otherwise abort.
if (!git.WaitForExit (1000 * 60 * 10)) {
GITUpdater.log.Error ("Getting commit info took more than 10 minutes, aborting.");
try {
git.Kill ();
git.WaitForExit (10000); // Give the process 10 more seconds to completely exit.
} catch (Exception ex) {
GITUpdater.log.ErrorFormat ("Aborting commit info retrieval failed: {0}", ex.ToString ());
}
}
if (git.HasExited && git.ExitCode == 0) {
GITUpdater.log.InfoFormat ("Got commit info successfully in {0} seconds", (DateTime.Now - git_start).TotalSeconds);
person = new DBPerson ();
person.fullname = author_name;
person.Emails = new string [] { author_email };
people.Add (person);
if (author_name != committer_name && !string.IsNullOrEmpty (committer_name)) {
person = new DBPerson ();
person.fullname = committer_name;
person.Emails = new string [] {committer_email};
people.Add (person);
}
GITUpdater.log.DebugFormat ("Git commit info for {0}: author_name = {1} author_email: {2} committer_name: {3} committer_email: {4}", revision.revision, author_name, author_email, committer_name, committer_email);
} else {
GITUpdater.log.ErrorFormat ("Didn't get commit info, HasExited: {0}, ExitCode: {1}", git.HasExited, git.HasExited ? git.ExitCode.ToString () : "N/A");
}
}
}
} catch (Exception ex) {
GITUpdater.log.ErrorFormat ("Exception while trying to get commit info: {0}", ex.ToString ());
}
}
public static DBRevisionWork Find (DB db, DBLane lane, DBHost host, DBRevision revision)
{
DBRevisionWork result;
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = "SELECT * FROM RevisionWork WHERE lane_id = @lane_id AND revision_id = @revision_id ";
if (host != null) {
cmd.CommandText += " AND host_id = @host_id;";
DB.CreateParameter (cmd, "host_id", host.id);
}
DB.CreateParameter (cmd, "lane_id", lane.id);
DB.CreateParameter (cmd, "revision_id", revision.id);
using (IDataReader reader = cmd.ExecuteReader ()) {
if (!reader.Read ())
return null;
result = new DBRevisionWork (reader);
if (reader.Read ())
throw new ApplicationException (string.Format ("Found more than one revision work for the specified lane/host/revision ({0}/{1}/{2})", lane.lane, host == null ? "null" : host.host, revision.revision));
return result;
}
}
}
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people)
{
if (lane.source_control == "git") {
GITUpdater.FindPeopleForCommit (lane, revision, people);
/*
} else if (lane.source_control == "svn") {
SVNUpdater.FindPeopleForCommit (lane, revision, people);
* */
} else {
log.ErrorFormat ("Unknown source control for lane {0}: {1}", lane.lane, lane.source_control);
}
}
public DBRevision GetRevision (string lane, int revision)
{
using (IDbCommand cmd = CreateCommand ()) {
cmd.CommandText = "SELECT * from revisions where lane = @lane AND revision = @revision";
DB.CreateParameter (cmd, "lane", lane);
DB.CreateParameter (cmd, "revision", revision.ToString ());
using (IDataReader reader = cmd.ExecuteReader ()) {
if (!reader.Read ())
return null;
if (reader.IsDBNull (0))
return null;
DBRevision rev = new DBRevision ();
rev.Load (reader);
return rev;
}
}
}
/// <summary>
/// Finds pending steps for the current revision
/// </summary>
/// <returns></returns>
public static List<DBWorkView2> GetNextWork (this DBRevisionWork rw, DB db, DBLane lane, DBHost host, DBRevision revision, bool multiple_work)
{
List<DBWorkView2> result = new List<DBWorkView2> (); ;
if (revision == null)
throw new ArgumentNullException ("revision");
if (lane == null)
throw new ArgumentNullException ("lane");
if (host == null)
throw new ArgumentNullException ("host");
rw.FilterPendingWork (db, db.GetWork (rw), result, multiple_work);
if (result.Count == 0 && !rw.completed) {
rw.completed = true;
rw.UpdateState (db);
}
return result;
}
private static void UpdateSVNDiff (object dummy)
{
try {
Logger.Log ("SVNDiff: Thread started.");
using (DB db = new DB (true)) {
using (DB db_save = new DB (true)) {
using (IDbCommand cmd = db.CreateCommand ()) {
cmd.CommandText = @"
SELECT Revision.*, Lane.repository, Lane.lane
FROM Revision
INNER JOIN Lane ON Lane.id = Revision.lane_id
WHERE (Revision.diff IS NULL OR Revision.diff = '') AND Revision.diff_file_id IS NULL;";
using (IDataReader reader = cmd.ExecuteReader ()) {
while (!quit_svn_diff && reader.Read ()) {
DBRevision revision = new DBRevision (reader);
string repositories = reader.GetString (reader.GetOrdinal ("repository"));
string lane = reader.GetString (reader.GetOrdinal ("lane"));
string diff = null;
foreach (string repository in repositories.Split (',')) {
diff = GetSVNDiff (lane, repository, revision.revision);
if (!string.IsNullOrEmpty (diff))
break;
}
if (string.IsNullOrEmpty (diff))
diff = "No diff";
revision.diff_file_id = db_save.UploadString (diff, ".log", false).id;
revision.Save (db_save);
Logger.Log ("SVNDiff: Got diff for lane '{0}', revision '{1}'", lane, revision.revision);
}
}
}
}
}
Logger.Log ("SVNDiff: Thread stopping. Done: {0}", !quit_svn_diff);
} catch (Exception ex) {
Logger.Log ("SVNDiff: Exception: {0} \n{1}", ex.Message, ex.StackTrace);
}
}
//public List<string> GetRevisions(DBLane lane)
//{
// List<string> result = new List<string>();
// using (IDbCommand cmd = Connection.CreateCommand())
// {
// cmd.CommandText = "SELECT DISTINCT revision FROM revisions WHERE lane = @lane ORDER BY revision DESC";
// DB.CreateParameter(cmd, "lane", lane);
// using (IDataReader reader = cmd.ExecuteReader())
// {
// while (reader.Read())
// result.Add(reader.GetString(0));
// }
// }
// return result;
//}
public Dictionary<string, DBRevision> GetDBRevisions (int lane_id)
{
Dictionary<string, DBRevision> result = new Dictionary<string, DBRevision> ();
DBRevision rev;
using (IDbCommand cmd = CreateCommand ()) {
cmd.CommandText = "SELECT * FROM Revision WHERE lane_id = @lane_id";
DB.CreateParameter (cmd, "lane_id", lane_id);
using (IDataReader reader = cmd.ExecuteReader ()) {
while (reader.Read ()) {
rev = new DBRevision ();
rev.Load (reader);
result.Add (rev.revision, rev);
}
}
}
return result;
}
void GenerateActionLink(StringBuilder header, DBLane lane, DBHost host, DBRevision dbr, string cmd, string short_label, string long_label, bool hidden = false)
{
var href = confirmViewLaneAction(lane, host, dbr, cmd, short_label);
if (hidden)
header.AppendFormat("- <a style='{0}' href='{1}'>{2}</a>", "display:none", href, long_label);
else
header.AppendFormat("- <a href='{0}'>{1}</a>", href, long_label);
}
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people)
{
if (lane.source_control == "git") {
GITUpdater.FindPeopleForCommit (lane, revision, people);
/*
} else if (lane.source_control == "svn") {
SVNUpdater.FindPeopleForCommit (lane, revision, people);
* */
} else {
Logger.Log ("FindPeopleForCommit (): unknown source control: '{0}'", lane.source_control);
}
}
string confirmViewLaneAction(DBLane lane, DBHost host, DBRevision dbr, string action, string command)
{
return String.Format("javascript:confirmViewLaneAction (\"ViewLane.aspx?lane_id={0}&host_id={2}&revision_id={1}&action={3}\", \"{4}\");", lane.id, dbr.id, host.id, action, command);
}
public List<DBRevision> GetDBRevisions (int lane_id, int limit)
{
List<DBRevision> result = new List<DBRevision> ();
DBRevision rev;
using (IDbCommand cmd = CreateCommand ()) {
cmd.CommandText = "SELECT Revision.*, CAST (revision as int) AS r FROM Revision WHERE lane_id = @lane_id ORDER BY r DESC";
if (limit > 0)
cmd.CommandText += " LIMIT " + limit.ToString ();
DB.CreateParameter (cmd, "lane_id", lane_id);
using (IDataReader reader = cmd.ExecuteReader ()) {
while (reader.Read ()) {
rev = new DBRevision ();
rev.Load (reader);
result.Add (rev);
}
}
}
return result;
}
public List<DBRevision> GetDBRevisionsWithoutWork (int lane_id, int host_id)
{
List<DBRevision> result = new List<DBRevision> ();
DBRevision rev;
using (IDbCommand cmd = CreateCommand ()) {
//cmd.CommandText = "SELECT * FROM Revision WHERE lane_id = @lane_id AND NOT EXISTS (SELECT 1 FROM Work WHERE lane_id = @lane_id AND host_id = @host_id AND revision_id = revision.id) ORDER BY date DESC";
cmd.CommandText = @"
SELECT Revision.*, C
FROM
(SELECT RevisionWork.id, RevisionWork.revision_id, Count(Work.revisionwork_id) AS C
FROM RevisionWork
LEFT JOIN work ON Work.revisionwork_id = RevisionWork.id
WHERE RevisionWork.lane_id = @lane_id AND RevisionWork.host_id = @host_id
GROUP BY RevisionWork.id, RevisionWork.revision_id) AS T
INNER JOIN Revision ON Revision.id = T.revision_id
INNER JOIN RevisionWork ON T.id = RevisionWork.id
WHERE C = 0 OR RevisionWork.state = 9
ORDER BY Revision.date DESC;
";
DB.CreateParameter (cmd, "lane_id", lane_id);
DB.CreateParameter (cmd, "host_id", host_id);
using (IDataReader reader = cmd.ExecuteReader ()) {
while (reader.Read ()) {
rev = new DBRevision ();
rev.Load (reader);
result.Add (rev);
}
}
}
return result;
}
public static string GenerateHeader (GetViewLaneDataResponse response, DBLane lane, DBHost host, DBRevision revision, string description)
{
if (!Authentication.IsInRole (response, MonkeyWrench.DataClasses.Logic.Roles.Administrator)) {
return string.Format (@"
<h2>{4} revision <a href='ViewLane.aspx?lane_id={0}&host_id={1}&revision_id={6}'>{5}</a> on lane '{2}' on '<a href='ViewHostHistory.aspx?host_id={1}'>{3}</a>'
(<a href='ViewTable.aspx?lane_id={0}&host_id={1}'>table</a>)</h2><br/>", lane.id, host.id, lane.lane, host.host, description, revision.revision, revision.id);
} else {
return string.Format (@"
<h2>{4} revision <a href='ViewLane.aspx?lane_id={0}&host_id={1}&revision_id={6}'>{5}</a> on lane '<a href='EditLane.aspx?lane_id={0}'>{2}</a>' on '<a href='ViewHostHistory.aspx?host_id={1}'>{3}</a>'
(<a href='ViewTable.aspx?lane_id={0}&host_id={1}'>table</a>)</h2><br/>", lane.id, host.id, lane.lane, host.host, description, revision.revision, revision.id);
}
}
protected override bool UpdateRevisionsInDBInternal (DB db, DBLane lane, string repository,Dictionary<string, DBRevision> revisions, List<DBHost> hosts, List<DBHostLane> hostlanes, string min_revision)
{
string revision;
XmlDocument svn_log;
bool update_steps = false;
DBRevision r;
int min_revision_int = string.IsNullOrEmpty (min_revision) ? 0 : int.Parse (min_revision);
int max_revision_int = int.MaxValue;
int current_revision;
string log;
XmlNode n;
XmlAttribute attrib;
Log ("Updating '{0}'", lane.lane);
if (min_revision_int == 0 && !string.IsNullOrEmpty (lane.min_revision))
min_revision_int = int.Parse (lane.min_revision);
if (!string.IsNullOrEmpty (lane.max_revision))
max_revision_int = int.Parse (lane.max_revision);
log = GetSVNLog (lane, repository, min_revision_int, max_revision_int);
if (string.IsNullOrEmpty (log)) {
Log ("Didn't get a svn log for '{0}'", repository);
return false;
}
svn_log = new XmlDocument ();
svn_log.PreserveWhitespace = true;
svn_log.Load (new StringReader (log));
foreach (XmlNode node in svn_log.SelectNodes ("/log/logentry")) {
revision = node.Attributes ["revision"].Value;
if (revisions.ContainsKey (revision))
continue;
try {
current_revision = int.Parse (revision);
if (current_revision < min_revision_int || current_revision > max_revision_int)
continue;
} catch {
continue;
}
r = new DBRevision ();
attrib = node.Attributes ["revision"];
if (attrib == null || string.IsNullOrEmpty (attrib.Value)) {
Log ("An entry without revision in {0}, skipping entry", repository);
continue;
}
r.revision = attrib.Value;
r.lane_id = lane.id;
n = node.SelectSingleNode ("author");
if (n != null) {
r.author = n.InnerText;
} else {
Log ("No author specified in r{0} in {1}", r.revision, repository);
r.author = "?";
}
n = node.SelectSingleNode ("date");
if (n != null) {
DateTime dt;
if (DateTime.TryParse (n.InnerText, out dt)) {
r.date = dt;
} else {
Log ("Could not parse the date '{0}' in r{1} in {2}", n.InnerText, r.revision, repository);
r.date = DateTime.MinValue;
}
} else {
Log ("No date specified in r{0} in {1}", r.revision, repository);
r.date = DateTime.MinValue;
}
n = node.SelectSingleNode ("msg");
if (n != null) {
r.log_file_id = db.UploadString (n.InnerText, ".log", false).id;
} else {
Log ("No msg specified in r{0} in {1}", r.revision, repository);
r.log_file_id = null;
}
r.Save (db);
update_steps = true;
Log ("Saved revision '{0}' for lane '{1}'", r.revision, lane.lane);
}
return update_steps;
}
public static int MoveFilesToFileSystem ()
{
long moved_bytes = 0;
Manager.log.Info ("MoveFilesToFileSystem: [START]");
using (DB db = new DB ()) {
using (DB download_db = new DB ()) {
while (true) {
using (IDbCommand cmd = db.CreateCommand ()) {
// execute this in chunks to avoid huge data transfers and slowdowns.
cmd.CommandText = "SELECT * FROM File WHERE NOT file_id IS NULL LIMIT 100";
using (IDataReader reader = cmd.ExecuteReader ()) {
if (!reader.Read ())
break;
do {
DBFile file = new DBFile (reader);
byte [] buffer = new byte [1024];
int oid = file.file_id.Value;
int read;
string fn = FileUtilities.CreateFilename (file.md5, file.compressed_mime == MimeTypes.GZ, true);
using (FileStream writer = new FileStream (fn, FileMode.Create, FileAccess.Write, FileShare.Read)) {
using (Stream str = download_db.Download (file)) {
while ((read = str.Read (buffer, 0, buffer.Length)) != 0)
writer.Write (buffer, 0, read);
}
}
IDbTransaction transaction = download_db.BeginTransaction ();
download_db.Manager.Delete (oid);
file.file_id = null;
file.Save (download_db);
transaction.Commit ();
moved_bytes += file.size;
log.InfoFormat ("MoveFilesToFileSystem: Moved oid {0} to {1} ({2} bytes, {3} total bytes moved)", oid, fn, file.size, moved_bytes);
} while (reader.Read ());
}
}
}
while (true) {
using (IDbCommand cmd = db.CreateCommand ()) {
// execute this in chunks to avoid huge data transfers and slowdowns.
cmd.CommandText = "SELECT * FROM Revision WHERE (diff_file_id IS NULL AND NOT diff = '') OR (log_file_id IS NULL AND NOT log = '') LIMIT 100";
using (IDataReader reader = cmd.ExecuteReader ()) {
if (!reader.Read ())
break;
do {
DBRevision revision = new DBRevision (reader);
string tmpfile = null;
if (!string.IsNullOrEmpty (revision.diff)) {
int length = 0;
if (revision.diff_file_id == null) {
try {
length = revision.diff.Length;
tmpfile = Path.GetTempFileName ();
File.WriteAllText (tmpfile, revision.diff);
DBFile diff = download_db.Upload (tmpfile, ".log", false, null);
revision.diff_file_id = diff.id;
revision.diff = null;
} finally {
try {
if (File.Exists (tmpfile))
File.Delete (tmpfile);
} catch (Exception ex) {
log.ErrorFormat ("error deleting temp file: {0}", ex);
}
}
moved_bytes += length;
log.InfoFormat ("MoveFilesToFileSystem: Moved revision {0}'s diff to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
}
}
if (!string.IsNullOrEmpty (revision.log)) {
int length = 0;
if (revision.log_file_id == null) {
try {
length = revision.log.Length;
tmpfile = Path.GetTempFileName ();
File.WriteAllText (tmpfile, revision.log);
DBFile log = download_db.Upload (tmpfile, ".log", false, null);
revision.log_file_id = log.id;
revision.log = null;
} finally {
try {
if (File.Exists (tmpfile))
File.Delete (tmpfile);
} catch (Exception ex) {
log.ErrorFormat ("error deleting temp file: {0}", ex);
}
}
moved_bytes += length;
Manager.log.InfoFormat ("MoveFilesToFileSystem: Moved revision {0}'s log to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
}
revision.log = null;
}
revision.Save (download_db);
} while (reader.Read ());
}
}
}
}
}
Manager.log.Info ("MoveFilesToFileSystem: [Done]");
return 0;
}
public static void FindPeopleForCommit (DBLane lane, DBRevision revision, List<DBPerson> people)
{
DBPerson person = new DBPerson ();
person.fullname = revision.author;
people.Add (person);
}
