protected override bool UpdateRevisionsInDBInternal(DB db, DBLane lane, string repository, Dictionary <string, DBRevision> revisions, List <DBHost> hosts, List <DBHostLane> hostlanes, string min_revision, string max_revision) { string revision; bool update_steps = false; List <DateTime> used_dates; DBRevision r; List <GitEntry> log; if (string.IsNullOrEmpty(max_revision)) { max_revision = "remotes/origin/master"; } Log("Updating lane: '{0}', repository: '{1}' min revision: '{2}' max revision: '{3}'", lane.lane, repository, min_revision, max_revision); log = GetGITLog(lane, repository, min_revision, max_revision); if (log == null || log.Count == 0) { Log("Didn't get a git log for '{0}'", repository); return(false); } Log("Got {0} log records", log.Count); used_dates = new List <DateTime> (); foreach (GitEntry entry in log) { string hash = entry.revision; string unix_timestamp_str = entry.timestamp; long unix_timestamp; string author = entry.author; string msg = entry.message; DateTime date; if (!long.TryParse(unix_timestamp_str, out unix_timestamp)) { /* here something is wrong, this way the commit shows up as the first one so that it's easy to spot and start investigating */ date = DateTime.Now.AddYears(20); Log("Could not parse timestamp '{0}' for revision '{1}' in lane '{2}' in repository {3}", unix_timestamp_str, entry.revision, lane.lane, repository); } else { const long EPOCH_DIFF = 0x019DB1DED53E8000; /* 116444736000000000 nsecs */ const long RATE_DIFF = 10000000; /* 100 nsecs */ date = DateTime.FromFileTimeUtc((unix_timestamp * RATE_DIFF) + EPOCH_DIFF); } /* * The timestamp resolution on my machine seems to be 1 second, * which means that if you commit fast enough you'll get * commits with the same date. This is a very bad thing since * the commits are order by the commit date, and if two commits * have the same date the order they're build / shown is random * (the db decides whatever it feels like). Work around this by * keeping a list of used dates and if the date has already * used, add a millisecond to it (and try again). Note that * there is still a possibility of duplicate dates: if there * already is a revision in the database with this date (from * a previous run of the scheduler). * * It may seem like there is a very small possibility of having * two commits within a second, but this happens all the time * for our test suite. */ while (used_dates.Contains(date)) { date = date.AddMilliseconds(1); } used_dates.Add(date); revision = hash; if (revision == null) { continue; } if (revisions.ContainsKey(revision)) { /* Check if we've saved the wrong date earlier and fix it */ if (revisions [revision].date > new DateTime(2030, 1, 1)) { /* Hopefully this code will not stay here for 20 years */ revisions [revision].date = date; revisions [revision].Save(db); Log("Detected wrong date in revision '{0}' in lane '{1}' in repository {2}, fixing it", revision, lane.lane, repository); } // Log (2, "Already got {0}", revision); continue; } if (!string.IsNullOrEmpty(lane.commit_filter)) { FetchFiles(entry, repository); if (DoesFilterExclude(entry, lane.commit_filter)) { continue; } } r = new DBRevision(); r.revision = revision; r.lane_id = lane.id; r.author = author; if (string.IsNullOrEmpty(r.author)) { Log("No author specified in r{0} in {1}", r.revision, repository); r.author = "?"; } r.date = date; if (!string.IsNullOrEmpty(msg)) { r.log_file_id = db.UploadString(msg, ".log", false).id; } else { Log("No msg specified in r{0} in {1}", r.revision, repository); r.log_file_id = null; } r.Save(db); update_steps = true; Log(1, "Saved revision '{0}' for lane '{1}' author: {2}, date: {3:yyyy/MM/dd HH:mm:ss.ffffff} {5} {6}", r.revision, lane.lane, r.author, r.date, msg, unix_timestamp, unix_timestamp_str); } return(update_steps); }
public static int MoveFilesToFileSystem() { long moved_bytes = 0; Manager.log.Info("MoveFilesToFileSystem: [START]"); using (DB db = new DB()) { using (DB download_db = new DB()) { while (true) { using (IDbCommand cmd = db.CreateCommand()) { // execute this in chunks to avoid huge data transfers and slowdowns. cmd.CommandText = "SELECT * FROM File WHERE NOT file_id IS NULL LIMIT 100"; using (IDataReader reader = cmd.ExecuteReader()) { if (!reader.Read()) { break; } do { DBFile file = new DBFile(reader); byte [] buffer = new byte [1024]; int oid = file.file_id.Value; int read; string fn = FileUtilities.CreateFilename(file.md5, file.compressed_mime == MimeTypes.GZ, true); using (FileStream writer = new FileStream(fn, FileMode.Create, FileAccess.Write, FileShare.Read)) { using (Stream str = download_db.Download(file)) { while ((read = str.Read(buffer, 0, buffer.Length)) != 0) { writer.Write(buffer, 0, read); } } } IDbTransaction transaction = download_db.BeginTransaction(); download_db.Manager.Delete(oid); file.file_id = null; file.Save(download_db); transaction.Commit(); moved_bytes += file.size; log.InfoFormat("MoveFilesToFileSystem: Moved oid {0} to {1} ({2} bytes, {3} total bytes moved)", oid, fn, file.size, moved_bytes); } while (reader.Read()); } } } while (true) { using (IDbCommand cmd = db.CreateCommand()) { // execute this in chunks to avoid huge data transfers and slowdowns. cmd.CommandText = "SELECT * FROM Revision WHERE (diff_file_id IS NULL AND NOT diff = '') OR (log_file_id IS NULL AND NOT log = '') LIMIT 100"; using (IDataReader reader = cmd.ExecuteReader()) { if (!reader.Read()) { break; } do { DBRevision revision = new DBRevision(reader); string tmpfile = null; if (!string.IsNullOrEmpty(revision.diff)) { int length = 0; if (revision.diff_file_id == null) { try { length = revision.diff.Length; tmpfile = Path.GetTempFileName(); File.WriteAllText(tmpfile, revision.diff); DBFile diff = download_db.Upload(tmpfile, ".log", false, null); revision.diff_file_id = diff.id; revision.diff = null; } finally { try { if (File.Exists(tmpfile)) { File.Delete(tmpfile); } } catch (Exception ex) { log.ErrorFormat("error deleting temp file: {0}", ex); } } moved_bytes += length; log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s diff to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes); } } if (!string.IsNullOrEmpty(revision.log)) { int length = 0; if (revision.log_file_id == null) { try { length = revision.log.Length; tmpfile = Path.GetTempFileName(); File.WriteAllText(tmpfile, revision.log); DBFile log = download_db.Upload(tmpfile, ".log", false, null); revision.log_file_id = log.id; revision.log = null; } finally { try { if (File.Exists(tmpfile)) { File.Delete(tmpfile); } } catch (Exception ex) { log.ErrorFormat("error deleting temp file: {0}", ex); } } moved_bytes += length; Manager.log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s log to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes); } revision.log = null; } revision.Save(download_db); } while (reader.Read()); } } } } } Manager.log.Info("MoveFilesToFileSystem: [Done]"); return(0); }