protected override bool UpdateRevisionsInDBInternal(DB db, DBLane lane, string repository, Dictionary <string, DBRevision> revisions, List <DBHost> hosts, List <DBHostLane> hostlanes, string min_revision, string max_revision)
{
string revision;
bool update_steps = false;
List <DateTime> used_dates;
DBRevision r;
List <GitEntry> log;
if (string.IsNullOrEmpty(max_revision))
{
max_revision = "remotes/origin/master";
}
Log("Updating lane: '{0}', repository: '{1}' min revision: '{2}' max revision: '{3}'", lane.lane, repository, min_revision, max_revision);
log = GetGITLog(lane, repository, min_revision, max_revision);
if (log == null || log.Count == 0)
{
Log("Didn't get a git log for '{0}'", repository);
return(false);
}
Log("Got {0} log records", log.Count);
used_dates = new List <DateTime> ();
foreach (GitEntry entry in log)
{
string hash = entry.revision;
string unix_timestamp_str = entry.timestamp;
long unix_timestamp;
string author = entry.author;
string msg = entry.message;
DateTime date;
if (!long.TryParse(unix_timestamp_str, out unix_timestamp))
{
/* here something is wrong, this way the commit shows up as the first one so that it's easy to spot and start investigating */
date = DateTime.Now.AddYears(20);
Log("Could not parse timestamp '{0}' for revision '{1}' in lane '{2}' in repository {3}", unix_timestamp_str, entry.revision, lane.lane, repository);
}
else
{
const long EPOCH_DIFF = 0x019DB1DED53E8000; /* 116444736000000000 nsecs */
const long RATE_DIFF = 10000000; /* 100 nsecs */
date = DateTime.FromFileTimeUtc((unix_timestamp * RATE_DIFF) + EPOCH_DIFF);
}
/*
* The timestamp resolution on my machine seems to be 1 second,
* which means that if you commit fast enough you'll get
* commits with the same date. This is a very bad thing since
* the commits are order by the commit date, and if two commits
* have the same date the order they're build / shown is random
* (the db decides whatever it feels like). Work around this by
* keeping a list of used dates and if the date has already
* used, add a millisecond to it (and try again). Note that
* there is still a possibility of duplicate dates: if there
* already is a revision in the database with this date (from
* a previous run of the scheduler).
*
* It may seem like there is a very small possibility of having
* two commits within a second, but this happens all the time
* for our test suite.
*/
while (used_dates.Contains(date))
{
date = date.AddMilliseconds(1);
}
used_dates.Add(date);
revision = hash;
if (revision == null)
{
continue;
}
if (revisions.ContainsKey(revision))
{
/* Check if we've saved the wrong date earlier and fix it */
if (revisions [revision].date > new DateTime(2030, 1, 1))
{
/* Hopefully this code will not stay here for 20 years */
revisions [revision].date = date;
revisions [revision].Save(db);
Log("Detected wrong date in revision '{0}' in lane '{1}' in repository {2}, fixing it", revision, lane.lane, repository);
}
// Log (2, "Already got {0}", revision);
continue;
}
if (!string.IsNullOrEmpty(lane.commit_filter))
{
FetchFiles(entry, repository);
if (DoesFilterExclude(entry, lane.commit_filter))
{
continue;
}
}
r = new DBRevision();
r.revision = revision;
r.lane_id = lane.id;
r.author = author;
if (string.IsNullOrEmpty(r.author))
{
Log("No author specified in r{0} in {1}", r.revision, repository);
r.author = "?";
}
r.date = date;
if (!string.IsNullOrEmpty(msg))
{
r.log_file_id = db.UploadString(msg, ".log", false).id;
}
else
{
Log("No msg specified in r{0} in {1}", r.revision, repository);
r.log_file_id = null;
}
r.Save(db);
update_steps = true;
Log(1, "Saved revision '{0}' for lane '{1}' author: {2}, date: {3:yyyy/MM/dd HH:mm:ss.ffffff} {5} {6}", r.revision, lane.lane, r.author, r.date, msg, unix_timestamp, unix_timestamp_str);
}
return(update_steps);
}
public static int MoveFilesToFileSystem()
{
long moved_bytes = 0;
Manager.log.Info("MoveFilesToFileSystem: [START]");
using (DB db = new DB()) {
using (DB download_db = new DB()) {
while (true)
{
using (IDbCommand cmd = db.CreateCommand()) {
// execute this in chunks to avoid huge data transfers and slowdowns.
cmd.CommandText = "SELECT * FROM File WHERE NOT file_id IS NULL LIMIT 100";
using (IDataReader reader = cmd.ExecuteReader()) {
if (!reader.Read())
{
break;
}
do
{
DBFile file = new DBFile(reader);
byte [] buffer = new byte [1024];
int oid = file.file_id.Value;
int read;
string fn = FileUtilities.CreateFilename(file.md5, file.compressed_mime == MimeTypes.GZ, true);
using (FileStream writer = new FileStream(fn, FileMode.Create, FileAccess.Write, FileShare.Read)) {
using (Stream str = download_db.Download(file)) {
while ((read = str.Read(buffer, 0, buffer.Length)) != 0)
{
writer.Write(buffer, 0, read);
}
}
}
IDbTransaction transaction = download_db.BeginTransaction();
download_db.Manager.Delete(oid);
file.file_id = null;
file.Save(download_db);
transaction.Commit();
moved_bytes += file.size;
log.InfoFormat("MoveFilesToFileSystem: Moved oid {0} to {1} ({2} bytes, {3} total bytes moved)", oid, fn, file.size, moved_bytes);
} while (reader.Read());
}
}
}
while (true)
{
using (IDbCommand cmd = db.CreateCommand()) {
// execute this in chunks to avoid huge data transfers and slowdowns.
cmd.CommandText = "SELECT * FROM Revision WHERE (diff_file_id IS NULL AND NOT diff = '') OR (log_file_id IS NULL AND NOT log = '') LIMIT 100";
using (IDataReader reader = cmd.ExecuteReader()) {
if (!reader.Read())
{
break;
}
do
{
DBRevision revision = new DBRevision(reader);
string tmpfile = null;
if (!string.IsNullOrEmpty(revision.diff))
{
int length = 0;
if (revision.diff_file_id == null)
{
try {
length = revision.diff.Length;
tmpfile = Path.GetTempFileName();
File.WriteAllText(tmpfile, revision.diff);
DBFile diff = download_db.Upload(tmpfile, ".log", false, null);
revision.diff_file_id = diff.id;
revision.diff = null;
} finally {
try {
if (File.Exists(tmpfile))
{
File.Delete(tmpfile);
}
} catch (Exception ex) {
log.ErrorFormat("error deleting temp file: {0}", ex);
}
}
moved_bytes += length;
log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s diff to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
}
}
if (!string.IsNullOrEmpty(revision.log))
{
int length = 0;
if (revision.log_file_id == null)
{
try {
length = revision.log.Length;
tmpfile = Path.GetTempFileName();
File.WriteAllText(tmpfile, revision.log);
DBFile log = download_db.Upload(tmpfile, ".log", false, null);
revision.log_file_id = log.id;
revision.log = null;
} finally {
try {
if (File.Exists(tmpfile))
{
File.Delete(tmpfile);
}
} catch (Exception ex) {
log.ErrorFormat("error deleting temp file: {0}", ex);
}
}
moved_bytes += length;
Manager.log.InfoFormat("MoveFilesToFileSystem: Moved revision {0}'s log to db/filesystem ({1} bytes, {2} total bytes moved)", revision.id, length, moved_bytes);
}
revision.log = null;
}
revision.Save(download_db);
} while (reader.Read());
}
}
}
}
}
Manager.log.Info("MoveFilesToFileSystem: [Done]");
return(0);
}
