private bool DoesFilterExclude(GitEntry entry, string filter)
{
string [] expressions;
Regex [] regexes;
bool include_all = false;
if (string.IsNullOrEmpty(filter))
{
return(false);
}
if (filter.StartsWith("ExcludeAllExcept:"))
{
include_all = false;
}
else if (filter.StartsWith("IncludeAllExcept:"))
{
include_all = true;
}
else
{
Log("Invalid commit filter: {0}, including all commits.", filter);
return(false);
}
expressions = filter.Substring(filter.IndexOf(':') + 1).Trim().Split(';');
if (expressions.Length > 0)
{
regexes = new Regex [expressions.Length];
for (int r = 0; r < regexes.Length; r++)
{
regexes [r] = new Regex(FileUtilities.GlobToRegExp(expressions [r].Trim()));
}
for (int f = 0; f < entry.files.Count; f++)
{
for (int r = 0; r < regexes.Length; r++)
{
if (regexes [r].IsMatch(entry.files [f]))
{
return(include_all);
}
}
}
}
return(!include_all);
}
private List<GitEntry> GetGITLog (DBLane dblane, string repository, string min_revision, string max_revision)
{
List<GitEntry> result = null;
try {
GITUpdater.log.InfoFormat ("Retrieving log for '{0}', repository: '{1}', min_revision: {2} max_revision: {3}", dblane.lane, repository, min_revision, max_revision);
// Updating the repository cache
string cache_dir = Configuration.GetSchedulerRepositoryCacheDirectory (repository);
if (!Directory.Exists (cache_dir))
Directory.CreateDirectory (cache_dir);
// Download/update the cache
using (Process git = new Process ()) {
DateTime git_start = DateTime.Now;
if (fetched_directories.Contains (repository)) {
GITUpdater.log.DebugFormat ("Not fetching repository '{0}', it has already been fetched in this run", repository);
} else {
git.StartInfo.FileName = "git";
if (!Directory.Exists (Path.Combine (cache_dir, ".git"))) {
git.StartInfo.Arguments = "clone -q --no-checkout " + repository + " .";
} else {
git.StartInfo.Arguments = "fetch";
}
git.StartInfo.WorkingDirectory = cache_dir;
git.StartInfo.UseShellExecute = false;
git.StartInfo.RedirectStandardOutput = true;
git.StartInfo.RedirectStandardError = true;
git.StartInfo.StandardOutputEncoding = System.Text.Encoding.UTF8;
git.StartInfo.StandardErrorEncoding = System.Text.Encoding.UTF8;
GITUpdater.log.DebugFormat ("Executing: '{0} {1}' in {2}", git.StartInfo.FileName, git.StartInfo.Arguments, cache_dir);
git.OutputDataReceived += delegate (object sender, DataReceivedEventArgs e)
{
if (e.Data == null)
return;
GITUpdater.log.DebugFormat ("FETCH: {0}", e.Data);
};
git.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs e)
{
if (e.Data == null)
return;
GITUpdater.log.WarnFormat ("FETCH STDERR: {0}", e.Data);
};
git.Start ();
git.BeginOutputReadLine ();
git.BeginErrorReadLine ();
if (!git.WaitForExit (1000 * 60 * 10 /* 10 minutes */)) {
GITUpdater.log.ErrorFormat ("Could not fetch repository, git didn't finish in 10 minutes.");
return null;
}
if (!git.HasExited || git.ExitCode != 0) {
GITUpdater.log.ErrorFormat ("Could not fetch repository, HasExited: {0}, ExitCode: {1}", git.HasExited, git.HasExited ? git.ExitCode.ToString () : "N/A");
return null;
}
fetched_directories.Add (repository);
GITUpdater.log.InfoFormat ("Fetched git repository in {0} seconds", (DateTime.Now - git_start).TotalSeconds);
}
}
string range = string.Empty;
if (string.IsNullOrEmpty (min_revision)) {
range = max_revision;
} else {
range = min_revision + "^.." + max_revision;
}
using (Process git = new Process ()) {
DateTime git_start = DateTime.Now;
git.StartInfo.FileName = "git";
// --reverse: git normally gives commits in newest -> oldest, we want to add them to the db in the reverse order
git.StartInfo.Arguments = "rev-list --reverse --header ";
if (!dblane.traverse_merge)
git.StartInfo.Arguments += "--first-parent ";
git.StartInfo.Arguments += range;
GITUpdater.log.DebugFormat ("Executing: '{0} {1}' in {2}", git.StartInfo.FileName, git.StartInfo.Arguments, cache_dir);
git.StartInfo.WorkingDirectory = cache_dir;
git.StartInfo.UseShellExecute = false;
git.StartInfo.RedirectStandardOutput = true;
git.StartInfo.RedirectStandardError = true;
git.StartInfo.WorkingDirectory = cache_dir;
Thread stdout = new Thread (delegate ()
{
StringBuilder builder = new StringBuilder ();
GitEntry current = new GitEntry ();
bool in_header = true;
bool done = false;
while (!done) {
int ch = 0;
if (git.StandardOutput.EndOfStream) {
done = true;
} else {
ch = git.StandardOutput.Read ();
}
if (ch == 0) {
/* end of record */
if (result == null)
result = new List<GitEntry> ();
result.Add (current);
current = new GitEntry ();
in_header = true;
builder.Length = 0;
} else if (in_header && ch == '\n') {
/* end of header line */
if (builder.Length == 0) {
/* entering log message */
in_header = false;
} else {
string header = builder.ToString ();
if (current.revision == null) {
current.revision = header;
} else if (header.StartsWith ("author ")) {
header = header.Substring ("author ".Length, header.IndexOf ('<') - "author ".Length - 1);
current.author = header;
} else if (header.StartsWith ("committer ")) {
header = header.Substring ("committer ".Length);
int gt = header.LastIndexOf ('>');
if (gt > 0) {
current.timestamp = header.Substring (gt + 1).Trim ();
current.timestamp = current.timestamp.Substring (0, current.timestamp.IndexOf (' ')).Trim ();
} else {
GITUpdater.log.Warn ("Could not find timestamp in committer line");
}
} else {
// do nothing
}
}
builder.Length = 0;
} else {
builder.Append ((char) ch);
}
}
});
var stderr_output = new StringBuilder ();
Thread stderr = new Thread (delegate ()
{
string line;
while (null != (line = git.StandardError.ReadLine ())) {
stderr_output.AppendLine (line);
}
});
git.Start ();
stdout.Start ();
stderr.Start ();
// Wait 10 minutes for git to finish, otherwise abort.
if (!git.WaitForExit (1000 * 60 * 10)) {
GITUpdater.log.ErrorFormat ("Getting log took more than 10 minutes, aborting.");
try {
git.Kill ();
git.WaitForExit (10000); // Give the process 10 more seconds to completely exit.
} catch (Exception ex) {
GITUpdater.log.ErrorFormat ("Aborting log retrieval failed: {0}", ex.ToString ());
}
}
stdout.Join ((int) TimeSpan.FromMinutes (1).TotalMilliseconds);
stderr.Join ((int) TimeSpan.FromMinutes (1).TotalMilliseconds);
if (git.HasExited && git.ExitCode == 0) {
GITUpdater.log.InfoFormat ("Got log successfully in {0} seconds", (DateTime.Now - git_start).TotalSeconds);
return result;
} else {
GITUpdater.log.ErrorFormat ("Couldn't get git log for lane {0} repository {1}. HasExited: {2} ExitCode: {3} StandardError: \n{4}", dblane.lane, repository, git.HasExited, git.HasExited ? git.ExitCode.ToString () : "N/A", stderr_output.ToString ());
return null;
}
}
} catch (Exception ex) {
GITUpdater.log.ErrorFormat ("Exception while trying to get git log: {0}", ex);
return null;
}
}
private void FetchFiles (GitEntry entry, string repository)
{
try {
string cache_dir = Configuration.GetSchedulerRepositoryCacheDirectory (repository);
StringBuilder stderr_log = new StringBuilder ();
entry.files = new List<string> ();
using (Process git = new Process ()) {
git.StartInfo.FileName = "git";
git.StartInfo.Arguments = "show --name-only --pretty='format:' " + entry.revision;
GITUpdater.log.DebugFormat ("Executing: '{0} {1}' in {2}", git.StartInfo.FileName, git.StartInfo.Arguments, cache_dir);
git.StartInfo.WorkingDirectory = cache_dir;
git.StartInfo.UseShellExecute = false;
git.StartInfo.RedirectStandardOutput = true;
git.StartInfo.RedirectStandardError = true;
git.StartInfo.WorkingDirectory = cache_dir;
Thread stdout = new Thread (delegate ()
{
string line;
while ((line = git.StandardOutput.ReadLine ()) != null) {
if (string.IsNullOrEmpty (line.Trim ()))
continue;
entry.files.Add (line);
}
});
Thread stderr = new Thread (delegate ()
{
string line;
while (null != (line = git.StandardError.ReadLine ())) {
Console.Error.WriteLine (line);
stderr_log.AppendLine (line);
}
});
git.Start ();
stdout.Start ();
stderr.Start ();
// Wait 10 minutes for git to finish, otherwise abort.
if (!git.WaitForExit (1000 * 60 * 10)) {
GITUpdater.log.ErrorFormat ("Getting files took more than 10 minutes, aborting.");
try {
git.Kill ();
git.WaitForExit (10000); // Give the process 10 more seconds to completely exit.
} catch (Exception ex) {
GITUpdater.log.ErrorFormat ("Aborting file retrieval failed: {0}", ex.ToString ());
}
}
stdout.Join ((int) TimeSpan.FromMinutes (1).TotalMilliseconds);
stderr.Join ((int) TimeSpan.FromMinutes (1).TotalMilliseconds);
if (git.HasExited && git.ExitCode == 0) {
GITUpdater.log.InfoFormat ("Got {0} files successfully", entry.files.Count);
} else {
GITUpdater.log.ErrorFormat ("Didn't get files, HasExited: {0}, ExitCode: {1}, stderr: {2}", git.HasExited, git.HasExited ? git.ExitCode.ToString () : "N/A", stderr_log.ToString ());
}
}
} catch (Exception ex) {
GITUpdater.log.ErrorFormat ("Exception while trying to get files for commit {1} {0}", ex.ToString (), entry.revision);
}
}
private bool DoesFilterExclude (GitEntry entry, string filter)
{
string [] expressions;
Regex [] regexes;
bool include_all = false;
if (string.IsNullOrEmpty (filter))
return false;
if (filter.StartsWith ("ExcludeAllExcept:")) {
include_all = false;
} else if (filter.StartsWith ("IncludeAllExcept:")) {
include_all = true;
} else {
GITUpdater.log.WarnFormat ("Invalid commit filter: {0}, including all commits.", filter);
return false;
}
expressions = filter.Substring (filter.IndexOf (':') + 1).Trim ().Split (';');
if (expressions.Length > 0) {
regexes = new Regex [expressions.Length];
for (int r = 0; r < regexes.Length; r++) {
regexes [r] = new Regex (FileUtilities.GlobToRegExp (expressions [r].Trim ()));
}
for (int f = 0; f < entry.files.Count; f++) {
for (int r = 0; r < regexes.Length; r++) {
if (regexes [r].IsMatch (entry.files [f])) {
return include_all;
}
}
}
}
return !include_all;
}
private List <GitEntry> GetGITLog(DBLane dblane, string repository, string min_revision, string max_revision)
{
List <GitEntry> result = null;
try {
Log("Retrieving log for '{0}', repository: '{1}', min_revision: {2} max_revision: {3}", dblane.lane, repository, min_revision, max_revision);
// Updating the repository cache
string cache_dir = Configuration.GetSchedulerRepositoryCacheDirectory(repository);
if (!Directory.Exists(cache_dir))
{
Directory.CreateDirectory(cache_dir);
}
// Download/update the cache
using (Process git = new Process()) {
DateTime git_start = DateTime.Now;
if (fetched_directories.Contains(repository))
{
Log("Not fetching repository '{0}', it has already been fetched in this run", repository);
}
else
{
git.StartInfo.FileName = "git";
if (!Directory.Exists(Path.Combine(cache_dir, ".git")))
{
git.StartInfo.Arguments = "clone --no-checkout " + repository + " .";
}
else
{
git.StartInfo.Arguments = "fetch";
}
git.StartInfo.WorkingDirectory = cache_dir;
git.StartInfo.UseShellExecute = false;
git.StartInfo.RedirectStandardOutput = true;
git.StartInfo.RedirectStandardError = true;
git.StartInfo.StandardOutputEncoding = System.Text.Encoding.UTF8;
git.StartInfo.StandardErrorEncoding = System.Text.Encoding.UTF8;
Log("Executing: '{0} {1}' in {2}", git.StartInfo.FileName, git.StartInfo.Arguments, cache_dir);
git.OutputDataReceived += delegate(object sender, DataReceivedEventArgs e)
{
if (e.Data == null)
{
return;
}
Log("FETCH: {0}", e.Data);
};
git.ErrorDataReceived += delegate(object sender, DataReceivedEventArgs e)
{
if (e.Data == null)
{
return;
}
Log("FETCH STDERR: {0}", e.Data);
};
git.Start();
git.BeginOutputReadLine();
git.BeginErrorReadLine();
if (!git.WaitForExit(1000 * 60 * 10 /* 10 minutes */))
{
Log("Could not fetch repository, git didn't finish in 10 minutes.");
return(null);
}
if (!git.HasExited || git.ExitCode != 0)
{
Log("Could not fetch repository, HasExited: {0}, ExitCode: {1}", git.HasExited, git.HasExited ? git.ExitCode.ToString() : "N/A");
return(null);
}
fetched_directories.Add(repository);
Log("Fetched git repository in {0} seconds", (DateTime.Now - git_start).TotalSeconds);
}
}
string range = string.Empty;
if (string.IsNullOrEmpty(min_revision))
{
range = max_revision;
}
else
{
range = min_revision + ".." + max_revision;
}
using (Process git = new Process()) {
DateTime git_start = DateTime.Now;
git.StartInfo.FileName = "git";
// --reverse: git normally gives commits in newest -> oldest, we want to add them to the db in the reverse order
git.StartInfo.Arguments = "rev-list --reverse --header ";
if (!dblane.traverse_merge)
{
git.StartInfo.Arguments += "--first-parent ";
}
git.StartInfo.Arguments += range;
Log("Executing: '{0} {1}' in {2}", git.StartInfo.FileName, git.StartInfo.Arguments, cache_dir);
git.StartInfo.WorkingDirectory = cache_dir;
git.StartInfo.UseShellExecute = false;
git.StartInfo.RedirectStandardOutput = true;
git.StartInfo.RedirectStandardError = true;
git.StartInfo.WorkingDirectory = cache_dir;
Thread stdout = new Thread(delegate()
{
StringBuilder builder = new StringBuilder();
GitEntry current = new GitEntry();
bool in_header = true;
bool done = false;
while (!done)
{
int ch = 0;
if (git.StandardOutput.EndOfStream)
{
done = true;
}
else
{
ch = git.StandardOutput.Read();
}
if (ch == 0)
{
/* end of record */
if (result == null)
{
result = new List <GitEntry> ();
}
current.message = builder.ToString();
result.Add(current);
current = new GitEntry();
in_header = true;
builder.Length = 0;
}
else if (in_header && ch == '\n')
{
/* end of header line */
if (builder.Length == 0)
{
/* entering log message */
in_header = false;
}
else
{
string header = builder.ToString();
if (current.revision == null)
{
current.revision = header;
}
else if (header.StartsWith("author "))
{
header = header.Substring("author ".Length, header.IndexOf('<') - "author ".Length - 1);
current.author = header;
}
else if (header.StartsWith("committer "))
{
header = header.Substring("committer ".Length);
int gt = header.LastIndexOf('>');
if (gt > 0)
{
current.timestamp = header.Substring(gt + 1).Trim();
current.timestamp = current.timestamp.Substring(0, current.timestamp.IndexOf(' ')).Trim();
}
else
{
Logger.Log("Could not find timestamp in committer line");
}
}
else
{
// do nothing
}
}
builder.Length = 0;
}
else
{
builder.Append((char)ch);
}
}
});
Thread stderr = new Thread(delegate()
{
string line;
while (null != (line = git.StandardError.ReadLine()))
{
Console.Error.WriteLine(line);
}
});
git.Start();
stdout.Start();
stderr.Start();
// Wait 10 minutes for git to finish, otherwise abort.
if (!git.WaitForExit(1000 * 60 * 10))
{
Log("Getting log took more than 10 minutes, aborting.");
try {
git.Kill();
git.WaitForExit(10000); // Give the process 10 more seconds to completely exit.
} catch (Exception ex) {
Log("Aborting log retrieval failed: {0}", ex.ToString());
}
}
stdout.Join((int)TimeSpan.FromMinutes(1).TotalMilliseconds);
stderr.Join((int)TimeSpan.FromMinutes(1).TotalMilliseconds);
if (git.HasExited && git.ExitCode == 0)
{
Log("Got log successfully in {0} seconds", (DateTime.Now - git_start).TotalSeconds);
return(result);
}
else
{
Log("Didn't get log, HasExited: {0}, ExitCode: {1}", git.HasExited, git.HasExited ? git.ExitCode.ToString() : "N/A");
return(null);
}
}
} catch (Exception ex) {
Log("Exception while trying to get svn log: {0}", ex.ToString());
return(null);
}
}
private void FetchFiles(GitEntry entry, string repository)
{
try {
string cache_dir = Configuration.GetSchedulerRepositoryCacheDirectory(repository);
StringBuilder stderr_log = new StringBuilder();
entry.files = new List <string> ();
using (Process git = new Process()) {
git.StartInfo.FileName = "git";
git.StartInfo.Arguments = "show --name-only --pretty='format:' " + entry.revision;
Log("Executing: '{0} {1}' in {2}", git.StartInfo.FileName, git.StartInfo.Arguments, cache_dir);
git.StartInfo.WorkingDirectory = cache_dir;
git.StartInfo.UseShellExecute = false;
git.StartInfo.RedirectStandardOutput = true;
git.StartInfo.RedirectStandardError = true;
git.StartInfo.WorkingDirectory = cache_dir;
Thread stdout = new Thread(delegate()
{
string line;
while ((line = git.StandardOutput.ReadLine()) != null)
{
if (string.IsNullOrEmpty(line.Trim()))
{
continue;
}
entry.files.Add(line);
}
});
Thread stderr = new Thread(delegate()
{
string line;
while (null != (line = git.StandardError.ReadLine()))
{
Console.Error.WriteLine(line);
stderr_log.AppendLine(line);
}
});
git.Start();
stdout.Start();
stderr.Start();
// Wait 10 minutes for git to finish, otherwise abort.
if (!git.WaitForExit(1000 * 60 * 10))
{
Log("Getting files took more than 10 minutes, aborting.");
try {
git.Kill();
git.WaitForExit(10000); // Give the process 10 more seconds to completely exit.
} catch (Exception ex) {
Log("Aborting file retrieval failed: {0}", ex.ToString());
}
}
stdout.Join((int)TimeSpan.FromMinutes(1).TotalMilliseconds);
stderr.Join((int)TimeSpan.FromMinutes(1).TotalMilliseconds);
if (git.HasExited && git.ExitCode == 0)
{
Log("Got {0} files successfully", entry.files.Count);
}
else
{
Log("Didn't get files, HasExited: {0}, ExitCode: {1}, stderr: {2}", git.HasExited, git.HasExited ? git.ExitCode.ToString() : "N/A", stderr_log.ToString());
}
}
} catch (Exception ex) {
Log("Exception while trying to get files for commit {1} {0}", ex.ToString(), entry.revision);
}
}