public Nightly()
{
_nightly = new Xml("nightly");
_failures = _nightly.Append("failures");
_leaks = _nightly.Append("leaks");
// Locate relevant directories.
var nightlyDir = GetNightlyDir();
_logDir = Path.Combine(nightlyDir, "Logs");
// First guess at working directory
_skylineTesterDir = Path.Combine(nightlyDir, "SkylineTesterForNightly");
// Default duration.
_duration = TimeSpan.FromHours(9);
}
public Nightly(RunMode runMode, string decorateSrcDirName = null)
{
_runMode = runMode;
_nightly = new Xml("nightly");
_failures = _nightly.Append("failures");
_leaks = _nightly.Append("leaks");
// Locate relevant directories.
var nightlyDir = GetNightlyDir();
_logDir = Path.Combine(nightlyDir, "Logs");
// Clean up after any old screengrab directories
var logDirScreengrabs = Path.Combine(_logDir, "NightlyScreengrabs");
if (Directory.Exists(logDirScreengrabs))
{
Directory.Delete(logDirScreengrabs, true);
}
// First guess at working directory - distinguish between run types for machines that do double duty
_skylineTesterDir = Path.Combine(nightlyDir, "SkylineTesterForNightly_" + runMode + (decorateSrcDirName ?? string.Empty));
// Default duration.
_duration = TimeSpan.FromHours(DEFAULT_DURATION_HOURS);
}
private int ParseTests(string log, bool storeXml = true)
{
var startTest = new Regex(@"\r\n\[(\d\d:\d\d)\] +(\d+).(\d+) +(\S+) +\((\w\w)\) ", RegexOptions.Compiled);
var endTestOld = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, (\d+) sec\.\r\n", RegexOptions.Compiled);
var endTest = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, ([\.\d]+)/([\.\d]+) handles, (\d+) sec\.\r\n", RegexOptions.Compiled);
string lastPass = null;
int testCount = 0;
for (var startMatch = startTest.Match(log); startMatch.Success; startMatch = startMatch.NextMatch())
{
var timestamp = startMatch.Groups[1].Value;
var passId = startMatch.Groups[2].Value;
var testId = startMatch.Groups[3].Value;
var name = startMatch.Groups[4].Value;
var language = startMatch.Groups[5].Value;
string userGdi = null, handles = null;
var endMatch = endTestOld.Match(log, startMatch.Index);
int durationIndex = 3;
if (!endMatch.Success)
{
endMatch = endTest.Match(log, startMatch.Index);
userGdi = endMatch.Groups[3].Value;
handles = endMatch.Groups[4].Value;
durationIndex = 5;
}
var managed = endMatch.Groups[1].Value;
var total = endMatch.Groups[2].Value;
var duration = endMatch.Groups[durationIndex].Value;
if (string.IsNullOrEmpty(managed) || string.IsNullOrEmpty(total) || string.IsNullOrEmpty(duration))
{
continue;
}
if (lastPass != passId)
{
lastPass = passId;
if (storeXml)
{
_pass = _nightly.Append("pass");
_pass["id"] = passId;
}
}
if (storeXml)
{
var test = _pass.Append("test");
test["id"] = testId;
test["name"] = name;
test["language"] = language;
test["timestamp"] = timestamp;
test["duration"] = duration;
test["managed"] = managed;
test["total"] = total;
if (!string.IsNullOrEmpty(userGdi))
{
test["user_gdi"] = userGdi;
}
if (!string.IsNullOrEmpty(handles))
{
test["handles"] = handles;
}
}
testCount++;
}
return(testCount);
}
/// <summary>
/// Run nightly build/test and report results to server.
/// </summary>
public string Run()
{
string result = string.Empty;
// Locate relevant directories.
var nightlyDir = GetNightlyDir();
var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");
bool withPerfTests = _runMode != RunMode.trunk && _runMode != RunMode.integration && _runMode != RunMode.release;
if (_runMode == RunMode.stress)
{
_duration = TimeSpan.FromHours(168); // Let it go as long as a week
}
else if (withPerfTests)
{
_duration = TimeSpan.FromHours(PERF_DURATION_HOURS); // Let it go a bit longer than standard 9 hours
}
// Kill any other instance of SkylineNightly, unless this is
// the StressTest mode, in which case assume that a previous invocation
// is still running and just exit to stay out of its way.
foreach (var process in Process.GetProcessesByName("skylinenightly"))
{
if (process.Id != Process.GetCurrentProcess().Id)
{
if (_runMode == RunMode.stress)
{
Application.Exit(); // Just let the already (long!) running process do its thing
}
else
{
process.Kill();
}
}
}
// Kill processes started within the proposed working directory - most likely SkylineTester and/or TestRunner.
// This keeps stuck tests around for 24 hours, which should be sufficient, but allows us to replace directory
// on a daily basis - otherwise we could fill the hard drive on smaller machines
foreach (var process in Process.GetProcesses())
{
try
{
if (process.Modules[0].FileName.StartsWith(_skylineTesterDir) &&
process.Id != Process.GetCurrentProcess().Id)
{
process.Kill();
}
}
// ReSharper disable once EmptyGeneralCatchClause
catch (Exception)
{
}
}
// Create place to put run logs
if (!Directory.Exists(_logDir))
{
Directory.CreateDirectory(_logDir);
}
// Start the nightly log file
StartLog(_runMode);
// Delete source tree and old SkylineTester.
Delete(skylineNightlySkytr);
Log("Delete SkylineTester");
var skylineTesterDirBasis = _skylineTesterDir; // Default name
const int maxRetry = 1000; // Something would have to be very wrong to get here, but better not to risk a hang
string nextDir = _skylineTesterDir;
for (var retry = 1; retry < maxRetry; retry++)
{
try
{
if (!Directory.Exists(nextDir))
{
break;
}
string deleteDir = nextDir;
// Keep going until a directory is found that does not exist
nextDir = skylineTesterDirBasis + "_" + retry;
Delete(deleteDir);
}
catch (Exception e)
{
if (Directory.Exists(_skylineTesterDir))
{
// Work around undeletable file that sometimes appears under Windows 10
Log("Unable to delete " + _skylineTesterDir + "(" + e + "), using " + nextDir + " instead.");
_skylineTesterDir = nextDir;
}
}
}
Log("buildRoot is " + PwizDir);
// We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
try
{
Delete(Path.Combine(nightlyDir, "pwiz"));
}
// ReSharper disable once EmptyGeneralCatchClause
catch
{
}
Directory.CreateDirectory(_skylineTesterDir);
// Download most recent build of SkylineTester.
var skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
const int attempts = 30;
string branchUrl = null;
for (int i = 0; i < attempts; i++)
{
try
{
DownloadSkylineTester(skylineTesterZip, _runMode);
}
catch (Exception ex)
{
Log("Exception while downloading SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
if (i == attempts - 1)
{
LogAndThrow("Unable to download SkylineTester");
}
Thread.Sleep(60 * 1000); // one minute
continue;
}
// Install SkylineTester.
if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
{
LogAndThrow("SkylineTester installation failed.");
}
try
{
// Delete zip file.
Log("Delete zip file " + skylineTesterZip);
File.Delete(skylineTesterZip);
// Figure out which branch we're working in - there's a file in the downloaded SkylineTester zip that tells us.
var branchLine = File.ReadAllLines(Path.Combine(_skylineTesterDir, "SkylineTester Files", "Version.cpp")).FirstOrDefault(l => l.Contains("Version::Branch"));
if (!string.IsNullOrEmpty(branchLine))
{
// Looks like std::string Version::Branch() {return "Skyline/skyline_9_7";}
var branch = branchLine.Split(new[] { "\"" }, StringSplitOptions.None)[1];
if (branch.Equals("master"))
{
branchUrl = GIT_MASTER_URL;
}
else
{
branchUrl = GIT_BRANCHES_URL + branch; // Looks like https://github.com/ProteoWizard/pwiz/tree/Skyline/skyline_9_7
}
}
break;
}
catch (Exception ex)
{
Log("Exception while unzipping SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
if (i == attempts - 1)
{
LogAndThrow("Unable to identify branch from Version.cpp in SkylineTester");
}
Thread.Sleep(60 * 1000); // one minute
}
}
// Create ".skytr" file to execute nightly build in SkylineTester.
var assembly = Assembly.GetExecutingAssembly();
const string resourceName = "SkylineNightly.SkylineNightly.skytr";
int durationSeconds;
using (var stream = assembly.GetManifestResourceStream(resourceName))
{
if (stream == null)
{
LogAndThrow(result = "Embedded resource is broken");
return(result);
}
using (var reader = new StreamReader(stream))
{
var skylineTester = Xml.FromString(reader.ReadToEnd());
skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
skylineTester.GetChild("nightlyDuration").Set(((int)_duration.TotalHours).ToString());
skylineTester.GetChild("nightlyRepeat").Set(_runMode == RunMode.stress ? "100" : "1");
skylineTester.GetChild("nightlyRandomize").Set(_runMode == RunMode.stress ? "true" : "false");
if (!string.IsNullOrEmpty(branchUrl) && branchUrl.Contains("tree"))
{
skylineTester.GetChild("nightlyBuildTrunk").Set("false");
skylineTester.GetChild("nightlyBranch").Set("true");
skylineTester.GetChild("nightlyBranchUrl").Set(branchUrl);
Log("Testing branch at " + branchUrl);
}
skylineTester.Save(skylineNightlySkytr);
var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60; // 30 minutes grace before we kill SkylineTester
}
}
// Start SkylineTester to do the build.
var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");
Log(string.Format("Starting {0} with config file {1}, which contains:", skylineTesterExe, skylineNightlySkytr));
foreach (var line in File.ReadAllLines(skylineNightlySkytr))
{
Log(line);
}
var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
{
WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
};
var startTime = DateTime.Now;
bool retryTester;
const int maxRetryMinutes = 60;
do
{
var skylineTesterProcess = Process.Start(processInfo);
if (skylineTesterProcess == null)
{
LogAndThrow(result = "SkylineTester did not start");
return(result);
}
Log("SkylineTester started");
if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
{
SaveErrorScreenshot();
Log(result = "SkylineTester has exceeded its " + durationSeconds +
" second WaitForExit timeout. You should investigate.");
}
else if (skylineTesterProcess.ExitCode == 0xDEAD)
{
// User killed, don't post
Log(result = SkylineTesterStoppedByUser);
return(result);
}
else
{
Log("SkylineTester finished");
}
_duration = DateTime.Now - startTime;
retryTester = _duration.TotalMinutes < maxRetryMinutes;
if (retryTester)
{
// Retry a very short test run if there is no log file or the log file does not contain any tests
string logFile = GetLatestLog();
if (logFile != null && File.Exists(logFile))
{
retryTester = ParseTests(File.ReadAllText(logFile), false) == 0;
}
if (retryTester)
{
Log("No tests run in " + Math.Round(_duration.TotalMinutes) + " minutes retrying.");
}
}
}while (retryTester);
return(result);
}
private int ParseTests(string log)
{
var startTest = new Regex(@"\r\n\[\d\d:\d\d\] +(\d+).(\d+) +(\S+) +\((\w\w)\) ", RegexOptions.Compiled);
var endTest = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, (\d+) sec\.\r\n", RegexOptions.Compiled);
string lastPass = null;
int testCount = 0;
for (var startMatch = startTest.Match(log); startMatch.Success; startMatch = startMatch.NextMatch())
{
var passId = startMatch.Groups[1].Value;
var testId = startMatch.Groups[2].Value;
var name = startMatch.Groups[3].Value;
var language = startMatch.Groups[4].Value;
var endMatch = endTest.Match(log, startMatch.Index);
var managed = endMatch.Groups[1].Value;
var total = endMatch.Groups[2].Value;
var duration = endMatch.Groups[3].Value;
if (string.IsNullOrEmpty(managed) || string.IsNullOrEmpty(total) || string.IsNullOrEmpty(duration))
continue;
if (lastPass != passId)
{
lastPass = passId;
_pass = _nightly.Append("pass");
_pass["id"] = passId;
}
var test = _pass.Append("test");
test["id"] = testId;
test["name"] = name;
test["language"] = language;
test["duration"] = duration;
test["managed"] = managed;
test["total"] = total;
testCount++;
}
return testCount;
}