/// <summary>
/// Run nightly build/test and report results to server.
/// </summary>
public void Run(bool withPerfTests)
{
// Locate relevant directories.
var nightlyDir = GetNightlyDir();
var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");
if (withPerfTests)
{
_duration = TimeSpan.FromHours(12);
}
// Kill any other instance of SkylineNightly.
foreach (var process in Process.GetProcessesByName("skylinenightly"))
{
if (process.Id != Process.GetCurrentProcess().Id)
{
process.Kill();
}
}
// Kill processes started within the nightly directory.
foreach (var process in Process.GetProcesses())
{
try
{
if (process.Modules[0].FileName.StartsWith(nightlyDir) &&
process.Id != Process.GetCurrentProcess().Id)
{
process.Kill();
}
}
// ReSharper disable once EmptyGeneralCatchClause
catch (Exception)
{
}
}
_startTime = DateTime.Now;
// Create log file.
if (!Directory.Exists(_logDir))
{
Directory.CreateDirectory(_logDir);
}
_logFile = Path.Combine(nightlyDir, "SkylineNightly.log");
Delete(_logFile);
Log(DateTime.Now.ToShortDateString());
// Delete source tree and old SkylineTester.
Delete(skylineNightlySkytr);
Log("Delete SkylineTester");
var skylineTesterDirBasis = _skylineTesterDir; // Default name
const int maxRetry = 1000; // Something would have to be very wrong to get here, but better not to risk a hang
for (var retry = 1; retry < maxRetry; retry++)
{
try
{
Delete(_skylineTesterDir);
break;
}
catch (Exception e)
{
// Work around undeletable file that sometimes appears under Windows 10
var newDir = skylineTesterDirBasis + "_" + retry;
Log("Unable to delete " + _skylineTesterDir + "(" + e + "), using " + newDir + " instead.");
_skylineTesterDir = newDir;
}
}
Log("buildRoot is " + PwizDir);
// We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
try
{
Delete(Path.Combine(nightlyDir, "pwiz"));
}
// ReSharper disable once EmptyGeneralCatchClause
catch
{
}
Directory.CreateDirectory(_skylineTesterDir);
// Download most recent build of SkylineTester.
var skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
const int attempts = 30;
for (int i = 0; i < attempts; i++)
{
try
{
DownloadSkylineTester(skylineTesterZip);
break;
}
catch (Exception ex)
{
Log("Exception while downloading SkylineTester: " + ex.Message);
if (i == attempts - 1)
{
LogAndThrow("Unable to download SkylineTester");
}
Thread.Sleep(60 * 1000); // one minute
}
}
// Install SkylineTester.
if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
{
LogAndThrow("SkylineTester installation failed.");
}
// Delete zip file.
Log("Delete zip file " + skylineTesterZip);
File.Delete(skylineTesterZip);
// Create ".skytr" file to execute nightly build in SkylineTester.
var assembly = Assembly.GetExecutingAssembly();
const string resourceName = "SkylineNightly.SkylineNightly.skytr";
int durationSeconds;
using (var stream = assembly.GetManifestResourceStream(resourceName))
{
if (stream == null)
{
LogAndThrow("Embedded resource is broken");
return; // Just so we don't get the "possible null" warning below
}
using (var reader = new StreamReader(stream))
{
var skylineTester = Xml.FromString(reader.ReadToEnd());
skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
skylineTester.GetChild("nightlyDuration").Set(_duration.Hours.ToString());
skylineTester.Save(skylineNightlySkytr);
var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60; // 30 minutes grace before we kill SkylineTester
}
}
// Start SkylineTester to do the build.
var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");
var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
{
WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
};
var startTime = DateTime.Now;
var skylineTesterProcess = Process.Start(processInfo);
if (skylineTesterProcess == null)
{
LogAndThrow("SkylineTester did not start");
return; // Just so we don't get the "possible null" warning below
}
Log("SkylineTester started");
if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
{
SaveErrorScreenshot();
skylineTesterProcess.Kill();
Log("SkylineTester killed after " + durationSeconds + " second WaitForExit timeout");
}
else
{
Log("SkylineTester finished");
}
_duration = DateTime.Now - startTime;
}
/// <summary>
/// Run nightly build/test and report results to server.
/// </summary>
public string Run()
{
string result = string.Empty;
// Locate relevant directories.
var nightlyDir = GetNightlyDir();
var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");
bool withPerfTests = _runMode != RunMode.trunk && _runMode != RunMode.integration && _runMode != RunMode.release;
if (_runMode == RunMode.stress)
{
_duration = TimeSpan.FromHours(168); // Let it go as long as a week
}
else if (withPerfTests)
{
_duration = TimeSpan.FromHours(PERF_DURATION_HOURS); // Let it go a bit longer than standard 9 hours
}
// Kill any other instance of SkylineNightly, unless this is
// the StressTest mode, in which case assume that a previous invocation
// is still running and just exit to stay out of its way.
foreach (var process in Process.GetProcessesByName("skylinenightly"))
{
if (process.Id != Process.GetCurrentProcess().Id)
{
if (_runMode == RunMode.stress)
{
Application.Exit(); // Just let the already (long!) running process do its thing
}
else
{
process.Kill();
}
}
}
// Kill processes started within the proposed working directory - most likely SkylineTester and/or TestRunner.
// This keeps stuck tests around for 24 hours, which should be sufficient, but allows us to replace directory
// on a daily basis - otherwise we could fill the hard drive on smaller machines
foreach (var process in Process.GetProcesses())
{
try
{
if (process.Modules[0].FileName.StartsWith(_skylineTesterDir) &&
process.Id != Process.GetCurrentProcess().Id)
{
process.Kill();
}
}
// ReSharper disable once EmptyGeneralCatchClause
catch (Exception)
{
}
}
// Create place to put run logs
if (!Directory.Exists(_logDir))
{
Directory.CreateDirectory(_logDir);
}
// Start the nightly log file
StartLog(_runMode);
// Delete source tree and old SkylineTester.
Delete(skylineNightlySkytr);
Log("Delete SkylineTester");
var skylineTesterDirBasis = _skylineTesterDir; // Default name
const int maxRetry = 1000; // Something would have to be very wrong to get here, but better not to risk a hang
string nextDir = _skylineTesterDir;
for (var retry = 1; retry < maxRetry; retry++)
{
try
{
if (!Directory.Exists(nextDir))
{
break;
}
string deleteDir = nextDir;
// Keep going until a directory is found that does not exist
nextDir = skylineTesterDirBasis + "_" + retry;
Delete(deleteDir);
}
catch (Exception e)
{
if (Directory.Exists(_skylineTesterDir))
{
// Work around undeletable file that sometimes appears under Windows 10
Log("Unable to delete " + _skylineTesterDir + "(" + e + "), using " + nextDir + " instead.");
_skylineTesterDir = nextDir;
}
}
}
Log("buildRoot is " + PwizDir);
// We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
try
{
Delete(Path.Combine(nightlyDir, "pwiz"));
}
// ReSharper disable once EmptyGeneralCatchClause
catch
{
}
Directory.CreateDirectory(_skylineTesterDir);
// Download most recent build of SkylineTester.
var skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
const int attempts = 30;
string branchUrl = null;
for (int i = 0; i < attempts; i++)
{
try
{
DownloadSkylineTester(skylineTesterZip, _runMode);
}
catch (Exception ex)
{
Log("Exception while downloading SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
if (i == attempts - 1)
{
LogAndThrow("Unable to download SkylineTester");
}
Thread.Sleep(60 * 1000); // one minute
continue;
}
// Install SkylineTester.
if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
{
LogAndThrow("SkylineTester installation failed.");
}
try
{
// Delete zip file.
Log("Delete zip file " + skylineTesterZip);
File.Delete(skylineTesterZip);
// Figure out which branch we're working in - there's a file in the downloaded SkylineTester zip that tells us.
var branchLine = File.ReadAllLines(Path.Combine(_skylineTesterDir, "SkylineTester Files", "Version.cpp")).FirstOrDefault(l => l.Contains("Version::Branch"));
if (!string.IsNullOrEmpty(branchLine))
{
// Looks like std::string Version::Branch() {return "Skyline/skyline_9_7";}
var branch = branchLine.Split(new[] { "\"" }, StringSplitOptions.None)[1];
if (branch.Equals("master"))
{
branchUrl = GIT_MASTER_URL;
}
else
{
branchUrl = GIT_BRANCHES_URL + branch; // Looks like https://github.com/ProteoWizard/pwiz/tree/Skyline/skyline_9_7
}
}
break;
}
catch (Exception ex)
{
Log("Exception while unzipping SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
if (i == attempts - 1)
{
LogAndThrow("Unable to identify branch from Version.cpp in SkylineTester");
}
Thread.Sleep(60 * 1000); // one minute
}
}
// Create ".skytr" file to execute nightly build in SkylineTester.
var assembly = Assembly.GetExecutingAssembly();
const string resourceName = "SkylineNightly.SkylineNightly.skytr";
int durationSeconds;
using (var stream = assembly.GetManifestResourceStream(resourceName))
{
if (stream == null)
{
LogAndThrow(result = "Embedded resource is broken");
return(result);
}
using (var reader = new StreamReader(stream))
{
var skylineTester = Xml.FromString(reader.ReadToEnd());
skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
skylineTester.GetChild("nightlyDuration").Set(((int)_duration.TotalHours).ToString());
skylineTester.GetChild("nightlyRepeat").Set(_runMode == RunMode.stress ? "100" : "1");
skylineTester.GetChild("nightlyRandomize").Set(_runMode == RunMode.stress ? "true" : "false");
if (!string.IsNullOrEmpty(branchUrl) && branchUrl.Contains("tree"))
{
skylineTester.GetChild("nightlyBuildTrunk").Set("false");
skylineTester.GetChild("nightlyBranch").Set("true");
skylineTester.GetChild("nightlyBranchUrl").Set(branchUrl);
Log("Testing branch at " + branchUrl);
}
skylineTester.Save(skylineNightlySkytr);
var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60; // 30 minutes grace before we kill SkylineTester
}
}
// Start SkylineTester to do the build.
var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");
Log(string.Format("Starting {0} with config file {1}, which contains:", skylineTesterExe, skylineNightlySkytr));
foreach (var line in File.ReadAllLines(skylineNightlySkytr))
{
Log(line);
}
var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
{
WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
};
var startTime = DateTime.Now;
bool retryTester;
const int maxRetryMinutes = 60;
do
{
var skylineTesterProcess = Process.Start(processInfo);
if (skylineTesterProcess == null)
{
LogAndThrow(result = "SkylineTester did not start");
return(result);
}
Log("SkylineTester started");
if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
{
SaveErrorScreenshot();
Log(result = "SkylineTester has exceeded its " + durationSeconds +
" second WaitForExit timeout. You should investigate.");
}
else if (skylineTesterProcess.ExitCode == 0xDEAD)
{
// User killed, don't post
Log(result = SkylineTesterStoppedByUser);
return(result);
}
else
{
Log("SkylineTester finished");
}
_duration = DateTime.Now - startTime;
retryTester = _duration.TotalMinutes < maxRetryMinutes;
if (retryTester)
{
// Retry a very short test run if there is no log file or the log file does not contain any tests
string logFile = GetLatestLog();
if (logFile != null && File.Exists(logFile))
{
retryTester = ParseTests(File.ReadAllText(logFile), false) == 0;
}
if (retryTester)
{
Log("No tests run in " + Math.Round(_duration.TotalMinutes) + " minutes retrying.");
}
}
}while (retryTester);
return(result);
}
/// <summary>
/// Run nightly build/test and report results to server.
/// </summary>
public string Run()
{
string result = string.Empty;
// Locate relevant directories.
var nightlyDir = GetNightlyDir();
var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");
bool withPerfTests = _runMode != RunMode.trunk && _runMode != RunMode.integration;
if (_runMode == RunMode.stress)
{
_duration = TimeSpan.FromHours(168); // Let it go as long as a week
}
else if (withPerfTests)
{
_duration = TimeSpan.FromHours(PERF_DURATION_HOURS); // Let it go a bit longer than standard 9 hours
}
// Kill any other instance of SkylineNightly, unless this is
// the StressTest mode, in which case assume that a previous invocation
// is still running and just exit to stay out of its way.
foreach (var process in Process.GetProcessesByName("skylinenightly"))
{
if (process.Id != Process.GetCurrentProcess().Id)
{
if (_runMode == RunMode.stress)
{
Application.Exit(); // Just let the already (long!) running process do its thing
}
else
{
process.Kill();
}
}
}
// Kill processes started within the proposed working directory - most likely SkylineTester and/or TestRunner.
// This keeps stuck tests around for 24 hours, which should be sufficient, but allows us to replace directory
// on a daily basis - otherwise we could fill the hard drive on smaller machines
foreach (var process in Process.GetProcesses())
{
try
{
if (process.Modules[0].FileName.StartsWith(_skylineTesterDir) &&
process.Id != Process.GetCurrentProcess().Id)
{
process.Kill();
}
}
// ReSharper disable once EmptyGeneralCatchClause
catch (Exception)
{
}
}
// Create place to put run logs
if (!Directory.Exists(_logDir))
{
Directory.CreateDirectory(_logDir);
}
// Start the nightly log file
StartLog(_runMode);
// Delete source tree and old SkylineTester.
Delete(skylineNightlySkytr);
Log("Delete SkylineTester");
var skylineTesterDirBasis = _skylineTesterDir; // Default name
const int maxRetry = 1000; // Something would have to be very wrong to get here, but better not to risk a hang
for (var retry = 1; retry < maxRetry; retry++)
{
try
{
Delete(_skylineTesterDir);
break;
}
catch (Exception e)
{
// Work around undeletable file that sometimes appears under Windows 10
var newDir = skylineTesterDirBasis + "_" + retry;
Log("Unable to delete " + _skylineTesterDir + "(" + e + "), using " + newDir + " instead.");
_skylineTesterDir = newDir;
}
}
Log("buildRoot is " + PwizDir);
// We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
try
{
Delete(Path.Combine(nightlyDir, "pwiz"));
}
// ReSharper disable once EmptyGeneralCatchClause
catch
{
}
Directory.CreateDirectory(_skylineTesterDir);
// Download most recent build of SkylineTester.
var skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
const int attempts = 30;
for (int i = 0; i < attempts; i++)
{
try
{
DownloadSkylineTester(skylineTesterZip, _runMode);
break;
}
catch (Exception ex)
{
Log("Exception while downloading SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
if (i == attempts - 1)
{
LogAndThrow("Unable to download SkylineTester");
}
Thread.Sleep(60 * 1000); // one minute
}
}
// Install SkylineTester.
if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
{
LogAndThrow("SkylineTester installation failed.");
}
// Delete zip file.
Log("Delete zip file " + skylineTesterZip);
File.Delete(skylineTesterZip);
// Now figure out which branch we're working in - there's a file in the downloaded SkylineTester zip that tells us.
string branchUrl = null;
var lines = File.ReadAllLines(Path.Combine(_skylineTesterDir, "SkylineTester Files", "SVN_info.txt"));
if (lines[1].Contains("branches"))
{
// Looks like $URL: https://svn.code.sf.net/p/proteowizard/code/branches/skyline_3_5/SVN_info.txt $
branchUrl =
lines[1].Split(new[] { "URL: " }, StringSplitOptions.None)[1].Split(new[] { "/SVN_info" }, StringSplitOptions.None)[0];
}
// Or if we are running the integration build, then we use the trunk SkylineTester, but have it build the branch TestRunner.exe
else if (_runMode == RunMode.integration)
{
branchUrl = SVN_INTEGRATION_BRANCH_URL;
}
// Create ".skytr" file to execute nightly build in SkylineTester.
var assembly = Assembly.GetExecutingAssembly();
const string resourceName = "SkylineNightly.SkylineNightly.skytr";
int durationSeconds;
using (var stream = assembly.GetManifestResourceStream(resourceName))
{
if (stream == null)
{
LogAndThrow(result = "Embedded resource is broken");
return(result);
}
using (var reader = new StreamReader(stream))
{
var skylineTester = Xml.FromString(reader.ReadToEnd());
skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
skylineTester.GetChild("nightlyDuration").Set(((int)_duration.TotalHours).ToString());
skylineTester.GetChild("nightlyRepeat").Set(_runMode == RunMode.stress ? "100" : "1");
skylineTester.GetChild("nightlyRandomize").Set(_runMode == RunMode.stress ? "true" : "false");
if (!string.IsNullOrEmpty(branchUrl) && !branchUrl.Contains("trunk"))
{
skylineTester.GetChild("nightlyBuildTrunk").Set("false");
skylineTester.GetChild("nightlyBranch").Set("true");
skylineTester.GetChild("nightlyBranchUrl").Set(branchUrl);
Log("Testing branch at " + branchUrl);
}
skylineTester.Save(skylineNightlySkytr);
var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60; // 30 minutes grace before we kill SkylineTester
}
}
// Start SkylineTester to do the build.
var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");
Log(string.Format("Starting {0} with config file {1}, which contains:", skylineTesterExe, skylineNightlySkytr));
foreach (var line in File.ReadAllLines(skylineNightlySkytr))
{
Log(line);
}
var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
{
WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
};
var startTime = DateTime.Now;
var skylineTesterProcess = Process.Start(processInfo);
if (skylineTesterProcess == null)
{
LogAndThrow(result = "SkylineTester did not start");
return(result);
}
Log("SkylineTester started");
if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
{
SaveErrorScreenshot();
Log(result = "SkylineTester has exceeded its " + durationSeconds + " second WaitForExit timeout. You should investigate.");
}
else
{
Log("SkylineTester finished");
}
_duration = DateTime.Now - startTime;
return(result);
}