public Nightly() { _nightly = new Xml("nightly"); _failures = _nightly.Append("failures"); _leaks = _nightly.Append("leaks"); // Locate relevant directories. var nightlyDir = GetNightlyDir(); _logDir = Path.Combine(nightlyDir, "Logs"); // First guess at working directory _skylineTesterDir = Path.Combine(nightlyDir, "SkylineTesterForNightly"); // Default duration. _duration = TimeSpan.FromHours(9); }
public Nightly(RunMode runMode, string decorateSrcDirName = null) { _runMode = runMode; _nightly = new Xml("nightly"); _failures = _nightly.Append("failures"); _leaks = _nightly.Append("leaks"); // Locate relevant directories. var nightlyDir = GetNightlyDir(); _logDir = Path.Combine(nightlyDir, "Logs"); // Clean up after any old screengrab directories var logDirScreengrabs = Path.Combine(_logDir, "NightlyScreengrabs"); if (Directory.Exists(logDirScreengrabs)) { Directory.Delete(logDirScreengrabs, true); } // First guess at working directory - distinguish between run types for machines that do double duty _skylineTesterDir = Path.Combine(nightlyDir, "SkylineTesterForNightly_" + runMode + (decorateSrcDirName ?? string.Empty)); // Default duration. _duration = TimeSpan.FromHours(DEFAULT_DURATION_HOURS); }
private int ParseTests(string log, bool storeXml = true) { var startTest = new Regex(@"\r\n\[(\d\d:\d\d)\] +(\d+).(\d+) +(\S+) +\((\w\w)\) ", RegexOptions.Compiled); var endTestOld = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, (\d+) sec\.\r\n", RegexOptions.Compiled); var endTest = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, ([\.\d]+)/([\.\d]+) handles, (\d+) sec\.\r\n", RegexOptions.Compiled); string lastPass = null; int testCount = 0; for (var startMatch = startTest.Match(log); startMatch.Success; startMatch = startMatch.NextMatch()) { var timestamp = startMatch.Groups[1].Value; var passId = startMatch.Groups[2].Value; var testId = startMatch.Groups[3].Value; var name = startMatch.Groups[4].Value; var language = startMatch.Groups[5].Value; string userGdi = null, handles = null; var endMatch = endTestOld.Match(log, startMatch.Index); int durationIndex = 3; if (!endMatch.Success) { endMatch = endTest.Match(log, startMatch.Index); userGdi = endMatch.Groups[3].Value; handles = endMatch.Groups[4].Value; durationIndex = 5; } var managed = endMatch.Groups[1].Value; var total = endMatch.Groups[2].Value; var duration = endMatch.Groups[durationIndex].Value; if (string.IsNullOrEmpty(managed) || string.IsNullOrEmpty(total) || string.IsNullOrEmpty(duration)) { continue; } if (lastPass != passId) { lastPass = passId; if (storeXml) { _pass = _nightly.Append("pass"); _pass["id"] = passId; } } if (storeXml) { var test = _pass.Append("test"); test["id"] = testId; test["name"] = name; test["language"] = language; test["timestamp"] = timestamp; test["duration"] = duration; test["managed"] = managed; test["total"] = total; if (!string.IsNullOrEmpty(userGdi)) { test["user_gdi"] = userGdi; } if (!string.IsNullOrEmpty(handles)) { test["handles"] = handles; } } testCount++; } return(testCount); }
/// <summary> /// Run nightly build/test and report results to server. /// </summary> public string Run() { string result = string.Empty; // Locate relevant directories. var nightlyDir = GetNightlyDir(); var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr"); bool withPerfTests = _runMode != RunMode.trunk && _runMode != RunMode.integration && _runMode != RunMode.release; if (_runMode == RunMode.stress) { _duration = TimeSpan.FromHours(168); // Let it go as long as a week } else if (withPerfTests) { _duration = TimeSpan.FromHours(PERF_DURATION_HOURS); // Let it go a bit longer than standard 9 hours } // Kill any other instance of SkylineNightly, unless this is // the StressTest mode, in which case assume that a previous invocation // is still running and just exit to stay out of its way. foreach (var process in Process.GetProcessesByName("skylinenightly")) { if (process.Id != Process.GetCurrentProcess().Id) { if (_runMode == RunMode.stress) { Application.Exit(); // Just let the already (long!) running process do its thing } else { process.Kill(); } } } // Kill processes started within the proposed working directory - most likely SkylineTester and/or TestRunner. // This keeps stuck tests around for 24 hours, which should be sufficient, but allows us to replace directory // on a daily basis - otherwise we could fill the hard drive on smaller machines foreach (var process in Process.GetProcesses()) { try { if (process.Modules[0].FileName.StartsWith(_skylineTesterDir) && process.Id != Process.GetCurrentProcess().Id) { process.Kill(); } } // ReSharper disable once EmptyGeneralCatchClause catch (Exception) { } } // Create place to put run logs if (!Directory.Exists(_logDir)) { Directory.CreateDirectory(_logDir); } // Start the nightly log file StartLog(_runMode); // Delete source tree and old SkylineTester. Delete(skylineNightlySkytr); Log("Delete SkylineTester"); var skylineTesterDirBasis = _skylineTesterDir; // Default name const int maxRetry = 1000; // Something would have to be very wrong to get here, but better not to risk a hang string nextDir = _skylineTesterDir; for (var retry = 1; retry < maxRetry; retry++) { try { if (!Directory.Exists(nextDir)) { break; } string deleteDir = nextDir; // Keep going until a directory is found that does not exist nextDir = skylineTesterDirBasis + "_" + retry; Delete(deleteDir); } catch (Exception e) { if (Directory.Exists(_skylineTesterDir)) { // Work around undeletable file that sometimes appears under Windows 10 Log("Unable to delete " + _skylineTesterDir + "(" + e + "), using " + nextDir + " instead."); _skylineTesterDir = nextDir; } } } Log("buildRoot is " + PwizDir); // We used to put source tree alongside SkylineTesterDir instead of under it, delete that too try { Delete(Path.Combine(nightlyDir, "pwiz")); } // ReSharper disable once EmptyGeneralCatchClause catch { } Directory.CreateDirectory(_skylineTesterDir); // Download most recent build of SkylineTester. var skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip"); const int attempts = 30; string branchUrl = null; for (int i = 0; i < attempts; i++) { try { DownloadSkylineTester(skylineTesterZip, _runMode); } catch (Exception ex) { Log("Exception while downloading SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)"); if (i == attempts - 1) { LogAndThrow("Unable to download SkylineTester"); } Thread.Sleep(60 * 1000); // one minute continue; } // Install SkylineTester. if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir)) { LogAndThrow("SkylineTester installation failed."); } try { // Delete zip file. Log("Delete zip file " + skylineTesterZip); File.Delete(skylineTesterZip); // Figure out which branch we're working in - there's a file in the downloaded SkylineTester zip that tells us. var branchLine = File.ReadAllLines(Path.Combine(_skylineTesterDir, "SkylineTester Files", "Version.cpp")).FirstOrDefault(l => l.Contains("Version::Branch")); if (!string.IsNullOrEmpty(branchLine)) { // Looks like std::string Version::Branch() {return "Skyline/skyline_9_7";} var branch = branchLine.Split(new[] { "\"" }, StringSplitOptions.None)[1]; if (branch.Equals("master")) { branchUrl = GIT_MASTER_URL; } else { branchUrl = GIT_BRANCHES_URL + branch; // Looks like https://github.com/ProteoWizard/pwiz/tree/Skyline/skyline_9_7 } } break; } catch (Exception ex) { Log("Exception while unzipping SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)"); if (i == attempts - 1) { LogAndThrow("Unable to identify branch from Version.cpp in SkylineTester"); } Thread.Sleep(60 * 1000); // one minute } } // Create ".skytr" file to execute nightly build in SkylineTester. var assembly = Assembly.GetExecutingAssembly(); const string resourceName = "SkylineNightly.SkylineNightly.skytr"; int durationSeconds; using (var stream = assembly.GetManifestResourceStream(resourceName)) { if (stream == null) { LogAndThrow(result = "Embedded resource is broken"); return(result); } using (var reader = new StreamReader(stream)) { var skylineTester = Xml.FromString(reader.ReadToEnd()); skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString()); skylineTester.GetChild("nightlyRoot").Set(nightlyDir); skylineTester.GetChild("buildRoot").Set(_skylineTesterDir); skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false"); skylineTester.GetChild("nightlyDuration").Set(((int)_duration.TotalHours).ToString()); skylineTester.GetChild("nightlyRepeat").Set(_runMode == RunMode.stress ? "100" : "1"); skylineTester.GetChild("nightlyRandomize").Set(_runMode == RunMode.stress ? "true" : "false"); if (!string.IsNullOrEmpty(branchUrl) && branchUrl.Contains("tree")) { skylineTester.GetChild("nightlyBuildTrunk").Set("false"); skylineTester.GetChild("nightlyBranch").Set("true"); skylineTester.GetChild("nightlyBranchUrl").Set(branchUrl); Log("Testing branch at " + branchUrl); } skylineTester.Save(skylineNightlySkytr); var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value); durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60; // 30 minutes grace before we kill SkylineTester } } // Start SkylineTester to do the build. var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe"); Log(string.Format("Starting {0} with config file {1}, which contains:", skylineTesterExe, skylineNightlySkytr)); foreach (var line in File.ReadAllLines(skylineNightlySkytr)) { Log(line); } var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr) { WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? "" }; var startTime = DateTime.Now; bool retryTester; const int maxRetryMinutes = 60; do { var skylineTesterProcess = Process.Start(processInfo); if (skylineTesterProcess == null) { LogAndThrow(result = "SkylineTester did not start"); return(result); } Log("SkylineTester started"); if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000)) { SaveErrorScreenshot(); Log(result = "SkylineTester has exceeded its " + durationSeconds + " second WaitForExit timeout. You should investigate."); } else if (skylineTesterProcess.ExitCode == 0xDEAD) { // User killed, don't post Log(result = SkylineTesterStoppedByUser); return(result); } else { Log("SkylineTester finished"); } _duration = DateTime.Now - startTime; retryTester = _duration.TotalMinutes < maxRetryMinutes; if (retryTester) { // Retry a very short test run if there is no log file or the log file does not contain any tests string logFile = GetLatestLog(); if (logFile != null && File.Exists(logFile)) { retryTester = ParseTests(File.ReadAllText(logFile), false) == 0; } if (retryTester) { Log("No tests run in " + Math.Round(_duration.TotalMinutes) + " minutes retrying."); } } }while (retryTester); return(result); }
private int ParseTests(string log) { var startTest = new Regex(@"\r\n\[\d\d:\d\d\] +(\d+).(\d+) +(\S+) +\((\w\w)\) ", RegexOptions.Compiled); var endTest = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, (\d+) sec\.\r\n", RegexOptions.Compiled); string lastPass = null; int testCount = 0; for (var startMatch = startTest.Match(log); startMatch.Success; startMatch = startMatch.NextMatch()) { var passId = startMatch.Groups[1].Value; var testId = startMatch.Groups[2].Value; var name = startMatch.Groups[3].Value; var language = startMatch.Groups[4].Value; var endMatch = endTest.Match(log, startMatch.Index); var managed = endMatch.Groups[1].Value; var total = endMatch.Groups[2].Value; var duration = endMatch.Groups[3].Value; if (string.IsNullOrEmpty(managed) || string.IsNullOrEmpty(total) || string.IsNullOrEmpty(duration)) continue; if (lastPass != passId) { lastPass = passId; _pass = _nightly.Append("pass"); _pass["id"] = passId; } var test = _pass.Append("test"); test["id"] = testId; test["name"] = name; test["language"] = language; test["duration"] = duration; test["managed"] = managed; test["total"] = total; testCount++; } return testCount; }