Example #1
0
        public Nightly()
        {
            _nightly = new Xml("nightly");
            _failures = _nightly.Append("failures");
            _leaks = _nightly.Append("leaks");

            // Locate relevant directories.
            var nightlyDir = GetNightlyDir();
            _logDir = Path.Combine(nightlyDir, "Logs");
            // First guess at working directory
            _skylineTesterDir = Path.Combine(nightlyDir, "SkylineTesterForNightly");

            // Default duration.
            _duration = TimeSpan.FromHours(9);
        }
Example #2
0
        public Nightly(RunMode runMode, string decorateSrcDirName = null)
        {
            _runMode  = runMode;
            _nightly  = new Xml("nightly");
            _failures = _nightly.Append("failures");
            _leaks    = _nightly.Append("leaks");

            // Locate relevant directories.
            var nightlyDir = GetNightlyDir();

            _logDir = Path.Combine(nightlyDir, "Logs");
            // Clean up after any old screengrab directories
            var logDirScreengrabs = Path.Combine(_logDir, "NightlyScreengrabs");

            if (Directory.Exists(logDirScreengrabs))
            {
                Directory.Delete(logDirScreengrabs, true);
            }
            // First guess at working directory - distinguish between run types for machines that do double duty
            _skylineTesterDir = Path.Combine(nightlyDir, "SkylineTesterForNightly_" + runMode + (decorateSrcDirName ?? string.Empty));

            // Default duration.
            _duration = TimeSpan.FromHours(DEFAULT_DURATION_HOURS);
        }
Example #3
0
        private int ParseTests(string log, bool storeXml = true)
        {
            var startTest  = new Regex(@"\r\n\[(\d\d:\d\d)\] +(\d+).(\d+) +(\S+) +\((\w\w)\) ", RegexOptions.Compiled);
            var endTestOld = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, (\d+) sec\.\r\n", RegexOptions.Compiled);
            var endTest    = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, ([\.\d]+)/([\.\d]+) handles, (\d+) sec\.\r\n", RegexOptions.Compiled);

            string lastPass  = null;
            int    testCount = 0;

            for (var startMatch = startTest.Match(log); startMatch.Success; startMatch = startMatch.NextMatch())
            {
                var timestamp = startMatch.Groups[1].Value;
                var passId    = startMatch.Groups[2].Value;
                var testId    = startMatch.Groups[3].Value;
                var name      = startMatch.Groups[4].Value;
                var language  = startMatch.Groups[5].Value;

                string userGdi = null, handles = null;
                var    endMatch      = endTestOld.Match(log, startMatch.Index);
                int    durationIndex = 3;
                if (!endMatch.Success)
                {
                    endMatch      = endTest.Match(log, startMatch.Index);
                    userGdi       = endMatch.Groups[3].Value;
                    handles       = endMatch.Groups[4].Value;
                    durationIndex = 5;
                }
                var managed  = endMatch.Groups[1].Value;
                var total    = endMatch.Groups[2].Value;
                var duration = endMatch.Groups[durationIndex].Value;

                if (string.IsNullOrEmpty(managed) || string.IsNullOrEmpty(total) || string.IsNullOrEmpty(duration))
                {
                    continue;
                }

                if (lastPass != passId)
                {
                    lastPass = passId;
                    if (storeXml)
                    {
                        _pass       = _nightly.Append("pass");
                        _pass["id"] = passId;
                    }
                }

                if (storeXml)
                {
                    var test = _pass.Append("test");
                    test["id"]        = testId;
                    test["name"]      = name;
                    test["language"]  = language;
                    test["timestamp"] = timestamp;
                    test["duration"]  = duration;
                    test["managed"]   = managed;
                    test["total"]     = total;
                    if (!string.IsNullOrEmpty(userGdi))
                    {
                        test["user_gdi"] = userGdi;
                    }
                    if (!string.IsNullOrEmpty(handles))
                    {
                        test["handles"] = handles;
                    }
                }

                testCount++;
            }
            return(testCount);
        }
Example #4
0
        /// <summary>
        /// Run nightly build/test and report results to server.
        /// </summary>
        public string Run()
        {
            string result = string.Empty;
            // Locate relevant directories.
            var nightlyDir          = GetNightlyDir();
            var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");

            bool withPerfTests = _runMode != RunMode.trunk && _runMode != RunMode.integration && _runMode != RunMode.release;

            if (_runMode == RunMode.stress)
            {
                _duration = TimeSpan.FromHours(168);  // Let it go as long as a week
            }
            else if (withPerfTests)
            {
                _duration = TimeSpan.FromHours(PERF_DURATION_HOURS); // Let it go a bit longer than standard 9 hours
            }

            // Kill any other instance of SkylineNightly, unless this is
            // the StressTest mode, in which case assume that a previous invocation
            // is still running and just exit to stay out of its way.
            foreach (var process in Process.GetProcessesByName("skylinenightly"))
            {
                if (process.Id != Process.GetCurrentProcess().Id)
                {
                    if (_runMode == RunMode.stress)
                    {
                        Application.Exit();  // Just let the already (long!) running process do its thing
                    }
                    else
                    {
                        process.Kill();
                    }
                }
            }

            // Kill processes started within the proposed working directory - most likely SkylineTester and/or TestRunner.
            // This keeps stuck tests around for 24 hours, which should be sufficient, but allows us to replace directory
            // on a daily basis - otherwise we could fill the hard drive on smaller machines
            foreach (var process in Process.GetProcesses())
            {
                try
                {
                    if (process.Modules[0].FileName.StartsWith(_skylineTesterDir) &&
                        process.Id != Process.GetCurrentProcess().Id)
                    {
                        process.Kill();
                    }
                }
                // ReSharper disable once EmptyGeneralCatchClause
                catch (Exception)
                {
                }
            }

            // Create place to put run logs
            if (!Directory.Exists(_logDir))
            {
                Directory.CreateDirectory(_logDir);
            }
            // Start the nightly log file
            StartLog(_runMode);

            // Delete source tree and old SkylineTester.
            Delete(skylineNightlySkytr);
            Log("Delete SkylineTester");
            var       skylineTesterDirBasis = _skylineTesterDir; // Default name
            const int maxRetry = 1000;                           // Something would have to be very wrong to get here, but better not to risk a hang
            string    nextDir  = _skylineTesterDir;

            for (var retry = 1; retry < maxRetry; retry++)
            {
                try
                {
                    if (!Directory.Exists(nextDir))
                    {
                        break;
                    }

                    string deleteDir = nextDir;
                    // Keep going until a directory is found that does not exist
                    nextDir = skylineTesterDirBasis + "_" + retry;

                    Delete(deleteDir);
                }
                catch (Exception e)
                {
                    if (Directory.Exists(_skylineTesterDir))
                    {
                        // Work around undeletable file that sometimes appears under Windows 10
                        Log("Unable to delete " + _skylineTesterDir + "(" + e + "),  using " + nextDir + " instead.");
                        _skylineTesterDir = nextDir;
                    }
                }
            }
            Log("buildRoot is " + PwizDir);

            // We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
            try
            {
                Delete(Path.Combine(nightlyDir, "pwiz"));
            }
            // ReSharper disable once EmptyGeneralCatchClause
            catch
            {
            }

            Directory.CreateDirectory(_skylineTesterDir);

            // Download most recent build of SkylineTester.
            var       skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
            const int attempts         = 30;
            string    branchUrl        = null;

            for (int i = 0; i < attempts; i++)
            {
                try
                {
                    DownloadSkylineTester(skylineTesterZip, _runMode);
                }
                catch (Exception ex)
                {
                    Log("Exception while downloading SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
                    if (i == attempts - 1)
                    {
                        LogAndThrow("Unable to download SkylineTester");
                    }
                    Thread.Sleep(60 * 1000);  // one minute
                    continue;
                }

                // Install SkylineTester.
                if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
                {
                    LogAndThrow("SkylineTester installation failed.");
                }
                try
                {
                    // Delete zip file.
                    Log("Delete zip file " + skylineTesterZip);
                    File.Delete(skylineTesterZip);

                    // Figure out which branch we're working in - there's a file in the downloaded SkylineTester zip that tells us.
                    var branchLine = File.ReadAllLines(Path.Combine(_skylineTesterDir, "SkylineTester Files", "Version.cpp")).FirstOrDefault(l => l.Contains("Version::Branch"));
                    if (!string.IsNullOrEmpty(branchLine))
                    {
                        // Looks like std::string Version::Branch()   {return "Skyline/skyline_9_7";}
                        var branch = branchLine.Split(new[] { "\"" }, StringSplitOptions.None)[1];
                        if (branch.Equals("master"))
                        {
                            branchUrl = GIT_MASTER_URL;
                        }
                        else
                        {
                            branchUrl = GIT_BRANCHES_URL + branch; // Looks like https://github.com/ProteoWizard/pwiz/tree/Skyline/skyline_9_7
                        }
                    }

                    break;
                }
                catch (Exception ex)
                {
                    Log("Exception while unzipping SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
                    if (i == attempts - 1)
                    {
                        LogAndThrow("Unable to identify branch from Version.cpp in SkylineTester");
                    }
                    Thread.Sleep(60 * 1000);  // one minute
                }
            }
            // Create ".skytr" file to execute nightly build in SkylineTester.
            var          assembly     = Assembly.GetExecutingAssembly();
            const string resourceName = "SkylineNightly.SkylineNightly.skytr";
            int          durationSeconds;

            using (var stream = assembly.GetManifestResourceStream(resourceName))
            {
                if (stream == null)
                {
                    LogAndThrow(result = "Embedded resource is broken");
                    return(result);
                }
                using (var reader = new StreamReader(stream))
                {
                    var skylineTester = Xml.FromString(reader.ReadToEnd());
                    skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
                    skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
                    skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
                    skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
                    skylineTester.GetChild("nightlyDuration").Set(((int)_duration.TotalHours).ToString());
                    skylineTester.GetChild("nightlyRepeat").Set(_runMode == RunMode.stress ? "100" : "1");
                    skylineTester.GetChild("nightlyRandomize").Set(_runMode == RunMode.stress ? "true" : "false");
                    if (!string.IsNullOrEmpty(branchUrl) && branchUrl.Contains("tree"))
                    {
                        skylineTester.GetChild("nightlyBuildTrunk").Set("false");
                        skylineTester.GetChild("nightlyBranch").Set("true");
                        skylineTester.GetChild("nightlyBranchUrl").Set(branchUrl);
                        Log("Testing branch at " + branchUrl);
                    }
                    skylineTester.Save(skylineNightlySkytr);
                    var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
                    durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60;  // 30 minutes grace before we kill SkylineTester
                }
            }


            // Start SkylineTester to do the build.
            var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");

            Log(string.Format("Starting {0} with config file {1}, which contains:", skylineTesterExe, skylineNightlySkytr));
            foreach (var line in File.ReadAllLines(skylineNightlySkytr))
            {
                Log(line);
            }

            var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
            {
                WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
            };

            var startTime = DateTime.Now;

            bool      retryTester;
            const int maxRetryMinutes = 60;

            do
            {
                var skylineTesterProcess = Process.Start(processInfo);
                if (skylineTesterProcess == null)
                {
                    LogAndThrow(result = "SkylineTester did not start");
                    return(result);
                }
                Log("SkylineTester started");
                if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
                {
                    SaveErrorScreenshot();
                    Log(result = "SkylineTester has exceeded its " + durationSeconds +
                                 " second WaitForExit timeout.  You should investigate.");
                }
                else if (skylineTesterProcess.ExitCode == 0xDEAD)
                {
                    // User killed, don't post
                    Log(result = SkylineTesterStoppedByUser);
                    return(result);
                }
                else
                {
                    Log("SkylineTester finished");
                }

                _duration   = DateTime.Now - startTime;
                retryTester = _duration.TotalMinutes < maxRetryMinutes;
                if (retryTester)
                {
                    // Retry a very short test run if there is no log file or the log file does not contain any tests
                    string logFile = GetLatestLog();
                    if (logFile != null && File.Exists(logFile))
                    {
                        retryTester = ParseTests(File.ReadAllText(logFile), false) == 0;
                    }
                    if (retryTester)
                    {
                        Log("No tests run in " + Math.Round(_duration.TotalMinutes) + " minutes retrying.");
                    }
                }
            }while (retryTester);

            return(result);
        }
Example #5
0
        private int ParseTests(string log)
        {
            var startTest = new Regex(@"\r\n\[\d\d:\d\d\] +(\d+).(\d+) +(\S+) +\((\w\w)\) ", RegexOptions.Compiled);
            var endTest = new Regex(@" \d+ failures, ([\.\d]+)/([\.\d]+) MB, (\d+) sec\.\r\n", RegexOptions.Compiled);

            string lastPass = null;
            int testCount = 0;
            for (var startMatch = startTest.Match(log); startMatch.Success; startMatch = startMatch.NextMatch())
            {
                var passId = startMatch.Groups[1].Value;
                var testId = startMatch.Groups[2].Value;
                var name = startMatch.Groups[3].Value;
                var language = startMatch.Groups[4].Value;

                var endMatch = endTest.Match(log, startMatch.Index);
                var managed = endMatch.Groups[1].Value;
                var total = endMatch.Groups[2].Value;
                var duration = endMatch.Groups[3].Value;

                if (string.IsNullOrEmpty(managed) || string.IsNullOrEmpty(total) || string.IsNullOrEmpty(duration))
                    continue;

                if (lastPass != passId)
                {
                    lastPass = passId;
                    _pass = _nightly.Append("pass");
                    _pass["id"] = passId;
                }

                var test = _pass.Append("test");
                test["id"] = testId;
                test["name"] = name;
                test["language"] = language;
                test["duration"] = duration;
                test["managed"] = managed;
                test["total"] = total;

                testCount++;
            }
            return testCount;
        }