Exemple #1
0
        /// <summary>
        /// Run nightly build/test and report results to server.
        /// </summary>
        public void Run(bool withPerfTests)
        {
            // Locate relevant directories.
            var nightlyDir          = GetNightlyDir();
            var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");

            if (withPerfTests)
            {
                _duration = TimeSpan.FromHours(12);
            }

            // Kill any other instance of SkylineNightly.
            foreach (var process in Process.GetProcessesByName("skylinenightly"))
            {
                if (process.Id != Process.GetCurrentProcess().Id)
                {
                    process.Kill();
                }
            }

            // Kill processes started within the nightly directory.
            foreach (var process in Process.GetProcesses())
            {
                try
                {
                    if (process.Modules[0].FileName.StartsWith(nightlyDir) &&
                        process.Id != Process.GetCurrentProcess().Id)
                    {
                        process.Kill();
                    }
                }
                // ReSharper disable once EmptyGeneralCatchClause
                catch (Exception)
                {
                }
            }

            _startTime = DateTime.Now;

            // Create log file.
            if (!Directory.Exists(_logDir))
            {
                Directory.CreateDirectory(_logDir);
            }
            _logFile = Path.Combine(nightlyDir, "SkylineNightly.log");
            Delete(_logFile);
            Log(DateTime.Now.ToShortDateString());

            // Delete source tree and old SkylineTester.
            Delete(skylineNightlySkytr);
            Log("Delete SkylineTester");
            var       skylineTesterDirBasis = _skylineTesterDir; // Default name
            const int maxRetry = 1000;                           // Something would have to be very wrong to get here, but better not to risk a hang

            for (var retry = 1; retry < maxRetry; retry++)
            {
                try
                {
                    Delete(_skylineTesterDir);
                    break;
                }
                catch (Exception e)
                {
                    // Work around undeletable file that sometimes appears under Windows 10
                    var newDir = skylineTesterDirBasis + "_" + retry;
                    Log("Unable to delete " + _skylineTesterDir + "(" + e + "),  using " + newDir + " instead.");
                    _skylineTesterDir = newDir;
                }
            }
            Log("buildRoot is " + PwizDir);

            // We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
            try
            {
                Delete(Path.Combine(nightlyDir, "pwiz"));
            }
            // ReSharper disable once EmptyGeneralCatchClause
            catch
            {
            }

            Directory.CreateDirectory(_skylineTesterDir);

            // Download most recent build of SkylineTester.
            var       skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
            const int attempts         = 30;

            for (int i = 0; i < attempts; i++)
            {
                try
                {
                    DownloadSkylineTester(skylineTesterZip);
                    break;
                }
                catch (Exception ex)
                {
                    Log("Exception while downloading SkylineTester: " + ex.Message);
                    if (i == attempts - 1)
                    {
                        LogAndThrow("Unable to download SkylineTester");
                    }
                    Thread.Sleep(60 * 1000);  // one minute
                }
            }

            // Install SkylineTester.
            if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
            {
                LogAndThrow("SkylineTester installation failed.");
            }

            // Delete zip file.
            Log("Delete zip file " + skylineTesterZip);
            File.Delete(skylineTesterZip);

            // Create ".skytr" file to execute nightly build in SkylineTester.
            var          assembly     = Assembly.GetExecutingAssembly();
            const string resourceName = "SkylineNightly.SkylineNightly.skytr";
            int          durationSeconds;

            using (var stream = assembly.GetManifestResourceStream(resourceName))
            {
                if (stream == null)
                {
                    LogAndThrow("Embedded resource is broken");
                    return; // Just so we don't get the "possible null" warning below
                }
                using (var reader = new StreamReader(stream))
                {
                    var skylineTester = Xml.FromString(reader.ReadToEnd());
                    skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
                    skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
                    skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
                    skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
                    skylineTester.GetChild("nightlyDuration").Set(_duration.Hours.ToString());
                    skylineTester.Save(skylineNightlySkytr);
                    var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
                    durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60;  // 30 minutes grace before we kill SkylineTester
                }
            }

            // Start SkylineTester to do the build.
            var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");

            var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
            {
                WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
            };

            var startTime            = DateTime.Now;
            var skylineTesterProcess = Process.Start(processInfo);

            if (skylineTesterProcess == null)
            {
                LogAndThrow("SkylineTester did not start");
                return; // Just so we don't get the "possible null" warning below
            }
            Log("SkylineTester started");
            if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
            {
                SaveErrorScreenshot();
                skylineTesterProcess.Kill();
                Log("SkylineTester killed after " + durationSeconds + " second WaitForExit timeout");
            }
            else
            {
                Log("SkylineTester finished");
            }

            _duration = DateTime.Now - startTime;
        }
Exemple #2
0
        /// <summary>
        /// Run nightly build/test and report results to server.
        /// </summary>
        public string Run()
        {
            string result = string.Empty;
            // Locate relevant directories.
            var nightlyDir          = GetNightlyDir();
            var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");

            bool withPerfTests = _runMode != RunMode.trunk && _runMode != RunMode.integration && _runMode != RunMode.release;

            if (_runMode == RunMode.stress)
            {
                _duration = TimeSpan.FromHours(168);  // Let it go as long as a week
            }
            else if (withPerfTests)
            {
                _duration = TimeSpan.FromHours(PERF_DURATION_HOURS); // Let it go a bit longer than standard 9 hours
            }

            // Kill any other instance of SkylineNightly, unless this is
            // the StressTest mode, in which case assume that a previous invocation
            // is still running and just exit to stay out of its way.
            foreach (var process in Process.GetProcessesByName("skylinenightly"))
            {
                if (process.Id != Process.GetCurrentProcess().Id)
                {
                    if (_runMode == RunMode.stress)
                    {
                        Application.Exit();  // Just let the already (long!) running process do its thing
                    }
                    else
                    {
                        process.Kill();
                    }
                }
            }

            // Kill processes started within the proposed working directory - most likely SkylineTester and/or TestRunner.
            // This keeps stuck tests around for 24 hours, which should be sufficient, but allows us to replace directory
            // on a daily basis - otherwise we could fill the hard drive on smaller machines
            foreach (var process in Process.GetProcesses())
            {
                try
                {
                    if (process.Modules[0].FileName.StartsWith(_skylineTesterDir) &&
                        process.Id != Process.GetCurrentProcess().Id)
                    {
                        process.Kill();
                    }
                }
                // ReSharper disable once EmptyGeneralCatchClause
                catch (Exception)
                {
                }
            }

            // Create place to put run logs
            if (!Directory.Exists(_logDir))
            {
                Directory.CreateDirectory(_logDir);
            }
            // Start the nightly log file
            StartLog(_runMode);

            // Delete source tree and old SkylineTester.
            Delete(skylineNightlySkytr);
            Log("Delete SkylineTester");
            var       skylineTesterDirBasis = _skylineTesterDir; // Default name
            const int maxRetry = 1000;                           // Something would have to be very wrong to get here, but better not to risk a hang
            string    nextDir  = _skylineTesterDir;

            for (var retry = 1; retry < maxRetry; retry++)
            {
                try
                {
                    if (!Directory.Exists(nextDir))
                    {
                        break;
                    }

                    string deleteDir = nextDir;
                    // Keep going until a directory is found that does not exist
                    nextDir = skylineTesterDirBasis + "_" + retry;

                    Delete(deleteDir);
                }
                catch (Exception e)
                {
                    if (Directory.Exists(_skylineTesterDir))
                    {
                        // Work around undeletable file that sometimes appears under Windows 10
                        Log("Unable to delete " + _skylineTesterDir + "(" + e + "),  using " + nextDir + " instead.");
                        _skylineTesterDir = nextDir;
                    }
                }
            }
            Log("buildRoot is " + PwizDir);

            // We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
            try
            {
                Delete(Path.Combine(nightlyDir, "pwiz"));
            }
            // ReSharper disable once EmptyGeneralCatchClause
            catch
            {
            }

            Directory.CreateDirectory(_skylineTesterDir);

            // Download most recent build of SkylineTester.
            var       skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
            const int attempts         = 30;
            string    branchUrl        = null;

            for (int i = 0; i < attempts; i++)
            {
                try
                {
                    DownloadSkylineTester(skylineTesterZip, _runMode);
                }
                catch (Exception ex)
                {
                    Log("Exception while downloading SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
                    if (i == attempts - 1)
                    {
                        LogAndThrow("Unable to download SkylineTester");
                    }
                    Thread.Sleep(60 * 1000);  // one minute
                    continue;
                }

                // Install SkylineTester.
                if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
                {
                    LogAndThrow("SkylineTester installation failed.");
                }
                try
                {
                    // Delete zip file.
                    Log("Delete zip file " + skylineTesterZip);
                    File.Delete(skylineTesterZip);

                    // Figure out which branch we're working in - there's a file in the downloaded SkylineTester zip that tells us.
                    var branchLine = File.ReadAllLines(Path.Combine(_skylineTesterDir, "SkylineTester Files", "Version.cpp")).FirstOrDefault(l => l.Contains("Version::Branch"));
                    if (!string.IsNullOrEmpty(branchLine))
                    {
                        // Looks like std::string Version::Branch()   {return "Skyline/skyline_9_7";}
                        var branch = branchLine.Split(new[] { "\"" }, StringSplitOptions.None)[1];
                        if (branch.Equals("master"))
                        {
                            branchUrl = GIT_MASTER_URL;
                        }
                        else
                        {
                            branchUrl = GIT_BRANCHES_URL + branch; // Looks like https://github.com/ProteoWizard/pwiz/tree/Skyline/skyline_9_7
                        }
                    }

                    break;
                }
                catch (Exception ex)
                {
                    Log("Exception while unzipping SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
                    if (i == attempts - 1)
                    {
                        LogAndThrow("Unable to identify branch from Version.cpp in SkylineTester");
                    }
                    Thread.Sleep(60 * 1000);  // one minute
                }
            }
            // Create ".skytr" file to execute nightly build in SkylineTester.
            var          assembly     = Assembly.GetExecutingAssembly();
            const string resourceName = "SkylineNightly.SkylineNightly.skytr";
            int          durationSeconds;

            using (var stream = assembly.GetManifestResourceStream(resourceName))
            {
                if (stream == null)
                {
                    LogAndThrow(result = "Embedded resource is broken");
                    return(result);
                }
                using (var reader = new StreamReader(stream))
                {
                    var skylineTester = Xml.FromString(reader.ReadToEnd());
                    skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
                    skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
                    skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
                    skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
                    skylineTester.GetChild("nightlyDuration").Set(((int)_duration.TotalHours).ToString());
                    skylineTester.GetChild("nightlyRepeat").Set(_runMode == RunMode.stress ? "100" : "1");
                    skylineTester.GetChild("nightlyRandomize").Set(_runMode == RunMode.stress ? "true" : "false");
                    if (!string.IsNullOrEmpty(branchUrl) && branchUrl.Contains("tree"))
                    {
                        skylineTester.GetChild("nightlyBuildTrunk").Set("false");
                        skylineTester.GetChild("nightlyBranch").Set("true");
                        skylineTester.GetChild("nightlyBranchUrl").Set(branchUrl);
                        Log("Testing branch at " + branchUrl);
                    }
                    skylineTester.Save(skylineNightlySkytr);
                    var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
                    durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60;  // 30 minutes grace before we kill SkylineTester
                }
            }


            // Start SkylineTester to do the build.
            var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");

            Log(string.Format("Starting {0} with config file {1}, which contains:", skylineTesterExe, skylineNightlySkytr));
            foreach (var line in File.ReadAllLines(skylineNightlySkytr))
            {
                Log(line);
            }

            var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
            {
                WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
            };

            var startTime = DateTime.Now;

            bool      retryTester;
            const int maxRetryMinutes = 60;

            do
            {
                var skylineTesterProcess = Process.Start(processInfo);
                if (skylineTesterProcess == null)
                {
                    LogAndThrow(result = "SkylineTester did not start");
                    return(result);
                }
                Log("SkylineTester started");
                if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
                {
                    SaveErrorScreenshot();
                    Log(result = "SkylineTester has exceeded its " + durationSeconds +
                                 " second WaitForExit timeout.  You should investigate.");
                }
                else if (skylineTesterProcess.ExitCode == 0xDEAD)
                {
                    // User killed, don't post
                    Log(result = SkylineTesterStoppedByUser);
                    return(result);
                }
                else
                {
                    Log("SkylineTester finished");
                }

                _duration   = DateTime.Now - startTime;
                retryTester = _duration.TotalMinutes < maxRetryMinutes;
                if (retryTester)
                {
                    // Retry a very short test run if there is no log file or the log file does not contain any tests
                    string logFile = GetLatestLog();
                    if (logFile != null && File.Exists(logFile))
                    {
                        retryTester = ParseTests(File.ReadAllText(logFile), false) == 0;
                    }
                    if (retryTester)
                    {
                        Log("No tests run in " + Math.Round(_duration.TotalMinutes) + " minutes retrying.");
                    }
                }
            }while (retryTester);

            return(result);
        }
Exemple #3
0
        /// <summary>
        /// Run nightly build/test and report results to server.
        /// </summary>
        public string Run()
        {
            string result = string.Empty;
            // Locate relevant directories.
            var nightlyDir          = GetNightlyDir();
            var skylineNightlySkytr = Path.Combine(nightlyDir, "SkylineNightly.skytr");

            bool withPerfTests = _runMode != RunMode.trunk && _runMode != RunMode.integration;

            if (_runMode == RunMode.stress)
            {
                _duration = TimeSpan.FromHours(168);  // Let it go as long as a week
            }
            else if (withPerfTests)
            {
                _duration = TimeSpan.FromHours(PERF_DURATION_HOURS); // Let it go a bit longer than standard 9 hours
            }

            // Kill any other instance of SkylineNightly, unless this is
            // the StressTest mode, in which case assume that a previous invocation
            // is still running and just exit to stay out of its way.
            foreach (var process in Process.GetProcessesByName("skylinenightly"))
            {
                if (process.Id != Process.GetCurrentProcess().Id)
                {
                    if (_runMode == RunMode.stress)
                    {
                        Application.Exit();  // Just let the already (long!) running process do its thing
                    }
                    else
                    {
                        process.Kill();
                    }
                }
            }

            // Kill processes started within the proposed working directory - most likely SkylineTester and/or TestRunner.
            // This keeps stuck tests around for 24 hours, which should be sufficient, but allows us to replace directory
            // on a daily basis - otherwise we could fill the hard drive on smaller machines
            foreach (var process in Process.GetProcesses())
            {
                try
                {
                    if (process.Modules[0].FileName.StartsWith(_skylineTesterDir) &&
                        process.Id != Process.GetCurrentProcess().Id)
                    {
                        process.Kill();
                    }
                }
                // ReSharper disable once EmptyGeneralCatchClause
                catch (Exception)
                {
                }
            }

            // Create place to put run logs
            if (!Directory.Exists(_logDir))
            {
                Directory.CreateDirectory(_logDir);
            }
            // Start the nightly log file
            StartLog(_runMode);

            // Delete source tree and old SkylineTester.
            Delete(skylineNightlySkytr);
            Log("Delete SkylineTester");
            var       skylineTesterDirBasis = _skylineTesterDir; // Default name
            const int maxRetry = 1000;                           // Something would have to be very wrong to get here, but better not to risk a hang

            for (var retry = 1; retry < maxRetry; retry++)
            {
                try
                {
                    Delete(_skylineTesterDir);
                    break;
                }
                catch (Exception e)
                {
                    // Work around undeletable file that sometimes appears under Windows 10
                    var newDir = skylineTesterDirBasis + "_" + retry;
                    Log("Unable to delete " + _skylineTesterDir + "(" + e + "),  using " + newDir + " instead.");
                    _skylineTesterDir = newDir;
                }
            }
            Log("buildRoot is " + PwizDir);

            // We used to put source tree alongside SkylineTesterDir instead of under it, delete that too
            try
            {
                Delete(Path.Combine(nightlyDir, "pwiz"));
            }
            // ReSharper disable once EmptyGeneralCatchClause
            catch
            {
            }

            Directory.CreateDirectory(_skylineTesterDir);

            // Download most recent build of SkylineTester.
            var       skylineTesterZip = Path.Combine(_skylineTesterDir, skylineTesterDirBasis + ".zip");
            const int attempts         = 30;

            for (int i = 0; i < attempts; i++)
            {
                try
                {
                    DownloadSkylineTester(skylineTesterZip, _runMode);
                    break;
                }
                catch (Exception ex)
                {
                    Log("Exception while downloading SkylineTester: " + ex.Message + " (Probably still being built, will retry every 60 seconds for 30 minutes.)");
                    if (i == attempts - 1)
                    {
                        LogAndThrow("Unable to download SkylineTester");
                    }
                    Thread.Sleep(60 * 1000);  // one minute
                }
            }

            // Install SkylineTester.
            if (!InstallSkylineTester(skylineTesterZip, _skylineTesterDir))
            {
                LogAndThrow("SkylineTester installation failed.");
            }

            // Delete zip file.
            Log("Delete zip file " + skylineTesterZip);
            File.Delete(skylineTesterZip);

            // Now figure out which branch we're working in - there's a file in the downloaded SkylineTester zip that tells us.
            string branchUrl = null;
            var    lines     = File.ReadAllLines(Path.Combine(_skylineTesterDir, "SkylineTester Files", "SVN_info.txt"));

            if (lines[1].Contains("branches"))
            {
                // Looks like  $URL: https://svn.code.sf.net/p/proteowizard/code/branches/skyline_3_5/SVN_info.txt $
                branchUrl =
                    lines[1].Split(new[] { "URL: " }, StringSplitOptions.None)[1].Split(new[] { "/SVN_info" }, StringSplitOptions.None)[0];
            }
            // Or if we are running the integration build, then we use the trunk SkylineTester, but have it build the branch TestRunner.exe
            else if (_runMode == RunMode.integration)
            {
                branchUrl = SVN_INTEGRATION_BRANCH_URL;
            }

            // Create ".skytr" file to execute nightly build in SkylineTester.
            var          assembly     = Assembly.GetExecutingAssembly();
            const string resourceName = "SkylineNightly.SkylineNightly.skytr";
            int          durationSeconds;

            using (var stream = assembly.GetManifestResourceStream(resourceName))
            {
                if (stream == null)
                {
                    LogAndThrow(result = "Embedded resource is broken");
                    return(result);
                }
                using (var reader = new StreamReader(stream))
                {
                    var skylineTester = Xml.FromString(reader.ReadToEnd());
                    skylineTester.GetChild("nightlyStartTime").Set(DateTime.Now.ToShortTimeString());
                    skylineTester.GetChild("nightlyRoot").Set(nightlyDir);
                    skylineTester.GetChild("buildRoot").Set(_skylineTesterDir);
                    skylineTester.GetChild("nightlyRunPerfTests").Set(withPerfTests?"true":"false");
                    skylineTester.GetChild("nightlyDuration").Set(((int)_duration.TotalHours).ToString());
                    skylineTester.GetChild("nightlyRepeat").Set(_runMode == RunMode.stress ? "100" : "1");
                    skylineTester.GetChild("nightlyRandomize").Set(_runMode == RunMode.stress ? "true" : "false");
                    if (!string.IsNullOrEmpty(branchUrl) && !branchUrl.Contains("trunk"))
                    {
                        skylineTester.GetChild("nightlyBuildTrunk").Set("false");
                        skylineTester.GetChild("nightlyBranch").Set("true");
                        skylineTester.GetChild("nightlyBranchUrl").Set(branchUrl);
                        Log("Testing branch at " + branchUrl);
                    }
                    skylineTester.Save(skylineNightlySkytr);
                    var durationHours = double.Parse(skylineTester.GetChild("nightlyDuration").Value);
                    durationSeconds = (int)(durationHours * 60 * 60) + 30 * 60;  // 30 minutes grace before we kill SkylineTester
                }
            }


            // Start SkylineTester to do the build.
            var skylineTesterExe = Path.Combine(_skylineTesterDir, "SkylineTester Files", "SkylineTester.exe");

            Log(string.Format("Starting {0} with config file {1}, which contains:", skylineTesterExe, skylineNightlySkytr));
            foreach (var line in File.ReadAllLines(skylineNightlySkytr))
            {
                Log(line);
            }

            var processInfo = new ProcessStartInfo(skylineTesterExe, skylineNightlySkytr)
            {
                WorkingDirectory = Path.GetDirectoryName(skylineTesterExe) ?? ""
            };

            var startTime            = DateTime.Now;
            var skylineTesterProcess = Process.Start(processInfo);

            if (skylineTesterProcess == null)
            {
                LogAndThrow(result = "SkylineTester did not start");
                return(result);
            }
            Log("SkylineTester started");
            if (!skylineTesterProcess.WaitForExit(durationSeconds * 1000))
            {
                SaveErrorScreenshot();
                Log(result = "SkylineTester has exceeded its " + durationSeconds + " second WaitForExit timeout.  You should investigate.");
            }
            else
            {
                Log("SkylineTester finished");
            }

            _duration = DateTime.Now - startTime;
            return(result);
        }