예제 #1
0
        static bool checkExists(NpgsqlConnection conn, string table, string column, string value)
        {
            PostgresRow row = new PostgresRow();

            row.Set(column, NpgsqlTypes.NpgsqlDbType.Varchar, value);
            var matches = PostgresInterface.Select(conn, "\"1\"." + table, new string[] { "1" }, column + " = :" + column, row);

            return(matches != null && matches.Any());
        }
예제 #2
0
        static void InsertWarned(NpgsqlConnection conn, long testRunSetId, string benchmark, bool faster)
        {
            var whereValues = new PostgresRow();

            whereValues.Set("name", NpgsqlTypes.NpgsqlDbType.Varchar, benchmark);
            var  rows        = PostgresInterface.Select(conn, "benchmark", new string[] { "id" }, "name = :name", whereValues);
            long benchmarkId = rows.First().GetValue <long> ("id").Value;

            var row = new PostgresRow();

            row.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
            row.Set("benchmark", NpgsqlTypes.NpgsqlDbType.Bigint, benchmarkId);
            row.Set("faster", NpgsqlTypes.NpgsqlDbType.Boolean, faster);
            PostgresInterface.Insert <long> (conn, "RegressionsWarned", row, "id");
        }
예제 #3
0
 static IDictionary <string, double[]> FetchResultsForRunSet(NpgsqlConnection conn, long runSetId)
 {
     if (!resultsForRunSetId.ContainsKey(runSetId))
     {
         var whereValues = new PostgresRow();
         whereValues.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, runSetId);
         whereValues.Set("metric", NpgsqlTypes.NpgsqlDbType.Varchar, "time");
         var rows = PostgresInterface.Select(conn, "\"1\".results",
                                             new string[] {
             "benchmark",
             "results"
         },
                                             "runSet = :runSet and metric = :metric and disabled is not true",
                                             whereValues);
         var dict = new Dictionary <string, double[]> ();
         foreach (var row in rows)
         {
             dict [row.GetReference <string> ("benchmark")] = row.GetReference <double[]> ("results");
         }
         resultsForRunSetId [runSetId] = dict;
     }
     return(resultsForRunSetId [runSetId]);
 }
예제 #4
0
        static async Task FindRegressions(NpgsqlConnection conn, string machineName, string configName, bool testRun, bool onlyNecessary)
        {
            const int    baselineWindowSize = 5;
            const int    testWindowSize     = 3;
            const double controlLimitSize   = 6;

            if (!checkExists(conn, "machine", "name", machineName))
            {
                Console.WriteLine("machine \"{0}\" unknown", machineName);
                UsageAndExit(false);
            }

            if (!checkExists(conn, "config", "name", configName))
            {
                Console.WriteLine("config \"{0}\" unknown", configName);
                UsageAndExit(false);
            }

            var summaryValues = new PostgresRow();

            summaryValues.Set("machine", NpgsqlTypes.NpgsqlDbType.Varchar, machineName);
            summaryValues.Set("config", NpgsqlTypes.NpgsqlDbType.Varchar, configName);
            summaryValues.Set("metric", NpgsqlTypes.NpgsqlDbType.Varchar, "time");
            var runSets = PostgresInterface.Select(conn, "\"1\".summary",
                                                   new string[] {
                "rs_id",
                "rs_startedAt",
                "c_hash",
                "rs_timedOutBenchmarks",
                "rs_crashedBenchmarks",
                "c_hash",
                "c_commitDate",
                "m_architecture",
                "averages",
                "variances"
            },
                                                   "m_name = :machine and cfg_name = :config and metric = :metric and rs_pullRequest is null",
                                                   summaryValues);
            var sortedRunSets = runSets.ToList();

            sortedRunSets.Sort((a, b) => {
                var aCommitDate = a.GetValue <DateTime> ("c_commitDate").Value;
                var bCommitDate = b.GetValue <DateTime> ("c_commitDate").Value;
                var result      = aCommitDate.CompareTo(bCommitDate);
                if (result != 0)
                {
                    return(result);
                }
                var aStartedDate = a.GetValue <DateTime> ("rs_startedAt").Value;
                var bStartedDate = b.GetValue <DateTime> ("rs_startedAt").Value;
                return(aStartedDate.CompareTo(bStartedDate));
            });
            var lastWarningIndex = new Dictionary <string, int> ();

            for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
            {
                var windowAverages  = new Dictionary <string, double> ();
                var windowVariances = new Dictionary <string, double> ();
                var benchmarkCounts = new Dictionary <string, int> ();

                for (var j = 1; j <= baselineWindowSize; ++j)
                {
                    var baselineRunSet = sortedRunSets [i - j];
                    var averages       = JsonMapToDictionary(baselineRunSet.GetReference <JObject> ("averages"));
                    var variances      = JsonMapToDictionary(baselineRunSet.GetReference <JObject> ("variances"));
                    foreach (var kvp in averages)
                    {
                        var name     = kvp.Key;
                        var average  = kvp.Value;
                        var variance = variances [name];
                        if (!windowAverages.ContainsKey(name))
                        {
                            windowAverages [name]  = 0.0;
                            windowVariances [name] = 0.0;
                            benchmarkCounts [name] = 0;
                        }
                        windowAverages [name]  += average;
                        windowVariances [name] += variance;
                        benchmarkCounts [name] += 1;
                    }
                }

                foreach (var kvp in benchmarkCounts)
                {
                    var name  = kvp.Key;
                    var count = kvp.Value;
                    windowAverages [name]  /= count;
                    windowVariances [name] /= count;
                }

                var testResults = new Dictionary <string, List <double> > ();
                for (var j = 0; j < testWindowSize; ++j)
                {
                    var results = FetchResultsForRunSet(conn, sortedRunSets [i + j].GetValue <long> ("rs_id").Value);
                    foreach (var kvp in results)
                    {
                        var benchmark = kvp.Key;
                        if (!testResults.ContainsKey(benchmark))
                        {
                            testResults.Add(benchmark, new List <double> ());
                        }
                        testResults [benchmark].AddRange(kvp.Value);
                    }
                }

                var testRunSet = sortedRunSets [i];

                var commitHash   = testRunSet.GetReference <string> ("c_hash");
                var testRunSetId = testRunSet.GetValue <long> ("rs_id").Value;
                Console.WriteLine("{0} {1}", testRunSetId, commitHash);

                var fasterBenchmarks = new List <string> ();
                var slowerBenchmarks = new List <string> ();

                foreach (var kvp in benchmarkCounts)
                {
                    var benchmark = kvp.Key;
                    if (kvp.Value < baselineWindowSize)
                    {
                        continue;
                    }
                    if (lastWarningIndex.ContainsKey(benchmark) && lastWarningIndex [benchmark] >= i - baselineWindowSize)
                    {
                        continue;
                    }
                    var average           = windowAverages [benchmark];
                    var variance          = windowVariances [benchmark];
                    var stdDev            = Math.Sqrt(variance);
                    var lowerControlLimit = average - controlLimitSize * stdDev;
                    var upperControlLimit = average + controlLimitSize * stdDev;
                    if (!testResults.ContainsKey(benchmark))
                    {
                        continue;
                    }
                    var results = testResults [benchmark];
                    if (results.Count < 5)
                    {
                        continue;
                    }
                    var numOutliersFaster = 0;
                    var numOutliersSlower = 0;
                    foreach (var elapsed in results)
                    {
                        if (elapsed < lowerControlLimit)
                        {
                            ++numOutliersFaster;
                        }
                        if (elapsed > upperControlLimit)
                        {
                            ++numOutliersSlower;
                        }
                    }
                    if (numOutliersFaster > results.Count * 3 / 4)
                    {
                        Console.WriteLine("+ regression in {0}: {1}/{2}", benchmark, numOutliersFaster, results.Count);
                        lastWarningIndex [benchmark] = i;
                        fasterBenchmarks.Add(benchmark);
                    }
                    else if (numOutliersSlower > results.Count * 3 / 4)
                    {
                        Console.WriteLine("- regression in {0}: {1}/{2}", benchmark, numOutliersSlower, results.Count);
                        lastWarningIndex [benchmark] = i;
                        slowerBenchmarks.Add(benchmark);
                    }

                    /*
                     * else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
                     *      Console.WriteLine ("  nothing in    {0}", name);
                     * } else {
                     *      Console.WriteLine ("? suspected in  {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
                     * }
                     */
                }

                if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
                {
                    var warnedFasterBenchmarks = new List <string> ();
                    var warnedSlowerBenchmarks = new List <string> ();

                    if (onlyNecessary)
                    {
                        var warningValues = new PostgresRow();
                        warningValues.Set("runset", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
                        var warnings = PostgresInterface.Select(conn, "RegressionsWarned",
                                                                new string[] {
                            "benchmark",
                            "faster"
                        },
                                                                "runSet = :runset", warningValues);

                        foreach (var row in warnings)
                        {
                            var benchmark = row.GetReference <string> ("benchmark");
                            if (row.GetValue <bool> ("faster").Value)
                            {
                                warnedFasterBenchmarks.Add(benchmark);
                            }
                            else
                            {
                                warnedSlowerBenchmarks.Add(benchmark);
                            }
                        }
                    }

                    var previousRunSet    = sortedRunSets [i - 1];
                    var newlyWarnedFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machineName, configName);

                    var newlyWarnedSlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machineName, configName);

                    if (!testRun)
                    {
                        foreach (var benchmark in newlyWarnedFaster)
                        {
                            InsertWarned(conn, testRunSetId, benchmark, true);
                        }
                        foreach (var benchmark in newlyWarnedSlower)
                        {
                            InsertWarned(conn, testRunSetId, benchmark, false);
                        }
                    }
                }
            }
        }