예제 #1
0
        public static void Main(string[] args)
        {
            LogManager.Adapter = new ConsoleOutLoggerFactoryAdapter();
            Logging.SetLogging(LogManager.GetLogger <MainClass> ());
            if (args.Length == 0)
            {
                UsageAndExit(true);
            }

            if (args [0] == "--find-regressions")
            {
                var machineId     = args [1];
                var configId      = args [2];
                var testRun       = args.Length > 3 && args [3] == "--test-run";
                var onlyNecessary = !testRun || (args.Length > 4 && args [4] == "--only-necessary");
                if (!testRun)
                {
                    var credentials = Accredit.GetCredentials("regressionSlack");
                    SlackHooksUrl = credentials ["hooksURL"].ToString();
                }
                var conn = PostgresInterface.Connect();
                AsyncContext.Run(() => FindRegressions(conn, machineId, configId, testRun, onlyNecessary));
            }
            else if (args [0] == "--delete-run-set")
            {
                models.HttpApi.AuthToken = Accredit.GetCredentials("httpAPITokens") ["default"].ToString();
                var runSetId = Int64.Parse(args [1]);
                var output   = AsyncContext.Run(() => models.HttpApi.DeleteRunset(runSetId));
                Console.WriteLine("server answer: " + output);
            }
            else
            {
                UsageAndExit(false);
            }
        }
예제 #2
0
        static bool checkExists(NpgsqlConnection conn, string table, string column, string value)
        {
            PostgresRow row = new PostgresRow();

            row.Set(column, NpgsqlTypes.NpgsqlDbType.Varchar, value);
            var matches = PostgresInterface.Select(conn, "\"1\"." + table, new string[] { "1" }, column + " = :" + column, row);

            return(matches != null && matches.Any());
        }
예제 #3
0
        static void InsertWarned(NpgsqlConnection conn, long testRunSetId, string benchmark, bool faster)
        {
            PostgresRow row = new PostgresRow();

            row.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
            row.Set("benchmark", NpgsqlTypes.NpgsqlDbType.Varchar, benchmark);
            row.Set("faster", NpgsqlTypes.NpgsqlDbType.Boolean, faster);
            PostgresInterface.Insert <long> (conn, "RegressionsWarned", row, "id");
        }
예제 #4
0
        static void InsertWarned(NpgsqlConnection conn, long testRunSetId, string benchmark, bool faster)
        {
            var whereValues = new PostgresRow();

            whereValues.Set("name", NpgsqlTypes.NpgsqlDbType.Varchar, benchmark);
            var  rows        = PostgresInterface.Select(conn, "benchmark", new string[] { "id" }, "name = :name", whereValues);
            long benchmarkId = rows.First().GetValue <long> ("id").Value;

            var row = new PostgresRow();

            row.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
            row.Set("benchmark", NpgsqlTypes.NpgsqlDbType.Bigint, benchmarkId);
            row.Set("faster", NpgsqlTypes.NpgsqlDbType.Boolean, faster);
            PostgresInterface.Insert <long> (conn, "RegressionsWarned", row, "id");
        }
        public static bool BuildWebsite(string databaseType, string databaseName)
        {
            switch (databaseType)
            {
            case "postgres":
                CoreApiGenerator.GenerateWebsite(PostgresInterface.GetWebsite(databaseName));
                Console.WriteLine("Website generated successfully.");
                break;

            default:
                Console.WriteLine($"Database type not recognized: {databaseType}");
                break;
            }
            return(true);
        }
예제 #6
0
 static IDictionary <string, double[]> FetchResultsForRunSet(NpgsqlConnection conn, long runSetId)
 {
     if (!resultsForRunSetId.ContainsKey(runSetId))
     {
         var whereValues = new PostgresRow();
         whereValues.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, runSetId);
         whereValues.Set("metric", NpgsqlTypes.NpgsqlDbType.Varchar, "time");
         var rows = PostgresInterface.Select(conn, "\"1\".results",
                                             new string[] {
             "benchmark",
             "results"
         },
                                             "runSet = :runSet and metric = :metric and disabled is not true",
                                             whereValues);
         var dict = new Dictionary <string, double[]> ();
         foreach (var row in rows)
         {
             dict [row.GetReference <string> ("benchmark")] = row.GetReference <double[]> ("results");
         }
         resultsForRunSetId [runSetId] = dict;
     }
     return(resultsForRunSetId [runSetId]);
 }
예제 #7
0
        static async Task FindRegressions(NpgsqlConnection conn, string machineName, string configName, bool testRun, bool onlyNecessary)
        {
            const int    baselineWindowSize = 5;
            const int    testWindowSize     = 3;
            const double controlLimitSize   = 6;

            if (!checkExists(conn, "machine", "name", machineName))
            {
                Console.WriteLine("machine \"{0}\" unknown", machineName);
                UsageAndExit(false);
            }

            if (!checkExists(conn, "config", "name", configName))
            {
                Console.WriteLine("config \"{0}\" unknown", configName);
                UsageAndExit(false);
            }

            var summaryValues = new PostgresRow();

            summaryValues.Set("machine", NpgsqlTypes.NpgsqlDbType.Varchar, machineName);
            summaryValues.Set("config", NpgsqlTypes.NpgsqlDbType.Varchar, configName);
            summaryValues.Set("metric", NpgsqlTypes.NpgsqlDbType.Varchar, "time");
            var runSets = PostgresInterface.Select(conn, "\"1\".summary",
                                                   new string[] {
                "rs_id",
                "rs_startedAt",
                "c_hash",
                "rs_timedOutBenchmarks",
                "rs_crashedBenchmarks",
                "c_hash",
                "c_commitDate",
                "m_architecture",
                "averages",
                "variances"
            },
                                                   "m_name = :machine and cfg_name = :config and metric = :metric and rs_pullRequest is null",
                                                   summaryValues);
            var sortedRunSets = runSets.ToList();

            sortedRunSets.Sort((a, b) => {
                var aCommitDate = a.GetValue <DateTime> ("c_commitDate").Value;
                var bCommitDate = b.GetValue <DateTime> ("c_commitDate").Value;
                var result      = aCommitDate.CompareTo(bCommitDate);
                if (result != 0)
                {
                    return(result);
                }
                var aStartedDate = a.GetValue <DateTime> ("rs_startedAt").Value;
                var bStartedDate = b.GetValue <DateTime> ("rs_startedAt").Value;
                return(aStartedDate.CompareTo(bStartedDate));
            });
            var lastWarningIndex = new Dictionary <string, int> ();

            for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
            {
                var windowAverages  = new Dictionary <string, double> ();
                var windowVariances = new Dictionary <string, double> ();
                var benchmarkCounts = new Dictionary <string, int> ();

                for (var j = 1; j <= baselineWindowSize; ++j)
                {
                    var baselineRunSet = sortedRunSets [i - j];
                    var averages       = JsonMapToDictionary(baselineRunSet.GetReference <JObject> ("averages"));
                    var variances      = JsonMapToDictionary(baselineRunSet.GetReference <JObject> ("variances"));
                    foreach (var kvp in averages)
                    {
                        var name     = kvp.Key;
                        var average  = kvp.Value;
                        var variance = variances [name];
                        if (!windowAverages.ContainsKey(name))
                        {
                            windowAverages [name]  = 0.0;
                            windowVariances [name] = 0.0;
                            benchmarkCounts [name] = 0;
                        }
                        windowAverages [name]  += average;
                        windowVariances [name] += variance;
                        benchmarkCounts [name] += 1;
                    }
                }

                foreach (var kvp in benchmarkCounts)
                {
                    var name  = kvp.Key;
                    var count = kvp.Value;
                    windowAverages [name]  /= count;
                    windowVariances [name] /= count;
                }

                var testResults = new Dictionary <string, List <double> > ();
                for (var j = 0; j < testWindowSize; ++j)
                {
                    var results = FetchResultsForRunSet(conn, sortedRunSets [i + j].GetValue <long> ("rs_id").Value);
                    foreach (var kvp in results)
                    {
                        var benchmark = kvp.Key;
                        if (!testResults.ContainsKey(benchmark))
                        {
                            testResults.Add(benchmark, new List <double> ());
                        }
                        testResults [benchmark].AddRange(kvp.Value);
                    }
                }

                var testRunSet = sortedRunSets [i];

                var commitHash   = testRunSet.GetReference <string> ("c_hash");
                var testRunSetId = testRunSet.GetValue <long> ("rs_id").Value;
                Console.WriteLine("{0} {1}", testRunSetId, commitHash);

                var fasterBenchmarks = new List <string> ();
                var slowerBenchmarks = new List <string> ();

                foreach (var kvp in benchmarkCounts)
                {
                    var benchmark = kvp.Key;
                    if (kvp.Value < baselineWindowSize)
                    {
                        continue;
                    }
                    if (lastWarningIndex.ContainsKey(benchmark) && lastWarningIndex [benchmark] >= i - baselineWindowSize)
                    {
                        continue;
                    }
                    var average           = windowAverages [benchmark];
                    var variance          = windowVariances [benchmark];
                    var stdDev            = Math.Sqrt(variance);
                    var lowerControlLimit = average - controlLimitSize * stdDev;
                    var upperControlLimit = average + controlLimitSize * stdDev;
                    if (!testResults.ContainsKey(benchmark))
                    {
                        continue;
                    }
                    var results = testResults [benchmark];
                    if (results.Count < 5)
                    {
                        continue;
                    }
                    var numOutliersFaster = 0;
                    var numOutliersSlower = 0;
                    foreach (var elapsed in results)
                    {
                        if (elapsed < lowerControlLimit)
                        {
                            ++numOutliersFaster;
                        }
                        if (elapsed > upperControlLimit)
                        {
                            ++numOutliersSlower;
                        }
                    }
                    if (numOutliersFaster > results.Count * 3 / 4)
                    {
                        Console.WriteLine("+ regression in {0}: {1}/{2}", benchmark, numOutliersFaster, results.Count);
                        lastWarningIndex [benchmark] = i;
                        fasterBenchmarks.Add(benchmark);
                    }
                    else if (numOutliersSlower > results.Count * 3 / 4)
                    {
                        Console.WriteLine("- regression in {0}: {1}/{2}", benchmark, numOutliersSlower, results.Count);
                        lastWarningIndex [benchmark] = i;
                        slowerBenchmarks.Add(benchmark);
                    }

                    /*
                     * else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
                     *      Console.WriteLine ("  nothing in    {0}", name);
                     * } else {
                     *      Console.WriteLine ("? suspected in  {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
                     * }
                     */
                }

                if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
                {
                    var warnedFasterBenchmarks = new List <string> ();
                    var warnedSlowerBenchmarks = new List <string> ();

                    if (onlyNecessary)
                    {
                        var warningValues = new PostgresRow();
                        warningValues.Set("runset", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
                        var warnings = PostgresInterface.Select(conn, "RegressionsWarned",
                                                                new string[] {
                            "benchmark",
                            "faster"
                        },
                                                                "runSet = :runset", warningValues);

                        foreach (var row in warnings)
                        {
                            var benchmark = row.GetReference <string> ("benchmark");
                            if (row.GetValue <bool> ("faster").Value)
                            {
                                warnedFasterBenchmarks.Add(benchmark);
                            }
                            else
                            {
                                warnedSlowerBenchmarks.Add(benchmark);
                            }
                        }
                    }

                    var previousRunSet    = sortedRunSets [i - 1];
                    var newlyWarnedFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machineName, configName);

                    var newlyWarnedSlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machineName, configName);

                    if (!testRun)
                    {
                        foreach (var benchmark in newlyWarnedFaster)
                        {
                            InsertWarned(conn, testRunSetId, benchmark, true);
                        }
                        foreach (var benchmark in newlyWarnedSlower)
                        {
                            InsertWarned(conn, testRunSetId, benchmark, false);
                        }
                    }
                }
            }
        }
예제 #8
0
        internal static void StartMiningCorePool(string configFile)
        {
            try
            {
                // Display Software Version Info
                var basePath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
                Console.WriteLine($"{Assembly.GetEntryAssembly().GetName().Name} - MinerNL build v{Assembly.GetEntryAssembly().GetName().Version}");
                Console.WriteLine($"Run location: {basePath}");
                Console.WriteLine(" ");

                // log unhandled program exception errors
                AppDomain currentDomain = AppDomain.CurrentDomain;
                currentDomain.UnhandledException += new UnhandledExceptionEventHandler(MC_UnhandledException);
                currentDomain.ProcessExit        += OnProcessExit;
                Console.CancelKeyPress           += new ConsoleCancelEventHandler(OnCancelKeyPress);

                // ValidateRuntimeEnvironment();
                // root check
                if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.UserName == "root")
                {
                    logger.Warn(() => "Running as root is discouraged!");
                }

                // require 64-bit Windows OS
                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && RuntimeInformation.ProcessArchitecture == Architecture.X86)
                {
                    throw new PoolStartupAbortException("Miningcore requires 64-Bit Windows");
                }

                // Read config.json file
                clusterConfig = PoolConfig.GetConfigContent(configFile);

                // Initialize Logging
                FileLogger.ConfigureLogging();

                // LogRuntimeInfo();
                //-----------------------------------------------------------------------------
                logger.Info(() => $"{RuntimeInformation.FrameworkDescription.Trim()} on {RuntimeInformation.OSDescription.Trim()} [{RuntimeInformation.ProcessArchitecture}]");

                // Bootstrap();
                //-----------------------------------------------------------------------------
                ZcashNetworks.Instance.EnsureRegistered();

                // Service collection
                var builder = new ContainerBuilder();
                builder.RegisterAssemblyModules(typeof(AutofacModule).GetTypeInfo().Assembly);
                builder.RegisterInstance(clusterConfig);
                builder.RegisterInstance(pools);
                builder.RegisterInstance(gcStats);

                // AutoMapper
                var amConf = new MapperConfiguration(cfg => { cfg.AddProfile(new AutoMapperProfile()); });
                builder.Register((ctx, parms) => amConf.CreateMapper());

                PostgresInterface.ConnectDatabase(builder);
                container = builder.Build();

                // Configure Equihash
                if (clusterConfig.EquihashMaxThreads.HasValue)
                {
                    EquihashSolver.MaxThreads = clusterConfig.EquihashMaxThreads.Value;
                }

                MonitorGarbageCollection();

                // Start Miningcore Pool Services
                if (!cts.IsCancellationRequested)
                {
                    StartMiningcorePoolServices().Wait(cts.Token);
                }
            }

            catch (PoolStartupAbortException ex)
            {
                if (!string.IsNullOrEmpty(ex.Message))
                {
                    Console.WriteLine(ex.Message);
                }

                Console.WriteLine("\nCluster cannot start. Good Bye!");
            }

            catch (JsonException)
            {
                // ignored
            }

            catch (IOException)
            {
                // ignored
            }

            catch (AggregateException ex)
            {
                if (!(ex.InnerExceptions.First() is PoolStartupAbortException))
                {
                    Console.WriteLine(ex);
                }

                Console.WriteLine("Cluster cannot start. Good Bye!");
            }

            catch (OperationCanceledException)
            {
                // Ctrl+C
            }

            catch (Exception ex)
            {
                Console.WriteLine(ex);

                Console.WriteLine("Cluster cannot start. Good Bye!");
            }

            finally
            {
                // Shutdown();
                Console.WriteLine("Miningcore is shuting down... bye!");
                logger?.Info(() => "Miningcore is shuting down... bye!");

                foreach (var poolToStop in pools.Values)
                {
                    Console.WriteLine($"Stopping pool {poolToStop}");
                    poolToStop.Stop();
                }

                shareRelay?.Stop();
                shareReceiver?.Stop();
                shareRecorder?.Stop();
                statsRecorder?.Stop();
                Process.GetCurrentProcess().Kill();
            }
        }
예제 #9
0
        public async Task General()
        {
            const int recordCount = 10;

            using (var dbGenerator = new DbGenerator())
            {
                var dbInterface = new PostgresInterface(dbGenerator.CreateDb());

                const string insertSql = "INSERT INTO frog (name, date_of_birth) VALUES (@Name, @DateOfBirth);" +
                                         "SELECT CAST(lastval() AS INT);";

                const string readSingleSql = "SELECT * FROM frog WHERE id = @Id";
                const string readAllSql    = "SELECT * FROM frog";
                const string updateSql     = "UPDATE frog SET name = @NewName WHERE id BETWEEN @FromId AND @ToId";

                var ids = new List <int>();
                for (var i = 1; i <= recordCount; i++)
                {
                    var nameParam        = new NpgsqlParameter("@Name", $"Kermit{i}");
                    var dateOfBirthParam = new NpgsqlParameter("@DateOfBirth", DateTime.Today.AddDays(-i));

                    var statement = new PostgresStatement(insertSql, nameParam, dateOfBirthParam);
                    var id        = (int)(await dbInterface.ExecuteScalarAsync(statement).ConfigureAwait(false));

                    // it's a new DB so should be the case, depending on this for following tests
                    Assert.Equal(i, id);
                    ids.Add(id);
                }

                Assert.Equal(recordCount, ids.Count);

                foreach (var id in ids)
                {
                    var idParam   = new NpgsqlParameter("@Id", id);
                    var statement = new PostgresStatement(readSingleSql, idParam);

                    using (var reader = await dbInterface.GetReaderAsync(statement).ConfigureAwait(false))
                    {
                        Assert.True(await reader.ReadAsync().ConfigureAwait(false));
                        Assert.Equal(id, reader["id"]);
                        Assert.Equal($"Kermit{id}", reader["name"]);
                        Assert.Equal(DateTime.Today.AddDays(-id), Convert.ToDateTime(reader["date_of_birth"]));
                    }
                }

                var updateStatement = new PostgresStatement(updateSql, new NpgsqlParameter("@NewName", "Frank"),
                                                            new NpgsqlParameter("@FromId", 5), new NpgsqlParameter("@ToId", 8));

                var affected = await dbInterface.ExecuteNonQueryAsync(updateStatement).ConfigureAwait(false);

                Assert.Equal(4, affected);

                var readAllStatement = new PostgresStatement(readAllSql);
                using (var reader = await dbInterface.GetReaderAsync(readAllStatement).ConfigureAwait(false))
                {
                    var count = 0;
                    while (await reader.ReadAsync().ConfigureAwait(false))
                    {
                        count++;
                        var id = (int)reader["id"];
                        if (id >= 5 && id <= 8)
                        {
                            Assert.Equal("Frank", reader["name"]);
                        }
                        else
                        {
                            Assert.Equal($"Kermit{id}", reader["name"]);
                        }
                    }

                    Assert.Equal(recordCount, count);
                }
            }
        }