public static void Main(string[] args)
{
LogManager.Adapter = new ConsoleOutLoggerFactoryAdapter();
Logging.SetLogging(LogManager.GetLogger <MainClass> ());
if (args.Length == 0)
{
UsageAndExit(true);
}
if (args [0] == "--find-regressions")
{
var machineId = args [1];
var configId = args [2];
var testRun = args.Length > 3 && args [3] == "--test-run";
var onlyNecessary = !testRun || (args.Length > 4 && args [4] == "--only-necessary");
if (!testRun)
{
var credentials = Accredit.GetCredentials("regressionSlack");
SlackHooksUrl = credentials ["hooksURL"].ToString();
}
var conn = PostgresInterface.Connect();
AsyncContext.Run(() => FindRegressions(conn, machineId, configId, testRun, onlyNecessary));
}
else if (args [0] == "--delete-run-set")
{
models.HttpApi.AuthToken = Accredit.GetCredentials("httpAPITokens") ["default"].ToString();
var runSetId = Int64.Parse(args [1]);
var output = AsyncContext.Run(() => models.HttpApi.DeleteRunset(runSetId));
Console.WriteLine("server answer: " + output);
}
else
{
UsageAndExit(false);
}
}
static bool checkExists(NpgsqlConnection conn, string table, string column, string value)
{
PostgresRow row = new PostgresRow();
row.Set(column, NpgsqlTypes.NpgsqlDbType.Varchar, value);
var matches = PostgresInterface.Select(conn, "\"1\"." + table, new string[] { "1" }, column + " = :" + column, row);
return(matches != null && matches.Any());
}
static void InsertWarned(NpgsqlConnection conn, long testRunSetId, string benchmark, bool faster)
{
PostgresRow row = new PostgresRow();
row.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
row.Set("benchmark", NpgsqlTypes.NpgsqlDbType.Varchar, benchmark);
row.Set("faster", NpgsqlTypes.NpgsqlDbType.Boolean, faster);
PostgresInterface.Insert <long> (conn, "RegressionsWarned", row, "id");
}
static void InsertWarned(NpgsqlConnection conn, long testRunSetId, string benchmark, bool faster)
{
var whereValues = new PostgresRow();
whereValues.Set("name", NpgsqlTypes.NpgsqlDbType.Varchar, benchmark);
var rows = PostgresInterface.Select(conn, "benchmark", new string[] { "id" }, "name = :name", whereValues);
long benchmarkId = rows.First().GetValue <long> ("id").Value;
var row = new PostgresRow();
row.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
row.Set("benchmark", NpgsqlTypes.NpgsqlDbType.Bigint, benchmarkId);
row.Set("faster", NpgsqlTypes.NpgsqlDbType.Boolean, faster);
PostgresInterface.Insert <long> (conn, "RegressionsWarned", row, "id");
}
public static bool BuildWebsite(string databaseType, string databaseName)
{
switch (databaseType)
{
case "postgres":
CoreApiGenerator.GenerateWebsite(PostgresInterface.GetWebsite(databaseName));
Console.WriteLine("Website generated successfully.");
break;
default:
Console.WriteLine($"Database type not recognized: {databaseType}");
break;
}
return(true);
}
static IDictionary <string, double[]> FetchResultsForRunSet(NpgsqlConnection conn, long runSetId)
{
if (!resultsForRunSetId.ContainsKey(runSetId))
{
var whereValues = new PostgresRow();
whereValues.Set("runSet", NpgsqlTypes.NpgsqlDbType.Bigint, runSetId);
whereValues.Set("metric", NpgsqlTypes.NpgsqlDbType.Varchar, "time");
var rows = PostgresInterface.Select(conn, "\"1\".results",
new string[] {
"benchmark",
"results"
},
"runSet = :runSet and metric = :metric and disabled is not true",
whereValues);
var dict = new Dictionary <string, double[]> ();
foreach (var row in rows)
{
dict [row.GetReference <string> ("benchmark")] = row.GetReference <double[]> ("results");
}
resultsForRunSetId [runSetId] = dict;
}
return(resultsForRunSetId [runSetId]);
}
static async Task FindRegressions(NpgsqlConnection conn, string machineName, string configName, bool testRun, bool onlyNecessary)
{
const int baselineWindowSize = 5;
const int testWindowSize = 3;
const double controlLimitSize = 6;
if (!checkExists(conn, "machine", "name", machineName))
{
Console.WriteLine("machine \"{0}\" unknown", machineName);
UsageAndExit(false);
}
if (!checkExists(conn, "config", "name", configName))
{
Console.WriteLine("config \"{0}\" unknown", configName);
UsageAndExit(false);
}
var summaryValues = new PostgresRow();
summaryValues.Set("machine", NpgsqlTypes.NpgsqlDbType.Varchar, machineName);
summaryValues.Set("config", NpgsqlTypes.NpgsqlDbType.Varchar, configName);
summaryValues.Set("metric", NpgsqlTypes.NpgsqlDbType.Varchar, "time");
var runSets = PostgresInterface.Select(conn, "\"1\".summary",
new string[] {
"rs_id",
"rs_startedAt",
"c_hash",
"rs_timedOutBenchmarks",
"rs_crashedBenchmarks",
"c_hash",
"c_commitDate",
"m_architecture",
"averages",
"variances"
},
"m_name = :machine and cfg_name = :config and metric = :metric and rs_pullRequest is null",
summaryValues);
var sortedRunSets = runSets.ToList();
sortedRunSets.Sort((a, b) => {
var aCommitDate = a.GetValue <DateTime> ("c_commitDate").Value;
var bCommitDate = b.GetValue <DateTime> ("c_commitDate").Value;
var result = aCommitDate.CompareTo(bCommitDate);
if (result != 0)
{
return(result);
}
var aStartedDate = a.GetValue <DateTime> ("rs_startedAt").Value;
var bStartedDate = b.GetValue <DateTime> ("rs_startedAt").Value;
return(aStartedDate.CompareTo(bStartedDate));
});
var lastWarningIndex = new Dictionary <string, int> ();
for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
{
var windowAverages = new Dictionary <string, double> ();
var windowVariances = new Dictionary <string, double> ();
var benchmarkCounts = new Dictionary <string, int> ();
for (var j = 1; j <= baselineWindowSize; ++j)
{
var baselineRunSet = sortedRunSets [i - j];
var averages = JsonMapToDictionary(baselineRunSet.GetReference <JObject> ("averages"));
var variances = JsonMapToDictionary(baselineRunSet.GetReference <JObject> ("variances"));
foreach (var kvp in averages)
{
var name = kvp.Key;
var average = kvp.Value;
var variance = variances [name];
if (!windowAverages.ContainsKey(name))
{
windowAverages [name] = 0.0;
windowVariances [name] = 0.0;
benchmarkCounts [name] = 0;
}
windowAverages [name] += average;
windowVariances [name] += variance;
benchmarkCounts [name] += 1;
}
}
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
var count = kvp.Value;
windowAverages [name] /= count;
windowVariances [name] /= count;
}
var testResults = new Dictionary <string, List <double> > ();
for (var j = 0; j < testWindowSize; ++j)
{
var results = FetchResultsForRunSet(conn, sortedRunSets [i + j].GetValue <long> ("rs_id").Value);
foreach (var kvp in results)
{
var benchmark = kvp.Key;
if (!testResults.ContainsKey(benchmark))
{
testResults.Add(benchmark, new List <double> ());
}
testResults [benchmark].AddRange(kvp.Value);
}
}
var testRunSet = sortedRunSets [i];
var commitHash = testRunSet.GetReference <string> ("c_hash");
var testRunSetId = testRunSet.GetValue <long> ("rs_id").Value;
Console.WriteLine("{0} {1}", testRunSetId, commitHash);
var fasterBenchmarks = new List <string> ();
var slowerBenchmarks = new List <string> ();
foreach (var kvp in benchmarkCounts)
{
var benchmark = kvp.Key;
if (kvp.Value < baselineWindowSize)
{
continue;
}
if (lastWarningIndex.ContainsKey(benchmark) && lastWarningIndex [benchmark] >= i - baselineWindowSize)
{
continue;
}
var average = windowAverages [benchmark];
var variance = windowVariances [benchmark];
var stdDev = Math.Sqrt(variance);
var lowerControlLimit = average - controlLimitSize * stdDev;
var upperControlLimit = average + controlLimitSize * stdDev;
if (!testResults.ContainsKey(benchmark))
{
continue;
}
var results = testResults [benchmark];
if (results.Count < 5)
{
continue;
}
var numOutliersFaster = 0;
var numOutliersSlower = 0;
foreach (var elapsed in results)
{
if (elapsed < lowerControlLimit)
{
++numOutliersFaster;
}
if (elapsed > upperControlLimit)
{
++numOutliersSlower;
}
}
if (numOutliersFaster > results.Count * 3 / 4)
{
Console.WriteLine("+ regression in {0}: {1}/{2}", benchmark, numOutliersFaster, results.Count);
lastWarningIndex [benchmark] = i;
fasterBenchmarks.Add(benchmark);
}
else if (numOutliersSlower > results.Count * 3 / 4)
{
Console.WriteLine("- regression in {0}: {1}/{2}", benchmark, numOutliersSlower, results.Count);
lastWarningIndex [benchmark] = i;
slowerBenchmarks.Add(benchmark);
}
/*
* else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
* Console.WriteLine (" nothing in {0}", name);
* } else {
* Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
* }
*/
}
if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
{
var warnedFasterBenchmarks = new List <string> ();
var warnedSlowerBenchmarks = new List <string> ();
if (onlyNecessary)
{
var warningValues = new PostgresRow();
warningValues.Set("runset", NpgsqlTypes.NpgsqlDbType.Bigint, testRunSetId);
var warnings = PostgresInterface.Select(conn, "RegressionsWarned",
new string[] {
"benchmark",
"faster"
},
"runSet = :runset", warningValues);
foreach (var row in warnings)
{
var benchmark = row.GetReference <string> ("benchmark");
if (row.GetValue <bool> ("faster").Value)
{
warnedFasterBenchmarks.Add(benchmark);
}
else
{
warnedSlowerBenchmarks.Add(benchmark);
}
}
}
var previousRunSet = sortedRunSets [i - 1];
var newlyWarnedFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machineName, configName);
var newlyWarnedSlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machineName, configName);
if (!testRun)
{
foreach (var benchmark in newlyWarnedFaster)
{
InsertWarned(conn, testRunSetId, benchmark, true);
}
foreach (var benchmark in newlyWarnedSlower)
{
InsertWarned(conn, testRunSetId, benchmark, false);
}
}
}
}
}
internal static void StartMiningCorePool(string configFile)
{
try
{
// Display Software Version Info
var basePath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
Console.WriteLine($"{Assembly.GetEntryAssembly().GetName().Name} - MinerNL build v{Assembly.GetEntryAssembly().GetName().Version}");
Console.WriteLine($"Run location: {basePath}");
Console.WriteLine(" ");
// log unhandled program exception errors
AppDomain currentDomain = AppDomain.CurrentDomain;
currentDomain.UnhandledException += new UnhandledExceptionEventHandler(MC_UnhandledException);
currentDomain.ProcessExit += OnProcessExit;
Console.CancelKeyPress += new ConsoleCancelEventHandler(OnCancelKeyPress);
// ValidateRuntimeEnvironment();
// root check
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.UserName == "root")
{
logger.Warn(() => "Running as root is discouraged!");
}
// require 64-bit Windows OS
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && RuntimeInformation.ProcessArchitecture == Architecture.X86)
{
throw new PoolStartupAbortException("Miningcore requires 64-Bit Windows");
}
// Read config.json file
clusterConfig = PoolConfig.GetConfigContent(configFile);
// Initialize Logging
FileLogger.ConfigureLogging();
// LogRuntimeInfo();
//-----------------------------------------------------------------------------
logger.Info(() => $"{RuntimeInformation.FrameworkDescription.Trim()} on {RuntimeInformation.OSDescription.Trim()} [{RuntimeInformation.ProcessArchitecture}]");
// Bootstrap();
//-----------------------------------------------------------------------------
ZcashNetworks.Instance.EnsureRegistered();
// Service collection
var builder = new ContainerBuilder();
builder.RegisterAssemblyModules(typeof(AutofacModule).GetTypeInfo().Assembly);
builder.RegisterInstance(clusterConfig);
builder.RegisterInstance(pools);
builder.RegisterInstance(gcStats);
// AutoMapper
var amConf = new MapperConfiguration(cfg => { cfg.AddProfile(new AutoMapperProfile()); });
builder.Register((ctx, parms) => amConf.CreateMapper());
PostgresInterface.ConnectDatabase(builder);
container = builder.Build();
// Configure Equihash
if (clusterConfig.EquihashMaxThreads.HasValue)
{
EquihashSolver.MaxThreads = clusterConfig.EquihashMaxThreads.Value;
}
MonitorGarbageCollection();
// Start Miningcore Pool Services
if (!cts.IsCancellationRequested)
{
StartMiningcorePoolServices().Wait(cts.Token);
}
}
catch (PoolStartupAbortException ex)
{
if (!string.IsNullOrEmpty(ex.Message))
{
Console.WriteLine(ex.Message);
}
Console.WriteLine("\nCluster cannot start. Good Bye!");
}
catch (JsonException)
{
// ignored
}
catch (IOException)
{
// ignored
}
catch (AggregateException ex)
{
if (!(ex.InnerExceptions.First() is PoolStartupAbortException))
{
Console.WriteLine(ex);
}
Console.WriteLine("Cluster cannot start. Good Bye!");
}
catch (OperationCanceledException)
{
// Ctrl+C
}
catch (Exception ex)
{
Console.WriteLine(ex);
Console.WriteLine("Cluster cannot start. Good Bye!");
}
finally
{
// Shutdown();
Console.WriteLine("Miningcore is shuting down... bye!");
logger?.Info(() => "Miningcore is shuting down... bye!");
foreach (var poolToStop in pools.Values)
{
Console.WriteLine($"Stopping pool {poolToStop}");
poolToStop.Stop();
}
shareRelay?.Stop();
shareReceiver?.Stop();
shareRecorder?.Stop();
statsRecorder?.Stop();
Process.GetCurrentProcess().Kill();
}
}
public async Task General()
{
const int recordCount = 10;
using (var dbGenerator = new DbGenerator())
{
var dbInterface = new PostgresInterface(dbGenerator.CreateDb());
const string insertSql = "INSERT INTO frog (name, date_of_birth) VALUES (@Name, @DateOfBirth);" +
"SELECT CAST(lastval() AS INT);";
const string readSingleSql = "SELECT * FROM frog WHERE id = @Id";
const string readAllSql = "SELECT * FROM frog";
const string updateSql = "UPDATE frog SET name = @NewName WHERE id BETWEEN @FromId AND @ToId";
var ids = new List <int>();
for (var i = 1; i <= recordCount; i++)
{
var nameParam = new NpgsqlParameter("@Name", $"Kermit{i}");
var dateOfBirthParam = new NpgsqlParameter("@DateOfBirth", DateTime.Today.AddDays(-i));
var statement = new PostgresStatement(insertSql, nameParam, dateOfBirthParam);
var id = (int)(await dbInterface.ExecuteScalarAsync(statement).ConfigureAwait(false));
// it's a new DB so should be the case, depending on this for following tests
Assert.Equal(i, id);
ids.Add(id);
}
Assert.Equal(recordCount, ids.Count);
foreach (var id in ids)
{
var idParam = new NpgsqlParameter("@Id", id);
var statement = new PostgresStatement(readSingleSql, idParam);
using (var reader = await dbInterface.GetReaderAsync(statement).ConfigureAwait(false))
{
Assert.True(await reader.ReadAsync().ConfigureAwait(false));
Assert.Equal(id, reader["id"]);
Assert.Equal($"Kermit{id}", reader["name"]);
Assert.Equal(DateTime.Today.AddDays(-id), Convert.ToDateTime(reader["date_of_birth"]));
}
}
var updateStatement = new PostgresStatement(updateSql, new NpgsqlParameter("@NewName", "Frank"),
new NpgsqlParameter("@FromId", 5), new NpgsqlParameter("@ToId", 8));
var affected = await dbInterface.ExecuteNonQueryAsync(updateStatement).ConfigureAwait(false);
Assert.Equal(4, affected);
var readAllStatement = new PostgresStatement(readAllSql);
using (var reader = await dbInterface.GetReaderAsync(readAllStatement).ConfigureAwait(false))
{
var count = 0;
while (await reader.ReadAsync().ConfigureAwait(false))
{
count++;
var id = (int)reader["id"];
if (id >= 5 && id <= 8)
{
Assert.Equal("Frank", reader["name"]);
}
else
{
Assert.Equal($"Kermit{id}", reader["name"]);
}
}
Assert.Equal(recordCount, count);
}
}
}
