/// <summary>
/// performs a specific run set
/// </summary>
/// <param name="server">
/// The WSUS Server
/// </param>
/// <param name="runSet">
/// A specific run set
/// </param>
/// <param name="isTest">
/// Whether we are in test mode
/// </param>
private static void DoRunSet(IUpdateServer server, RunSet runSet, bool isTest)
{
Console.WriteLine("Performing Run Set: " + runSet.Name);
// get the target group rules
Configuration.CheckIsValidRunSet(server, runSet);
var alreadyProcessed = new List <IComputerTargetGroup>();
if (runSet.TargetGroups.ElementInformation.IsPresent)
{
Console.WriteLine("Target Groups specified in Run Set.");
TargetGroupCollection targetGroups = runSet.TargetGroups;
foreach (TargetGroup targetGroup in targetGroups)
{
DoRunSetTargetGroup(server, targetGroup, isTest, alreadyProcessed);
}
}
else if (runSet.AllTargetGroups.ElementInformation.IsPresent)
{
AllTargetGroups allTargetGroups = runSet.AllTargetGroups;
DoRunSetAllTargetGroups(server, allTargetGroups, isTest, alreadyProcessed);
}
else
{
throw new ConfigurationErrorsException(
"Couldn't find a \"targetGroup\" or \"allTargetGroups\" element in the runset.");
}
}
public void TargetGroupsTest()
{
RunSet target = new RunSet();
TargetGroupCollection expected = new TargetGroupCollection();
target.TargetGroups = expected;
TargetGroupCollection actual = target.TargetGroups;
Assert.AreEqual(expected, actual);
}
public void CommandLineNull()
{
var element = new RunSet { Name = "Normal" };
var xmlConfigurationSection = new DataManagerMocking.Model.ConfigurationSection.RunSet();
xmlConfigurationSection.RunSets.Add(element);
Xunit.Assert.Throws<System.ArgumentNullException>(
() =>
Dhgms.DataManager.Model.Helper.Configuration.GetRunSet<CommandLineSettings, RunSet, DataManagerMocking.Model.ConfigurationSection.RunSet>(null, xmlConfigurationSection));
}
public void RunSetNotFound()
{
var element = new RunSet { Name = "Normal" };
var xmlConfigurationSection = new DataManagerMocking.Model.ConfigurationSection.RunSet();
xmlConfigurationSection.RunSets.Add(element);
var commandLine = new CommandLineSettings { RunSet = "Missing" };
Xunit.Assert.Throws<Dhgms.DataManager.Model.Exception.RunSetNotFoundException>(() =>
Dhgms.DataManager.Model.Helper.Configuration.GetRunSet<CommandLineSettings, RunSet, DataManagerMocking.Model.ConfigurationSection.RunSet>(commandLine, xmlConfigurationSection));
}
internal static void CreateBenchviewReport(string submissionType, RunSet runSet)
{
Console.WriteLine("Creating benchview results...");
// Serialize the xamarin/benchmarker object to a file.
var jsonConvertedSerializedRunSet = JsonConvert.SerializeObject(runSet);
using (var sw = new StreamWriter(s_runSetJsonFileName))
sw.Write(jsonConvertedSerializedRunSet);
var result = ConvertToMeasurement(s_runSetJsonFileName);
CreateBenchViewSubmission(submissionType, runSet);
}
private static void CreateBenchViewSubmission(string submissionType, RunSet runSet)
{
Console.WriteLine("Creating BenchView submission json");
var arguments = new string[]
{
$"\"{s_submissionPy}\"",
$"\"{s_measurementJson}\"",
$"--metadata=\"{s_submissionMetadataJson}\"",
$"--build=\"{s_buildJson}\"",
$"--machine-data=\"{s_machinedataJson}\"",
$"--group=\"{s_group}\"",
$"--type=\"{submissionType}\"",
$"--config-name=\"{runSet.Config.Name}\"",
$"--config MonoOptions \\\"{string.Join(" ", runSet.Config.MonoOptions)}\\\"",
$"--architecture=\"{runSet.Machine.Architecture}\"",
$"--machinepool=\"{runSet.Machine.Name}\"",
$"-o=\"{s_submissionJson}\""
};
ShellOutVital(s_pythonProcessName, string.Join(" ", arguments));
}
/// <summary>
///
/// </summary>
public static void CheckIsValidRunSet(
Microsoft.UpdateServices.Administration.IUpdateServer server,
RunSet runSet
)
{
TargetGroupCollection targetGroups = runSet.TargetGroups;
AllTargetGroups allTargetGroups = runSet.AllTargetGroups;
if (
targetGroups.Count == 0
&& allTargetGroups.ElementInformation.IsPresent == false
)
{
throw new System.ArgumentException(
"\"AllTargetGroups\" or \"TargetGroups\" missing from the runset \"" + runSet.Name + "\"."
);
}
if(
targetGroups.Count > 0
&& allTargetGroups.ElementInformation.IsPresent
)
{
throw new System.ArgumentException(
"\"AllTargetGroups\" and \"TargetGroups\" are both specified in runset \"" + runSet.Name + "\". You must specify one or the other."
);
}
if (targetGroups.Count > 0)
{
CheckIsValidTargetGroups(server, targetGroups, runSet.Name);
}
else
{
CheckIsValidAllTargetGroups(server, allTargetGroups, runSet.Name);
}
}
/// <summary>
///
/// </summary>
static public void CheckIsValidRunSet(
Microsoft.UpdateServices.Administration.IUpdateServer server,
RunSet runSet
)
{
TargetGroupCollection targetGroups = runSet.TargetGroups;
AllTargetGroups allTargetGroups = runSet.AllTargetGroups;
if (
targetGroups.Count == 0 &&
allTargetGroups.ElementInformation.IsPresent == false
)
{
throw new System.ArgumentException(
"\"AllTargetGroups\" or \"TargetGroups\" missing from the runset \"" + runSet.Name + "\"."
);
}
if (
targetGroups.Count > 0 &&
allTargetGroups.ElementInformation.IsPresent
)
{
throw new System.ArgumentException(
"\"AllTargetGroups\" and \"TargetGroups\" are both specified in runset \"" + runSet.Name + "\". You must specify one or the other."
);
}
if (targetGroups.Count > 0)
{
CheckIsValidTargetGroups(server, targetGroups, runSet.Name);
}
else
{
CheckIsValidAllTargetGroups(server, allTargetGroups, runSet.Name);
}
}
/// <summary>
/// This is run when there are a group of runsets in the app.config
/// Run Sets allow for different options to be run on different occasions
/// The runset is specified on the command line
/// </summary>
/// <param name="server">
/// The WSUS Server
/// </param>
/// <param name="runSets">
/// Group of runsets
/// </param>
/// <param name="commandLine">
/// Information on what was specified on the command line
/// </param>
/// <param name="isTest">
/// Whether we are in test mode
/// </param>
private static void DoRunSets(
// Model.ApplicationSettings settings,
IUpdateServer server,
RunSetCollection runSets,
CommandLine commandLine,
bool isTest)
{
// we need to work out which runset is being done
// we'll limit the command line to one runset
string requestedRunSet = commandLine.GetRunSetName();
RunSet requiredRunSet =
runSets.Cast <RunSet>()
.FirstOrDefault(runSet => requestedRunSet.Equals(runSet.Name, StringComparison.OrdinalIgnoreCase));
if (requiredRunSet == null)
{
throw new ArgumentException(
"The RunSet '" + requestedRunSet
+ "' as requested on the command line is not defined in the app.config.");
}
DoRunSet(server, requiredRunSet, isTest);
}
public static int Main(string[] args)
{
IEnumerable <string> benchmarkNames = null;
//var pausetime = false;
var timeout = -1;
string rootFromCmdline = null;
string buildURL = null;
string logURL = null;
string pullRequestURL = null;
string monoRepositoryPath = null;
long? runSetId = null;
long? runId = null;
string configFile = null;
string machineName = null;
bool justCreateRunSet = false;
bool justListBenchmarks = false;
string valgrindBinary = null;
ValgrindTool? valgrindTool = null;
string valgrindOutputFilename = null;
string grepBinprotPath = null;
string binprotFilePath = null;
bool jitStats = false;
Commit mainCommit = null;
List <Commit> secondaryCommits = new List <Commit> ();
var exeLocation = System.Reflection.Assembly.GetEntryAssembly().Location;
var exeName = Path.GetFileName(exeLocation);
var exeDir = Path.GetDirectoryName(exeLocation);
if (exeName != "compare.exe")
{
Console.Error.WriteLine("Error: Executable is not `compare.exe`. Please specify all paths manually.");
Environment.Exit(1);
}
if (Path.GetFileName(exeDir) != "tools")
{
Console.Error.WriteLine("Error: Executable is not in the `tools` directory. Please specify all paths manually.");
Environment.Exit(1);
}
var root = Path.GetDirectoryName(exeDir);
var testsDir = Path.Combine(root, "tests");
var benchmarksDir = Path.Combine(root, "benchmarks");
var machinesDir = Path.Combine(root, "machines");
var productsDir = Path.Combine(root, "products");
var optindex = 0;
for (; optindex < args.Length; ++optindex)
{
if (args [optindex] == "-b" || args [optindex] == "--benchmarks")
{
var newNames = args [++optindex].Split(',').Select(s => s.Trim());
if (benchmarkNames == null)
{
benchmarkNames = newNames.ToArray();
}
else
{
benchmarkNames = newNames.Union(benchmarkNames).ToArray();
}
}
else if (args [optindex] == "-c" || args [optindex] == "--config-file")
{
configFile = args [++optindex];
}
else if (args [optindex] == "-l" || args [optindex] == "--list-benchmarks")
{
justListBenchmarks = true;
}
else if (args [optindex] == "--machine")
{
machineName = args [++optindex];
}
else if (args [optindex] == "--build-url")
{
buildURL = args [++optindex];
}
else if (args [optindex] == "--log-url")
{
logURL = args [++optindex];
}
else if (args [optindex] == "--pull-request-url")
{
pullRequestURL = args [++optindex];
}
else if (args [optindex] == "--mono-repository")
{
monoRepositoryPath = args [++optindex];
}
else if (args [optindex] == "--create-run-set")
{
justCreateRunSet = true;
}
else if (args [optindex] == "--run-set-id")
{
runSetId = Int64.Parse(args [++optindex]);
}
else if (args [optindex] == "--run-id")
{
runId = Int64.Parse(args [++optindex]);
}
else if (args [optindex] == "--root")
{
rootFromCmdline = args [++optindex];
}
else if (args [optindex] == "--main-product")
{
var name = args [++optindex];
var hash = args [++optindex];
if (mainCommit != null)
{
Console.Error.WriteLine("Error: Only one --main-product is supported.");
UsageAndExit();
}
var product = compare.Utils.LoadProductFromFile(name, productsDir);
mainCommit = new Commit {
Product = product, Hash = hash
};
}
else if (args [optindex] == "--secondary-product")
{
var name = args [++optindex];
var hash = args [++optindex];
var product = compare.Utils.LoadProductFromFile(name, productsDir);
secondaryCommits.Add(new Commit {
Product = product, Hash = hash
});
}
else if (args [optindex] == "--valgrind-massif")
{
if (valgrindBinary != null)
{
Console.Error.WriteLine("Error: More than one Valgrind option given.");
UsageAndExit();
}
valgrindBinary = args [++optindex];
valgrindOutputFilename = args [++optindex];
valgrindTool = ValgrindTool.Massif;
}
else if (args [optindex] == "--valgrind-cachegrind")
{
if (valgrindBinary != null)
{
Console.Error.WriteLine("Error: More than one Valgrind option given.");
UsageAndExit();
}
valgrindBinary = args [++optindex];
valgrindOutputFilename = args [++optindex];
valgrindTool = ValgrindTool.Cachegrind;
}
else if (args [optindex] == "-t" || args [optindex] == "--timeout")
{
timeout = Int32.Parse(args [++optindex]);
timeout = timeout <= 0 ? -1 : timeout;
}
else if (args [optindex] == "--sgen-grep-binprot")
{
grepBinprotPath = args [++optindex];
}
else if (args [optindex] == "--upload-pause-times")
{
binprotFilePath = args [++optindex];
}
else if (args [optindex] == "--jit-stats")
{
jitStats = true;
}
else if (args [optindex].StartsWith("--help"))
{
UsageAndExit();
}
else if (args [optindex] == "--")
{
optindex += 1;
break;
}
else if (args [optindex].StartsWith("-"))
{
Console.Error.WriteLine("unknown parameter {0}", args [optindex]);
UsageAndExit();
}
else
{
break;
}
}
var configFileFromCommandLine = configFile != null;
if (!configFileFromCommandLine)
{
configFile = Path.Combine(root, "configs", "default-sgen.conf");
}
var config = compare.Utils.LoadConfigFromFile(configFile, rootFromCmdline, !(justListBenchmarks || binprotFilePath != null));
if (justCreateRunSet && runSetId != null)
{
Console.Error.WriteLine("Error: --create-run-set and --run-set-id are incompatible.");
Environment.Exit(1);
}
if (justListBenchmarks && benchmarkNames != null)
{
Console.Error.WriteLine("Error: -b/--benchmarks and -l/--list-benchmarks are incompatible.");
Environment.Exit(1);
}
if (justListBenchmarks && !configFileFromCommandLine)
{
Console.Error.WriteLine("Error: -l/--list-benchmarks requires --config-file.");
Environment.Exit(1);
}
if (args.Length - optindex != 0)
{
return(UsageAndExit(null, 1));
}
if (binprotFilePath != null && (runId == null || grepBinprotPath == null))
{
Console.Error.WriteLine("Error: --upload-pause-times also requires --run-id and --sgen-grep-binprot.");
Environment.Exit(1);
}
if (benchmarkNames == null)
{
benchmarkNames = config.Benchmarks;
}
var benchmarks = compare.Utils.LoadAllBenchmarksFrom(benchmarksDir, benchmarkNames);
if (benchmarks == null)
{
Console.Error.WriteLine("Error: Could not load all benchmarks.");
Environment.Exit(1);
}
if (justListBenchmarks)
{
if (machineName != null)
{
var listMachine = compare.Utils.LoadMachineFromFile(machineName, machinesDir);
if (listMachine == null)
{
Console.Error.WriteLine("Error: Could not load machine `{0}`.", machineName);
Environment.Exit(1);
}
if (listMachine.ExcludeBenchmarks != null)
{
benchmarks = benchmarks.Where(b => !listMachine.ExcludeBenchmarks.Contains(b.Name)).ToList();
}
}
foreach (var benchmark in benchmarks.OrderBy(b => b.Name))
{
Console.Out.WriteLine(benchmark.Name);
}
Environment.Exit(0);
}
InitCommons();
if (binprotFilePath != null)
{
var success = AsyncContext.Run(() => UploadPauseTimes(binprotFilePath, grepBinprotPath, runId.Value));
Environment.Exit(success ? 0 : 1);
}
if (mainCommit == null)
{
mainCommit = new Commit {
Product = compare.Utils.LoadProductFromFile("mono", productsDir)
}
}
;
var gitHubClient = GitHubInterface.GitHubClient;
Machine machine = null;
if (machineName == null)
{
machine = compare.Utils.LoadMachineCurrentFrom(machinesDir);
}
else
{
machine = compare.Utils.LoadMachineFromFile(machineName, machinesDir);
}
if (machine != null && machine.ExcludeBenchmarks != null)
{
benchmarks = benchmarks.Where(b => !machine.ExcludeBenchmarks.Contains(b.Name)).ToList();
}
if (machine == null) // couldn't find machine file
{
var hostarch = compare.Utils.LocalHostnameAndArch();
machine = new Machine();
machine.Name = hostarch.Item1;
machine.Architecture = hostarch.Item2;
}
foreach (var commit in new Commit[] { mainCommit }.Concat(secondaryCommits))
{
if (!AsyncContext.Run(() => compare.Utils.CompleteCommit(config, commit)))
{
Console.Error.WriteLine("Error: Could not get commit for product {0}.", commit.Product.Name);
Environment.Exit(1);
}
}
RunSet runSet;
if (runSetId != null)
{
if (pullRequestURL != null)
{
Console.Error.WriteLine("Error: Pull request URL cannot be specified for an existing run set.");
Environment.Exit(1);
}
runSet = AsyncContext.Run(() => RunSet.FromId(machine, runSetId.Value, config, mainCommit, secondaryCommits, buildURL, logURL));
if (runSet == null)
{
Console.Error.WriteLine("Error: Could not get run set.");
Environment.Exit(1);
}
}
else
{
long?pullRequestBaselineRunSetId = null;
if (pullRequestURL != null)
{
if (monoRepositoryPath == null)
{
Console.Error.WriteLine("Error: Must specify a mono repository path to test a pull request.");
Environment.Exit(1);
}
var repo = new compare.Repository(monoRepositoryPath);
var baselineResult = AsyncContext.Run(() => GetPullRequestBaselineRunSetId(mainCommit.Product, pullRequestURL, repo, config));
if (baselineResult == null)
{
Console.Error.WriteLine("Error: No appropriate baseline run set found.");
Environment.Exit(1);
}
pullRequestBaselineRunSetId = baselineResult.Item1;
mainCommit.MergeBaseHash = baselineResult.Item2;
}
runSet = new RunSet {
StartDateTime = DateTime.Now,
Machine = machine,
Config = config,
Commit = mainCommit,
SecondaryCommits = secondaryCommits,
BuildURL = buildURL,
LogURL = logURL,
PullRequestURL = pullRequestURL,
PullRequestBaselineRunSetId = pullRequestBaselineRunSetId
};
Console.Error.WriteLine("Set start time to {0}", runSet.StartDateTime.ToString(RunSet.DATETIME_PRETTY));
}
var reportFailure = false;
if (!justCreateRunSet)
{
var someSuccess = false;
var runTool = valgrindBinary;
string runToolArguments = null;
if (runTool != null)
{
switch (valgrindTool)
{
case ValgrindTool.Massif:
runToolArguments = string.Format("--tool=massif --massif-out-file={0} --max-snapshots=1000 --detailed-freq=100 --pages-as-heap=yes", valgrindOutputFilename);
break;
case ValgrindTool.Cachegrind:
runToolArguments = string.Format("--tool=cachegrind --cachegrind-out-file={0} --cache-sim=yes --branch-sim=yes", valgrindOutputFilename);
break;
default:
Console.Error.WriteLine("Error: Unsupported Valgrind tool.");
Environment.Exit(1);
break;
}
}
int binaryProtocolIndex = 0;
foreach (var benchmark in benchmarks.OrderBy(b => b.Name))
{
// Run the benchmarks
if (config.Count <= 0)
{
throw new ArgumentOutOfRangeException(String.Format("configs [\"{0}\"].Count <= 0", config.Name));
}
Console.Out.WriteLine("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name);
var runner = new compare.UnixRunner(testsDir, config, benchmark, machine, timeout, runTool, runToolArguments);
var haveTimedOut = false;
var haveCrashed = false;
var count = valgrindBinary == null ? config.Count + 1 : 1;
var successCount = 0;
for (var i = 0; i < count; ++i)
{
bool timedOut;
string stdoutOutput;
if (valgrindBinary == null)
{
Console.Out.Write("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format("({0}/{1})", i, config.Count));
}
string binaryProtocolFile = null;
string workingDirectory = Path.Combine(testsDir, benchmark.TestDirectory);
if (config.ProducesBinaryProtocol)
{
do
{
++binaryProtocolIndex;
binaryProtocolFile = Path.Combine(workingDirectory, string.Format("binprot.{0}", binaryProtocolIndex));
} while (File.Exists(binaryProtocolFile));
}
var elapsedMilliseconds = runner.Run(binaryProtocolFile, out timedOut, out stdoutOutput);
// if running for time, the first one is the dry run
if (valgrindBinary == null && i == 0)
{
continue;
}
if (elapsedMilliseconds != null)
{
var run = new Run {
Benchmark = benchmark,
BinaryProtocolFilename = binaryProtocolFile == null ? null : Path.Combine(workingDirectory, binaryProtocolFile)
};
if (valgrindBinary == null)
{
run.RunMetrics.Add(new RunMetric {
Metric = RunMetric.MetricType.Time,
Value = TimeSpan.FromMilliseconds(elapsedMilliseconds.Value)
});
if (jitStats)
{
foreach (var phase in ParseJITPhases(stdoutOutput))
{
run.RunMetrics.Add(phase);
}
}
}
else
{
switch (valgrindTool)
{
case ValgrindTool.Massif:
{
var results = MemoryIntegral(valgrindOutputFilename);
run.RunMetrics.Add(new RunMetric {
Metric = RunMetric.MetricType.MemoryIntegral,
Value = results.Item1
});
run.RunMetrics.Add(new RunMetric {
Metric = RunMetric.MetricType.Instructions,
Value = results.Item2
});
}
break;
case ValgrindTool.Cachegrind:
{
var results = CacheAndBranches(valgrindOutputFilename);
run.RunMetrics.Add(new RunMetric {
Metric = RunMetric.MetricType.CachegrindResults,
Value = results.Item1
});
run.RunMetrics.Add(new RunMetric {
Metric = RunMetric.MetricType.CacheMissRate,
Value = results.Item2
});
run.RunMetrics.Add(new RunMetric {
Metric = RunMetric.MetricType.BranchMispredictionRate,
Value = results.Item3
});
}
break;
}
}
runSet.Runs.Add(run);
successCount++;
someSuccess = true;
}
else
{
if (timedOut)
{
haveTimedOut = true;
}
else
{
haveCrashed = true;
}
}
}
if (haveTimedOut)
{
runSet.TimedOutBenchmarks.Add(benchmark.Name);
}
if (haveCrashed)
{
runSet.CrashedBenchmarks.Add(benchmark.Name);
}
if (haveTimedOut || successCount == 0)
{
reportFailure = true;
}
}
if (!someSuccess)
{
Console.WriteLine("all runs failed.");
}
}
runSet.FinishDateTime = DateTime.Now;
Console.Error.WriteLine("Start time is {0} - finish time is {1}", runSet.StartDateTime.ToString(RunSet.DATETIME_PRETTY), runSet.FinishDateTime.ToString(RunSet.DATETIME_PRETTY));
Console.WriteLine(JsonConvert.SerializeObject(runSet.AsDict()));
var uploadResult = AsyncContext.Run(() => runSet.Upload());
if (uploadResult == null)
{
Console.Error.WriteLine("Error: Could not upload run set.");
Environment.Exit(1);
}
Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/runset.html#id={0}", uploadResult.RunSetId);
if (pullRequestURL != null)
{
Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/pullrequest.html#id={0}", uploadResult.PullRequestId.Value);
}
Console.Write("{{ \"runSetId\": \"{0}\"", uploadResult.RunSetId);
if (pullRequestURL != null)
{
Console.Write(", \"pullRequestId\": \"{0}\"", uploadResult.PullRequestId.Value);
}
Console.Write(", \"runs\": [ ");
var runStrings = new List <string> ();
var allRuns = runSet.Runs.ToList();
for (var i = 0; i < allRuns.Count; i++)
{
var run = allRuns [i];
var id = uploadResult.RunIds [i];
var str = string.Format("\"id\": {0}", id);
if (run.BinaryProtocolFilename != null)
{
str = string.Format("{0}, \"binaryProtocolFile\": \"{1}\"", str, run.BinaryProtocolFilename);
}
runStrings.Add("{ " + str + " }");
}
Console.Write(string.Join(", ", runStrings));
Console.Write(" ]");
Console.WriteLine(" }");
if (reportFailure)
{
Console.Error.WriteLine("Error: Some benchmarks timed out or failed completely.");
return(1);
}
return(0);
}
}
public static void Main (string[] args)
{
string[] benchmarkNames = null;
//var pausetime = false;
var timeout = -1;
string commitFromCmdline = null;
string gitRepoDir = null;
string rootFromCmdline = null;
string buildURL = null;
string logURL = null;
string pullRequestURL = null;
string monoRepositoryPath = null;
string runSetId = null;
string configFile = null;
bool justCreateRunSet = false;
bool justListBenchmarks = false;
var optindex = 0;
for (; optindex < args.Length; ++optindex) {
if (args [optindex] == "-b" || args [optindex] == "--benchmarks") {
var newNames = args [++optindex].Split (',').Select (s => s.Trim ());
if (benchmarkNames == null)
benchmarkNames = newNames.ToArray ();
else
benchmarkNames = newNames.Union (benchmarkNames).ToArray ();
} else if (args [optindex] == "-c" || args [optindex] == "--config-file") {
configFile = args [++optindex];
} else if (args [optindex] == "-l" || args [optindex] == "--list-benchmarks") {
justListBenchmarks = true;
} else if (args [optindex] == "--commit") {
commitFromCmdline = args [++optindex];
} else if (args [optindex] == "--git-repo") {
gitRepoDir = args [++optindex];
} else if (args [optindex] == "--build-url") {
buildURL = args [++optindex];
} else if (args [optindex] == "--log-url") {
logURL = args [++optindex];
} else if (args [optindex] == "--pull-request-url") {
pullRequestURL = args [++optindex];
} else if (args [optindex] == "--mono-repository") {
monoRepositoryPath = args [++optindex];
} else if (args [optindex] == "--create-run-set") {
justCreateRunSet = true;
} else if (args [optindex] == "--run-set-id") {
runSetId = args [++optindex];
} else if (args [optindex] == "--root") {
rootFromCmdline = args [++optindex];
} else if (args [optindex] == "-t" || args [optindex] == "--timeout") {
timeout = Int32.Parse (args [++optindex]);
timeout = timeout <= 0 ? -1 : timeout;
// } else if (args [optindex] == "-p" || args [optindex] == "--pause-time") {
// pausetime = Boolean.Parse (args [++optindex]);
} else if (args [optindex].StartsWith ("--help")) {
UsageAndExit ();
} else if (args [optindex] == "--") {
optindex += 1;
break;
} else if (args [optindex].StartsWith ("-")) {
Console.Error.WriteLine ("unknown parameter {0}", args [optindex]);
UsageAndExit ();
} else {
break;
}
}
if (justCreateRunSet && runSetId != null) {
Console.Error.WriteLine ("Error: --create-run-set and --run-set-id are incompatible.");
Environment.Exit (1);
}
if (justListBenchmarks && benchmarkNames != null) {
Console.Error.WriteLine ("Error: -b/--benchmarks and -l/--list-benchmarks are incompatible.");
Environment.Exit (1);
}
string testsDir, benchmarksDir, machinesDir;
if (args.Length - optindex == 4) {
if (configFile != null) {
Console.Error.WriteLine ("Error: You must not specify the config file twice.");
Environment.Exit (1);
}
testsDir = args [optindex++];
benchmarksDir = args [optindex++];
machinesDir = args [optindex++];
configFile = args [optindex++];
} else if (args.Length - optindex == 0) {
var exeLocation = System.Reflection.Assembly.GetEntryAssembly ().Location;
var exeName = Path.GetFileName (exeLocation);
var exeDir = Path.GetDirectoryName (exeLocation);
if (exeName != "compare.exe") {
Console.Error.WriteLine ("Error: Executable is not `compare.exe`. Please specify all paths manually.");
Environment.Exit (1);
}
if (Path.GetFileName (exeDir) != "tools") {
Console.Error.WriteLine ("Error: Executable is not in the `tools` directory. Please specify all paths manually.");
Environment.Exit (1);
}
var root = Path.GetDirectoryName (exeDir);
testsDir = Path.Combine (root, "tests");
benchmarksDir = Path.Combine (root, "benchmarks");
machinesDir = Path.Combine (root, "machines");
if (configFile == null)
configFile = Path.Combine (root, "configs", "default-sgen.conf");
} else {
UsageAndExit (null, 1);
return;
}
var benchmarks = Benchmark.LoadAllFrom (benchmarksDir, benchmarkNames);
if (benchmarks == null) {
Console.WriteLine ("Error: Could not load all benchmarks.");
Environment.Exit (1);
}
if (justListBenchmarks) {
foreach (var benchmark in benchmarks.OrderBy (b => b.Name)) {
Console.Out.WriteLine (benchmark.Name);
}
Environment.Exit (0);
}
var gitHubClient = GitHubInterface.GitHubClient;
if (!ParseInterface.Initialize ()) {
Console.Error.WriteLine ("Error: Could not initialize Parse interface.");
Environment.Exit (1);
}
var config = Config.LoadFrom (configFile, rootFromCmdline);
var machine = Machine.LoadCurrentFrom (machinesDir);
var commit = AsyncContext.Run (() => config.GetCommit (commitFromCmdline, gitRepoDir));
if (commit == null) {
Console.WriteLine ("Error: Could not get commit");
Environment.Exit (1);
}
if (commit.CommitDate == null) {
Console.WriteLine ("Error: Could not get a commit date.");
Environment.Exit (1);
}
RunSet runSet;
if (runSetId != null) {
if (pullRequestURL != null) {
Console.WriteLine ("Error: Pull request URL cannot be specified for an existing run set.");
Environment.Exit (1);
}
runSet = AsyncContext.Run (() => RunSet.FromId (runSetId, config, commit, buildURL, logURL));
if (runSet == null) {
Console.WriteLine ("Error: Could not get run set.");
Environment.Exit (1);
}
} else {
ParseObject pullRequestBaselineRunSet = null;
if (pullRequestURL != null) {
if (monoRepositoryPath == null) {
Console.Error.WriteLine ("Error: Must specify a mono repository path to test a pull request.");
Environment.Exit (1);
}
var repo = new Benchmarker.Common.Git.Repository (monoRepositoryPath);
pullRequestBaselineRunSet = AsyncContext.Run (() => GetPullRequestBaselineRunSet (pullRequestURL, repo, config));
}
runSet = new RunSet {
StartDateTime = DateTime.Now,
Config = config,
Commit = commit,
BuildURL = buildURL,
LogURL = logURL,
PullRequestURL = pullRequestURL,
PullRequestBaselineRunSet = pullRequestBaselineRunSet
};
}
if (!justCreateRunSet) {
var someSuccess = false;
foreach (var benchmark in benchmarks.OrderBy (b => b.Name)) {
/* Run the benchmarks */
if (config.Count <= 0)
throw new ArgumentOutOfRangeException (String.Format ("configs [\"{0}\"].Count <= 0", config.Name));
Console.Out.WriteLine ("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name);
var runner = new Runner (testsDir, config, benchmark, machine, timeout);
var result = new Result {
DateTime = DateTime.Now,
Benchmark = benchmark,
Config = config,
};
var haveTimedOut = false;
var haveCrashed = false;
for (var i = 0; i < config.Count + 1; ++i) {
bool timedOut;
Console.Out.Write ("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format ("({0}/{1})", i, config.Count));
var run = runner.Run (out timedOut);
// skip first one
if (i == 0)
continue;
if (run != null) {
result.Runs.Add (run);
someSuccess = true;
} else {
if (timedOut)
haveTimedOut = true;
else
haveCrashed = true;
}
}
if (haveTimedOut)
runSet.TimedOutBenchmarks.Add (benchmark);
if (haveCrashed)
runSet.CrashedBenchmarks.Add (benchmark);
// FIXME: implement pausetime
//if (pausetime)
// throw new NotImplementedException ();
runSet.Results.Add (result);
}
if (!someSuccess)
Console.WriteLine ("all runs failed.");
}
runSet.FinishDateTime = DateTime.Now;
Console.WriteLine ("uploading");
try {
var parseObject = AsyncContext.Run (() => runSet.UploadToParse ());
Console.WriteLine ("http://xamarin.github.io/benchmarker/front-end/runset.html#{0}", parseObject.ObjectId);
ParseObject pullRequestObject = null;
if (pullRequestURL != null) {
pullRequestObject = parseObject.Get<ParseObject> ("pullRequest");
Console.WriteLine ("http://xamarin.github.io/benchmarker/front-end/pullrequest.html#{0}", pullRequestObject.ObjectId);
}
Console.Write ("{{ \"runSetId\": \"{0}\"", parseObject.ObjectId);
if (pullRequestURL != null)
Console.Write (", \"pullRequestId\": \"{0}\"", pullRequestObject.ObjectId);
Console.WriteLine (" }");
} catch (Exception exc) {
Console.WriteLine ("Error: Failure uploading data: " + exc);
Environment.Exit (1);
}
}
static async Task <ParseObject> GetPullRequestBaselineRunSet(string pullRequestURL, Benchmarker.Common.Git.Repository repository, Config config)
{
var gitHubClient = GitHubInterface.GitHubClient;
var match = Regex.Match(pullRequestURL, @"^https?://github\.com/mono/mono/pull/(\d+)/?$");
if (match == null)
{
Console.WriteLine("Error: Cannot parse pull request URL.");
Environment.Exit(1);
}
var pullRequestNumber = Int32.Parse(match.Groups [1].Value);
Console.WriteLine("pull request {0}", pullRequestNumber);
var pullRequest = await gitHubClient.PullRequest.Get("mono", "mono", pullRequestNumber);
var prRepo = pullRequest.Head.Repository.SshUrl;
var prBranch = pullRequest.Head.Ref;
var prSha = repository.Fetch(prRepo, prBranch);
if (prSha == null)
{
Console.Error.WriteLine("Error: Could not fetch pull request branch {0} from repo {1}", prBranch, prRepo);
Environment.Exit(1);
}
var masterSha = repository.Fetch("[email protected]:mono/mono.git", "master");
if (masterSha == null)
{
Console.Error.WriteLine("Error: Could not fetch master.");
Environment.Exit(1);
}
var baseSha = repository.MergeBase(prSha, masterSha);
if (baseSha == null)
{
Console.Error.WriteLine("Error: Could not determine merge base of pull request.");
Environment.Exit(1);
}
Console.WriteLine("Merge base sha is {0}", baseSha);
var revList = repository.RevList(baseSha);
if (revList == null)
{
Console.Error.WriteLine("Error: Could not get rev-list for merge base {0}.", baseSha);
Environment.Exit(1);
}
Console.WriteLine("{0} commits in rev-list", revList.Length);
var configObj = await config.GetFromParse();
if (configObj == null)
{
Console.Error.WriteLine("Error: The config does not exist.");
Environment.Exit(1);
}
var machineObj = await RunSet.GetMachineFromParse();
if (machineObj == null)
{
Console.Error.WriteLine("Error: The machine does not exist.");
Environment.Exit(1);
}
var runSets = await ParseInterface.PageQueryWithRetry(() => ParseObject.GetQuery ("RunSet")
.WhereEqualTo ("machine", machineObj)
.WhereEqualTo ("config", configObj)
.WhereDoesNotExist ("pullRequest")
.Include("commit"));
Console.WriteLine("{0} run sets", runSets.Count());
var runSetsByCommits = new Dictionary <string, ParseObject> ();
foreach (var runSet in runSets)
{
var sha = runSet.Get <ParseObject> ("commit").Get <string> ("hash");
if (runSetsByCommits.ContainsKey(sha))
{
// FIXME: select between them?
continue;
}
runSetsByCommits.Add(sha, runSet);
}
foreach (var sha in revList)
{
if (runSetsByCommits.ContainsKey(sha))
{
Console.WriteLine("tested base commit is {0}", sha);
return(runSetsByCommits [sha]);
}
}
return(null);
}
public static void Main(string[] args)
{
string[] benchmarkNames = null;
//var pausetime = false;
var timeout = -1;
string commitFromCmdline = null;
string gitRepoDir = null;
string rootFromCmdline = null;
string buildURL = null;
string logURL = null;
string pullRequestURL = null;
string monoRepositoryPath = null;
string runSetId = null;
string configFile = null;
bool justCreateRunSet = false;
bool justListBenchmarks = false;
var optindex = 0;
for (; optindex < args.Length; ++optindex)
{
if (args [optindex] == "-b" || args [optindex] == "--benchmarks")
{
var newNames = args [++optindex].Split(',').Select(s => s.Trim());
if (benchmarkNames == null)
{
benchmarkNames = newNames.ToArray();
}
else
{
benchmarkNames = newNames.Union(benchmarkNames).ToArray();
}
}
else if (args [optindex] == "-c" || args [optindex] == "--config-file")
{
configFile = args [++optindex];
}
else if (args [optindex] == "-l" || args [optindex] == "--list-benchmarks")
{
justListBenchmarks = true;
}
else if (args [optindex] == "--commit")
{
commitFromCmdline = args [++optindex];
}
else if (args [optindex] == "--git-repo")
{
gitRepoDir = args [++optindex];
}
else if (args [optindex] == "--build-url")
{
buildURL = args [++optindex];
}
else if (args [optindex] == "--log-url")
{
logURL = args [++optindex];
}
else if (args [optindex] == "--pull-request-url")
{
pullRequestURL = args [++optindex];
}
else if (args [optindex] == "--mono-repository")
{
monoRepositoryPath = args [++optindex];
}
else if (args [optindex] == "--create-run-set")
{
justCreateRunSet = true;
}
else if (args [optindex] == "--run-set-id")
{
runSetId = args [++optindex];
}
else if (args [optindex] == "--root")
{
rootFromCmdline = args [++optindex];
}
else if (args [optindex] == "-t" || args [optindex] == "--timeout")
{
timeout = Int32.Parse(args [++optindex]);
timeout = timeout <= 0 ? -1 : timeout;
// } else if (args [optindex] == "-p" || args [optindex] == "--pause-time") {
// pausetime = Boolean.Parse (args [++optindex]);
}
else if (args [optindex].StartsWith("--help"))
{
UsageAndExit();
}
else if (args [optindex] == "--")
{
optindex += 1;
break;
}
else if (args [optindex].StartsWith("-"))
{
Console.Error.WriteLine("unknown parameter {0}", args [optindex]);
UsageAndExit();
}
else
{
break;
}
}
if (justCreateRunSet && runSetId != null)
{
Console.Error.WriteLine("Error: --create-run-set and --run-set-id are incompatible.");
Environment.Exit(1);
}
if (justListBenchmarks && benchmarkNames != null)
{
Console.Error.WriteLine("Error: -b/--benchmarks and -l/--list-benchmarks are incompatible.");
Environment.Exit(1);
}
string testsDir, benchmarksDir, machinesDir;
if (args.Length - optindex == 4)
{
if (configFile != null)
{
Console.Error.WriteLine("Error: You must not specify the config file twice.");
Environment.Exit(1);
}
testsDir = args [optindex++];
benchmarksDir = args [optindex++];
machinesDir = args [optindex++];
configFile = args [optindex++];
}
else if (args.Length - optindex == 0)
{
var exeLocation = System.Reflection.Assembly.GetEntryAssembly().Location;
var exeName = Path.GetFileName(exeLocation);
var exeDir = Path.GetDirectoryName(exeLocation);
if (exeName != "compare.exe")
{
Console.Error.WriteLine("Error: Executable is not `compare.exe`. Please specify all paths manually.");
Environment.Exit(1);
}
if (Path.GetFileName(exeDir) != "tools")
{
Console.Error.WriteLine("Error: Executable is not in the `tools` directory. Please specify all paths manually.");
Environment.Exit(1);
}
var root = Path.GetDirectoryName(exeDir);
testsDir = Path.Combine(root, "tests");
benchmarksDir = Path.Combine(root, "benchmarks");
machinesDir = Path.Combine(root, "machines");
if (configFile == null)
{
configFile = Path.Combine(root, "configs", "default-sgen.conf");
}
}
else
{
UsageAndExit(null, 1);
return;
}
var benchmarks = Benchmark.LoadAllFrom(benchmarksDir, benchmarkNames);
if (benchmarks == null)
{
Console.WriteLine("Error: Could not load all benchmarks.");
Environment.Exit(1);
}
if (justListBenchmarks)
{
foreach (var benchmark in benchmarks.OrderBy(b => b.Name))
{
Console.Out.WriteLine(benchmark.Name);
}
Environment.Exit(0);
}
var gitHubClient = GitHubInterface.GitHubClient;
if (!ParseInterface.Initialize())
{
Console.Error.WriteLine("Error: Could not initialize Parse interface.");
Environment.Exit(1);
}
var config = Config.LoadFrom(configFile, rootFromCmdline);
var machine = Machine.LoadCurrentFrom(machinesDir);
var commit = AsyncContext.Run(() => config.GetCommit(commitFromCmdline, gitRepoDir));
if (commit == null)
{
Console.WriteLine("Error: Could not get commit");
Environment.Exit(1);
}
if (commit.CommitDate == null)
{
Console.WriteLine("Error: Could not get a commit date.");
Environment.Exit(1);
}
RunSet runSet;
if (runSetId != null)
{
if (pullRequestURL != null)
{
Console.WriteLine("Error: Pull request URL cannot be specified for an existing run set.");
Environment.Exit(1);
}
runSet = AsyncContext.Run(() => RunSet.FromId(runSetId, config, commit, buildURL, logURL));
if (runSet == null)
{
Console.WriteLine("Error: Could not get run set.");
Environment.Exit(1);
}
}
else
{
ParseObject pullRequestBaselineRunSet = null;
if (pullRequestURL != null)
{
if (monoRepositoryPath == null)
{
Console.Error.WriteLine("Error: Must specify a mono repository path to test a pull request.");
Environment.Exit(1);
}
var repo = new Benchmarker.Common.Git.Repository(monoRepositoryPath);
pullRequestBaselineRunSet = AsyncContext.Run(() => GetPullRequestBaselineRunSet(pullRequestURL, repo, config));
}
runSet = new RunSet {
StartDateTime = DateTime.Now,
Config = config,
Commit = commit,
BuildURL = buildURL,
LogURL = logURL,
PullRequestURL = pullRequestURL,
PullRequestBaselineRunSet = pullRequestBaselineRunSet
};
}
if (!justCreateRunSet)
{
var someSuccess = false;
foreach (var benchmark in benchmarks.OrderBy(b => b.Name))
{
/* Run the benchmarks */
if (config.Count <= 0)
{
throw new ArgumentOutOfRangeException(String.Format("configs [\"{0}\"].Count <= 0", config.Name));
}
Console.Out.WriteLine("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name);
var runner = new Runner(testsDir, config, benchmark, machine, timeout);
var result = new Result {
DateTime = DateTime.Now,
Benchmark = benchmark,
Config = config,
};
var haveTimedOut = false;
var haveCrashed = false;
for (var i = 0; i < config.Count + 1; ++i)
{
bool timedOut;
Console.Out.Write("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format("({0}/{1})", i, config.Count));
var run = runner.Run(out timedOut);
// skip first one
if (i == 0)
{
continue;
}
if (run != null)
{
result.Runs.Add(run);
someSuccess = true;
}
else
{
if (timedOut)
{
haveTimedOut = true;
}
else
{
haveCrashed = true;
}
}
}
if (haveTimedOut)
{
runSet.TimedOutBenchmarks.Add(benchmark);
}
if (haveCrashed)
{
runSet.CrashedBenchmarks.Add(benchmark);
}
// FIXME: implement pausetime
//if (pausetime)
// throw new NotImplementedException ();
runSet.Results.Add(result);
}
if (!someSuccess)
{
Console.WriteLine("all runs failed.");
}
}
runSet.FinishDateTime = DateTime.Now;
Console.WriteLine("uploading");
try {
var parseObject = AsyncContext.Run(() => runSet.UploadToParse());
Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/runset.html#{0}", parseObject.ObjectId);
ParseObject pullRequestObject = null;
if (pullRequestURL != null)
{
pullRequestObject = parseObject.Get <ParseObject> ("pullRequest");
Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/pullrequest.html#{0}", pullRequestObject.ObjectId);
}
Console.Write("{{ \"runSetId\": \"{0}\"", parseObject.ObjectId);
if (pullRequestURL != null)
{
Console.Write(", \"pullRequestId\": \"{0}\"", pullRequestObject.ObjectId);
}
Console.WriteLine(" }");
} catch (Exception exc) {
Console.WriteLine("Error: Failure uploading data: " + exc);
Environment.Exit(1);
}
}
public void RunSetConstructorTest()
{
//SmartApprove.Model.ApplicationSettings appSettings = SmartApprove.Program.LoadSettings();
RunSet target = new RunSet();
}
public void ShouldSucceed()
{
var element = new RunSet { Name = "Normal" };
var xmlConfigurationSection = new DataManagerMocking.Model.ConfigurationSection.RunSet();
xmlConfigurationSection.RunSets.Add(element);
var commandLine = new CommandLineSettings { RunSet = "Normal" };
Dhgms.DataManager.Model.Helper.Configuration.GetRunSet<CommandLineSettings, RunSet, DataManagerMocking.Model.ConfigurationSection.RunSet>(
commandLine, xmlConfigurationSection);
}
