private void SetupChart()
{
model = new PlotModel()
{
};
var linearAxis = new OxyPlot.Axes.LinearAxis()
{
MinimumPadding = 0,
MaximumPadding = 0.1,
};
var axis = new OxyPlot.Axes.CategoryAxis()
{
};
axis.Labels.AddRange(new[] { "Video A", "Video B" });
model.Axes.Add(linearAxis);
model.Axes.Add(axis);
series = new ErrorColumnSeries()
{
StrokeThickness = 1,
};
model.Series.Add(series);
chart.Children.Add(new PlotView()
{
Model = model,
Width = chart.Width,
Height = chart.Height,
});
}
Task RenderErrorColumn(Series plot)
{
var errorColumn = new ErrorColumnSeries();
foreach (var item in plot.Data)
{
errorColumn.Items.Add(new ColumnItem(item.Value, item.CategoryIndex));
}
OxyplotModel.Series.Add(errorColumn);
return(Task.CompletedTask);
}
public static PlotModel GetErrorColumnSeries()
{
var model = new PlotModel("ErrorColumnSeries")
{
LegendPlacement = LegendPlacement.Outside,
LegendPosition = LegendPosition.BottomCenter,
LegendOrientation = LegendOrientation.Horizontal,
LegendBorderThickness = 0
};
var s1 = new ErrorColumnSeries { Title = "Series 1", IsStacked = false, StrokeColor = OxyColors.Black, StrokeThickness = 1 };
s1.Items.Add(new ErrorColumnItem { Value = 25, Error = 2 });
s1.Items.Add(new ErrorColumnItem { Value = 137, Error = 25 });
s1.Items.Add(new ErrorColumnItem { Value = 18, Error = 4 });
s1.Items.Add(new ErrorColumnItem { Value = 40, Error = 29 });
var s2 = new ErrorColumnSeries { Title = "Series 2", IsStacked = false, StrokeColor = OxyColors.Black, StrokeThickness = 1 };
s2.Items.Add(new ErrorColumnItem { Value = 35, Error = 20 });
s2.Items.Add(new ErrorColumnItem { Value = 17, Error = 7 });
s2.Items.Add(new ErrorColumnItem { Value = 118, Error = 44 });
s2.Items.Add(new ErrorColumnItem { Value = 49, Error = 29 });
var categoryAxis = new CategoryAxis { Position = AxisPosition.Bottom };
categoryAxis.Labels.Add("Category A");
categoryAxis.Labels.Add("Category B");
categoryAxis.Labels.Add("Category C");
categoryAxis.Labels.Add("Category D");
var valueAxis = new LinearAxis(AxisPosition.Left) { MinimumPadding = 0, MaximumPadding = 0.06, AbsoluteMinimum = 0 };
model.Series.Add(s1);
model.Series.Add(s2);
model.Axes.Add(categoryAxis);
model.Axes.Add(valueAxis);
return model;
}
public static PlotModel GetErrorColumnSeries()
{
var model = new PlotModel
{
Title = "ErrorColumnSeries",
LegendPlacement = LegendPlacement.Outside,
LegendPosition = LegendPosition.BottomCenter,
LegendOrientation = LegendOrientation.Horizontal,
LegendBorderThickness = 0
};
var s1 = new ErrorColumnSeries {
Title = "Series 1", IsStacked = false, StrokeColor = OxyColors.Black, StrokeThickness = 1
};
s1.Items.Add(new ErrorColumnItem {
Value = 25, Error = 2
});
s1.Items.Add(new ErrorColumnItem {
Value = 137, Error = 25
});
s1.Items.Add(new ErrorColumnItem {
Value = 18, Error = 4
});
s1.Items.Add(new ErrorColumnItem {
Value = 40, Error = 29
});
var s2 = new ErrorColumnSeries {
Title = "Series 2", IsStacked = false, StrokeColor = OxyColors.Black, StrokeThickness = 1
};
s2.Items.Add(new ErrorColumnItem {
Value = 35, Error = 20
});
s2.Items.Add(new ErrorColumnItem {
Value = 17, Error = 7
});
s2.Items.Add(new ErrorColumnItem {
Value = 118, Error = 44
});
s2.Items.Add(new ErrorColumnItem {
Value = 49, Error = 29
});
var categoryAxis = new CategoryAxis {
Position = AxisPosition.Bottom
};
categoryAxis.Labels.Add("Category A");
categoryAxis.Labels.Add("Category B");
categoryAxis.Labels.Add("Category C");
categoryAxis.Labels.Add("Category D");
var valueAxis = new LinearAxis {
Position = AxisPosition.Left, MinimumPadding = 0, MaximumPadding = 0.06, AbsoluteMinimum = 0
};
model.Series.Add(s1);
model.Series.Add(s2);
model.Axes.Add(categoryAxis);
model.Axes.Add(valueAxis);
return(model);
}
//bool showXAxis = true, showYAxix = true, isZoomable = true, isMovable = true;
//IPlotType chart;
//public bool ShowXAxis
//{
// get => showXAxis;
// set
// {
// if (showXAxis == value) return;
// showXAxis = value;
// }
//}
//public bool ShowYAxix
//{
// get => showYAxix;
// set
// {
// if (showYAxix == value) return;
// showYAxix = value;
// }
//}
//public bool IsZoomable
//{
// get => isZoomable;
// set
// {
// if (isZoomable == value) return;
// isZoomable = value;
// }
//}
//public bool IsMovable
//{
// get => isMovable;
// set
// {
// if (isMovable == value) return;
// isMovable = value;
// }
//}
public async Task Add(PlotModel plotModel)
{
this.oxyplotModel = new OxyPlot.PlotModel();
this.oxyplotModel.Title = plotModel.Title;
foreach (var chart in plotModel.Series)
{
if (chart is TwoColorArea) //it should be placed before Area if clause
{
var twoColorAreaSeries = new TwoColorAreaSeries
{
Color = ChartColor.CastToOxyColor(((TwoColorArea)chart).Color),
Color2 = ChartColor.CastToOxyColor(((TwoColorArea)chart).Color2),
Limit = ((TwoColorArea)chart).Limit
};
foreach (var item in ((TwoColorArea)chart).Data)
{
twoColorAreaSeries.Points.Add(item);
}
this.oxyplotModel.Series.Add(twoColorAreaSeries);
}
else if (chart is Area)
{
var areaSeries = new AreaSeries();
foreach (var point in ((Area)chart).Data)
{
areaSeries.Points.Add(point);
}
this.oxyplotModel.Series.Add(areaSeries);
}
else if (chart is TwoColorLine)//it should be placed before line if clause
{
var twoColorLineSeries = new TwoColorLineSeries
{
Color = ChartColor.CastToOxyColor(((TwoColorLine)chart).Color),
Color2 = ChartColor.CastToOxyColor(((TwoColorLine)chart).Color2),
Limit = ((TwoColorLine)chart).Limit
};
foreach (var item in ((TwoColorLine)chart).Data)
{
twoColorLineSeries.Points.Add(item);
}
this.oxyplotModel.Series.Add(twoColorLineSeries);
}
else if (chart is Line)
{
var lineSeries = new LineSeries
{
MarkerType = MarkerType.Circle,
MarkerSize = 4,
MarkerStroke = OxyColors.White
};
foreach (var point in ((Line)chart).Data)
{
lineSeries.Points.Add(point);
}
this.oxyplotModel.Series.Add(lineSeries);
}
else if (chart is Pie)
{
var pieSeries = new PieSeries();
foreach (var slice in ((Pie)chart).Data)
{
pieSeries.Slices.Add(slice);
}
this.oxyplotModel.Series.Add(pieSeries);
}
else if (chart is Bar)
{
var barSeries = new BarSeries();
foreach (var item in ((Bar)chart).Data)
{
barSeries.Items.Add(item);
}
this.oxyplotModel.Series.Add(barSeries);
}
else if (chart is ErrorColumn)
{
var errorColumn = new ErrorColumnSeries();
foreach (ErrorColumnItem item in ((ErrorColumn)chart).Data)
{
errorColumn.Items.Add((OxyPlot.Series.ErrorColumnItem)item);
}
this.oxyplotModel.Series.Add(errorColumn);
}
else if (chart is Column)
{
var barSeries = new ColumnSeries();
foreach (var item in ((Column)chart).Data)
{
barSeries.Items.Add(item);
}
this.oxyplotModel.Series.Add(barSeries);
}
else if (chart is Box)
{
var boxSeries = new BoxPlotSeries();
foreach (var item in ((Box)chart).Data)
{
boxSeries.Items.Add(item);
}
this.oxyplotModel.Series.Add(boxSeries);
}
else if (chart is Contour)
{
var contourSeries = new ContourSeries
{
Data = ((Contour)chart).Data,
ColumnCoordinates = ((Contour)chart).ColumnCoordinates,
RowCoordinates = ((Contour)chart).RowCoordinates
};
this.oxyplotModel.Series.Add(contourSeries);
}
else if (chart is RectangleBar)
{
var rectangleBarSeries = new RectangleBarSeries
{
Title = ((RectangleBar)chart).Title
};
foreach (var item in ((RectangleBar)chart).Data)
{
rectangleBarSeries.Items.Add(item);
}
this.oxyplotModel.Series.Add(rectangleBarSeries);
}
else if (chart is CandleStick)
{
var candleStickSeries = new CandleStickSeries();
foreach (var item in ((CandleStick)chart).Data)
{
candleStickSeries.Items.Add(item);
}
this.oxyplotModel.Series.Add(candleStickSeries);
}
else if (chart is HeatMap)
{
var heatMapSeries = new HeatMapSeries()
{
Data = ((HeatMap)chart).Data,
X0 = ((HeatMap)chart).X0,
X1 = ((HeatMap)chart).X1,
Y0 = ((HeatMap)chart).Y0,
Y1 = ((HeatMap)chart).Y1,
Interpolate = ((HeatMap)chart).Interpolate,
RenderMethod = ((HeatMap)chart).RenderMethod
};
this.oxyplotModel.Series.Add(heatMapSeries);
}
else if (chart is HighLow)
{
var highLowSeries = new HighLowSeries();
foreach (var item in ((HighLow)chart).Data)
{
highLowSeries.Items.Add(item);
}
this.oxyplotModel.Series.Add(highLowSeries);
}
else if (chart is IntervalBar)
{
var intervalBarSeries = new IntervalBarSeries();
foreach (var item in ((IntervalBar)chart).Data)
{
intervalBarSeries.Items.Add(item);
}
this.oxyplotModel.Series.Add(intervalBarSeries);
}
else if (chart is Scatter)
{
var scatterSeries = new ScatterSeries();
foreach (var item in ((Scatter)chart).Data)
{
scatterSeries.Points.Add(item);
}
this.oxyplotModel.Series.Add(scatterSeries);
}
}
foreach (var axis in plotModel.Axes)
{
this.oxyplotModel.Axes.Add(axis);
}
}
public static void Main(string[] args)
{
var benchmarksnames = new string[0];
var timeout = Int32.MaxValue;
var pausetime = false;
var monoexe = String.Empty;
var graph = "graph.svg";
var loadresultfrom = new string [0];
var nograph = false;
var norun = false;
var counter = String.Empty;
var geomean = false;
var minimum = Double.NaN;
var optindex = 0;
for (; optindex < args.Length; ++optindex)
{
if (args [optindex] == "-b" || args [optindex] == "--benchmarks")
{
benchmarksnames = args [++optindex].Split(',').Select(s => s.Trim()).Union(benchmarksnames).ToArray();
}
else if (args [optindex] == "-t" || args [optindex] == "--timeout")
{
timeout = Int32.Parse(args [++optindex]) * 1000;
timeout = timeout == 0 ? Int32.MaxValue : timeout;
// } else if (args [optindex] == "-p" || args [optindex] == "--pause-time") {
// pausetime = Boolean.Parse (args [++optindex]);
}
else if (args [optindex] == "-m" || args [optindex] == "--mono-exe")
{
monoexe = args [++optindex];
}
else if (args [optindex] == "-g" || args [optindex] == "--graph")
{
graph = args [++optindex];
}
else if (args [optindex] == "-l" || args [optindex] == "--load-from")
{
loadresultfrom = args [++optindex].Split(',').Select(s => s.Trim()).Union(loadresultfrom).ToArray();
}
else if (args [optindex] == "--no-graph")
{
nograph = true;
}
else if (args [optindex] == "--no-run")
{
norun = true;
}
else if (args [optindex] == "-c" || args [optindex] == "--counter")
{
counter = args [++optindex];
}
else if (args [optindex] == "--geomean")
{
geomean = true;
}
else if (args [optindex] == "--minimum")
{
minimum = Double.Parse(args [++optindex]);
}
else if (args [optindex].StartsWith("--help"))
{
UsageAndExit();
}
else if (args [optindex] == "--")
{
optindex += 1;
break;
}
else if (args [optindex].StartsWith("-"))
{
Console.Error.WriteLine("unknown parameter {0}", args [optindex]);
UsageAndExit();
}
else
{
break;
}
}
if (norun && nograph)
{
UsageAndExit("You cannot disable run and graph at the same time", 1);
}
if (!norun && loadresultfrom.Length > 0)
{
UsageAndExit("You cannot load a run from a file if you run the benchmarks", 1);
}
if (args.Length - optindex < (norun ? 3 : 4))
{
UsageAndExit(null, 1);
}
var testsdir = args [optindex++];
var resultsdir = args [optindex++];
var benchmarksdir = args [optindex++];
var configfiles = args.Skip(optindex).ToArray();
List <Result> results = new List <Result> ();
if (norun)
{
foreach (var resultfile in (loadresultfrom.Length > 0 ? loadresultfrom : Directory.EnumerateFiles(resultsdir, "*.json")))
{
var result = Result.LoadFrom(resultfile);
if (result == null)
{
throw new InvalidDataException(String.Format("Cannot load Result from {0}", resultfile));
}
if (results.Any(r => r.Benchmark.Equals(result.Benchmark) && r.Config.Equals(result.Config)))
{
continue;
}
// If we have been given configuration names, the result's
// configuration must match one of them.
if (configfiles.Length > 0 && !configfiles.Any(n => n == result.Config.Name))
{
continue;
}
results.Add(result);
}
}
else
{
var configs = configfiles.Select(c => Config.LoadFrom(c)).ToList();
/* Run or load the benchmarks */
foreach (var benchmark in Benchmark.LoadAllFrom(benchmarksdir, benchmarksnames).OrderBy(b => b.Name))
{
foreach (var config in configs)
{
var runfileprefix = String.Join("_", benchmark.Name, config.Name, "");
var version = String.Empty;
/* Run the benchmarks */
if (config.Count <= 0)
{
throw new ArgumentOutOfRangeException(String.Format("configs [\"{0}\"].Count <= 0", config.Name));
}
Console.Out.WriteLine("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name);
var info = new ProcessStartInfo()
{
WorkingDirectory = Path.Combine(testsdir, benchmark.TestDirectory),
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
if (config.NoMono)
{
info.FileName = Path.Combine(info.WorkingDirectory, benchmark.CommandLine [0]);
benchmark.CommandLine = benchmark.CommandLine.Skip(1).ToArray();
}
else
{
info.FileName = !String.IsNullOrEmpty(monoexe) ? monoexe : !String.IsNullOrEmpty(config.Mono) ? config.Mono : "mono";
}
foreach (var env in config.MonoEnvironmentVariables)
{
info.EnvironmentVariables.Add(env.Key, env.Value);
}
var envvar = String.Join(" ", config.MonoEnvironmentVariables.Select(kv => kv.Key + "=" + kv.Value));
var arguments = String.Join(" ", config.MonoOptions.Union(benchmark.CommandLine));
if (!config.NoMono)
{
/* Run without timing with --version */
info.Arguments = "--version " + arguments;
Console.Out.WriteLine("\t$> {0} {1} {2}", envvar, info.FileName, info.Arguments);
var process1 = Process.Start(info);
version = Task.Run(() => new StreamReader(process1.StandardOutput.BaseStream).ReadToEnd()).Result;
var versionerror = Task.Run(() => new StreamReader(process1.StandardError.BaseStream).ReadToEnd());
process1.WaitForExit();
}
else
{
info.Arguments = arguments;
}
/* Run with timing */
if (!config.NoMono)
{
info.Arguments = "--stats " + arguments;
}
var result = new Result {
DateTime = DateTime.Now,
Benchmark = benchmark,
Config = config,
Version = version,
Timedout = false,
Runs = new Result.Run [config.Count]
};
for (var i = 0; i < config.Count + 1; ++i)
{
Console.Out.WriteLine("\t$> {0} {1} {2}", envvar, info.FileName, info.Arguments);
Console.Out.Write("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format("({0}/{1})", i, config.Count));
timeout = benchmark.Timeout > 0 ? benchmark.Timeout : timeout;
var sw = Stopwatch.StartNew();
var process = Process.Start(info);
var stdout = Task.Factory.StartNew(() => new StreamReader(process.StandardOutput.BaseStream).ReadToEnd(), TaskCreationOptions.LongRunning);
var stderr = Task.Factory.StartNew(() => new StreamReader(process.StandardError.BaseStream).ReadToEnd(), TaskCreationOptions.LongRunning);
var success = process.WaitForExit(timeout < 0 ? -1 : (Math.Min(Int32.MaxValue / 1000, timeout) * 1000));
sw.Stop();
if (!success)
{
process.Kill();
}
Console.Out.WriteLine(success ? sw.ElapsedMilliseconds.ToString() + "ms" : "timeout!");
// skip first one
if (i > 0)
{
result.Runs [i - 1] = new Result.Run {
WallClockTime = success ? TimeSpan.FromTicks(sw.ElapsedTicks) : TimeSpan.Zero,
Output = success ? stdout.Result : null,
Error = success ? stderr.Result : null
};
result.Timedout = result.Timedout || !success;
}
process.Close();
}
// FIXME: implement pausetime
if (pausetime)
{
throw new NotImplementedException();
}
result.StoreTo(Path.Combine(resultsdir, runfileprefix + DateTime.Now.ToString("s").Replace(':', '-') + ".json"));
results.Add(result);
}
}
}
/* Generate the graph */
if (!nograph)
{
Console.WriteLine("Generate graph in \"{0}\"", graph);
var plot = new PlotModel {
LegendPlacement = LegendPlacement.Outside,
LegendPosition = LegendPosition.TopCenter,
LegendOrientation = LegendOrientation.Horizontal,
LegendBorderThickness = 0,
Padding = new OxyThickness(10, 0, 0, 75),
DefaultColors = XamarinColors,
};
var categoryaxis = new CategoryAxis {
Position = AxisPosition.Bottom, Angle = 90
};
var valueaxis = new LinearAxis {
Position = AxisPosition.Left,
MinorGridlineStyle = LineStyle.Automatic,
MajorGridlineStyle = LineStyle.Automatic,
};
plot.Axes.Add(categoryaxis);
plot.Axes.Add(valueaxis);
var benchmarks = results
.GroupBy(r => r.Benchmark)
.Select(benchmark => {
var benchmarkconfigs = benchmark.Select(r => r.Config).ToArray();
Debug.Assert(benchmarkconfigs.Length == benchmarkconfigs.Distinct().Count(), "There are duplicate configs for benchmark \"{0}\" : {1}",
benchmark.Key.Name, String.Join(", ", benchmarkconfigs.OrderBy(c => c.Name).Select(c => c.Name)));
if (benchmark.Any(r => r.Runs.Any(ru => ru.WallClockTime == TimeSpan.Zero)))
{
Console.WriteLine("Don't have data for benchmark \"{0}\" in all configs - removing", benchmark.Key.Name);
return(null);
}
double[][] values;
try {
counter = String.IsNullOrWhiteSpace(counter) ? "Time" : counter;
values = benchmark.Select(r => r.Runs.Select(ru => ExtractCounterValue(counter, ru)).ToArray()).ToArray();
} catch (FormatException) {
Console.Error.WriteLine("Could not parse value for counter \"{1}\"", counter);
return(null);
}
double[] means = values.Select(vs => vs.Sum() / vs.Length).ToArray();
double[] errors = values.Zip(means, (vs, m) => m - vs.Min()).ToArray();
if (minimum != Double.NaN && means.Any(m => m < minimum))
{
Console.WriteLine("Mean value for benchmark \"{0}\" below minimum - removing", benchmark.Key.Name);
return(null);
}
double ratio = means.ElementAt(0);
double[] nmeans = means.Select(m => m / ratio).ToArray();
double[] nerrors = errors.Select(e => e / ratio).ToArray();
Debug.Assert(benchmarkconfigs.Length == nmeans.Length);
Debug.Assert(benchmarkconfigs.Length == nerrors.Length);
return(new { Benchmark = benchmark.Key, Configs = benchmarkconfigs, NormalizedMeans = nmeans, NormalizedErrors = nerrors });
})
.Where(t => t != null)
.ToArray();
foreach (var n in benchmarks.Select(v => v.Benchmark.Name).Distinct())
{
categoryaxis.Labels.Add(n);
}
var configs = benchmarks
.SelectMany(b => b.Configs.Select((c, i) => new { Benchmark = b.Benchmark, Config = c, NormalizedMean = b.NormalizedMeans [i], NormalizedError = b.NormalizedErrors [i] }))
.GroupBy(v => v.Config);
foreach (var config in configs)
{
var serie = new ErrorColumnSeries {
Title = config.Key.Name, LabelFormatString = "{0:F2}", StrokeThickness = 1
};
var nmeans = config.Select(c => c.NormalizedMean).ToArray();
var nerrors = config.Select(c => c.NormalizedError).ToArray();
for (int i = 0, l = nmeans.Length; i < l; ++i)
{
serie.Items.Add(new ErrorColumnItem {
Value = nmeans [i], Error = nerrors [i], Color = OxyColors.Automatic
});
}
plot.Series.Add(serie);
}
if (geomean)
{
var geomeanserie = new ColumnSeries {
Title = "geomean", LabelFormatString = "{0:F2}", StrokeThickness = 1
};
foreach (var v in benchmarks)
{
geomeanserie.Items.Add(new ColumnItem {
Value = Math.Pow(v.NormalizedMeans.Aggregate(1d, (a, m) => m * a), 1d / v.NormalizedMeans.Length),
Color = OxyColors.Automatic,
});
}
plot.Series.Add(geomeanserie);
}
valueaxis.AbsoluteMinimum = valueaxis.Minimum = benchmarks.Aggregate(Double.MaxValue, (a, v) => Math.Min(a, v.NormalizedMeans.Zip(v.NormalizedErrors, (m, e) => m - e).Min())) * 0.99;
valueaxis.AbsoluteMaximum = valueaxis.Maximum = benchmarks.Aggregate(Double.MinValue, (a, v) => Math.Max(a, v.NormalizedMeans.Zip(v.NormalizedErrors, (m, e) => m + e).Max())) * 1.01;
using (var stream = new FileStream(graph, FileMode.Create))
SvgExporter.Export(plot, stream, 1024, 768, true);
}
}
