public FastAndDirtyConfig()
{
Add(Job.Default
.WithLaunchCount(1)
.WithIterationTime(TimeInterval.FromSeconds(1))
.WithWarmupCount(5)
.WithTargetCount(5)
);
}
public int CreateTerminalTester(string uart, string prompt = null, int timeout = 120, string machine = null)
{
return(CreateNewTester(uartObject =>
{
var tester = new TerminalTester(TimeInterval.FromSeconds((uint)timeout), prompt);
tester.Terminal.AttachTo(uartObject);
return tester;
}, uart, machine));
}
public TerminalTesterResult WaitForNextLineOnUart(uint?timeout = null, int?testerId = null)
{
GetTesterOrThrowException(testerId).WaitUntilLineExpr(
x => true,
out string line,
out var time,
timeout == null ? (TimeInterval?)null : TimeInterval.FromSeconds(timeout.Value)
);
return(new TerminalTesterResult(line, time.TotalMilliseconds));
}
public void TestIfUartIsIdle(ulong timeInSeconds, int?testerId = null)
{
var tester = GetTesterOrThrowException(testerId);
var result = tester.IsIdle(TimeInterval.FromSeconds(timeInSeconds));
if (!result)
{
OperationFail(tester);
}
}
public TerminalTester(TimeInterval?timeout = null, EndLineOption endLineOption = EndLineOption.TreatLineFeedAsEndLine, bool removeColors = true)
{
GlobalTimeout = timeout ?? TimeInterval.FromSeconds(DefaultSecondsTimeout);
this.endLineOption = endLineOption;
this.removeColors = removeColors;
charEvent = new AutoResetEvent(false);
lines = new List <Line>();
currentLineBuffer = new SafeStringBuilder();
sgrDecodingBuffer = new SafeStringBuilder();
report = new SafeStringBuilder();
}
public int CreateTerminalTester(string uart, int timeout = 120, string machine = null, string endLineOption = null)
{
return(CreateNewTester(uartObject =>
{
TerminalTester tester;
if (Enum.TryParse <EndLineOption>(endLineOption, out var result))
{
tester = new TerminalTester(TimeInterval.FromSeconds((uint)timeout), result);
}
else
{
tester = new TerminalTester(TimeInterval.FromSeconds((uint)timeout));
}
tester.AttachTo(uartObject);
return tester;
}, uart, machine));
}
public static void Main(string[] args)
{
BenchmarkSwitcher.FromTypes(new [] { typeof(BaselineBenchmark), typeof(NoSamplingBenchmark), typeof(DefaultBenchmark) }).RunAllJoined(
DefaultConfig.Instance
.With(
new Job()
.With(RunStrategy.Monitoring)
.WithLaunchCount(3)
.WithWarmupCount(1)
.WithIterationTime(TimeInterval.FromSeconds(10))
.WithCustomBuildConfiguration("ReleaseV3")
.WithOutlierMode(OutlierMode.DontRemove)
)
.With(MemoryDiagnoser.Default)
.With(HardwareCounter.TotalCycles)
);
}
public void ShouldObserveShorterPeriodClockAfterAdd()
{
var clockSource = new BaseClockSource();
var counterA = 0;
var counterB = 0;
Action handlerA = () => counterA++;
Action handlerB = () => counterB++;
ClockEntry entryA = new ClockEntry(1000, 1, handlerA, null, String.Empty)
{
Value = 0
};
ClockEntry entryB = new ClockEntry(100, 1, handlerB, null, String.Empty)
{
Value = 0
};
clockSource.AddClockEntry(entryA);
clockSource.Advance(TimeInterval.FromSeconds(500));
clockSource.AddClockEntry(entryB);
entryA = clockSource.GetClockEntry(handlerA);
entryB = clockSource.GetClockEntry(handlerB);
Assert.AreEqual(entryA.Value, 500);
Assert.AreEqual(entryA.Period, 1000);
Assert.AreEqual(entryB.Value, 0);
Assert.AreEqual(entryB.Period, 100);
clockSource.Advance(TimeInterval.FromSeconds(50));
entryA = clockSource.GetClockEntry(handlerA);
entryB = clockSource.GetClockEntry(handlerB);
Assert.AreEqual(counterA, 0);
Assert.AreEqual(counterB, 0);
Assert.AreEqual(entryA.Value, 550);
Assert.AreEqual(entryA.Period, 1000);
Assert.AreEqual(entryB.Value, 50);
Assert.AreEqual(entryB.Period, 100);
clockSource.Advance(TimeInterval.FromSeconds(50));
entryA = clockSource.GetClockEntry(handlerA);
entryB = clockSource.GetClockEntry(handlerB);
Assert.AreEqual(counterA, 0);
Assert.AreEqual(counterB, 1);
Assert.AreEqual(entryA.Value, 600);
Assert.AreEqual(entryA.Period, 1000);
Assert.AreEqual(entryB.Value, 0);
Assert.AreEqual(entryB.Period, 100);
}
public void ShouldTickWithOneHandler()
{
var clocksource = new BaseClockSource();
var counter = 0;
clocksource.AddClockEntry(new ClockEntry(2, 1, () => counter++, null, String.Empty)
{
Value = 0
});
clocksource.Advance(TimeInterval.FromSeconds(1));
Assert.AreEqual(0, counter);
clocksource.Advance(TimeInterval.FromSeconds(1));
Assert.AreEqual(1, counter);
clocksource.Advance(TimeInterval.FromSeconds(1));
Assert.AreEqual(1, counter);
clocksource.Advance(TimeInterval.FromSeconds(2));
Assert.AreEqual(2, counter);
}
public TerminalTesterResult WaitForLineOnUart(string content, ulong?timeout = null, int?testerId = null, bool treatAsRegex = false, bool includeUnfinishedLine = false)
{
TimeInterval?timeInterval = null;
if (timeout.HasValue)
{
timeInterval = TimeInterval.FromSeconds(timeout.Value);
}
var tester = GetTesterOrThrowException(testerId);
var result = tester.WaitFor(content, timeInterval, treatAsRegex, includeUnfinishedLine);
if (result == null)
{
OperationFail(tester);
}
return(result);
}
public TerminalTesterResult WaitForNextLineOnUart(uint?timeout = null, int?testerId = null)
{
TimeInterval?timeInterval = null;
if (timeout.HasValue)
{
timeInterval = TimeInterval.FromSeconds(timeout.Value);
}
var tester = GetTesterOrThrowException(testerId);
var result = tester.NextLine(timeInterval);
if (result == null)
{
OperationFail(tester);
}
return(result);
}
public int CreateTerminalTester(string uart, float?timeout = null, string machine = null, string endLineOption = null)
{
return(CreateNewTester(uartObject =>
{
var timeoutInSeconds = timeout ?? globalTimeout;
TerminalTester tester;
if (Enum.TryParse <EndLineOption>(endLineOption, out var result))
{
tester = new TerminalTester(TimeInterval.FromSeconds(timeoutInSeconds), result);
}
else
{
tester = new TerminalTester(TimeInterval.FromSeconds(timeoutInSeconds));
}
tester.AttachTo(uartObject);
return tester;
}, uart, machine));
}
public static Threshold Create(ThresholdUnit unit, double value)
{
switch (unit)
{
case ThresholdUnit.Ratio: return(new RelativeThreshold(value));
case ThresholdUnit.Nanoseconds: return(new AbsoluteTimeThreshold(TimeInterval.FromNanoseconds(value)));
case ThresholdUnit.Microseconds: return(new AbsoluteTimeThreshold(TimeInterval.FromMicroseconds(value)));
case ThresholdUnit.Milliseconds: return(new AbsoluteTimeThreshold(TimeInterval.FromMilliseconds(value)));
case ThresholdUnit.Seconds: return(new AbsoluteTimeThreshold(TimeInterval.FromSeconds(value)));
case ThresholdUnit.Minutes: return(new AbsoluteTimeThreshold(TimeInterval.FromMinutes(value)));
default: throw new ArgumentOutOfRangeException(nameof(unit), unit, null);
}
}
public int CreateTerminalTester(string uart, string prompt = null, int timeout = 30, string machine = null)
{
lock (testers)
{
Machine machineObject;
if (machine == null)
{
if (!EmulationManager.Instance.CurrentEmulation.Machines.Any())
{
throw new KeywordException("There is no machine in the emulation. Could not create tester for peripheral: {0}", uart);
}
machineObject = EmulationManager.Instance.CurrentEmulation.Machines.Count() == 1
? EmulationManager.Instance.CurrentEmulation.Machines.First()
: null;
if (machineObject == null)
{
throw new KeywordException("No machine name provided. Don't know which one to choose. Available machines: [{0}]",
string.Join(", ", EmulationManager.Instance.CurrentEmulation.Machines.Select(x => EmulationManager.Instance.CurrentEmulation[x])));
}
}
else if (!EmulationManager.Instance.CurrentEmulation.TryGetMachineByName(machine, out machineObject))
{
throw new KeywordException("Machine with name {0} not found. Available machines: [{1}]", machine,
string.Join(", ", EmulationManager.Instance.CurrentEmulation.Machines.Select(x => EmulationManager.Instance.CurrentEmulation[x])));
}
if (!machineObject.TryGetByName(uart, out IUART uartObject))
{
throw new KeywordException("Peripheral for machine {0} not found or of wrong type: {1}", machine, uart);
}
if (uartsWithTesters.Contains(uartObject))
{
throw new KeywordException("Terminal tester for peripheral {0} in machine {1} already exists", uart, machine);
}
var tester = new TerminalTester(TimeInterval.FromSeconds((uint)timeout), prompt);
tester.Terminal.AttachTo(uartObject);
uartsWithTesters.Add(uartObject);
testers.Add(uartsWithTesters.Count, tester);
}
return(uartsWithTesters.Count);
}
public void ShouldHaveHandlersInSync()
{
// we test here whether handler executed by the slower clock entry
// always "sees" value of the faster one as ten times its own value
var clockSource = new BaseClockSource();
Action firstHandler = () =>
{
};
var values = new List <ulong>();
clockSource.AddClockEntry(new ClockEntry(10000, 10, firstHandler, null, String.Empty));
clockSource.AddClockEntry(new ClockEntry(10, 1, () => values.Add(clockSource.GetClockEntry(firstHandler).Value), null, String.Empty));
clockSource.Advance(TimeInterval.FromSeconds(9), true);
clockSource.Advance(TimeInterval.FromSeconds(8), true);
clockSource.Advance(TimeInterval.FromSeconds(20), true);
CollectionAssert.AreEqual(new [] { 100, 200, 300 }, values);
}
static void Main(string[] args)
{
var config = ManualConfig.Create(DefaultConfig.Instance)
.With(BenchmarkDotNet.Analysers.EnvironmentAnalyser.Default)
.With(BenchmarkDotNet.Exporters.MarkdownExporter.GitHub)
.With(BenchmarkDotNet.Diagnosers.MemoryDiagnoser.Default)
.With(StatisticColumn.Mean)
.With(StatisticColumn.Median)
.With(StatisticColumn.StdDev)
.With(StatisticColumn.OperationsPerSecond)
.With(BaselineRatioColumn.RatioMean)
.With(RankColumn.Arabic)
.With(Job.Core
.WithIterationCount(10)
.WithInvocationCount(10)
.WithIterationTime(TimeInterval.FromSeconds(2))
.WithWarmupCount(4)
.WithLaunchCount(1));
BenchmarkRunner.Run <BenchmarkTests>(config);
}
private static void runBenchmarks()
{
var config = ManualConfig.Create(DefaultConfig.Instance)
.With(BenchmarkDotNet.Analysers.EnvironmentAnalyser.Default)
.With(BenchmarkDotNet.Exporters.MarkdownExporter.GitHub)
.With(BenchmarkDotNet.Diagnosers.MemoryDiagnoser.Default)
.With(StatisticColumn.Mean)
.With(StatisticColumn.Median)
.With(StatisticColumn.StdDev)
.With(StatisticColumn.OperationsPerSecond)
.With(BaselineRatioColumn.RatioMean)
.With(RankColumn.Arabic)
.With(Job.Default.With(CoreRuntime.Core31)
.WithIterationCount(300)
.WithInvocationCount(160)
.WithIterationTime(TimeInterval.FromSeconds(1000))
.WithWarmupCount(4)
.WithLaunchCount(1));
BenchmarkRunner.Run <BenchmarkTestsForStringReplace>(config);
}
public TerminalTesterResult WaitForLineOnUart(string content, uint?timeout = null, int?testerId = null, bool treatAsRegex = false)
{
var groups = new string[0];
GetTesterOrThrowException(testerId).WaitUntilLineFunc(
x =>
{
if (!treatAsRegex)
{
return(x.Contains(content));
}
var match = Regex.Match(x, content);
groups = match.Success ? match.Groups.Cast <Group>().Skip(1).Select(y => y.Value).ToArray() : new string[0];
return(match.Success);
},
out string line,
out var time,
timeout == null ? (TimeInterval?)null : TimeInterval.FromSeconds(timeout.Value)
);
return(new TerminalTesterResult(line, time.TotalMilliseconds, groups));
}
public IEngine CreateReadyToRun(EngineParameters engineParameters)
{
if (engineParameters.WorkloadActionNoUnroll == null)
{
throw new ArgumentNullException(nameof(engineParameters.WorkloadActionNoUnroll));
}
if (engineParameters.WorkloadActionUnroll == null)
{
throw new ArgumentNullException(nameof(engineParameters.WorkloadActionUnroll));
}
if (engineParameters.Dummy1Action == null)
{
throw new ArgumentNullException(nameof(engineParameters.Dummy1Action));
}
if (engineParameters.Dummy2Action == null)
{
throw new ArgumentNullException(nameof(engineParameters.Dummy2Action));
}
if (engineParameters.Dummy3Action == null)
{
throw new ArgumentNullException(nameof(engineParameters.Dummy3Action));
}
if (engineParameters.OverheadActionNoUnroll == null)
{
throw new ArgumentNullException(nameof(engineParameters.OverheadActionNoUnroll));
}
if (engineParameters.OverheadActionUnroll == null)
{
throw new ArgumentNullException(nameof(engineParameters.OverheadActionUnroll));
}
if (engineParameters.TargetJob == null)
{
throw new ArgumentNullException(nameof(engineParameters.TargetJob));
}
engineParameters.GlobalSetupAction?.Invoke(); // whatever the settings are, we MUST call global setup here, the global cleanup is part of Engine's Dispose
if (!engineParameters.NeedsJitting) // just create the engine, do NOT jit
{
return(CreateMultiActionEngine(engineParameters));
}
int jitIndex = 0;
if (engineParameters.HasInvocationCount || engineParameters.HasUnrollFactor) // it's a job with explicit configuration, just create the engine and jit it
{
var warmedUpMultiActionEngine = CreateMultiActionEngine(engineParameters);
DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(warmedUpMultiActionEngine, ++jitIndex, invokeCount: engineParameters.UnrollFactor, unrollFactor: engineParameters.UnrollFactor));
return(warmedUpMultiActionEngine);
}
var singleActionEngine = CreateSingleActionEngine(engineParameters);
var singleInvocationTime = Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1);
if (singleInvocationTime > engineParameters.IterationTime && singleInvocationTime < TimeInterval.FromSeconds(1.0))
{
// if the Jitting took more than IterationTime but still less than 1s (a magic number based on observations of the reported bug)
// we call it one more time to see if Jitting itself has not dominated the first invocation
// if it did, it shoud NOT be a single invocation engine (see #837, #1337 and #1338)
singleInvocationTime = Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1);
}
if (singleInvocationTime > engineParameters.IterationTime)
{
return(singleActionEngine); // executing once takes longer than iteration time => long running benchmark, needs no pilot and no overhead
}
int defaultUnrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, EngineParameters.DefaultResolver);
double timesPerIteration = engineParameters.IterationTime / singleInvocationTime; // how many times can we run given benchmark per iteration
if (timesPerIteration < 1.5) // example: IterationTime is 0.5s, but single invocation takes 0.4s => we don't want to run it twice per iteration
{
return(singleActionEngine);
}
int roundedUpTimesPerIteration = (int)Math.Ceiling(timesPerIteration);
if (roundedUpTimesPerIteration < defaultUnrollFactor) // if we run it defaultUnrollFactor times per iteration, it's going to take longer than IterationTime
{
var needsPilot = engineParameters.TargetJob
.WithUnrollFactor(1) // we don't want to use unroll factor!
.WithMinInvokeCount(2) // the minimum is 2 (not the default 4 which can be too much and not 1 which we already know is not enough)
.WithEvaluateOverhead(false); // it's something very time consuming, it overhead is too small compared to total time
return(CreateEngine(engineParameters, needsPilot, engineParameters.OverheadActionNoUnroll, engineParameters.WorkloadActionNoUnroll));
}
var multiActionEngine = CreateMultiActionEngine(engineParameters);
DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine, ++jitIndex, invokeCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor));
return(multiActionEngine);
}
public void ToFolderNameStructTest()
{
Assert.Equal("0-42", FolderNameHelper.ToFolderName(0.42m));
Assert.Equal("1234000000ns", FolderNameHelper.ToFolderName(TimeInterval.FromSeconds(1.234)));
}
public void TestIfUartIsIdle(uint timeInSeconds, int?testerId = null)
{
GetTesterOrThrowException(testerId).CheckIfUartIsIdle(TimeInterval.FromSeconds(timeInSeconds));
}
public static void CreateTerminalTester(this Emulation emulation, string name, uint timeoutInSeconds = 300, string prompt = @"/ # ")
{
var tester = new TerminalTester(TimeInterval.FromSeconds(timeoutInSeconds), prompt);
emulation.ExternalsManager.AddExternal(tester, name);
}
public TerminalTesterResult WaitForPromptOnUart(int?testerId = null, uint?timeout = null)
{
return(new TerminalTesterResult(
GetTesterOrThrowException(testerId).ReadToPrompt(out var time, timeout == null ? (TimeInterval?)null : TimeInterval.FromSeconds(timeout.Value)),
time.TotalMilliseconds
));
}
static void Main(string[] args)
{
Assembly thisAssembly = Assembly.GetCallingAssembly();
List <Type> benchmarks = new List <Type>();
if (args.Length > 0)
{
// Set the argument as the crash dump. We can't just set CrashDump here because it needs to be read from child processes.
Environment.SetEnvironmentVariable(DumpFileEnv, args[0]);
foreach (string benchmark in args.Skip(1))
{
Type benchmarkType = thisAssembly.GetType($"Benchmarks.{benchmark}", throwOnError: false, ignoreCase: true);
if (benchmarkType == null)
{
Console.WriteLine($"Benchmark '{benchmark}' not found.");
Environment.Exit(2);
}
benchmarks.Add(benchmarkType);
}
}
// We want to run this even if we don't use the result to make sure we can successfully load 'CrashDump'.
int targetPointerSize = GetTargetPointerSize();
ManualConfig benchmarkConfiguration = ManualConfig.Create(DefaultConfig.Instance);
// Windows supports x86 and x64 so we need to choose the correct version of .Net.
Job job;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
string dotnetPath = GetDotnetPath(targetPointerSize);
if (targetPointerSize == 4)
{
job = Job.RyuJitX86.With(CsProjCoreToolchain.From(NetCoreAppSettings.NetCoreApp31.WithCustomDotNetCliPath(dotnetPath)));
}
else
{
job = Job.RyuJitX64.With(CsProjCoreToolchain.From(NetCoreAppSettings.NetCoreApp31.WithCustomDotNetCliPath(dotnetPath)));
}
}
else
{
job = Job.Default;
}
string id = $"{RuntimeInformation.OSDescription} {RuntimeInformation.FrameworkDescription} {(targetPointerSize == 4 ? "32bit" : "64bit")}";
job = job.WithId(id)
.WithWarmupCount(1)
.WithIterationTime(TimeInterval.FromSeconds(1))
.WithMinIterationCount(10)
.WithMaxIterationCount(20)
.DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
benchmarkConfiguration.Add(job);
if (benchmarks.Count == 0)
{
BenchmarkRunner.Run(thisAssembly, benchmarkConfiguration);
}
else
{
foreach (Type t in benchmarks)
{
BenchmarkRunner.Run(t, benchmarkConfiguration);
}
}
}
public WelchTTestAbsoluteSecondsAttribute(double threshold)
: base(BaselineScaledColumn.CreateWelchTTest(new AbsoluteHypothesis(TimeInterval.FromSeconds(threshold))))
{
}
public TerminalTesterResult WaitForPromptOnUart(string prompt = null, int?testerId = null, uint?timeout = null)
{
var tester = GetTesterOrThrowException(testerId);
string previousPrompt = null;
if (prompt != null)
{
previousPrompt = tester.Terminal.Prompt;
tester.Terminal.Prompt = prompt;
}
var result = new TerminalTesterResult(
tester.ReadToPrompt(out var time, timeout == null ? (TimeInterval?)null : TimeInterval.FromSeconds(timeout.Value)),
time.TotalMilliseconds
);
if (previousPrompt != null)
{
tester.Terminal.Prompt = previousPrompt;
}
return(result);
}
public void AdvanceBySeconds(ulong seconds)
{
Advance(TimeInterval.FromSeconds(seconds));
}
