// Registering at least one aggregation forces worker threads to copy/produce measurements after each iteration.
// Registering many aggregations shouldn't impact test performance more compared to just a single aggregation.
//
//
// While this test can achieve 1+million/sec speed (w/o debugger and release build):
// current HistogramAggregator implementation won't be able to process this massive amount of incoming data in real time.
// as result of it, data gets queued up in memory and this test will easily eat ~5+ GB's of ram in first test run,
// so be sure that there is enough of RAM available before running this one.
//
// This also shows how Run() won't exit past 13seconds. While test is no longer running, engine waits until all data gets processed.
//
// This example runs test twice:
// * One with simple histogram setup
// * And other without histogram but receiving a direct stream of raw measurements and just counting them.
// - This one should be able to count results in the real time
public static void Run()
{
TimeSpan duration = TimeSpan.FromSeconds(13); // 2x tests
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(4)))
.Add(new CountMetric())
.Add(new TransactionsPerSecMetric())
.Add(new GlobalTimerPeriodMetric());
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario <BlankScenario>()
.SetThreading(new FixedThreadCount(4))
.SetLimit(new TimeLimit(duration))
.SetAggregator(aggregator);
TimeSpan durationWithHistogram = MeassuredRun(strategy.Build());
// Ease use of ram ASAP.
GC.Collect(1, GCCollectionMode.Forced, true);
int iterations = 0;
strategy.SetAggregator(new StreamAggregator(result => iterations = iterations + 1));
TimeSpan durationWithStream = MeassuredRun(strategy.Build());
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
Console.WriteLine($"Time took for [HistogramAggregator] to process whats left in the buffer: {durationWithHistogram - duration:g}");
Console.WriteLine($"Time took for [StreamAggregator] to count whats left in the buffer: {durationWithStream - duration:g}");
}
public static void Run()
{
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(10)))
.Add(new FuncMetric <int>("Working Threads", 0, (i, r) => Math.Max(r.CreatedThreads - r.IdleThreads, i)))
.Add(new CountMetric(Checkpoint.NotMeassuredCheckpoints))
.Add(new TransactionsPerSecMetric());
KpiPrinterAggregator kpi = new KpiPrinterAggregator(
TimeSpan.FromSeconds(4),
new FuncMetric <string>("T+", "???", (s, result) => result.IterationFinished.ToString("g")),
new FuncMetric <int>("Created Threads", 0, (i, r) => Math.Max(r.CreatedThreads, i)),
new FuncMetric <int>("Working Threads", 0, (i, r) => Math.Max(r.CreatedThreads - r.IdleThreads, i)),
new CountMetric(Checkpoint.NotMeassuredCheckpoints),
new CountMetric(i => i / 2.0, Checkpoint.NotMeassuredCheckpoints)
{
Prefix = "TPS "
}
);
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario <LimitConcurrencyAndTpsDemo>()
.AddSpeed(new LimitWorkingThreads(11))
.AddSpeed(new IncrementalSpeed(1, TimeSpan.FromSeconds(20), 10))
.SetThreading(new IncrementalThreadCount(250, TimeSpan.FromSeconds(10), -50))
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(52)))
.SetAggregator(aggregator, kpi);
strategy.Build().Run();
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
}
public static void Run()
{
HistogramAggregator histogram = new HistogramAggregator()
.Add(new FuncDimension("ThreadIterationId", r => r.ThreadIterationId.ToString()))
.Add(new DistinctListMetric <IResult, string>("Sleeps", r => $"{r.ThreadId}:{((int) r.UserData)}"))
.Add(new DistinctListMetric <IResult, string>("TStarts", r => $"{r.ThreadId}:{(int)r.IterationStarted.TotalMilliseconds}"))
.Add(new DistinctListMetric <IResult, string>("TEnds", r => $"{r.ThreadId}:{(int)r.IterationFinished.TotalMilliseconds}"))
.Add(new FilterMetric <IResult>(r => r.ThreadId == 0, new ValueMetric <IResult>("Tr#0", r => (int)r.IterationFinished.TotalMilliseconds)))
.Add(new FilterMetric <IResult>(r => r.ThreadId == 1, new ValueMetric <IResult>("Tr#1", r => (int)r.IterationFinished.TotalMilliseconds)))
.Add(new FilterMetric <IResult>(r => r.ThreadId == 2, new ValueMetric <IResult>("Tr#2", r => (int)r.IterationFinished.TotalMilliseconds)))
.Add(new FilterMetric <IResult>(r => r.ThreadId == 3, new ValueMetric <IResult>("Tr#3", r => (int)r.IterationFinished.TotalMilliseconds)))
.Add(new FuncMetric <int>("Min TStart", Int32.MaxValue, (i, r) => Math.Min((int)r.IterationStarted.TotalMilliseconds, i)))
.Add(new FuncMetric <int>("Max Sleep", -1, (i, r) => Math.Max(((int)r.UserData), i)))
.Add(new FuncMetric <int>("Max TEnd", Int32.MinValue, (i, r) => Math.Max((int)r.IterationFinished.TotalMilliseconds, i)));
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario <BatchAndWaitDemo>()
.AddSpeed(new BatchBySlowestSpeed(ThreadCount))
.SetThreading(new FixedThreadCount(ThreadCount))
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(20)))
.SetAggregator(histogram);
strategy.Build().Run();
Console.WriteLine(JsonConvert.SerializeObject(histogram.BuildResultsObjects(), Formatting.Indented));
}
public static void Run()
{
// Initialize LoadRunnerEngine by providing:
// * Type of class which implements ILoadTestScenario (e.g DemoTestScenario)
// * LoadRunnerParameters
// * As many aggregators as you like
HistogramAggregator histogramAggregator = AggregationSetup.Build();
LoadRunnerParameters parameters = ParametersSetup.Build();
LoadRunnerEngine loadRunner = LoadRunnerEngine.Create <DemoTestScenario>(parameters, histogramAggregator);
// Run test (blocking call)
loadRunner.Run();
// Once finished, extract information from used aggregators, and do some exceling :)
// BuildResultsObjects() will produce results in structure compatible with JSON -> CSV converters. (See ~20 lines below)
IEnumerable <object> defaultResults = histogramAggregator.BuildResultsObjects();
Console.WriteLine(JsonConvert.SerializeObject(defaultResults, Formatting.Indented));
Console.ReadKey();
//Alternative export way is
// HistogramResults results = histogramAggregator.BuildResults();
//
// results will be presented in this structure:
// * string[] ColumnNames;
// * object[][] Values;
}
// HDD/SSD intensive, beware
public static void Run()
{
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(2)))
.Add(new CountMetric())
.Add(new TransactionsPerSecMetric());
HistogramAggregator aggregagorOriginal = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(2)))
.Add(new CountMetric())
.Add(new TransactionsPerSecMetric());
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario <BlankScenario>()
.SetThreading(new FixedThreadCount(3))
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(5)))
.SetAggregator(new JsonStreamAggregator("Raw.json"), aggregagorOriginal);
strategy.Build().Run();
JsonStreamAggregator.Replay("Raw.json", aggregator);
Console.WriteLine(JsonConvert.SerializeObject(aggregagorOriginal.BuildResultsObjects(), Formatting.Indented));
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
}
public static void Run()
{
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new FuncDimension("Iteration", result => result.GlobalIterationId.ToString()))
.Add(new FuncDimension("Time", result => result.IterationStarted.TotalSeconds.ToString(CultureInfo.InvariantCulture)))
.Add(new FuncDimension("Data", result => result.UserData?.ToString()))
.Add(new FuncMetric <int>("CThreads", 0, (i, r) => r.CreatedThreads))
.Add(new FuncMetric <int>("IThreads", 0, (i, r) => r.IdleThreads));
ReplayStrategyBuilder <string> settings = new ReplayStrategyBuilder <string>()
.SetAggregator(aggregator)
.SetData(DataGenerator.Create(5, 1, 3, 3).ToArray())
.SetScenario <ReplayScenario>()
.SetThreadCount(30)
.SetSpeed(1);
// UI
//LoadRunnerUi engineUi = settings.BuildUi(new DataItem(TimeSpan.Zero, "Validation demo"));
//engineUi.StartWindow();
// Non UI blocking
LoadRunnerEngine engine = settings.Build();
//engine.Run();
// Non UI Async
engine.RunAsync();
engine.Wait();
object defaultResults = aggregator.BuildResultsObjects();
Console.WriteLine(JsonConvert.SerializeObject(defaultResults, Formatting.Indented));
//Console.ReadKey();
}
public static void Run()
{
// Initialize aggregator
string[] ignoredCheckpoints =
{
Checkpoint.Names.Setup,
Checkpoint.Names.TearDown
};
HistogramAggregator histogramAggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(10)))
.Add(new FuncMetric <TimeSpan>("TMin", TimeSpan.MaxValue,
(span, result) => span > result.IterationStarted ? result.IterationStarted : span))
.Add(new FuncMetric <TimeSpan>("TMax", TimeSpan.MinValue,
(span, result) => span < result.IterationFinished ? result.IterationFinished : span))
.Add(new FuncMetric <int>("Working Threads", 0,
(i, result) => result.CreatedThreads - result.IdleThreads))
//.Add(new MinDurationMetric(ignoredCheckpoints))
.Add(new AvgDurationMetric(ignoredCheckpoints))
.Add(new MaxDurationMetric(ignoredCheckpoints))
//.Add(new PercentileMetric(new[] {0.99999}, ignoredCheckpoints))
.Add(new CountMetric(ignoredCheckpoints))
.Add(new TransactionsPerSecMetric())
.Add(new ErrorCountMetric())
.Alias($"Min: {Checkpoint.Names.Iteration}", "Min (ms)")
.Alias($"Avg: {Checkpoint.Names.Iteration}", "Avg (ms)")
.Alias($"Max: {Checkpoint.Names.Iteration}", "Max (ms)")
.Alias($"50%: {Checkpoint.Names.Iteration}", "50% (ms)")
.Alias($"80%: {Checkpoint.Names.Iteration}", "80% (ms)")
.Alias($"90%: {Checkpoint.Names.Iteration}", "90% (ms)")
.Alias($"95%: {Checkpoint.Names.Iteration}", "95% (ms)")
.Alias($"99.99%: {Checkpoint.Names.Iteration}", "99.99% (ms)")
.Alias($"Count: {Checkpoint.Names.Iteration}", "Success: Count");
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario <TestScenario>()
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(10)))
.SetThreading(new FixedThreadCount(40))
.SetSpeed(new FixedSpeed(2000)) // Tps is lower in results due to failed iterations not being counted
.SetFinishTimeout(TimeSpan.FromSeconds(60))
.SetAggregator(histogramAggregator);
IStrategyExecutor engine = strategy.Build();
engine.Run();
object defaultResults = histogramAggregator.BuildResultsObjects();
Console.WriteLine(JsonConvert.SerializeObject(defaultResults, Formatting.Indented));
}
public static HistogramAggregator BuildHistogram()
{
// This demo shows this new HistogramAggregator which isn't currently documented anywhere else. (TODO)
//
// This preset shown below more or less should work for most of the test cases, except for TimeDimension,
// which you might want to adjust for bigger chunks or remove it to get totals [no dimensions == no grouping by].
//
// HistogramAggregator is a modular Dimension/Metric style aggregator tool, which can be expanded by implementing IMetric or IDimension
// * Metrics are aggregation functions like COUNT, SUM, etc..
//
// Dimensions are like row keys (Adding multiple dimensions would multiply result row count)
// * In SQL it would be like GROUP BY function
// Available IDimension's:
// TimeDimension(TimeSpan interval, string dimensionName = "Time (s)")
// FuncDimension(string dimensionName, Func<IResult,string> dimensionValueSelector)
//
// Metrics are like values, which will be meassured in test execution
// In SQL it would be like aggregation function, like SUM, COUNT, etc.
// Available IMetric's:
// AvgDurationMetric(params string[] ignoredCheckpoints)
// BreakByMetric(IDimension subDimension, params IMetric[] actualMetrics)
// CountMetric(params string[] ignoredCheckpoints)
// ErrorCountMetric(bool includeTotals = true)
// ErrorRatioMetric(params string[] ignoredCheckpoints)
// FuncMetric(string keyName, TValue initialValue, Func<TValue, IResult, TValue> metricFunc)
// FuncMultiMetric(Action<FlexiRow<string,TValue>, IResult> metricProcedure, Func<TValue> cellBuilderFunc)
// MaxDurationMetric(params string[] ignoredCheckpoints)
// MinDurationMetric(params string[] ignoredCheckpoints)
// PercentileMetric(double[] percentiles, string[] ignoredCheckpoints)
// TransactionsPerSecMetric()
HistogramAggregator histogramAggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(10)))
.Add(new MinDurationMetric())
.Add(new AvgDurationMetric())
.Add(new MaxDurationMetric())
.Add(new PercentileMetric(new[] { 0.95, 0.99 }))
.Add(new CountMetric(Checkpoint.NotMeassuredCheckpoints))
.Add(new ErrorCountMetric())
.Add(new FuncMetric <int>("Created Threads", 0, (i, result) => result.CreatedThreads))
.Alias($"Min: {Checkpoint.Names.Iteration}", "Min (ms)")
.Alias($"Avg: {Checkpoint.Names.Iteration}", "Avg (ms)")
.Alias($"Max: {Checkpoint.Names.Iteration}", "Max (ms)")
.Alias($"95%: {Checkpoint.Names.Iteration}", "95% (ms)")
.Alias($"99%: {Checkpoint.Names.Iteration}", "99% (ms)")
.Alias($"Count: {Checkpoint.Names.Iteration}", "Success: Count")
.Alias($"Errors: {Checkpoint.Names.Setup}", "Errors: Setup")
.Alias($"Errors: {Checkpoint.Names.Iteration}", "Errors: Iteration")
.Alias($"Errors: {Checkpoint.Names.TearDown}", "Errors: Teardown");
return(histogramAggregator);
}
// All IAggregator implementations receive un-aggregated raw measurements (later raw-data) from workers.
// - (TODO: More info in IAggregator customization, But IAggregator ir pretty self explanatory anyway :))
// Main idea behind Raw-Data is that one gets un-aggregated measurements in IResult form
// and one can do anything with it, preferably post-load-test.
//
//
// Having this unlocks few advantages:
// * Save this raw-data somewhere(file on disk, array in memory) and do any aggregations after the test.
// - I personally persist raw-data from all of the tests I do,
// because one will never know what aggregation might be needed until one runs the test and sees initial results.
// - There is a RnD JsonStreamAggregator which saves to JSON array file. Though it can take up some space on disk as JSON has lots of overhead.
// * Or one can write own custom specialized aggregation ignoring provided HistogramAggregator totally.
//
//
// This demo contains:
// * Load-test with one aggregation:
// - [Tools nuget: JsonStreamAggregator] Write data to raw.json file for later aggregation
// * Replay of raw measurements after the test and do some more aggregating post-load-test.
// - Two histogram aggregators: Timeline and KPI.
// - One StreamAggregator instance which gives IEnumerable of IResult's for custom inline aggregation.
public static void Run()
{
// JsonStreamAggregator dumps everything it receives to file.
JsonStreamAggregator jsonAggregator = new JsonStreamAggregator("raw.json");
StrategyBuilder builder = new StrategyBuilder()
.AddLimit(new TimeLimit(TimeSpan.FromSeconds(4)))
.SetThreading(new FixedThreadCount(4))
.SetScenario(new SleepingScenarioFactory(TimeSpan.FromMilliseconds(100))) // Use scenario from common which sleeps every iteration
.SetAggregator(jsonAggregator); // Register JsonAggregator to be only one to receive results
// A quick run
builder.Build().Run();
// Now lets do actual aggregation post testing.
// First - just configure HistogramAggregator or any other aggregator in same way one would do for the live test.
HistogramAggregator histogramTimeline = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(1)))
.Add(new CountMetric())
.Add(new GlobalTimerMinValueMetric())
.Add(new GlobalTimerMaxValueMetric())
.Add(new GlobalTimerPeriodMetric());
HistogramAggregator histogramKpi = new HistogramAggregator()
.Add(new CountMetric())
.Add(new GlobalTimerMinValueMetric())
.Add(new GlobalTimerMaxValueMetric())
.Add(new GlobalTimerPeriodMetric());
// Lets define another more specialized aggregation just for lols.
TimeSpan lastIterationEnded = TimeSpan.Zero;
StreamAggregator streamAggregator = new StreamAggregator(
results =>
{
lastIterationEnded = results.Max(r => r.IterationFinished);
}
);
// now just replay saved raw-data stream to those later-defined aggregations.
JsonStreamAggregator.Replay("raw.json", histogramTimeline, histogramKpi, streamAggregator);
Console.WriteLine($"StreamAggregator, last iteration finished: {lastIterationEnded:g}");
Console.WriteLine("---------------- Histogram KPI -----------");
Console.WriteLine(JsonConvert.SerializeObject(histogramKpi.BuildResultsObjects(), Formatting.Indented));
Console.WriteLine("---------------- Histogram timeline -----------");
Console.WriteLine(JsonConvert.SerializeObject(histogramTimeline.BuildResultsObjects(), Formatting.Indented));
}
// ***
private static void AggregateFromMultipleSources()
{
IEnumerable <ReplayResult <object> > pc1 = JsonStreamAggregator.Load <object>("masterdata.json");
IEnumerable <ReplayResult <object> > pc2 = JsonStreamAggregator.Load <object>("masterdata.json");
IEnumerable <ReplayResult <object> > merged = pc1.Concat(pc2);
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new CountMetric())
.Add(new TransactionsPerSecMetric());
StreamAggregator.Replay(merged);
Console.WriteLine("## Merged demo ##");
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
Console.ReadKey();
}
static void Main()
{
ReadmeDemo.Run();
return;
HistogramAggregator histo = new HistogramAggregator();
histo
.Add(new TimeDimension(TimeSpan.FromSeconds(1)))
.Add(new CountMetric())
.Add(new AvgDurationMetric())
.Add(new PercentileMetric(0.95));
JsonStreamAggregator.Replay("d:\\test.stream.json", histo);
}
public static void Run()
{
// [2] Results aggregation (Or raw measurement collection, see RawDataMeasurementsDemo.cs)
// Define how data gets aggregated.
// Dimensions are like GROUP BY keys in SQL
// Metrics are aggregation functions like COUNT, SUM, etc..
// Extensive HistogramAggregator demo now WiP
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(5)))
.Add(new CountMetric())
.Add(new ErrorCountMetric())
.Add(new TransactionsPerSecMetric())
.Add(new PercentileMetric(0.95, 0.99));
// Secondary aggregation just to monitor key metrics.
KpiPrinterAggregator kpiPrinter = new KpiPrinterAggregator(
TimeSpan.FromSeconds(5),
new CountMetric(Checkpoint.NotMeassuredCheckpoints),
new ErrorCountMetric(false),
new TransactionsPerSecMetric()
);
// [3] Execution settings
// Using StrategyBuilder put defined aggregation, scenario, and execution settings into one object
StrategyBuilder strategy = new StrategyBuilder()
.SetAggregator(aggregator, kpiPrinter) // Optional
.SetScenario <BlankScenario>() // Required
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(20))) // Optional, but if not provided, execution will never stop - unless running test with RunAsync() and stopping later with CancelAsync(true)
.SetSpeed(new FixedSpeed(100000)) // Optional (Skip for maximum throughput)
.SetThreading(new FixedThreadCount(4)); // Required
// [4] Execution
// All that's left is Build(), run and wait for completion and print out measured results.
LoadRunnerEngine engine = strategy.Build();
engine.Run();
IEnumerable <object> result = aggregator.BuildResultsObjects();
Console.WriteLine(JsonConvert.SerializeObject(result, Formatting.Indented));
}
public static void Run()
{
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(5)))
.Add(new CountMetric())
.Add(new GlobalTimerPeriodMetric())
.Add(new TransactionsPerSecMetric());
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario <BatchStrategyDemo>()
.AddSpeed(new BatchByTimeIntervalSpeed(TimeSpan.FromSeconds(5), 10))
.SetThreading(new FixedThreadCount(15))
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(20)))
.SetAggregator(aggregator);
strategy.Build().Run();
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
}
public static void Aggregate()
{
// Feature disabled?
if (!File.Exists("masterdata.json"))
{
return;
}
// Any random aggregator
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new FuncDimension("Created Threads", i => i.CreatedThreads.ToString()))
.Add(new CountMetric());
// [TUserData] - Type of UserData object in your tests (If not using it, leave object)
// Correct type is required to prevent deserialization problems.
JsonStreamAggregator.Replay <object>("masterdata.json", aggregator);
Console.WriteLine("## JsonStreamAggregator demo ##");
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
Console.ReadKey();
}
public static void Run()
{
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new CountMetric())
.Add(new FuncMetric <TimeSpan>("max", TimeSpan.Zero, (s, result) => s < result.IterationStarted ? result.IterationStarted : s));
AssertIterationIdsAggregator idsValidator = new AssertIterationIdsAggregator();
int streamCount = 0;
StreamAggregator streamAggregator = new StreamAggregator(results => streamCount = results.Count());
CountingScenarioFactory factory = new CountingScenarioFactory();
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario(factory)
.SetThreading(new FixedThreadCount(16))
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(12)))
.SetAggregator(aggregator, idsValidator, streamAggregator);
Stopwatch sw = new Stopwatch();
sw.Start();
strategy.Build().Run();
sw.Stop();
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
factory.PrintSum();
int expected = factory.GetSum();
int actual = (int)aggregator.BuildResults().Data[0][1];
TimeSpan lastIteration = (TimeSpan)aggregator.BuildResults().Data[0][3];
Console.WriteLine($@"TPS {expected / lastIteration.TotalSeconds:N}");
Console.WriteLine($@"Last iteration ended at: {lastIteration:g}");
Console.WriteLine($@"Aggregator catchup took: {(sw.Elapsed-lastIteration):g}");
Console.WriteLine();
Console.WriteLine($@"{expected}/{actual} & {streamCount} ? {expected==actual} && {expected==streamCount}");
Console.WriteLine();
idsValidator.PrintResults();
}
public static HistogramAggregator Build()
{
// This demo shows this new HistogramAggregator which isn't currently documented anywhere else. (TODO)
//
// Though there are older ones "Legacy" which are somewhat documented
// https://github.com/Vycka/LoadRunner/wiki/IResultsAggregator but avoid using them until HistogramAggregator is not enough (e.g. if wanting to make Exceptions dump)
// This preset shown below more or less should work for most of the test cases, except for TimeDimension,
// which you might want to adjust for bigger chunks or remove it to get totals [no dimensions == no grouping by].
//
// HistogramAggregator is a modular Dimension/Metric style aggregator tool, which can be expanded by implementing IMetric or IDimension
//
// Dimensions are like row keys (Adding multiple dimensions would multiply result row count)
// * Currently only TimeDimension and FuncDimension is available
// Metrics are like values, which will be meassured in test execution
// * Most usable metrics are all defined below
HistogramAggregator histogramAggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(10)))
.Add(new MinDurationMetric(IgnoredCheckpoints))
.Add(new AvgDurationMetric(IgnoredCheckpoints))
.Add(new MaxDurationMetric(IgnoredCheckpoints))
.Add(new PercentileMetric(new[] { 0.5, 0.8, 0.9, 0.95, 0.99 }, IgnoredCheckpoints))
.Add(new CountMetric(IgnoredCheckpoints))
.Add(new ErrorCountMetric())
.Add(new FuncMetric <int>("Created Threads", 0, (i, result) => result.CreatedThreads))
.Alias($"Min: {Checkpoint.IterationEndCheckpointName}", "Min (ms)")
.Alias($"Avg: {Checkpoint.IterationEndCheckpointName}", "Avg (ms)")
.Alias($"Max: {Checkpoint.IterationEndCheckpointName}", "Max (ms)")
.Alias($"50%: {Checkpoint.IterationEndCheckpointName}", "50% (ms)")
.Alias($"80%: {Checkpoint.IterationEndCheckpointName}", "80% (ms)")
.Alias($"90%: {Checkpoint.IterationEndCheckpointName}", "90% (ms)")
.Alias($"95%: {Checkpoint.IterationEndCheckpointName}", "95% (ms)")
.Alias($"99%: {Checkpoint.IterationEndCheckpointName}", "99% (ms)")
.Alias($"Count: {Checkpoint.IterationEndCheckpointName}", "Success: Count")
.Alias($"Errors: {Checkpoint.IterationSetupCheckpointName}", "Errors: Setup")
.Alias($"Errors: {Checkpoint.IterationStartCheckpointName}", "Errors: Iteration")
.Alias($"Errors: {Checkpoint.IterationTearDownCheckpointName}", "Errors: Teardown");
return(histogramAggregator);
}
public static void Run()
{
// Proper demo is being written ATM.
// So for time being below a common preset is provided.
// HistogramAggregator is a modular Dimension/Metric style aggregator tool,
// which can be expanded by implementing own custom IMetric or IDimension.
//
// Dimensions are like row keys (Adding multiple dimensions would also multiply result row count)
// * In SQL it would be like GROUP BY function, or GroupBy in Linq
// * Dimensions are optional, which would result data aggregated into single KPI row.
// Metrics are aggregation functions which will aggregate data for each slice presented by Dimensions
// * In SQL it would be aggregation functions like COUNT, SUM, AVERAGE, etc..
// In this preset data is grouped by 10 second periods.
// One can change period below, or totally remove TimeDimension to get KPI result.
HistogramAggregator histogramAggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(10)))
.Add(new MinDurationMetric())
.Add(new AvgDurationMetric())
.Add(new MaxDurationMetric())
.Add(new PercentileMetric(new[] { 0.95, 0.99 }))
.Add(new CountMetric(Checkpoint.NotMeassuredCheckpoints))
.Add(new TransactionsPerSecMetric())
.Add(new ErrorCountMetric())
.Add(new FuncMetric <int>("Created Threads", 0, (i, result) => result.CreatedThreads))
.Alias($"Min: {Checkpoint.Names.Iteration}", "Min (ms)")
.Alias($"Avg: {Checkpoint.Names.Iteration}", "Avg (ms)")
.Alias($"Max: {Checkpoint.Names.Iteration}", "Max (ms)")
.Alias($"95%: {Checkpoint.Names.Iteration}", "95% (ms)")
.Alias($"99%: {Checkpoint.Names.Iteration}", "99% (ms)")
.Alias($"Count: {Checkpoint.Names.Iteration}", "Success: Count")
.Alias($"Errors: {Checkpoint.Names.Setup}", "Errors: Setup")
.Alias($"Errors: {Checkpoint.Names.Iteration}", "Errors: Iteration")
.Alias($"Errors: {Checkpoint.Names.TearDown}", "Errors: Teardown");
}
public static void Run()
{
HistogramAggregator aggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(4)))
.Add(new CountMetric())
.Add(new TransactionsPerSecMetric());
//var kpiAggregator = new KpiPrinterAggregator(
// TimeSpan.FromSeconds(1),
// new MaxDurationMetric(),
// new CountMetric(Checkpoint.NotMeassuredCheckpoints),
// new TransactionsPerSecMetric());
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario(new CountingScenarioFactory())
.SetThreading(new FixedThreadCount(4))
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(13)))
.SetAggregator(aggregator);
strategy.Build().Run();
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
}
public static void Run()
{
IEnumerable <IResult> rawResults = null;
StreamAggregator streamAggregator = new StreamAggregator(results => rawResults = results);
HistogramAggregator aggregator = CreateAggregator();
HistogramAggregator aggregagorOriginal = CreateAggregator();
StrategyBuilder strategy = new StrategyBuilder()
.SetScenario <BlankScenario>()
.SetThreading(new FixedThreadCount(3))
.SetLimit(new TimeLimit(TimeSpan.FromSeconds(5)))
.SetAggregator(streamAggregator, aggregagorOriginal);
strategy.Build().Run();
StreamAggregator.Replay(rawResults, aggregator);
Console.WriteLine(@"-------- FROM LIVE AGGREGATION --------");
Console.WriteLine(JsonConvert.SerializeObject(aggregagorOriginal.BuildResultsObjects(), Formatting.Indented));
Console.WriteLine(@"-------- FROM STREAM --------");
Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented));
}
public static void Run()
{
// LoadRunnerParameters initializes defaults shown below
LoadRunnerParameters loadRunnerParameters = new LoadRunnerParameters
{
Limits = new ExecutionLimits
{
// Maximum LoadTest duration threshold, after which test is stopped
MaxDuration = TimeSpan.FromSeconds(30),
// Maximum executed iterations count threshold, after which test is stopped
MaxIterationsCount = Int32.MaxValue,
// Once LoadTest execution finishes because of [maxDuration] or [maxIterationsCount] limit
// coordinating thread will wait [FinishTimeout] amount of time before
// terminating them with Thread.Abort()
//
// Aborted threads won't get the chance to call IterationTearDown() or ScenarioTearDown()
// neither it will broadcast TestContextResultReceived() to aggregators with the state as it is after abort.
FinishTimeout = TimeSpan.FromSeconds(60)
},
// [ISpeedStrategy] defines maximum allowed load by dampening executed Iterations per second count
// * Other existing version of [ISpeedStrategy]
// - IncremantalSpeed(initialRequestsPerSec: 1.0, increasePeriod: TimeSpan.FromSeconds(10), increaseStep: 3.0)
SpeedStrategy = new FixedSpeed(maxIterationsPerSec: Double.MaxValue),
//[IThreadingStrategy] defines allowed worker thread count
// * SemiAutoThreading initializes [minThreadCount] at begining
// It will be increased if not enough threads are available to reach [ISpeedStrategy] limits
ThreadingStrategy = new IncrementalThreadCount(15, TimeSpan.FromSeconds(10), 15)
};
// Initialize aggregator
string[] ignoredCheckpoints =
{
Checkpoint.IterationSetupCheckpointName,
Checkpoint.IterationStartCheckpointName,
Checkpoint.IterationTearDownCheckpointName
};
HistogramAggregator histogramAggregator = new HistogramAggregator()
.Add(new TimeDimension(TimeSpan.FromSeconds(10)))
.Add(new MinDurationMetric(ignoredCheckpoints))
.Add(new AvgDurationMetric(ignoredCheckpoints))
.Add(new MaxDurationMetric(ignoredCheckpoints))
.Add(new PercentileMetric(new[] { 0.5, 0.8, 0.9, 0.95, 0.99 }, ignoredCheckpoints))
.Add(new CountMetric(ignoredCheckpoints))
.Add(new ErrorCountMetric())
.Add(new FuncMetric <int>("Created Threads", 0, (i, result) => result.CreatedThreads))
.Alias($"Min: {Checkpoint.IterationEndCheckpointName}", "Min (ms)")
.Alias($"Avg: {Checkpoint.IterationEndCheckpointName}", "Avg (ms)")
.Alias($"Max: {Checkpoint.IterationEndCheckpointName}", "Max (ms)")
.Alias($"50%: {Checkpoint.IterationEndCheckpointName}", "50% (ms)")
.Alias($"80%: {Checkpoint.IterationEndCheckpointName}", "80% (ms)")
.Alias($"90%: {Checkpoint.IterationEndCheckpointName}", "90% (ms)")
.Alias($"95%: {Checkpoint.IterationEndCheckpointName}", "95% (ms)")
.Alias($"99%: {Checkpoint.IterationEndCheckpointName}", "99% (ms)")
.Alias($"Count: {Checkpoint.IterationEndCheckpointName}", "Success: Count")
.Alias($"Errors: {Checkpoint.IterationSetupCheckpointName}", "Errors: Setup")
.Alias($"Errors: {Checkpoint.IterationStartCheckpointName}", "Errors: Iteration")
.Alias($"Errors: {Checkpoint.IterationTearDownCheckpointName}", "Errors: Teardown");
JsonStreamAggregator _jsonStreamAggregator = new JsonStreamAggregator(() => DateTime.Now.ToString("HH_mm_ss__ffff") + ".json");
//TotalsResultsAggregator resultsAggregator = new TotalsResultsAggregator();
// Initializing LoadTest Client
//LoadRunnerEngine loadRunner = LoadRunnerEngine.Create<TestScenario>(loadRunnerParameters, histogramAggregator, _jsonStreamAggregator);
LoadRunnerUi loadRunnerUi = LoadRunnerUi.Create <TestScenario>(loadRunnerParameters, histogramAggregator, _jsonStreamAggregator);
Application.Run(loadRunnerUi);
return;
// Run test (blocking call)
//loadRunner.Run();
//loadRunner.RunAsync();
//Console.WriteLine("Async started");
//loadRunner.Wait();
//object defaultResults = histogramAggregator.BuildResultsObjects();
//Console.WriteLine(JsonConvert.SerializeObject(defaultResults, Formatting.Indented));
//Console.ReadKey();
}
