public static void Run() { HistogramAggregator histogram = new HistogramAggregator() .Add(new FuncDimension("ThreadIterationId", r => r.ThreadIterationId.ToString())) .Add(new DistinctListMetric <IResult, string>("Sleeps", r => $"{r.ThreadId}:{((int) r.UserData)}")) .Add(new DistinctListMetric <IResult, string>("TStarts", r => $"{r.ThreadId}:{(int)r.IterationStarted.TotalMilliseconds}")) .Add(new DistinctListMetric <IResult, string>("TEnds", r => $"{r.ThreadId}:{(int)r.IterationFinished.TotalMilliseconds}")) .Add(new FilterMetric <IResult>(r => r.ThreadId == 0, new ValueMetric <IResult>("Tr#0", r => (int)r.IterationFinished.TotalMilliseconds))) .Add(new FilterMetric <IResult>(r => r.ThreadId == 1, new ValueMetric <IResult>("Tr#1", r => (int)r.IterationFinished.TotalMilliseconds))) .Add(new FilterMetric <IResult>(r => r.ThreadId == 2, new ValueMetric <IResult>("Tr#2", r => (int)r.IterationFinished.TotalMilliseconds))) .Add(new FilterMetric <IResult>(r => r.ThreadId == 3, new ValueMetric <IResult>("Tr#3", r => (int)r.IterationFinished.TotalMilliseconds))) .Add(new FuncMetric <int>("Min TStart", Int32.MaxValue, (i, r) => Math.Min((int)r.IterationStarted.TotalMilliseconds, i))) .Add(new FuncMetric <int>("Max Sleep", -1, (i, r) => Math.Max(((int)r.UserData), i))) .Add(new FuncMetric <int>("Max TEnd", Int32.MinValue, (i, r) => Math.Max((int)r.IterationFinished.TotalMilliseconds, i))); StrategyBuilder strategy = new StrategyBuilder() .SetScenario <BatchAndWaitDemo>() .AddSpeed(new BatchBySlowestSpeed(ThreadCount)) .SetThreading(new FixedThreadCount(ThreadCount)) .SetLimit(new TimeLimit(TimeSpan.FromSeconds(20))) .SetAggregator(histogram); strategy.Build().Run(); Console.WriteLine(JsonConvert.SerializeObject(histogram.BuildResultsObjects(), Formatting.Indented)); }
public static void Run() { // Initialize LoadRunnerEngine by providing: // * Type of class which implements ILoadTestScenario (e.g DemoTestScenario) // * LoadRunnerParameters // * As many aggregators as you like HistogramAggregator histogramAggregator = AggregationSetup.Build(); LoadRunnerParameters parameters = ParametersSetup.Build(); LoadRunnerEngine loadRunner = LoadRunnerEngine.Create <DemoTestScenario>(parameters, histogramAggregator); // Run test (blocking call) loadRunner.Run(); // Once finished, extract information from used aggregators, and do some exceling :) // BuildResultsObjects() will produce results in structure compatible with JSON -> CSV converters. (See ~20 lines below) IEnumerable <object> defaultResults = histogramAggregator.BuildResultsObjects(); Console.WriteLine(JsonConvert.SerializeObject(defaultResults, Formatting.Indented)); Console.ReadKey(); //Alternative export way is // HistogramResults results = histogramAggregator.BuildResults(); // // results will be presented in this structure: // * string[] ColumnNames; // * object[][] Values; }
// Registering at least one aggregation forces worker threads to copy/produce measurements after each iteration. // Registering many aggregations shouldn't impact test performance more compared to just a single aggregation. // // // While this test can achieve 1+million/sec speed (w/o debugger and release build): // current HistogramAggregator implementation won't be able to process this massive amount of incoming data in real time. // as result of it, data gets queued up in memory and this test will easily eat ~5+ GB's of ram in first test run, // so be sure that there is enough of RAM available before running this one. // // This also shows how Run() won't exit past 13seconds. While test is no longer running, engine waits until all data gets processed. // // This example runs test twice: // * One with simple histogram setup // * And other without histogram but receiving a direct stream of raw measurements and just counting them. // - This one should be able to count results in the real time public static void Run() { TimeSpan duration = TimeSpan.FromSeconds(13); // 2x tests HistogramAggregator aggregator = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(4))) .Add(new CountMetric()) .Add(new TransactionsPerSecMetric()) .Add(new GlobalTimerPeriodMetric()); StrategyBuilder strategy = new StrategyBuilder() .SetScenario <BlankScenario>() .SetThreading(new FixedThreadCount(4)) .SetLimit(new TimeLimit(duration)) .SetAggregator(aggregator); TimeSpan durationWithHistogram = MeassuredRun(strategy.Build()); // Ease use of ram ASAP. GC.Collect(1, GCCollectionMode.Forced, true); int iterations = 0; strategy.SetAggregator(new StreamAggregator(result => iterations = iterations + 1)); TimeSpan durationWithStream = MeassuredRun(strategy.Build()); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); Console.WriteLine($"Time took for [HistogramAggregator] to process whats left in the buffer: {durationWithHistogram - duration:g}"); Console.WriteLine($"Time took for [StreamAggregator] to count whats left in the buffer: {durationWithStream - duration:g}"); }
public static void Run() { HistogramAggregator aggregator = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(10))) .Add(new FuncMetric <int>("Working Threads", 0, (i, r) => Math.Max(r.CreatedThreads - r.IdleThreads, i))) .Add(new CountMetric(Checkpoint.NotMeassuredCheckpoints)) .Add(new TransactionsPerSecMetric()); KpiPrinterAggregator kpi = new KpiPrinterAggregator( TimeSpan.FromSeconds(4), new FuncMetric <string>("T+", "???", (s, result) => result.IterationFinished.ToString("g")), new FuncMetric <int>("Created Threads", 0, (i, r) => Math.Max(r.CreatedThreads, i)), new FuncMetric <int>("Working Threads", 0, (i, r) => Math.Max(r.CreatedThreads - r.IdleThreads, i)), new CountMetric(Checkpoint.NotMeassuredCheckpoints), new CountMetric(i => i / 2.0, Checkpoint.NotMeassuredCheckpoints) { Prefix = "TPS " } ); StrategyBuilder strategy = new StrategyBuilder() .SetScenario <LimitConcurrencyAndTpsDemo>() .AddSpeed(new LimitWorkingThreads(11)) .AddSpeed(new IncrementalSpeed(1, TimeSpan.FromSeconds(20), 10)) .SetThreading(new IncrementalThreadCount(250, TimeSpan.FromSeconds(10), -50)) .SetLimit(new TimeLimit(TimeSpan.FromSeconds(52))) .SetAggregator(aggregator, kpi); strategy.Build().Run(); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); }
// HDD/SSD intensive, beware public static void Run() { HistogramAggregator aggregator = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(2))) .Add(new CountMetric()) .Add(new TransactionsPerSecMetric()); HistogramAggregator aggregagorOriginal = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(2))) .Add(new CountMetric()) .Add(new TransactionsPerSecMetric()); StrategyBuilder strategy = new StrategyBuilder() .SetScenario <BlankScenario>() .SetThreading(new FixedThreadCount(3)) .SetLimit(new TimeLimit(TimeSpan.FromSeconds(5))) .SetAggregator(new JsonStreamAggregator("Raw.json"), aggregagorOriginal); strategy.Build().Run(); JsonStreamAggregator.Replay("Raw.json", aggregator); Console.WriteLine(JsonConvert.SerializeObject(aggregagorOriginal.BuildResultsObjects(), Formatting.Indented)); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); }
public static void Run() { HistogramAggregator aggregator = new HistogramAggregator() .Add(new FuncDimension("Iteration", result => result.GlobalIterationId.ToString())) .Add(new FuncDimension("Time", result => result.IterationStarted.TotalSeconds.ToString(CultureInfo.InvariantCulture))) .Add(new FuncDimension("Data", result => result.UserData?.ToString())) .Add(new FuncMetric <int>("CThreads", 0, (i, r) => r.CreatedThreads)) .Add(new FuncMetric <int>("IThreads", 0, (i, r) => r.IdleThreads)); ReplayStrategyBuilder <string> settings = new ReplayStrategyBuilder <string>() .SetAggregator(aggregator) .SetData(DataGenerator.Create(5, 1, 3, 3).ToArray()) .SetScenario <ReplayScenario>() .SetThreadCount(30) .SetSpeed(1); // UI //LoadRunnerUi engineUi = settings.BuildUi(new DataItem(TimeSpan.Zero, "Validation demo")); //engineUi.StartWindow(); // Non UI blocking LoadRunnerEngine engine = settings.Build(); //engine.Run(); // Non UI Async engine.RunAsync(); engine.Wait(); object defaultResults = aggregator.BuildResultsObjects(); Console.WriteLine(JsonConvert.SerializeObject(defaultResults, Formatting.Indented)); //Console.ReadKey(); }
public static void Run() { // Initialize aggregator string[] ignoredCheckpoints = { Checkpoint.Names.Setup, Checkpoint.Names.TearDown }; HistogramAggregator histogramAggregator = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(10))) .Add(new FuncMetric <TimeSpan>("TMin", TimeSpan.MaxValue, (span, result) => span > result.IterationStarted ? result.IterationStarted : span)) .Add(new FuncMetric <TimeSpan>("TMax", TimeSpan.MinValue, (span, result) => span < result.IterationFinished ? result.IterationFinished : span)) .Add(new FuncMetric <int>("Working Threads", 0, (i, result) => result.CreatedThreads - result.IdleThreads)) //.Add(new MinDurationMetric(ignoredCheckpoints)) .Add(new AvgDurationMetric(ignoredCheckpoints)) .Add(new MaxDurationMetric(ignoredCheckpoints)) //.Add(new PercentileMetric(new[] {0.99999}, ignoredCheckpoints)) .Add(new CountMetric(ignoredCheckpoints)) .Add(new TransactionsPerSecMetric()) .Add(new ErrorCountMetric()) .Alias($"Min: {Checkpoint.Names.Iteration}", "Min (ms)") .Alias($"Avg: {Checkpoint.Names.Iteration}", "Avg (ms)") .Alias($"Max: {Checkpoint.Names.Iteration}", "Max (ms)") .Alias($"50%: {Checkpoint.Names.Iteration}", "50% (ms)") .Alias($"80%: {Checkpoint.Names.Iteration}", "80% (ms)") .Alias($"90%: {Checkpoint.Names.Iteration}", "90% (ms)") .Alias($"95%: {Checkpoint.Names.Iteration}", "95% (ms)") .Alias($"99.99%: {Checkpoint.Names.Iteration}", "99.99% (ms)") .Alias($"Count: {Checkpoint.Names.Iteration}", "Success: Count"); StrategyBuilder strategy = new StrategyBuilder() .SetScenario <TestScenario>() .SetLimit(new TimeLimit(TimeSpan.FromSeconds(10))) .SetThreading(new FixedThreadCount(40)) .SetSpeed(new FixedSpeed(2000)) // Tps is lower in results due to failed iterations not being counted .SetFinishTimeout(TimeSpan.FromSeconds(60)) .SetAggregator(histogramAggregator); IStrategyExecutor engine = strategy.Build(); engine.Run(); object defaultResults = histogramAggregator.BuildResultsObjects(); Console.WriteLine(JsonConvert.SerializeObject(defaultResults, Formatting.Indented)); }
// All IAggregator implementations receive un-aggregated raw measurements (later raw-data) from workers. // - (TODO: More info in IAggregator customization, But IAggregator ir pretty self explanatory anyway :)) // Main idea behind Raw-Data is that one gets un-aggregated measurements in IResult form // and one can do anything with it, preferably post-load-test. // // // Having this unlocks few advantages: // * Save this raw-data somewhere(file on disk, array in memory) and do any aggregations after the test. // - I personally persist raw-data from all of the tests I do, // because one will never know what aggregation might be needed until one runs the test and sees initial results. // - There is a RnD JsonStreamAggregator which saves to JSON array file. Though it can take up some space on disk as JSON has lots of overhead. // * Or one can write own custom specialized aggregation ignoring provided HistogramAggregator totally. // // // This demo contains: // * Load-test with one aggregation: // - [Tools nuget: JsonStreamAggregator] Write data to raw.json file for later aggregation // * Replay of raw measurements after the test and do some more aggregating post-load-test. // - Two histogram aggregators: Timeline and KPI. // - One StreamAggregator instance which gives IEnumerable of IResult's for custom inline aggregation. public static void Run() { // JsonStreamAggregator dumps everything it receives to file. JsonStreamAggregator jsonAggregator = new JsonStreamAggregator("raw.json"); StrategyBuilder builder = new StrategyBuilder() .AddLimit(new TimeLimit(TimeSpan.FromSeconds(4))) .SetThreading(new FixedThreadCount(4)) .SetScenario(new SleepingScenarioFactory(TimeSpan.FromMilliseconds(100))) // Use scenario from common which sleeps every iteration .SetAggregator(jsonAggregator); // Register JsonAggregator to be only one to receive results // A quick run builder.Build().Run(); // Now lets do actual aggregation post testing. // First - just configure HistogramAggregator or any other aggregator in same way one would do for the live test. HistogramAggregator histogramTimeline = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(1))) .Add(new CountMetric()) .Add(new GlobalTimerMinValueMetric()) .Add(new GlobalTimerMaxValueMetric()) .Add(new GlobalTimerPeriodMetric()); HistogramAggregator histogramKpi = new HistogramAggregator() .Add(new CountMetric()) .Add(new GlobalTimerMinValueMetric()) .Add(new GlobalTimerMaxValueMetric()) .Add(new GlobalTimerPeriodMetric()); // Lets define another more specialized aggregation just for lols. TimeSpan lastIterationEnded = TimeSpan.Zero; StreamAggregator streamAggregator = new StreamAggregator( results => { lastIterationEnded = results.Max(r => r.IterationFinished); } ); // now just replay saved raw-data stream to those later-defined aggregations. JsonStreamAggregator.Replay("raw.json", histogramTimeline, histogramKpi, streamAggregator); Console.WriteLine($"StreamAggregator, last iteration finished: {lastIterationEnded:g}"); Console.WriteLine("---------------- Histogram KPI -----------"); Console.WriteLine(JsonConvert.SerializeObject(histogramKpi.BuildResultsObjects(), Formatting.Indented)); Console.WriteLine("---------------- Histogram timeline -----------"); Console.WriteLine(JsonConvert.SerializeObject(histogramTimeline.BuildResultsObjects(), Formatting.Indented)); }
// *** private static void AggregateFromMultipleSources() { IEnumerable <ReplayResult <object> > pc1 = JsonStreamAggregator.Load <object>("masterdata.json"); IEnumerable <ReplayResult <object> > pc2 = JsonStreamAggregator.Load <object>("masterdata.json"); IEnumerable <ReplayResult <object> > merged = pc1.Concat(pc2); HistogramAggregator aggregator = new HistogramAggregator() .Add(new CountMetric()) .Add(new TransactionsPerSecMetric()); StreamAggregator.Replay(merged); Console.WriteLine("## Merged demo ##"); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); Console.ReadKey(); }
public static void Run() { // [2] Results aggregation (Or raw measurement collection, see RawDataMeasurementsDemo.cs) // Define how data gets aggregated. // Dimensions are like GROUP BY keys in SQL // Metrics are aggregation functions like COUNT, SUM, etc.. // Extensive HistogramAggregator demo now WiP HistogramAggregator aggregator = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(5))) .Add(new CountMetric()) .Add(new ErrorCountMetric()) .Add(new TransactionsPerSecMetric()) .Add(new PercentileMetric(0.95, 0.99)); // Secondary aggregation just to monitor key metrics. KpiPrinterAggregator kpiPrinter = new KpiPrinterAggregator( TimeSpan.FromSeconds(5), new CountMetric(Checkpoint.NotMeassuredCheckpoints), new ErrorCountMetric(false), new TransactionsPerSecMetric() ); // [3] Execution settings // Using StrategyBuilder put defined aggregation, scenario, and execution settings into one object StrategyBuilder strategy = new StrategyBuilder() .SetAggregator(aggregator, kpiPrinter) // Optional .SetScenario <BlankScenario>() // Required .SetLimit(new TimeLimit(TimeSpan.FromSeconds(20))) // Optional, but if not provided, execution will never stop - unless running test with RunAsync() and stopping later with CancelAsync(true) .SetSpeed(new FixedSpeed(100000)) // Optional (Skip for maximum throughput) .SetThreading(new FixedThreadCount(4)); // Required // [4] Execution // All that's left is Build(), run and wait for completion and print out measured results. LoadRunnerEngine engine = strategy.Build(); engine.Run(); IEnumerable <object> result = aggregator.BuildResultsObjects(); Console.WriteLine(JsonConvert.SerializeObject(result, Formatting.Indented)); }
public static void Run() { HistogramAggregator aggregator = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(5))) .Add(new CountMetric()) .Add(new GlobalTimerPeriodMetric()) .Add(new TransactionsPerSecMetric()); StrategyBuilder strategy = new StrategyBuilder() .SetScenario <BatchStrategyDemo>() .AddSpeed(new BatchByTimeIntervalSpeed(TimeSpan.FromSeconds(5), 10)) .SetThreading(new FixedThreadCount(15)) .SetLimit(new TimeLimit(TimeSpan.FromSeconds(20))) .SetAggregator(aggregator); strategy.Build().Run(); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); }
public static void Aggregate() { // Feature disabled? if (!File.Exists("masterdata.json")) { return; } // Any random aggregator HistogramAggregator aggregator = new HistogramAggregator() .Add(new FuncDimension("Created Threads", i => i.CreatedThreads.ToString())) .Add(new CountMetric()); // [TUserData] - Type of UserData object in your tests (If not using it, leave object) // Correct type is required to prevent deserialization problems. JsonStreamAggregator.Replay <object>("masterdata.json", aggregator); Console.WriteLine("## JsonStreamAggregator demo ##"); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); Console.ReadKey(); }
public static void Run() { HistogramAggregator aggregator = new HistogramAggregator() .Add(new CountMetric()) .Add(new FuncMetric <TimeSpan>("max", TimeSpan.Zero, (s, result) => s < result.IterationStarted ? result.IterationStarted : s)); AssertIterationIdsAggregator idsValidator = new AssertIterationIdsAggregator(); int streamCount = 0; StreamAggregator streamAggregator = new StreamAggregator(results => streamCount = results.Count()); CountingScenarioFactory factory = new CountingScenarioFactory(); StrategyBuilder strategy = new StrategyBuilder() .SetScenario(factory) .SetThreading(new FixedThreadCount(16)) .SetLimit(new TimeLimit(TimeSpan.FromSeconds(12))) .SetAggregator(aggregator, idsValidator, streamAggregator); Stopwatch sw = new Stopwatch(); sw.Start(); strategy.Build().Run(); sw.Stop(); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); factory.PrintSum(); int expected = factory.GetSum(); int actual = (int)aggregator.BuildResults().Data[0][1]; TimeSpan lastIteration = (TimeSpan)aggregator.BuildResults().Data[0][3]; Console.WriteLine($@"TPS {expected / lastIteration.TotalSeconds:N}"); Console.WriteLine($@"Last iteration ended at: {lastIteration:g}"); Console.WriteLine($@"Aggregator catchup took: {(sw.Elapsed-lastIteration):g}"); Console.WriteLine(); Console.WriteLine($@"{expected}/{actual} & {streamCount} ? {expected==actual} && {expected==streamCount}"); Console.WriteLine(); idsValidator.PrintResults(); }
public static void Run() { HistogramAggregator aggregator = new HistogramAggregator() .Add(new TimeDimension(TimeSpan.FromSeconds(4))) .Add(new CountMetric()) .Add(new TransactionsPerSecMetric()); //var kpiAggregator = new KpiPrinterAggregator( // TimeSpan.FromSeconds(1), // new MaxDurationMetric(), // new CountMetric(Checkpoint.NotMeassuredCheckpoints), // new TransactionsPerSecMetric()); StrategyBuilder strategy = new StrategyBuilder() .SetScenario(new CountingScenarioFactory()) .SetThreading(new FixedThreadCount(4)) .SetLimit(new TimeLimit(TimeSpan.FromSeconds(13))) .SetAggregator(aggregator); strategy.Build().Run(); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); }
public static void Run() { IEnumerable <IResult> rawResults = null; StreamAggregator streamAggregator = new StreamAggregator(results => rawResults = results); HistogramAggregator aggregator = CreateAggregator(); HistogramAggregator aggregagorOriginal = CreateAggregator(); StrategyBuilder strategy = new StrategyBuilder() .SetScenario <BlankScenario>() .SetThreading(new FixedThreadCount(3)) .SetLimit(new TimeLimit(TimeSpan.FromSeconds(5))) .SetAggregator(streamAggregator, aggregagorOriginal); strategy.Build().Run(); StreamAggregator.Replay(rawResults, aggregator); Console.WriteLine(@"-------- FROM LIVE AGGREGATION --------"); Console.WriteLine(JsonConvert.SerializeObject(aggregagorOriginal.BuildResultsObjects(), Formatting.Indented)); Console.WriteLine(@"-------- FROM STREAM --------"); Console.WriteLine(JsonConvert.SerializeObject(aggregator.BuildResultsObjects(), Formatting.Indented)); }