/// <summary>
/// Generate new threshold colorcode values for all intermediate rows
/// </summary>
/// <param name="greenColorcode"></param>
/// <param name="yellowColorcode"></param>
/// <param name="redColorcode"></param>
public void GenerateThresholdValues(string greenColorcode, string yellowColorcode, string redColorcode, bool storeMetrics = false)
{
Log.WriteLine("generate threshold values...");
// load threshold config from database
Thresholds thresholds = new Thresholds(project);
// generate color values
Intermediate thValues = thresholds.GenerateThresholdValuesForTransactions(intermediate, greenColorcode, yellowColorcode, redColorcode);
// merge threshold colortransactions with dataset
Log.WriteLine(string.Format("adding {0} threshold entries...", thValues.Count));
intermediate.Add(thValues);
// count threshold violations (add in separate series) by transactionname patter + red color
Log.WriteLine("aggregate threshold violations...");
Intermediate thresholdViolations = thValues.AggregateCount(THRESHOLDVIOLATIONSKEY, @"\d\d_.*_c$", redColorcode); // only evaluate script transactions!
// store these newly generated metrics back to the database
if (storeMetrics)
{
thresholdViolations.SaveToDatabase(this.project, this.testrun, Category.Transaction, Entity.None);
}
Log.WriteLine("adding threshold violations: " + thresholdViolations.GetValue(THRESHOLDVIOLATIONSKEY));
this.intermediate.Add(thresholdViolations);
}
/// <summary>
/// Identify break in ramp up trend
/// </summary>
private void ExtractRampupBreak()
{
Log.WriteLine("calculating ramp up break on [" + OVERALLTRANSACTIONSKEY + "] by reference [" + OVERALLUSERSKEY + "]");
try
{
// get timeseries
// double[] throughput_values = _measureDetails.GetValuesAsDoubleArray(OVERALLTRANSACTIONSKEY);
double[] responsetimes_values = _measureDetails.GetValuesAsDoubleArray(OVERALLRESPONSETIMEKEY);
double[] users_values = _measureDetails.GetValuesAsDoubleArray(OVERALLUSERSKEY);
TrendAnalyzer trendAnalyzer = new TrendAnalyzer();
trendAnalyzer.ReferenceSeries = users_values;
// search for trend break if stable ramp-up is expected (stress testing)
// trendAnalyzer.DetectTrendBreak_Rampup_Throughput(throughput_values); // based on stable rise of throughput (needs more complexity, first try alternative)
trendAnalyzer.DetectTrendBreak_Rampup_Responsetimes(responsetimes_values); // based on stability of response times
_variables.Add(TRENDBREAKRAMPUPPRCKEY, trendAnalyzer.GetBreakPercentage_Reference().ToString("0"));
_variables.Add(TRENDBREAKRAMPUPUSRKEY, trendAnalyzer.GetBreakReferenceValue().ToString());
Log.WriteLine(string.Format("trend break RAMP UP: break on percentage={0:0}% users={1}", trendAnalyzer.GetBreakPercentage_Reference(), trendAnalyzer.GetBreakReferenceValue()));
}
catch (Exception e)
{
Log.WriteLine("ERROR calculation of ramp-up trend break failed");
Log.WriteLine(e.Message);
}
}
public const string DURATIONTIMEFORMAT = @"hh\:mm\:ss"; // time format (testrun duration)
public void Parse(ParamInterpreter p)
{
ReadLinesFromFile(p);
// rptgendatetime, no data needed
ParseRptGenDateTime();
_variables.Add(PRINTABLETIMEKEY, ParsePrintableTime()); // for backward compatibility
Log.WriteLine(PRINTABLETIMEKEY + "=" + _variables[PRINTABLETIMEKEY]);
_variables.Add(TESTRUNDATETIMEKEY, _variables[PRINTABLETIMEKEY]); // duplicate PrintableTime to proper variable name
Log.WriteLine(TESTRUNDATETIMEKEY + "=" + _variables[TESTRUNDATETIMEKEY]);
_variables.Add(TESTDURATIONKEY, ParseTestDuration(_variables[TESTRUNDATETIMEKEY]));
Log.WriteLine(TESTDURATIONKEY + "=" + _variables[TESTDURATIONKEY]);
_variables.Add(MEREGEDUSERSKEY, ParseMergedUsers());
Log.WriteLine(MEREGEDUSERSKEY + "=" + _variables[MEREGEDUSERSKEY]);
_variables.Add(LGTYPEKEY, ParseLoadgenType());
Log.WriteLine(LGTYPEKEY + "=" + _variables[LGTYPEKEY]);
_variables.Add(FAULTPERCENTAGEKEY, ParseFaultPercentage());
Log.WriteLine(FAULTPERCENTAGEKEY + "=" + _variables[FAULTPERCENTAGEKEY]);
WriteIntermediate(p.Value("intermediatefile"));
}
/// <summary>
/// Perform baselining and generate extra baseline tags
/// ${Appbeheer_01_Inloggen_br // reference (baseline value)
/// ${Appbeheer_01_Inloggen_be // evaluated value (difference from baseline in %)
/// ${Appbeheer_01_Inloggen_be_c // colorcode, mark when current value exceeds threshold th1, green when diff > -15%, red when diff > +15%
/// </summary>
/// <param name="currentTestrun"></param>
/// <param name="baselineTestrun"></param>
/// <param name="colorcodeBetter"></param>
/// <param name="colorcodeWorse"></param>
public void GenerateBaselineValues(string currentTestrun, string baselineTestrun, string colorcodeBetter, string colorcodeWorse, bool storeMetrics = false)
{
Thresholds thresholds = new Thresholds(this.project);
// First, collect current and baseline run info and threshold violations:
// lees baseline data, kies alternatief als baseline run niet gezet
Log.WriteLine("read baseline values...");
Intermediate baselineValues = ReadBaselineData(currentTestrun, baselineTestrun);
Log.WriteLine("generate threshold evaluation on baseline run...");
Intermediate baselineThresholdValues = thresholds.GenerateThresholdValuesForTransactions(baselineValues, Baselining.belowLowThreshold, "y", "r");
// baseline: merge values with threshold colorcodes
baselineThresholdValues.Add(baselineValues);
Log.WriteLine("generate threshold evaluation on current run...");
Intermediate currentThresholdValues = thresholds.GenerateThresholdValuesForTransactions(this.intermediate, Baselining.belowLowThreshold, "y", "r");
// current testrun: merge values (this.intermediate) with threshold colorcodes
currentThresholdValues.Add(this.intermediate);
// Second, compare current run with baseline run:
Log.WriteLine("generate baseline evaluation values...");
Baselining baselining = new Baselining();
Intermediate baselineEvaluation = baselining.GenerateBaselineEvaluationValues(currentThresholdValues, baselineThresholdValues, baselineValues, colorcodeBetter, colorcodeWorse);
this.intermediate.Add(baselineEvaluation);
Log.WriteLine("entries: " + baselineEvaluation.Count);
// Third: generate aggregated metrics
// Count baseline violations (add in separate series)
Log.WriteLine("aggregate baseline warnings...");
Intermediate baselineWarnings = baselineEvaluation.AggregateCount(BASELINEWARNINGSKEY, @"\d\d_.*_be_c", colorcodeWorse); // only evaluate script transactions!
Log.WriteLine("adding baseline warnings: " + baselineWarnings.GetValue(BASELINEWARNINGSKEY));
this.intermediate.Add(baselineWarnings);
// store generated high-level stats
if (storeMetrics)
{
// store baseline warning variables
baselineWarnings.SaveToDatabase(this.project, this.testrun, Category.Transaction, Entity.None);
// store calculated baseline reference chosen
this.intermediate.SaveOneToDatabase(this.project, this.testrun, Category.Variable, Entity.Generic, BASELINEREFVARNAME);
}
}
/// <summary>
/// Generate aggregate data (total faults, average of percentiles...)
/// </summary>
/// <param name="parameters"></param>
public void GenerateAggregates(ParamInterpreter parameters)
{
int cnt = 0;
Log.WriteLine("generate aggregated data...");
// scrape and aggregate data from transaction data
TransactionValueAggregate transactionAggregate = new TransactionValueAggregate();
// foreach transactionlines (evaluate only successful transactions, leave faulty ones away from aggregation)
foreach (string transactionName in _transactionNames)
{
// only include in agggregation if transactionname matches 'report transacton name pattern'
if (IsSummarizeTransaction(transactionName) && _transactionDetails.items.ContainsKey(transactionName))
{
TransactionValue trs = new TransactionValue(_transactionDetails.items[transactionName]);
transactionAggregate.AddTransaction(trs);
cnt++;
}
}
// give up if no summarizable transactions found (try to prevent weird crashes lateron)
if (0 == cnt)
{
string msg = "no transaction names found that meet naming convention (" + REPORTTRANSACTIONNAMEPATTERN + ")";
Log.WriteLine("SEVERE: " + msg);
throw new Exception(msg);
}
transactionAggregate.Aggregate();
// write aggregated transaction line
_transactionDetails.Add(AGGREGATEDTRSNAME, transactionAggregate.ToString());
// isolate most important values as variable for history graph -> measure format (later to variable category?)
_variables.Add(AVGOF90PERCENTILES, Utils.ToMeasureFormatString(transactionAggregate.p90));
_variables.Add(AVGOFMEDIAN, Utils.ToMeasureFormatString(transactionAggregate.median));
_variables.Add(AVGOF95PERCENTILES, Utils.ToMeasureFormatString(transactionAggregate.p95));
_variables.Add(AVGOFAVERAGES, Utils.ToMeasureFormatString(transactionAggregate.avg));
_variables.Add(TRANSACTIONSFAILED, transactionAggregate.fail);
// old, can be deleted in time (replaced by transactionssuccess)
// _variables.Add(TRANSACTIONSTOTAL, transactionAggregate.cnt);
_variables.Add(TRANSACTIONSTOTAL, AddIntStrings(transactionAggregate.cnt, transactionAggregate.fail));
_variables.Add(TRANSACTIONSSUCCESS, transactionAggregate.cnt); // is new
Log.WriteLine(string.Format("{0} of {1} transactions aggregated", cnt, _transactionNames.Length));
}