public static TestContext Start(IMessageLogger messageLogger, AdapterSettings settings)
{
var context = new TestContext(messageLogger, settings);
TestContext.context.Value = context;
return(context);
}
public static TestContext Start(PerformanceTestCase testCase, AdapterSettings settings)
{
var context = new TestContext(testCase, settings);
TestContext.context.Value = context;
return(context);
}
private TestContext(IMessageLogger messageLogger, AdapterSettings settings)
{
this.sb = stringBuilderPool.Get();
this.Properties = settings.TestProperties;
IsWarmup = false;
this.messageLogger = messageLogger;
}
private TestContext(PerformanceTestCase testCase, AdapterSettings settings)
{
this.sb = stringBuilderPool.Get();
this.Properties = settings.TestProperties;
IsWarmup = false;
TestCase = testCase;
}
public TestLogger InitSettings(AdapterSettings settings)
{
adapterSettings = settings;
Verbosity = adapterSettings.Verbosity;
return(this);
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
var dir = Directory.GetCurrentDirectory();
var folder = Path.Combine(dir, "TestResults");
Directory.CreateDirectory(folder);
var file = Path.Combine(folder, $"{Environment.UserName}_{Environment.MachineName}_{DateTime.Now:yyyy-MM-dd_HH_mm_ss}.csv");
using (var fs = File.OpenWrite(file))
using (var tw = new StreamWriter(fs))
{
tw.WriteCsvLine("Test Name", "Iteration", "Is Warmup", "Duration", "Iteration Status", "Run Status");
var logger = new TestLogger(frameworkHandle);
var settings = new AdapterSettings(logger);
settings.Load(runContext.RunSettings.SettingsXml);
logger.InitSettings(settings);
frameworkHandle.EnableShutdownAfterTestRun = true;
var toRun = Convert(tests);
var missing = tests.Except(toRun.Select(x => x.testCase));
foreach (var m in missing)
{
frameworkHandle.RecordEnd(m, TestOutcome.NotFound);
}
var lifecycleEvents = TestLifeCyclesCallbacks(tests.Select(x => x.Source).Distinct().ToList());
var beforeAll = lifecycleEvents.OfType <ITestLifecycleBeforeAllTests>().ToArray();
var afterAll = lifecycleEvents.OfType <ITestLifecycleAfterAllTests>().ToArray();
// generate report details for all runs etc of all calls
// parallel etc in here
using (var globalCtx = TestContext.Start(frameworkHandle, settings))
{
foreach (var evnt in beforeAll)
{
evnt.BeforeAllTests(globalCtx).GetAwaiter().GetResult();
}
}
foreach (var t in toRun)
{
var testResult = new TestResult(t.testCase);
if (t.perfTest.Skipped)
{
testResult.Outcome = TestOutcome.Skipped;
frameworkHandle.RecordResult(testResult);
tw.WriteCsvLine(t.perfTest.Name, "-", "-", "-", "Skipped");
continue;
}
frameworkHandle.RecordStart(t.testCase);
using (var context = TestContext.Start(t.perfTest, settings))
{
var sw = Stopwatch.StartNew();
var task = t.perfTest.ExecuteAsync(context);
Task.WaitAll(task);
sw.Stop();
var result = task.Result;
var errors = result.Select(x => x.Error).Where(x => x != null).ToList();
if (errors.Any())
{
testResult.ErrorStackTrace = string.Join("\n\n-------\n\n", errors.Select(x => x.StackTrace));
testResult.ErrorMessage = string.Join("\n\n-------\n\n", errors.Select(x => x.Message));
testResult.Outcome = TestOutcome.Failed;
}
else
{
testResult.Outcome = TestOutcome.Passed;
}
int counter = 0;
foreach (var r in result.Where(x => x.IsWarmup))
{
tw.WriteCsvLine(t.perfTest.Name, ++counter, r.IsWarmup, r.Duration.TotalSeconds, r.Error == null ? TestOutcome.Passed : TestOutcome.Failed, testResult.Outcome);
}
counter = 0;
foreach (var r in result.Where(x => !x.IsWarmup))
{
tw.WriteCsvLine(t.perfTest.Name, ++counter, r.IsWarmup, r.Duration.TotalSeconds, r.Error == null ? TestOutcome.Passed : TestOutcome.Failed, testResult.Outcome);
}
// process the results here
testResult.Duration = sw.Elapsed;
var runs = result.Where(x => x.IsWarmup == false).Select(x => x.Duration);
var warmups = result.Where(x => x.IsWarmup == true).Select(x => x.Duration);
var mean = TimeSpanStatistics.Mean(runs);
var standardDeviation = TimeSpanStatistics.StandardDeviation(runs);
// format a table of output results here
var msg = $@"Warm up Count : {warmups.Count()}
Warm up Duration : {new TimeSpan(warmups.Sum(x => x.Ticks))}
Executed : {runs.Count()}
Mean Duration: {mean}
Standard Deviation Duration: {standardDeviation}
";
testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, msg));
testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, context.Output));
foreach (var r in result.Where(x => !string.IsNullOrWhiteSpace(x.Output)))
{
testResult.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, r.Output));
}
frameworkHandle.RecordResult(testResult);
}
}
using (var globalCtx = TestContext.Start(frameworkHandle, settings))
{
foreach (var evnt in afterAll)
{
evnt.AfterAllTests(globalCtx).GetAwaiter().GetResult();
}
}
}
}