public async Task Experiment_ShouldBeFinishedWithoutAnyExceptions(string path, ExperimentInfo experimentInfo)
{
// Arrange
_outputHelper.WriteLine($"Running test with data from '{path}'");
var experiment = new Experiment
{
Id = Guid.NewGuid(),
TestsAmount = 1
};
var generatorMock = new Mock <IExperimentGenerator>();
generatorMock.Setup(g =>
g.GenerateDataForTest(It.Is <Experiment>(ex => ex.Id == experiment.Id), It.IsAny <int>()))
.Returns(() => experimentInfo);
ExperimentInfo resultInfo = null;
var experimentTestResultService = new Mock <IExperimentTestResultService>();
experimentTestResultService
.Setup(e => e.SaveExperimentTestResult(It.Is <Guid>(id => id == experiment.Id),
It.IsAny <ExperimentInfo>()))
.Callback <Guid, ExperimentInfo>((id, info) => resultInfo = info);
ExperimentReport experimentReport = null;
var reportServiceMock = new Mock <IReportsService>();
reportServiceMock.Setup(x => x.Save(It.Is <ExperimentReport>(r => r.ExperimentId == experiment.Id)))
.Callback <ExperimentReport>((report) => experimentReport = report);
var experimentPipeline = new ExperimentPipeline(
generatorMock.Object,
Mock.Of <IWorkerExperimentService>(),
experimentTestResultService.Object,
reportServiceMock.Object,
Mock.Of <ILogger <ExperimentPipeline> >(),
Mock.Of <IOptions <DbSettings> >(),
new OnlineExecutor());
// Act
await experimentPipeline.Run(new[]
{
experiment
}, false, true);
// Assert
resultInfo.Should().NotBeNull();
experimentReport.Should().NotBeNull();
}
public static void Main(string[] args)
{
var experiment = ReadExperimentFromConsole();
if (experiment == null)
{
return;
}
var reportsServiceMock = new Mock <IReportsService>();
ExperimentReport report = null;
reportsServiceMock
.Setup(r => r.Save(It.Is <ExperimentReport>(e => e.ExperimentId == experiment.Id)))
.Callback <ExperimentReport>(rep => report = rep);
var pipeline = new ExperimentPipeline(
new ExperimentGenerator(new RandomizeService()),
Mock.Of <IWorkerExperimentService>(),
new ExperimentTestResultFileService(),
reportsServiceMock.Object,
Mock.Of <ILogger <ExperimentPipeline> >(),
Mock.Of <IOptions <DbSettings> >(),
new OnlineExecutor());
var stopwatch = Stopwatch.StartNew();
pipeline.Run(new List <Experiment> {
experiment
}).ConfigureAwait(false);
stopwatch.Stop();
Console.ForegroundColor = ConsoleColor.DarkGreen;
Console.WriteLine("\n\n--------- RESULTS OF EXPERIMENTS' EXECUTION ---------");
Console.WriteLine(
$"Time of execution experiments: {TimeSpan.FromMilliseconds(stopwatch.ElapsedMilliseconds):%m' minute(s) '%s' second(s)'}\n\n");
Console.WriteLine($"Amounts of tests: {experiment.TestsAmount}");
Console.WriteLine(
$"Amounts of resolved tests in offline mode: {experiment.Results.Count}, {experiment.Results.Count * 100 / (double) experiment.TestsAmount:0.###}%");
Console.WriteLine($"Amounts of resolved tests in STOP 1.1: {report?.Stop1Percentage}%");
Console.WriteLine($"Amounts of resolved tests in STOP 1.2: {report?.Stop2Percentage}%");
Console.WriteLine($"Amounts of resolved tests in STOP 1.3: {report?.Stop3Percentage}%");
Console.WriteLine($"Amounts of resolved tests in STOP 1.4: {report?.Stop4Percentage}%");
Console.ReadKey();
public void Save(ExperimentReport report)
{
ResultRepository.Add(Mapper.Map <ExperimentResult>(report));
}
private async Task RunTests(Experiment experiment, bool stopOnException = false)
{
var experimentReport = new ExperimentReport
{
ExperimentId = experiment.Id
};
var stop1 = 0;
var stop2 = 0;
var stop3 = 0;
var stop4 = 0;
var downtimeAmount = 0;
var sumOfDeltaCmax = 0.0;
var deltaCmaxMax = 0.0f;
var offlineResolvedConflictAmount = 0;
var onlineResolvedConflictAmount = 0;
var onlineUnResolvedConflictAmount = 0;
var onlineExecutionTimeInMilliseconds = 0.0d;
var offlineExecutionTimeInMilliseconds = 0.0d;
var aggregationResult = new Dictionary <int, ResultInfo>();
foreach (var testRun in Enumerable.Range(0, experiment.TestsAmount))
{
using var _ = Logger.BeginScope("{TestNumber}", testRun + 1);
Logger.LogInformation("Start to execute test in experiment.");
var experimentInfo = Generator.GenerateDataForTest(experiment, testRun + 1);
try
{
RunTest(experimentInfo, ref stop1, ref stop2, ref stop3, ref stop4, ref sumOfDeltaCmax,
ref deltaCmaxMax, ref offlineExecutionTimeInMilliseconds, ref onlineExecutionTimeInMilliseconds,
ref offlineResolvedConflictAmount,
ref onlineResolvedConflictAmount, ref onlineUnResolvedConflictAmount, ref downtimeAmount);
}
catch (Exception ex)
{
if (stopOnException)
{
throw;
}
Logger.LogCritical(ex,
"Exception occurred during test run in scope of experiment. {@ExperimentInfo}", experimentInfo);
await using var db = new Database(Settings).Open();
await db.GetTable <ExperimentFailure>()
.InsertAsync(() => new ExperimentFailure
{
ExperimentId = experiment.Id,
ErrorMessage = JsonConvert.SerializeObject(ex),
ExperimentInfo = JsonConvert.SerializeObject(experimentInfo,
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Auto
})
});
}
Logger.LogInformation("Test in experiment was executed.");
await ResultService.SaveExperimentTestResult(experiment.Id, experimentInfo);
aggregationResult.Add(experimentInfo.TestNumber, experimentInfo.Result);
}
experimentReport.OfflineExecutionTime = TimeSpan.FromMilliseconds(offlineExecutionTimeInMilliseconds);
experimentReport.OnlineExecutionTime = TimeSpan.FromMilliseconds(onlineExecutionTimeInMilliseconds);
experimentReport.DeltaCmaxMax = deltaCmaxMax;
experimentReport.OfflineResolvedConflictAmount = offlineResolvedConflictAmount;
experimentReport.OnlineResolvedConflictAmount = onlineResolvedConflictAmount;
experimentReport.OnlineUnResolvedConflictAmount = onlineUnResolvedConflictAmount;
experimentReport.Stop1Percentage = (float)Math.Round(stop1 / (float)experiment.TestsAmount * 100, 1);
experimentReport.Stop2Percentage = (float)Math.Round(stop2 / (float)experiment.TestsAmount * 100, 1);
experimentReport.Stop3Percentage = (float)Math.Round(stop3 / (float)experiment.TestsAmount * 100, 1);
experimentReport.Stop4Percentage = (float)Math.Round(stop4 / (float)experiment.TestsAmount * 100, 1);
experimentReport.DowntimeAmount = downtimeAmount;
if (stop4 != 0)
{
experimentReport.DeltaCmaxAverage = (float)sumOfDeltaCmax / experiment.TestsAmount;
}
else
{
experimentReport.DeltaCmaxAverage = 0;
}
await ResultService.SaveAggregatedResult(experiment.Id, aggregationResult);
ReportService.Save(experimentReport);
}
