public async Task Test1(
IEnumerable <Detail> j1,
IEnumerable <Detail> j2,
IEnumerable <LaboriousDetail> j12,
IEnumerable <LaboriousDetail> j21)
{
// Arrange
var experimentInfo = new ExperimentInfo();
experimentInfo.J1.AddRange(j1);
experimentInfo.J2.AddRange(j2);
experimentInfo.J12.AddRange(j12);
experimentInfo.J21.AddRange(j21);
var experiment = new Experiment
{
Id = Guid.NewGuid(),
TestsAmount = 1
};
var generatorMock = new Mock <IExperimentGenerator>();
generatorMock.Setup(g =>
g.GenerateDataForTest(It.Is <Experiment>(ex => ex.Id == experiment.Id), It.IsAny <int>()))
.Returns(() => experimentInfo);
var originalGenerator = new ExperimentGenerator(new RandomizeService());
generatorMock.Setup(g => g.GenerateP(It.IsAny <IOnlineChainNode>()))
.Callback <IOnlineChainNode>(node => originalGenerator.GenerateP(node));
ExperimentInfo resultInfo = null;
var experimentTestResultService = new Mock <IExperimentTestResultService>();
experimentTestResultService
.Setup(e => e.SaveExperimentTestResult(It.Is <Guid>(id => id == experiment.Id),
It.IsAny <ExperimentInfo>()))
.Callback <Guid, ExperimentInfo>((id, info) => resultInfo = info);
var experimentPipeline = new ExperimentPipeline(
generatorMock.Object,
Mock.Of <IWorkerExperimentService>(),
experimentTestResultService.Object,
Mock.Of <IReportsService>(),
Mock.Of <ILogger <ExperimentPipeline> >(),
Mock.Of <IOptions <DbSettings> >(),
new OnlineExecutor());
// Act
await experimentPipeline.Run(Enumerable.Empty <Experiment>().Append(experiment));
// Assert
Assert.NotNull(resultInfo);
}
public async Task Experiment_ShouldBeFinishedWithoutAnyExceptions(string path, ExperimentInfo experimentInfo)
{
// Arrange
_outputHelper.WriteLine($"Running test with data from '{path}'");
var experiment = new Experiment
{
Id = Guid.NewGuid(),
TestsAmount = 1
};
var generatorMock = new Mock <IExperimentGenerator>();
generatorMock.Setup(g =>
g.GenerateDataForTest(It.Is <Experiment>(ex => ex.Id == experiment.Id), It.IsAny <int>()))
.Returns(() => experimentInfo);
ExperimentInfo resultInfo = null;
var experimentTestResultService = new Mock <IExperimentTestResultService>();
experimentTestResultService
.Setup(e => e.SaveExperimentTestResult(It.Is <Guid>(id => id == experiment.Id),
It.IsAny <ExperimentInfo>()))
.Callback <Guid, ExperimentInfo>((id, info) => resultInfo = info);
ExperimentReport experimentReport = null;
var reportServiceMock = new Mock <IReportsService>();
reportServiceMock.Setup(x => x.Save(It.Is <ExperimentReport>(r => r.ExperimentId == experiment.Id)))
.Callback <ExperimentReport>((report) => experimentReport = report);
var experimentPipeline = new ExperimentPipeline(
generatorMock.Object,
Mock.Of <IWorkerExperimentService>(),
experimentTestResultService.Object,
reportServiceMock.Object,
Mock.Of <ILogger <ExperimentPipeline> >(),
Mock.Of <IOptions <DbSettings> >(),
new OnlineExecutor());
// Act
await experimentPipeline.Run(new[]
{
experiment
}, false, true);
// Assert
resultInfo.Should().NotBeNull();
experimentReport.Should().NotBeNull();
}
private async Task RunTests(Experiment experiment, bool stopOnException = false)
{
var experimentReport = new ExperimentReport
{
ExperimentId = experiment.Id
};
var stop1 = 0;
var stop2 = 0;
var stop3 = 0;
var stop4 = 0;
var downtimeAmount = 0;
var sumOfDeltaCmax = 0.0;
var deltaCmaxMax = 0.0f;
var offlineResolvedConflictAmount = 0;
var onlineResolvedConflictAmount = 0;
var onlineUnResolvedConflictAmount = 0;
var onlineExecutionTimeInMilliseconds = 0.0d;
var offlineExecutionTimeInMilliseconds = 0.0d;
var aggregationResult = new Dictionary <int, ResultInfo>();
foreach (var testRun in Enumerable.Range(0, experiment.TestsAmount))
{
using var _ = Logger.BeginScope("{TestNumber}", testRun + 1);
Logger.LogInformation("Start to execute test in experiment.");
var experimentInfo = Generator.GenerateDataForTest(experiment, testRun + 1);
try
{
RunTest(experimentInfo, ref stop1, ref stop2, ref stop3, ref stop4, ref sumOfDeltaCmax,
ref deltaCmaxMax, ref offlineExecutionTimeInMilliseconds, ref onlineExecutionTimeInMilliseconds,
ref offlineResolvedConflictAmount,
ref onlineResolvedConflictAmount, ref onlineUnResolvedConflictAmount, ref downtimeAmount);
}
catch (Exception ex)
{
if (stopOnException)
{
throw;
}
Logger.LogCritical(ex,
"Exception occurred during test run in scope of experiment. {@ExperimentInfo}", experimentInfo);
await using var db = new Database(Settings).Open();
await db.GetTable <ExperimentFailure>()
.InsertAsync(() => new ExperimentFailure
{
ExperimentId = experiment.Id,
ErrorMessage = JsonConvert.SerializeObject(ex),
ExperimentInfo = JsonConvert.SerializeObject(experimentInfo,
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Auto
})
});
}
Logger.LogInformation("Test in experiment was executed.");
await ResultService.SaveExperimentTestResult(experiment.Id, experimentInfo);
aggregationResult.Add(experimentInfo.TestNumber, experimentInfo.Result);
}
experimentReport.OfflineExecutionTime = TimeSpan.FromMilliseconds(offlineExecutionTimeInMilliseconds);
experimentReport.OnlineExecutionTime = TimeSpan.FromMilliseconds(onlineExecutionTimeInMilliseconds);
experimentReport.DeltaCmaxMax = deltaCmaxMax;
experimentReport.OfflineResolvedConflictAmount = offlineResolvedConflictAmount;
experimentReport.OnlineResolvedConflictAmount = onlineResolvedConflictAmount;
experimentReport.OnlineUnResolvedConflictAmount = onlineUnResolvedConflictAmount;
experimentReport.Stop1Percentage = (float)Math.Round(stop1 / (float)experiment.TestsAmount * 100, 1);
experimentReport.Stop2Percentage = (float)Math.Round(stop2 / (float)experiment.TestsAmount * 100, 1);
experimentReport.Stop3Percentage = (float)Math.Round(stop3 / (float)experiment.TestsAmount * 100, 1);
experimentReport.Stop4Percentage = (float)Math.Round(stop4 / (float)experiment.TestsAmount * 100, 1);
experimentReport.DowntimeAmount = downtimeAmount;
if (stop4 != 0)
{
experimentReport.DeltaCmaxAverage = (float)sumOfDeltaCmax / experiment.TestsAmount;
}
else
{
experimentReport.DeltaCmaxAverage = 0;
}
await ResultService.SaveAggregatedResult(experiment.Id, aggregationResult);
ReportService.Save(experimentReport);
}
