public async Task SaveTestProviderResults_WhenItemsPasssed_ThenTelemetryLogged()
{
// Arrange
ITestResultsRepository testResultsRepository = CreateTestResultsRepository();
ISearchRepository <TestScenarioResultIndex> searchRepository = CreateSearchRespository();
ITelemetry telemetry = CreateTelemetry();
ITestResultsService service = CreateTestResultsService(testResultsRepository, searchRepository, telemetry: telemetry);
IEnumerable <ProviderResult> providerResults = Enumerable.Empty <ProviderResult>();
List <TestScenarioResult> itemsToUpdate = new List <TestScenarioResult>();
TestScenarioResult testScenarioResult = CreateTestScenarioResult();
itemsToUpdate.Add(testScenarioResult);
testResultsRepository
.SaveTestProviderResults(Arg.Any <IEnumerable <TestScenarioResult> >())
.Returns(HttpStatusCode.Created);
// Act
HttpStatusCode result = await service.SaveTestProviderResults(itemsToUpdate, providerResults);
// Assert
result.Should().Be(HttpStatusCode.Created);
telemetry
.Received(1)
.TrackEvent(
Arg.Is("UpdateTestScenario"),
Arg.Is <IDictionary <string, string> >(p =>
p.ContainsKey("SpecificationId") &&
p["SpecificationId"] == testScenarioResult.Specification.Id
),
Arg.Is <IDictionary <string, double> >(
m => m.ContainsKey("update-testscenario-elapsedMilliseconds") &&
m.ContainsKey("update-testscenario-recordsUpdated") &&
m["update-testscenario-elapsedMilliseconds"] > 0 &&
m["update-testscenario-recordsUpdated"] == 1
)
);
}
public async Task RunTests(Message message)
{
Stopwatch runTestsStopWatch = Stopwatch.StartNew();
string specificationId = message.UserProperties["specificationId"].ToString();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("Null or empty specification id provided");
return;
}
BuildProject buildProject = await _builProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectRepository.GetBuildProjectBySpecificationId(specificationId));
if (buildProject == null)
{
_logger.Error("A null build project was provided to UpdateAllocations");
throw new ArgumentNullException(nameof(buildProject));
}
string cacheKey = message.UserProperties["providerResultsCacheKey"].ToString();
if (string.IsNullOrWhiteSpace(cacheKey))
{
_logger.Error("Null or empty cache key provided");
return;
}
Stopwatch providerResultsQueryStopwatch = Stopwatch.StartNew();
IEnumerable <ProviderResult> providerResults = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{cacheKey}"));
providerResultsQueryStopwatch.Stop();
if (providerResults.IsNullOrEmpty())
{
_logger.Error($"No provider results found in cache for key: {cacheKey}");
return;
}
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.RemoveAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{cacheKey}"));
Stopwatch testScenariosStopwatch = Stopwatch.StartNew();
IEnumerable <TestScenario> testScenarios = await _scenariosRepositoryPolicy.ExecuteAsync(() => _scenariosRepository.GetTestScenariosBySpecificationId(specificationId));
testScenariosStopwatch.Stop();
if (testScenarios.IsNullOrEmpty())
{
_logger.Warning($"No test scenarios found for specification id: {specificationId}");
return;
}
Stopwatch specificationLookupStopwatch = Stopwatch.StartNew();
SpecificationSummary specification = await _specificationRepositoryPolicy.ExecuteAsync(() => _specificationRepository.GetSpecificationSummaryById(specificationId));
specificationLookupStopwatch.Stop();
if (specification == null)
{
_logger.Error($"No specification found for specification id: {specificationId}");
return;
}
IEnumerable <string> providerIds = providerResults.Select(m => m.Provider.Id);
Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew();
IEnumerable <ProviderSourceDataset> sourceDatasets = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() =>
_providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIds, specificationId));
providerSourceDatasetsStopwatch.Stop();
if (sourceDatasets.IsNullOrEmpty())
{
_logger.Error($"No source datasets found for specification id: {specificationId}");
return;
}
byte[] assembly = await _calculationsRepository.GetAssemblyBySpecificationId(specificationId);
if (assembly.IsNullOrEmpty())
{
_logger.Error($"No assemblyfor specification id: {specificationId}");
return;
}
buildProject.Build.Assembly = assembly;
Stopwatch existingTestResultsStopwatch = Stopwatch.StartNew();
IEnumerable <TestScenarioResult> testScenarioResults = await _testResultsRepositoryPolicy.ExecuteAsync(() => _testResultsRepository.GetCurrentTestResults(providerIds, specificationId));
existingTestResultsStopwatch.Stop();
Stopwatch runTestsStopwatch = Stopwatch.StartNew();
IEnumerable <TestScenarioResult> results = await _testEngine.RunTests(testScenarios, providerResults, sourceDatasets, testScenarioResults.ToList(), specification, buildProject);
runTestsStopwatch.Stop();
Stopwatch saveResultsStopwatch = new Stopwatch();
if (results.Any())
{
saveResultsStopwatch.Start();
HttpStatusCode status = await _testResultsService.SaveTestProviderResults(results, providerResults);
saveResultsStopwatch.Stop();
if (!status.IsSuccess())
{
_logger.Error($"Failed to save test results with status code: {status.ToString()}");
}
}
runTestsStopWatch.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "tests-run-totalMs", runTestsStopWatch.ElapsedMilliseconds },
{ "tests-run-testScenarioQueryMs", testScenariosStopwatch.ElapsedMilliseconds },
{ "tests-run-numberOfTestScenarios", testScenarios.Count() },
{ "tests-run-providersResultsQueryMs", providerResultsQueryStopwatch.ElapsedMilliseconds },
{ "tests-run-totalProvidersProcessed", providerIds.Count() },
{ "tests-run-specificationQueryMs", specificationLookupStopwatch.ElapsedMilliseconds },
{ "tests-run-providerSourceDatasetsQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds },
{ "tests-run-existingTestsQueryMs", existingTestResultsStopwatch.ElapsedMilliseconds },
{ "tests-run-existingTestScenarioResultsTotal", testScenarioResults.Count() },
{ "tests-run-runTestsMs", runTestsStopwatch.ElapsedMilliseconds },
};
if (results.Any())
{
metrics.Add("tests-run-saveTestResultsMs", saveResultsStopwatch.ElapsedMilliseconds);
metrics.Add("tests-run-numberOfSavedResults", results.Count());
}
_telemetry.TrackEvent("RunTests",
new Dictionary <string, string>()
{
{ "specificationId", specificationId },
{ "buildProjectId", buildProject.Id },
{ "cacheKey", cacheKey },
},
metrics
);
}
