public async Task <IEnumerable <TestScenarioResult> > RunTests(IEnumerable <TestScenario> testScenarios, IEnumerable <ProviderResult> providerResults,
IEnumerable <ProviderSourceDataset> sourceDatasets, IEnumerable <TestScenarioResult> currentResults, SpecModel.SpecificationSummary specification, BuildProject buildProject)
{
Guard.ArgumentNotNull(testScenarios, nameof(testScenarios));
Guard.ArgumentNotNull(providerResults, nameof(providerResults));
Guard.ArgumentNotNull(sourceDatasets, nameof(sourceDatasets));
Guard.ArgumentNotNull(currentResults, nameof(currentResults));
Guard.ArgumentNotNull(specification, nameof(specification));
Guard.ArgumentNotNull(buildProject, nameof(buildProject));
IList <TestScenarioResult> scenarioResults = new List <TestScenarioResult>();
if (!providerResults.Any())
{
_logger.Warning("No provider results were supplied to execute tests");
}
else if (!testScenarios.Any())
{
_logger.Warning("No test scenarios were supplied to execute tests");
}
else
{
foreach (ProviderResult providerResult in providerResults)
{
IEnumerable <ProviderSourceDataset> providerSourceDatasets = sourceDatasets.Where(m => m.ProviderId == providerResult.Provider.Id);
IEnumerable <ScenarioResult> testResults = await RunTests(testScenarios, providerResult, providerSourceDatasets, buildProject);
if (!testResults.IsNullOrEmpty())
{
foreach (ScenarioResult testResult in testResults)
{
TestResult status = (testResult.StepsExecuted == 0 || testResult.StepsExecuted < testResult.TotalSteps)
? TestResult.Ignored
: testResult.HasErrors
? TestResult.Failed
: TestResult.Passed;
TestScenarioResult filteredCurrentResults = currentResults.FirstOrDefault(m => m.Provider.Id == providerResult.Provider.Id && m.TestScenario.Id == testResult.Scenario.Id && m.TestResult == status);
if (filteredCurrentResults == null)
{
scenarioResults.Add(new TestScenarioResult
{
TestResult = status,
Specification = new Reference(specification.Id, specification.Name),
TestScenario = new Reference(testResult.Scenario.Id, testResult.Scenario.Name),
Provider = new Reference(providerResult.Provider.Id, providerResult.Provider.Name)
});
}
}
}
else
{
_logger.Warning($"No test results generated for provider: {providerResult.Provider?.Id} on specification: {providerResult.SpecificationId}");
}
}
}
return(scenarioResults);
}
public async Task RunTests_GivenNoTestResults_LogsAndreturnsEmptyResults()
{
//Arrange
IEnumerable <ProviderResult> providerResults = new[] { new ProviderResult {
Provider = new ProviderSummary {
Id = ProviderId
}, SpecificationId = SpecificationId
} };
IEnumerable <TestScenario> scenarios = new[] { new TestScenario() };
IEnumerable <ProviderSourceDataset> providerSourceDatasets = new ProviderSourceDataset[0];
IEnumerable <TestScenarioResult> testScenarioResults = new TestScenarioResult[0];
SpecificationSummary specificationSummary = new SpecificationSummary();
BuildProject buildProject = new BuildProject();
ILogger logger = CreateLogger();
TestEngine testEngine = CreateTestEngine(logger: logger);
//Act
IEnumerable <TestScenarioResult> results = await testEngine.RunTests(scenarios, providerResults, providerSourceDatasets,
testScenarioResults, specificationSummary, buildProject);
results
.Count()
.Should()
.Be(0);
logger
.Received(1)
.Warning(Arg.Is($"No test results generated for provider: {ProviderId} on specification: {SpecificationId}"));
}
public async Task RunTests_GivenNoTestScenarios_LogsAndreturnsEmptyResults()
{
//Arrange
IEnumerable <ProviderResult> providerResults = new[] { new ProviderResult() };
IEnumerable <TestScenario> scenarios = new TestScenario[0];
IEnumerable <ProviderSourceDataset> providerSourceDatasets = new ProviderSourceDataset[0];
IEnumerable <TestScenarioResult> testScenarioResults = new TestScenarioResult[0];
SpecificationSummary specificationSummary = new SpecificationSummary();
BuildProject buildProject = new BuildProject();
ILogger logger = CreateLogger();
TestEngine testEngine = CreateTestEngine(logger: logger);
//Act
IEnumerable <TestScenarioResult> results = await testEngine.RunTests(scenarios, providerResults, providerSourceDatasets,
testScenarioResults, specificationSummary, buildProject);
results
.Count()
.Should()
.Be(0);
logger
.Received(1)
.Warning(Arg.Is("No test scenarios were supplied to execute tests"));
}
public async Task <IActionResult> ReIndex(HttpRequest request)
{
IEnumerable <DocumentEntity <TestScenarioResult> > testScenarioResults = await _testResultsRepository.GetAllTestResults();
IEnumerable <ProviderSummary> summaries = await _providerService.FetchCoreProviderData();
IList <TestScenarioResultIndex> searchItems = new List <TestScenarioResultIndex>();
foreach (DocumentEntity <TestScenarioResult> documentEnity in testScenarioResults)
{
TestScenarioResult testScenarioResult = documentEnity.Content;
TestScenarioResultIndex testScenarioResultIndex = new TestScenarioResultIndex
{
TestResult = testScenarioResult.TestResult.ToString(),
SpecificationId = testScenarioResult.Specification.Id,
SpecificationName = testScenarioResult.Specification.Name,
TestScenarioId = testScenarioResult.TestScenario.Id,
TestScenarioName = testScenarioResult.TestScenario.Name,
ProviderName = testScenarioResult.Provider.Name,
ProviderId = testScenarioResult.Provider.Id,
LastUpdatedDate = documentEnity.UpdatedAt
};
ProviderSummary providerSummary = summaries.FirstOrDefault(m => m.Id == testScenarioResult.Provider.Id);
if (providerSummary != null)
{
testScenarioResultIndex.EstablishmentNumber = providerSummary.EstablishmentNumber;
testScenarioResultIndex.UKPRN = providerSummary.UKPRN;
testScenarioResultIndex.UPIN = providerSummary.UPIN;
testScenarioResultIndex.URN = providerSummary.URN;
testScenarioResultIndex.LocalAuthority = providerSummary.Authority;
testScenarioResultIndex.ProviderType = providerSummary.ProviderType;
testScenarioResultIndex.ProviderSubType = providerSummary.ProviderSubType;
testScenarioResultIndex.OpenDate = providerSummary.DateOpened;
}
searchItems.Add(testScenarioResultIndex);
}
for (int i = 0; i < searchItems.Count; i += 100)
{
IEnumerable <TestScenarioResultIndex> partitionedResults = searchItems.Skip(i).Take(100);
IEnumerable <IndexError> errors = await _searchRepository.Index(partitionedResults);
if (errors.Any())
{
return(new StatusCodeResult(500));
}
}
return(new NoContentResult());
}
public async Task RunTests_GivenTestResultsReturnedFromExecutorWhereNoStepsExecuted_ReturnsOneIgnoreTestResult()
{
//Arrange
ProviderResult providerResult = new ProviderResult {
Provider = new ProviderSummary {
Id = ProviderId, Name = "any provider"
}, SpecificationId = SpecificationId
};
IEnumerable <ProviderResult> providerResults = new[] { providerResult };
IEnumerable <TestScenario> scenarios = new[] { new TestScenario() };
IEnumerable <ProviderSourceDataset> providerSourceDatasets = new ProviderSourceDataset[0];
IEnumerable <TestScenarioResult> testScenarioResults = new TestScenarioResult[0];
SpecModel.SpecificationSummary specificationSummary = new SpecModel.SpecificationSummary {
Id = SpecificationId, Name = "spec-name"
};
BuildProject buildProject = new BuildProject();
IEnumerable <ScenarioResult> scenarioResults = new[]
{
new ScenarioResult {
Scenario = new Reference("sceanrio=id", "scenario name")
}
};
ILogger logger = CreateLogger();
IGherkinExecutor gherkinExecutor = CreateGherkinExecutor();
gherkinExecutor
.Execute(Arg.Any <ProviderResult>(), Arg.Any <IEnumerable <ProviderSourceDataset> >(), Arg.Any <IEnumerable <TestScenario> >(), Arg.Any <BuildProject>())
.Returns(scenarioResults);
TestEngine testEngine = CreateTestEngine(gherkinExecutor, logger);
//Act
IEnumerable <TestScenarioResult> results = await testEngine.RunTests(scenarios, providerResults, providerSourceDatasets,
testScenarioResults, specificationSummary, buildProject);
results
.Count()
.Should()
.Be(1);
results
.First()
.TestResult
.Should()
.Be(TestResult.Ignored);
}
public void TestScenarioResult_WhenPropertiesAreInvalid_ThenValidIsFalse()
{
// Arrange
TestScenarioResult result = new TestScenarioResult()
{
};
// Act
bool isValid = result.IsValid();
// Assert
isValid.Should().BeFalse();
}
public void TestScenarioResult_WhenPropertiesValid_ThenValidationPasses()
{
// Arrange
TestScenarioResult result = new TestScenarioResult()
{
Provider = new Reference("Ref ID", "Reference Name"),
Specification = new Reference("SpecificationId", "Specification Name"),
TestResult = TestResult.Passed,
TestScenario = new Reference("TestScenarioId", "Test Scenario Name")
};
// Act
bool isValid = result.IsValid();
// Assert
isValid.Should().BeTrue();
}
public async Task SaveTestProviderResults_WhenItemsPasssed_ThenTelemetryLogged()
{
// Arrange
ITestResultsRepository testResultsRepository = CreateTestResultsRepository();
ISearchRepository <TestScenarioResultIndex> searchRepository = CreateSearchRespository();
ITelemetry telemetry = CreateTelemetry();
ITestResultsService service = CreateTestResultsService(testResultsRepository, searchRepository, telemetry: telemetry);
IEnumerable <ProviderResult> providerResults = Enumerable.Empty <ProviderResult>();
List <TestScenarioResult> itemsToUpdate = new List <TestScenarioResult>();
TestScenarioResult testScenarioResult = CreateTestScenarioResult();
itemsToUpdate.Add(testScenarioResult);
testResultsRepository
.SaveTestProviderResults(Arg.Any <IEnumerable <TestScenarioResult> >())
.Returns(HttpStatusCode.Created);
// Act
HttpStatusCode result = await service.SaveTestProviderResults(itemsToUpdate, providerResults);
// Assert
result.Should().Be(HttpStatusCode.Created);
telemetry
.Received(1)
.TrackEvent(
Arg.Is("UpdateTestScenario"),
Arg.Is <IDictionary <string, string> >(p =>
p.ContainsKey("SpecificationId") &&
p["SpecificationId"] == testScenarioResult.Specification.Id
),
Arg.Is <IDictionary <string, double> >(
m => m.ContainsKey("update-testscenario-elapsedMilliseconds") &&
m.ContainsKey("update-testscenario-recordsUpdated") &&
m["update-testscenario-elapsedMilliseconds"] > 0 &&
m["update-testscenario-recordsUpdated"] == 1
)
);
}
public async Task CleanupTestResultsForSpecificationProviders_GivenCurrentTestResults_ThenCallsDelete()
{
//Arrange
TestScenarioResult testScenarioResult = CreateTestScenarioResult();
ITestResultsRepository testResultsRepository = CreateTestResultsRepository();
testResultsRepository
.GetCurrentTestResults(Arg.Any <IEnumerable <string> >(), Arg.Is <string>(testScenarioResult.Specification.Id))
.Returns(new TestScenarioResult[] { testScenarioResult });
TestResultsService service = CreateTestResultsService(testResultsRepository: testResultsRepository);
SpecificationProviders specificationProviders = new SpecificationProviders {
SpecificationId = testScenarioResult.Specification.Id, Providers = new string[] { testScenarioResult.Provider.Id }
};
Dictionary <string, string> properties = new Dictionary <string, string>
{
{ "specificationId", testScenarioResult.Specification.Id },
{ "sfa-correlationId", Guid.NewGuid().ToString() }
};
Message message = new Message {
Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(specificationProviders))
};
message.UserProperties["specificationId"] = testScenarioResult.Specification.Id;
//Act
await service.CleanupTestResultsForSpecificationProviders(message);
//Assert
await testResultsRepository
.Received(1)
.DeleteCurrentTestScenarioTestResults(Arg.Is <IEnumerable <TestScenarioResult> >(x => x.First().Provider.Id == testScenarioResult.Provider.Id));
}
public async Task <HttpStatusCode> SaveTestProviderResults(IEnumerable <TestScenarioResult> providerResult)
{
Guard.ArgumentNotNull(providerResult, nameof(providerResult));
List <TestScenarioResult> items = new List <TestScenarioResult>(providerResult);
List <KeyValuePair <string, TestScenarioResult> > resultItems = new List <KeyValuePair <string, TestScenarioResult> >(items.Count);
for (int i = 0; i < items.Count; i++)
{
TestScenarioResult result = items[i];
if (result == null)
{
_logger.Error("Result {i} provided was null", i);
throw new InvalidOperationException($"Result {i} provided was null");
}
if (!result.IsValid())
{
_logger.Error("Result {i} provided was not valid", i);
throw new InvalidOperationException($"Result {i} provided was valid");
}
resultItems.Add(new KeyValuePair <string, TestScenarioResult>(result.Provider.Id, result));
}
if (resultItems.Any())
{
await _cosmosRepository.BulkUpsertAsync <TestScenarioResult>(resultItems, degreeOfParallelism : _engineSettings.SaveTestProviderResultsDegreeOfParallelism);
}
else
{
return(HttpStatusCode.NotModified);
}
return(HttpStatusCode.Created);
}
public async Task SaveTestProviderResults_WhenItemsPasssed_ThenSearchResultsMapped()
{
// Arrange
ITestResultsRepository testResultsRepository = CreateTestResultsRepository();
ISearchRepository <TestScenarioResultIndex> searchRepository = CreateSearchRespository();
TestResultsService service = CreateTestResultsService(testResultsRepository, searchRepository);
IEnumerable <ProviderResult> providerResults = new[]
{
new ProviderResult
{
Provider = new ProviderSummary
{
UKPRN = "111",
UPIN = "222",
URN = "333",
EstablishmentNumber = "123",
DateOpened = DateTimeOffset.Now,
Authority = "authority",
ProviderSubType = "provider sub type",
ProviderType = "provider type",
Id = "ProviderId"
}
}
};
List <TestScenarioResult> itemsToUpdate = new List <TestScenarioResult>();
TestScenarioResult testScenarioResult = CreateTestScenarioResult();
itemsToUpdate.Add(testScenarioResult);
testResultsRepository
.SaveTestProviderResults(Arg.Any <IEnumerable <TestScenarioResult> >())
.Returns(HttpStatusCode.Created);
// Act
HttpStatusCode result = await service.SaveTestProviderResults(itemsToUpdate, providerResults);
// Assert
result.Should().Be(HttpStatusCode.Created);
await testResultsRepository
.Received(1)
.SaveTestProviderResults(itemsToUpdate);
await searchRepository
.Received(1)
.Index(Arg.Is <IEnumerable <TestScenarioResultIndex> >(c =>
c.First().Id == testScenarioResult.Id &&
c.First().ProviderId == testScenarioResult.Provider.Id &&
c.First().ProviderName == testScenarioResult.Provider.Name &&
c.First().SpecificationId == testScenarioResult.Specification.Id &&
c.First().SpecificationName == testScenarioResult.Specification.Name &&
c.First().TestResult == Enum.GetName(typeof(Models.Results.TestResult), testScenarioResult.TestResult) &&
c.First().TestScenarioId == testScenarioResult.TestScenario.Id &&
c.First().TestScenarioName == testScenarioResult.TestScenario.Name &&
c.First().LastUpdatedDate > DateTimeOffset.Now.AddDays(-1) &&
c.First().EstablishmentNumber == "123" &&
c.First().UKPRN == "111" &&
c.First().UPIN == "222" &&
c.First().URN == "333" &&
c.First().LocalAuthority == "authority" &&
c.First().ProviderType == "provider type" &&
c.First().ProviderSubType == "provider sub type" &&
c.First().OpenDate.HasValue
));
}
private static void WriteTestScenarioResults(IScenario scenario, Dictionary <TestCase, Task <TestCaseResult> > testCaseResults)
{
TestScenarioResult _scenarioResult = new TestScenarioResult();
_scenarioResult.JobName = Environment.GetEnvironmentVariable("JOB_NAME");
_scenarioResult.BuildNumber = Environment.GetEnvironmentVariable("BUILD_NUMBER");
_scenarioResult.TestScenario = scenario.Id;
_scenarioResult.TestSite = testsite_name;
_scenarioResult.TestTargetUrl = Configuration.Current.ServiceProviders[target_name].Data.Url;
_scenarioResult.TestTarget = target_name;
_scenarioResult.ZoneOffset = DateTime.Now.ToString("zz");
try // fails on macOS if file sharing is disabled (macOS can't resolve its own hostname then)
{
_scenarioResult.HostName = Dns.GetHostName();
_scenarioResult.HostAddress = Dns.GetHostAddresses(_scenarioResult.HostName)[0].ToString();
}
catch (Exception)
{
_scenarioResult.HostName = "localhost";
_scenarioResult.HostAddress = "127.0.0.1";
}
_scenarioResult.TestCaseResults = new TestCaseResult[testCaseResults.Count];
Schemas.testsuites _testSuites = new Schemas.testsuites();
_testSuites.testsuite = new Schemas.testsuite[1];
_testSuites.testsuite[0] = new Schemas.testsuite();
_testSuites.testsuite[0].id = scenario.Id;
_testSuites.testsuite[0].name = scenario.Title;
_testSuites.testsuite[0].testcase = new Schemas.testcase[testCaseResults.Count];
int _testCasesErrors = 0;
for (int i = 0; i < testCaseResults.Count; i++)
{
try
{
_scenarioResult.TestCaseResults[i] = testCaseResults.ElementAt(i).Value.Result;
_testSuites.testsuite[0].testcase[i] = new Schemas.testcase();
_testSuites.testsuite[0].testcase[i].name = _scenarioResult.TestCaseResults[i].TestName;
_testSuites.testsuite[0].testcase[i].status = "OK";
_testSuites.testsuite[0].testcase[i].classname = _scenarioResult.TestCaseResults[i].ClassName;
}
catch (AggregateException e)
{
log.WarnFormat("Error reading Test Case {0} results: {1}", testCaseResults.ElementAt(i).Key.Id, e.InnerException.Message);
log.Debug(e.InnerException.StackTrace);
_testCasesErrors++;
_scenarioResult.TestCaseResults[i] = new TestCaseResult(testCaseResults.ElementAt(i).Key.Id, new List <IMetric>(), testCaseResults.ElementAt(i).Key.StartTime, testCaseResults.ElementAt(i).Key.EndTime);
_testSuites.testsuite[0].testcase[i] = new Schemas.testcase();
_testSuites.testsuite[0].testcase[i].name = testCaseResults.ElementAt(i).Key.Title;
_testSuites.testsuite[0].testcase[i].classname = testCaseResults.ElementAt(i).Key.GetType().ToString();
_testSuites.testsuite[0].testcase[i].status = "ERROR";
_testSuites.testsuite[0].testcase[i].error = new Schemas.error[1];
_testSuites.testsuite[0].testcase[i].error[0] = new Schemas.error();
_testSuites.testsuite[0].testcase[i].error[0].message = e.Message;
_testSuites.testsuite[0].testcase[i].error[0].type = e.GetType().ToString();
_testSuites.testsuite[0].testcase[i].classname = testCaseResults.ElementAt(i).Key.GetType().ToString();
}
}
_testSuites.testsuite[0].errors = _testCasesErrors.ToString();
File.WriteAllText(string.Format("{0}Results.json", scenario.Id), JsonConvert.SerializeObject(_scenarioResult, new JsonSerializerSettings {
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
XmlSerializer ser = new XmlSerializer(typeof(Schemas.testsuites));
FileStream fs = new FileStream(string.Format("junit.xml"), FileMode.Create, FileAccess.Write);
ser.Serialize(fs, _testSuites);
fs.Close();
}