public void GetResultById_should_return_expected_result()
{
// given
TestsClient client = Helpers.CreateTestsClient();
TestFile testFile = Helpers.CreateTestFileAndTest(client);
var repository = ServiceStarter.Container.GetInstance <ITestFileResultRepository>();
var result1 = new TestFileResult()
{
StartTime = DateTime.Now,
EndTime = DateTime.Now.AddSeconds(1),
Filename = testFile.Filename
};
var result2 = new TestFileResult()
{
StartTime = DateTime.Now,
EndTime = DateTime.Now.AddSeconds(1),
Filename = testFile.Filename
};
repository.Add(result1).Wait();
repository.Add(result2).Wait();
// when
TestFileResult actualResult = client.GetResultById(result2.Id);
// then
Assert.That(actualResult, Is.Not.Null);
Assert.That(actualResult.Id, Is.EqualTo(result2.Id));
}
public async Task Run_should_verify_positive_and_negative_items_when_httpcode_passes()
{
// given
TestFileRunner runner = CreateRunner();
_httpClientMock.Response.StatusCode = HttpStatusCode.OK;
_httpClientMock.Response.Content = "some content";
var testFile = CreateTestFile(new[]
{
new Test()
{
Url = "foo1",
ExpectedHttpStatusCode = HttpStatusCode.OK,
Assertions = new List <Assertion>()
{
new Assertion("positive-1", "some content", AssertionType.Positive, AssertionMethod.Regex),
new Assertion("negative-1", "no text like this", AssertionType.Negative, AssertionMethod.Regex)
}
},
});
// when
TestFileResult session = await runner.RunAsync(testFile, "development", "bob");
// then
var result = session.TestResults.Single();
Assert.That(result.ResultState, Is.EqualTo(TestResultState.Success));
Assert.That(result.AssertionResults.Where(x => x.AssertionType == AssertionType.Positive).Count, Is.EqualTo(1));
Assert.That(result.AssertionResults[0].Success, Is.True);
Assert.That(result.AssertionResults.Where(x => x.AssertionType == AssertionType.Negative).Count, Is.EqualTo(1));
Assert.That(result.AssertionResults[0].Success, Is.True);
}
public async Task Run_should_ignore_null_scriptevaluator_output()
{
// given
const string environment = "knights-of-the-white-table";
const string httpContent = "im lowercase";
TestFileRunner runner = CreateRunner();
_httpClientMock.Response.Content = httpContent;
_httpClientMock.Response.StatusCode = HttpStatusCode.OK;
var testFile = CreateTestFile(new[]
{
new Test()
{
Url = "test1",
ExpectedHttpStatusCode = HttpStatusCode.OK,
}
});
// when
TestFileResult session = await runner.RunAsync(testFile, environment, "bob");
// then
Assert.That(testFile, Is.Not.Null);
Assert.That(testFile.Tests.FirstOrDefault(), Is.Not.Null);
Assert.That(session.TestResults.First().HttpResponse, Is.Not.Null);
}
public void GetSummaries_should_return_all_results()
{
// given
TestsClient client = Helpers.CreateTestsClient();
TestFile testFile = Helpers.CreateTestFileAndTest(client);
var repository = ServiceStarter.Container.GetInstance <ITestFileResultRepository>();
var result1 = new TestFileResult()
{
StartTime = DateTime.Now,
EndTime = DateTime.Now.AddSeconds(1),
Filename = testFile.Filename
};
var result2 = new TestFileResult()
{
StartTime = DateTime.Now,
EndTime = DateTime.Now.AddSeconds(1),
Filename = testFile.Filename
};
repository.Add(result1).Wait();
repository.Add(result2).Wait();
// when
TestFileResultSummaryCollection results = client.GetSummaries(It.IsAny <DateTime>()).Result;
// then
Assert.That(results.TotalFileResults, Is.EqualTo(2));
}
public async Task Run_should_populate_StartTime_and_EndTime_and_TotalRunTime()
{
// given
var beforeStart = DateTime.UtcNow;
var response = new HttpResponse();
response.ResponseTime = TimeSpan.FromSeconds(5);
HttpClientMock httpClient = new HttpClientMock(response);
var runner = new TestFileRunner(httpClient, GetRepositoryFactory(), new JsonConfiguration(), _capturedVariableProviderFactory.Object, GetTestFileRunnerLoggerFactory());
var testFile = CreateTestFile(new[]
{
new Test()
{
Url = "foo1"
},
});
// when
TestFileResult session = await runner.RunAsync(testFile, "development", "bob");
// then
Assert.That(session.StartTime, Is.GreaterThanOrEqualTo(beforeStart));
Assert.That(session.EndTime, Is.GreaterThanOrEqualTo(session.StartTime));
Assert.That(session.TotalRunTime, Is.EqualTo(session.EndTime - session.StartTime));
}
public ActionResult Delete(Guid id)
{
TestFileResult session = _testsClient.GetResultById(id);
_testsClient.DeleteResult(session.Id);
return(RedirectToAction("Index"));
}
private void AddResult(TestFileResult session, TestResult result)
{
session.TestResults.Add(result);
lock (_currentResults)
{
_currentResults.Add(result);
}
NotifySubscribersOfAddedResult(result);
}
public ActionResult ViewLog(Guid testFileResultId, int resultId)
{
TestFileResult testFileResult = _testsClient.GetResultById(testFileResultId);
TestResult result = testFileResult.TestResults.ElementAtOrDefault(resultId);
if (result != null)
{
return(Content(result.Log, "text/plain"));
}
return(Content("Result Id not found"));
}
public void TotalAssertionsFailed_should_count_failed_tests()
{
// given
var builder = new TestFileResultsBuilder()
.New().AddPositiveVerify().Add()
.New().AddPositiveVerify(false).Add()
.New().AddNegativeVerify(false).Add();
var testFileResult = new TestFileResult();
testFileResult.TestResults = builder.GetCollection();
// when + then
Assert.That(testFileResult.TotalAssertionsFailed, Is.EqualTo(2));
}
public async Task Run_should_set_MinResponseTime_and_MaxResponseTime_from_http_response_times()
{
// given
var response = new HttpResponse();
response.ResponseTime = TimeSpan.FromSeconds(5);
HttpClientMock httpClient = new HttpClientMock(response);
httpClient.ResponseTimes = new List <TimeSpan>()
{
// Deliberately mixed up order
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(88),
TimeSpan.FromSeconds(3),
TimeSpan.FromSeconds(10)
};
httpClient.Response = response;
var runner = new TestFileRunner(httpClient, GetRepositoryFactory(), new JsonConfiguration(), _capturedVariableProviderFactory.Object, GetTestFileRunnerLoggerFactory());
var testFile = CreateTestFile(new[]
{
new Test()
{
Url = "foo1"
},
new Test()
{
Url = "foo2"
},
new Test()
{
Url = "foo3"
},
new Test()
{
Url = "foo4"
},
});
// when
TestFileResult session = await runner.RunAsync(testFile, "development", "bob");
// then
Assert.That(session.MinResponseTime, Is.EqualTo(TimeSpan.FromSeconds(3)));
Assert.That(session.MaxResponseTime, Is.EqualTo(TimeSpan.FromSeconds(88)));
}
public ActionResult ViewHtml(Guid testFileResultId, int resultId)
{
TestFileResult testFileResult = _testsClient.GetResultById(testFileResultId);
TestResult result = testFileResult.TestResults.ElementAtOrDefault(resultId);
if (result != null)
{
string html = result.HttpContent;
string baseUrl = _urlHelper.GetBaseUrl(result.ActualUrl);
html = _urlHelper.AddUrlBase(baseUrl, html);
return(Content(html));
}
return(Content("Result Id not found"));
}
public void TotalTestsPassed_should_count_passed_tests()
{
// given
var builder = new TestFileResultsBuilder()
.New().WithSuccess().Add()
.New().WithSuccess().Add()
.New().WithSuccess().Add()
.New().WithFail().Add()
.New().WithFail().Add();
var testFileResult = new TestFileResult();
testFileResult.TestResults = builder.GetCollection();
// when + then
Assert.That(testFileResult.TotalTestsPassed, Is.EqualTo(3));
}
static ProcessState CalcBeforeCheckoutState(TestFileResult result)
{
switch (result)
{
case TestFileResult.Ok:
return(ProcessState.Default);
case TestFileResult.Fail:
return(ProcessState.Failed);
case TestFileResult.Ignore:
return(ProcessState.Ignored);
default:
throw new Exception("result");
}
}
public async Task should_parse_capturedvariables()
{
// Arrange
var httpClient = new HttpClient(new RestClient());
string xml = TestHelpers.ReadEmbeddedFile("capturedvariables.xml", XmlExamplesFolder);
var stringReader = new StringReader(xml);
var reader = new TestFileReader();
TestFile testFile = reader.Read(stringReader);
var runner = new TestFileRunner(httpClient, GetRepository(), new JsonConfiguration());
// Act
TestFileResult result = await runner.RunAsync(testFile);
// Assert
Assert.That(result, Is.Not.Null);
Assert.That(result.TestResults.Count, Is.EqualTo(2));
}
public async Task GetById_should_return_session()
{
// Arrange
var fixture = new Fixture();
var expectedSession = fixture.Create <TestFileResult>();
MongoTestFileResultRepository repository = GetTestFileResultRepository();
await repository.Add(expectedSession);
// Act
TestFileResult actualSession = repository.GetById(expectedSession.Id);
// Assert
Assert.That(actualSession, Is.Not.Null, "couldn't find the session");
string actual = GetAsJson(actualSession);
string expected = GetAsJson(expectedSession);
Assert.That(actual, Is.EqualTo(expected));
}
public async Task Run_should_set_capturedvariables()
{
// given
const string environment = "big-daddy-doo-dah";
const string httpContent = "THIS IS SOME CONTENT - content coming to you 24/7, 365, every and all init yeah.";
TestFileRunner runner = CreateRunner();
_httpClientMock.Response.Content = httpContent;
_httpClientMock.Response.StatusCode = HttpStatusCode.OK;
var testFile = CreateTestFile(new[]
{
new Test()
{
Url = "test1",
ExpectedHttpStatusCode = HttpStatusCode.OK,
CapturedVariables = new List <CapturedVariable>()
{
new CapturedVariable("var1", "some content (.*?every)")
},
Assertions = new List <Assertion>()
{
new Assertion("positive-1", httpContent.Substring(0, 10), AssertionType.Positive, AssertionMethod.Regex)
},
}
});
// when
TestFileResult session = await runner.RunAsync(testFile, environment, "bob");
// then
var addedVariable = _capturedVariableProvider.Variables.FirstOrDefault(x => x.Name == "var1");
Assert.That(addedVariable, Is.Not.Null);
Assert.That(addedVariable.Value, Is.EqualTo("- content coming to you 24/7, 365, every"));
var testResult = session.TestResults.Single();
Assert.That(testResult.AssertionResults[0].Success, Is.True);
}
public async Task Run_should_set_result_state_and_response_when_httpcode_fails()
{
// given
TestFileRunner runner = CreateRunner();
_httpClientMock.Response.StatusCode = HttpStatusCode.OK;
var testFile = CreateTestFile(new[]
{
new Test()
{
Url = "foo1",
ExpectedHttpStatusCode = HttpStatusCode.Ambiguous
},
});
// when
TestFileResult session = await runner.RunAsync(testFile, "development", "bob");
// then
Assert.That(session.TestResults.Single().ResultState, Is.EqualTo(TestResultState.Failed));
Assert.That(session.TestResults.Single().HttpResponse, Is.EqualTo(_httpClientMock.Response));
}
public bool ProcessCheckout(IEnumerable <SyncItem> items, bool ignoreSharedFiles, TrackBranch branch)
{
var list = items.ToList();
var sharedFiles = list.Where(item => IsSharedFile(item.VcsPath)).ToList();
sharedFiles.ForEach(x => CheckIsSingleSharedFile(list, x));
list.ForEach(x => {
TestFileResult result = ProcessBeforeCheckout(x, ignoreSharedFiles, branch);
x.State = CalcBeforeCheckoutState(result);
});
if (list.Any(x => x.State == ProcessState.Failed))
{
return(false);
}
list.ForEach(x => {
TestFileResult result = ProcessCheckoutItem(x, x.Comment.ToString());
x.State = CalcCheckoutStateAfterCheckout(result);
});
return(list.All(x => x.State == ProcessState.Modified || x.State == ProcessState.Ignored));
}
public void DeleteResultAsync_should_delete_expected_result()
{
// given
TestsClient client = Helpers.CreateTestsClient();
TestFile testFile = Helpers.CreateTestFileAndTest(client);
var repository = ServiceStarter.Container.GetInstance <ITestFileResultRepository>();
var result1 = new TestFileResult()
{
StartTime = DateTime.Now,
EndTime = DateTime.Now.AddSeconds(1),
Filename = testFile.Filename
};
var result2 = new TestFileResult()
{
StartTime = DateTime.Now,
EndTime = DateTime.Now.AddSeconds(1),
Filename = testFile.Filename
};
repository.Add(result1).Wait();
repository.Add(result2).Wait();
// when
var result = client.DeleteResult(result2.Id);
// then
Assert.That(result, Is.True);
TestFileResult deletedResult = client.GetResultById(result2.Id);
Assert.That(deletedResult, Is.Null);
TestFileResult otherResult = client.GetResultById(result1.Id);
Assert.That(otherResult, Is.Not.Null);
}
public async Task Run_should_skip_test_if_it_is_set_to_ignore_environment()
{
// given
TestFileRunner runner = CreateRunner();
_httpClientMock.Response.StatusCode = HttpStatusCode.OK;
var testFile = CreateTestFile(new[]
{
new Test()
{
TestConditions = new TestConditions
{
RequiredEnvironments = { "not-development" }
}
},
});
// when
TestFileResult session = await runner.RunAsync(testFile, "development", "bob");
// then
Assert.That(session.TestResults.Single().ResultState, Is.EqualTo(TestResultState.Skipped));
}
public async Task <TestFileResult> RunAsync(TestFile testFile, string environment, string username)
{
_isStopPending = false;
lock (_currentResults)
{
_currentResults = new List <TestResult>();
}
var testFileResult = new TestFileResult
{
Id = SessionId,
Filename = testFile.Filename,
StartTime = DateTime.UtcNow,
Environment = environment,
Username = username
};
// Add all config variables and ones in this <test>
ICapturedVariableProvider variables = _capturedVariableProviderFactory.Create(environment);
variables.AddOrUpdateVariables(testFile.Variables);
var verificationsMatcher = new AssertionsMatcher(variables, _loggerFactory.CreateLogger());
List <Test> tests = testFile.Tests.ToList();
TimeSpan minResponseTime = TimeSpan.MaxValue;
TimeSpan maxResponseTime = TimeSpan.MinValue;
int totalTestsRun = 0;
TestsRun = 0;
TotalTests = tests.Count;
bool shouldSave = true;
for (int i = 0; i < tests.Count; i++)
{
if (_isStopPending)
{
break;
}
try
{
TestResult result = await RunTestAsync(tests.ElementAt(i), i, variables, verificationsMatcher);
AddResult(testFileResult, result);
if (result.ResponseTime < minResponseTime)
{
minResponseTime = result.ResponseTime;
}
if (result.ResponseTime > maxResponseTime)
{
maxResponseTime = result.ResponseTime;
}
}
catch (Exception ex)
{
ReportError(ex);
}
finally
{
totalTestsRun++;
TestsRun++;
}
if (_isStopPending)
{
shouldSave = false;
break;
}
}
testFileResult.EndTime = DateTime.UtcNow;
testFileResult.TotalRunTime = testFileResult.EndTime - testFileResult.StartTime;
testFileResult.TotalTestsRun = totalTestsRun;
testFileResult.MinResponseTime = minResponseTime;
testFileResult.MaxResponseTime = maxResponseTime;
if (shouldSave)
{
using (ITestFileResultRepository repository = RepositoryFactory.GetRepository())
{
await repository.Add(testFileResult);
}
}
NotifySubscribersOfCompletion(testFileResult.Id);
return(testFileResult);
}
public Task Add(TestFileResult testFileResult)
{
SavedTestFileResult = testFileResult;
return(Task.FromResult <object>(null));
}
static ProcessState CalcCheckoutStateAfterCheckout(TestFileResult result)
{
switch (result) {
case TestFileResult.Ok:
return ProcessState.Modified;
case TestFileResult.Fail:
return ProcessState.Failed;
case TestFileResult.Ignore:
return ProcessState.Ignored;
default:
throw new Exception("result");
}
}
public async Task Add(TestFileResult testFileResult)
{
await _collection.InsertOneAsync(testFileResult);
}
static ProcessState CalcBeforeCheckoutState(TestFileResult result)
{
switch (result) {
case TestFileResult.Ok:
return ProcessState.Default;
case TestFileResult.Fail:
return ProcessState.Failed;
case TestFileResult.Ignore:
return ProcessState.Ignored;
default:
throw new Exception("result");
}
}
