public void MochaExecutionResult_WithUnitTests_ShouldReturnCorrectUserTestsCount()
{
var jsonString = @"{stats: {passes: 1, tests: 1}, tests: [{fullTitle: ""Test 0 "", err: {}}]}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.AreEqual(1, result.UsersTestCount, "Json Execution Result does not return correct number of UserTests.");
}
public void MochaExecutionResultParseShouldReturnCorrectErrorMessageWhenTheInputDoesNotContainAllRequiredFields()
{
var jsonString = "{stats: {passes: 0, tests: 3}}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.AreEqual(ParseErrorMessage, result.Error, "Incorrect Error message set.");
}
public void MochaExecutionResult_WithUnitTests_ShouldReturnCorrect()
{
var jsonString = @"{stats: {passes: 1, tests: 4}, tests: [{fullTitle: ""Test 0 "", err: {}}, {fullTitle: ""Test 1 "", err: {}}, {fullTitle: ""Test 1 "", err: { stack: ""Some Error""}}, {fullTitle: ""Test 1 "", err: { stack: ""Some Other error""}}]}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.AreEqual(1, result.InitialPassingTests, "Json Execution Result does not return the correct number of initial passing tests.");
}
public void MochaExecutionResultParseShouldReturnCorrectErrorMessageWhenTheInputCannotBeParsed()
{
var jsonString = "{stats: {passes: 0}, f";
var result = JsonExecutionResult.Parse(jsonString);
Assert.AreEqual(ParseErrorMessage, result.Error, "Incorrect Error message set.");
}
public void MochaExecutionResultParseShouldReturnNotPassedWithCorrectErrorDoubleCommentInJSONString()
{
var jsonString = "{stats: {passes: 1}} //** comment **// {stats: {passes: 1}}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.IsFalse(result.Passed);
Assert.AreEqual(ParseErrorMessage, result.Error);
}
public void MochaExecutionResultParseShouldReturnNotPassedWithCorrectErrorInvalidTwoObjectsInJSONString()
{
var jsonString = "{stats: {passes: 1}} {stats: {passes: 1}}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.IsFalse(result.Passed);
Assert.AreEqual(ParseErrorMessage, result.Error);
}
public void MochaExecutionResultParseShouldReturnNotPassedWithCorrectErrorWithCorrectNoPassesAndFailiureJSONString()
{
var jsonString = "{stats: {passes: 0}, failures: [{ err: { message: \"Custom error\" } }]}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.IsFalse(result.Passed);
Assert.AreEqual("Custom error", result.Error);
}
public void MochaExecutionResultParseShouldReturnPassedWithCorrectPassesJSONString()
{
var jsonString = "{stats: {passes: 1}}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.IsTrue(result.Passed);
Assert.IsNullOrEmpty(result.Error);
}
public void MochaExecutionResultParseShouldFailParsingWithDoubleCommentInJsonString()
{
var jsonString = @"{stats: {passes: 1, tests:2}, tests: []} //** comment **// {stats: {passes: 2, tests:3}, tests: [{err: {message: ""Error: expected 1 to equal 2"", stack: ""Some stack""}}]}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.AreEqual(0, result.TotalPassingTests, "Incorrect number of Passed Tests returned.");
Assert.AreEqual(0, result.TotalTests, "Incorrect number of Total Tests returned");
Assert.AreEqual(ParseErrorMessage, result.Error, "Incorrect Error message set.");
}
public void MochaExecutionResultParseShouldFailParsingWithCommentInJsonString()
{
var jsonString = "{stats: {passes: 1, tests: 6}, tests: []} /* comment */ {stats: {passes: 1, tests: 1}, tests: []}";
var result = JsonExecutionResult.Parse(jsonString);
Assert.AreEqual(0, result.TotalPassingTests, "Incorrect number of Passed Tests returned.");
Assert.AreEqual(0, result.TotalTests, "Incorrect number of Total Tests returned");
Assert.AreEqual(ParseErrorMessage, result.Error, "Incorrect Error message set.");
}
public void MochaExecutionResultParseShouldReturnCorrectNumberOfPassedTests()
{
var jsonString = "{stats: {passes: 3, tests: 4}, tests: []}";
var result = JsonExecutionResult.Parse(jsonString);
var expected = 3;
Assert.AreEqual(expected, result.TotalPassingTests, "Incorrect ammount of passed tests returned.");
Assert.IsNullOrEmpty(result.Error, "Error field was not empty with a correct output");
}
public void MochaExecutionResultParseShouldReturnCorrectErrorMessages()
{
var jsonString = @"{stats: {passes: 1, tests: 3}, tests:[ {fullTitle: ""Test 0 "", err: {message: ""Error: expected 1 to equal 2"", stack: ""Some stack""}}, {fullTitle: ""Test 1 "", err:{}}, {fullTitle: ""Test 2 "", err: {message: ""Error: expected 'a' to equal 'b'"", stack: ""Another stack trace""}}]}";
var result = JsonExecutionResult.Parse(jsonString);
var errorMessage1 = "Error: expected 1 to equal 2";
var errorMessage2 = "Error: expected 'a' to equal 'b'";
Assert.AreEqual(errorMessage1, result.TestErrors[0], "Incorrect Error message returned");
Assert.IsNull(result.TestErrors[1], "Error message was returned on a passing test");
Assert.AreEqual(errorMessage2, result.TestErrors[2], "Incorrect Error message returned");
Assert.IsNullOrEmpty(result.Error, "Error field was not empty with a correct output");
}
protected override List <TestResult> ProcessTests(
IExecutionContext <TestsInputModel> executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(this.AdditionalExecutionArguments);
var processExecutionResult = executor.Execute(
this.NodeJsExecutablePath,
string.Empty,
executionContext.TimeLimit,
executionContext.MemoryLimit,
arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var currentTest = 0;
foreach (var test in executionContext.Input.Tests)
{
var message = "yes";
if (!string.IsNullOrEmpty(mochaResult.Error))
{
message = mochaResult.Error;
}
else if (mochaResult.TestErrors[currentTest] != null)
{
message = $"Unexpected error: {mochaResult.TestErrors[currentTest]}";
}
var testResult = this.CheckAndGetTestResult(
test,
processExecutionResult,
checker,
message);
currentTest++;
testResults.Add(testResult);
}
return(testResults);
}
protected override List <TestResult> ProcessTests(
IExecutionContext <TestsInputModel> executionContext,
IExecutor executor,
IChecker checker,
string codeSavePath)
{
var testResults = new List <TestResult>();
var arguments = new List <string>();
arguments.Add(this.MochaModulePath);
arguments.Add(codeSavePath);
arguments.AddRange(this.AdditionalExecutionArguments);
var testCount = 0;
var processExecutionResult = executor.Execute(
this.NodeJsExecutablePath,
string.Empty,
executionContext.TimeLimit,
executionContext.MemoryLimit,
arguments);
var mochaResult = JsonExecutionResult.Parse(processExecutionResult.ReceivedOutput);
var numberOfUserTests = mochaResult.UsersTestCount;
var correctSolutionTestPasses = mochaResult.InitialPassingTests;
// an offset for tracking the current subset of tests (by convention we always have 2 Zero tests)
var testOffset = numberOfUserTests * 2;
foreach (var test in executionContext.Input.Tests)
{
var message = TestPassedMessage;
TestResult testResult = null;
if (testCount == 0)
{
var minTestCount = int.Parse(
Regex.Match(
test.Input,
"<minTestCount>(\\d+)</minTestCount>").Groups[1].Value);
if (numberOfUserTests < minTestCount)
{
message = $"Insufficient amount of tests, you have to have atleast {minTestCount} tests!";
}
testResult = this.CheckAndGetTestResult(
test,
processExecutionResult,
checker,
message);
}
else if (testCount == 1)
{
if (numberOfUserTests == 0)
{
message = "The submitted code was either incorrect or contained no tests!";
}
else if (correctSolutionTestPasses != numberOfUserTests)
{
message = "Error: Some tests failed while running the correct solution!";
}
testResult = this.CheckAndGetTestResult(
test,
processExecutionResult,
checker,
message);
}
else
{
var numberOfPasses = mochaResult.TestErrors.Skip(testOffset).Take(numberOfUserTests).Count(x => x == null);
if (numberOfPasses >= correctSolutionTestPasses)
{
message = "No unit test covering this functionality!";
}
testResult = this.CheckAndGetTestResult(
test,
processExecutionResult,
checker,
message);
testOffset += numberOfUserTests;
}
testCount++;
testResults.Add(testResult);
}
return(testResults);
}
