public void Compare(ref IList <AnnotatedTest> annotatedTests, ref TraceTests instructor, ref TraceTests student, ref FeedbackDto feedback)
{
foreach (var coverage in instructor.TestCoverages)
{
var annotation = FindAnnotation(ref annotatedTests, coverage);
var studentTests = GetStudentTests(coverage, ref student);
TestStatusEnum testStatus;
if (!coverage.Pass)
{
testStatus = TestStatusEnum.Failed;
}
else
{
var studentCoverage = FindCoverage(coverage, ref student);
testStatus = studentCoverage == null ? TestStatusEnum.Uncovered : TestStatusEnum.Covered;
}
feedback.InstructorTests.Add(
new InstructorTestDto
{
Name = coverage.Test.TestName,
Concepts = (annotation == null) ? new string[] { } : annotation.Concepts.ToArray(),
EquivalenceClass = (annotation == null) ? "NONE" : annotation.EquivalanceClass,
TestStatus = testStatus,
StudentTests = studentTests
});
}
}
public TestCoverage FindCoverage(TestCoverage coverage, ref TraceTests against)
{
foreach (var testCoverage in against.TestCoverages)
{
if (testCoverage.Pass != coverage.Pass)
{
continue;
}
if (CompareModuleCoverages(testCoverage.ModuleCoverages,
coverage.ModuleCoverages))
{
return(testCoverage);
}
}
return(null);
}
private bool CompareTraceTest(TraceTests expected, TraceTests actual)
{
if (expected.TestCoverages.Count != actual.TestCoverages.Count)
{
return(false);
}
for (int i = 0; i < expected.TestCoverages.Count; i++)
{
var expectedCoverage = expected.TestCoverages[i];
var actualCoverage = actual.TestCoverages[i];
if (expectedCoverage.ModuleCoverages.Count != actualCoverage.ModuleCoverages.Count)
{
return(false);
}
if (!Equals(expectedCoverage.Test, actualCoverage.Test))
{
return(false);
}
if (expectedCoverage.Pass != actualCoverage.Pass)
{
return(false);
}
for (int j = 0; j < expectedCoverage.ModuleCoverages.Count; j++)
{
var expectedModule = expectedCoverage.ModuleCoverages[j];
var actualModule = actualCoverage.ModuleCoverages[j];
if (expectedModule.ModuleName != actualModule.ModuleName)
{
return(false);
}
}
}
return(true);
}
public IList <StudentTestDto> GetStudentTests(TestCoverage instructor, ref TraceTests student)
{
var studentTests = new List <StudentTestDto>();
var testStatus = TestStatusEnum.Covered;
foreach (var coverage in student.TestCoverages)
{
if (CompareModuleCoverages(instructor.ModuleCoverages, coverage.ModuleCoverages))
{
studentTests.Add(
new StudentTestDto()
{
TestStatus = testStatus,
Name = coverage.Test.TestName,
Passed = coverage.Pass
});
testStatus = TestStatusEnum.Redundant;
}
}
return(studentTests);
}
public void GetTestCoverageFromSingleFile_ShouldPass()
{
// Arrange
var file = Path.Combine(CurrentDirectory, "my_test.py");
var coverDir = Path.Combine(CurrentDirectory, "CoverDir");
Directory.CreateDirectory(coverDir);
File.WriteAllText(file,
"def test_one():\r\n" +
" assert 2 == 2\r\n" +
"def test_two():\r\n" +
" assert \"d\" in \"abc\"");
// Act
var actual = Tracer.GetTestCoverage(CurrentDirectory,
coverDir,
new List <IndividualTest>()
{
new IndividualTest("my_test.py::test_one"),
new IndividualTest("my_test.py::test_two"),
},
new List <ModuleName>()
{
new ModuleName()
{
Name = "my_test.py"
}
});
// Assert
var expected = new TraceTests
{
TestCoverages = new List <TestCoverage>()
{
new TestCoverage()
{
Test = new IndividualTest("my_test.py::test_one"),
Pass = true,
ModuleCoverages = new List <ModuleCoverage>()
{
new ModuleCoverage()
{
ModuleName = new ModuleName()
{
Name = "my_test.py"
},
Contents = new[] { "" }
}
}
},
new TestCoverage()
{
Test = new IndividualTest("my_test.py::test_two"),
Pass = false,
ModuleCoverages = new List <ModuleCoverage>()
{
new ModuleCoverage()
{
ModuleName = new ModuleName()
{
Name = "my_test.py"
},
Contents = new[] { "" }
}
}
}
}
};
Assert.IsTrue(CompareTraceTest(expected, actual));
}
public List <Tuple <List <int>, string> > GetInstructorCoverage(List <IEnumerable <MethodMetaData> > methodMetaDatas, ref TraceTests instructor)
{
List <Tuple <List <int>, string> > list = new List <Tuple <List <int>, string> >();
foreach (var method in methodMetaDatas)
{
foreach (var i in instructor.TestCoverages)
{
var m = method.Where(x => x.TestName == i.Test.TestName);
if (m != null)
{
var returnVal = FindCoverage(m);
var names = list
.Select(x => x.Item2);
if (!names.Contains(i.Test.TestName) && returnVal.Count() != 0)
{
list.Add(new Tuple <List <int>, string>(returnVal, i.Test.TestName));
}
}
}
}
return(list);
}
public void Compare(ref IList <AnnotatedTest> annotatedTests, ref TraceTests instructor,
ref TraceTests student, ref FeedbackDto feedback, EngineWorkingDirectories workingDirectories)
{
studentDirectory = workingDirectories.StudentDllPath;
instructorDirectory = workingDirectories.InstructorDllPath;
var InstructorMethodData = GetMethodMetaDatas(instructor.TestCoverages);
var instCov = GetInstructorCoverage(InstructorMethodData, ref instructor);
var StudentMethodData = GetMethodMetaDatas(student.TestCoverages);
foreach (var coverage in instructor.TestCoverages)
{
var annotation = FindAnnotation(ref annotatedTests, coverage);
var studentTests = new List <StudentTestDto>();
TestStatusEnum testStatus = TestStatusEnum.Redundant;
if (!coverage.Pass)
{
testStatus = TestStatusEnum.Failed;
}
else
{
var testData = StudentMethodData
.SelectMany(x => x
.Where(y => y.TestName == coverage.Test.TestName));
var studentCoverage = FindCoverage(testData);
int count = 0;
foreach (var test in instCov)
{
if (test.Item1.Count() == studentCoverage.Count())
{
var areEq = test.Item1.SequenceEqual(studentCoverage);
var instDto = feedback.InstructorTests.FirstOrDefault(x => x.Name == test.Item2);
if (instDto != null)
{
if (instDto.TestStatus == TestStatusEnum.Covered)
{
studentTests.Add(
new StudentTestDto()
{
TestStatus = TestStatusEnum.Redundant,
Name = coverage.Test.TestName,
Passed = coverage.Pass
});
testStatus = TestStatusEnum.Redundant;
}
}
else if (areEq && count == 0)
{
studentTests.Add(
new StudentTestDto()
{
TestStatus = TestStatusEnum.Covered,
Name = coverage.Test.TestName,
Passed = coverage.Pass
});
count++;
testStatus = TestStatusEnum.Covered;
}
else if (!areEq)
{
studentTests.Add(
new StudentTestDto()
{
TestStatus = TestStatusEnum.Uncovered,
Name = coverage.Test.TestName,
Passed = coverage.Pass
});
testStatus = TestStatusEnum.Uncovered;
}
}
}
}
foreach (var test in feedback.InstructorTests)
{
if (test.StudentTests.Count() == 0)
{
test.TestStatus = TestStatusEnum.Uncovered;
}
if (test.StudentTests.Count() > 1)
{
test.TestStatus = TestStatusEnum.Redundant;
var t = test.StudentTests.ToArray().Where(x => x.Name == x.Name);
if (t.Count() == test.StudentTests.Count())
{
test.TestStatus = TestStatusEnum.Covered;
}
}
}
feedback.InstructorTests.Add(
new InstructorTestDto
{
Name = coverage.Test.TestName,
Concepts = (annotation == null) ? new string[] { } : annotation.Concepts.ToArray(),
EquivalenceClass = (annotation == null) ? "NONE" : annotation.EquivalanceClass,
TestStatus = testStatus,
StudentTests = studentTests
});
}
var coverageSession = GetTotalCoverage(workingDirectories);
//OpenCover generates reports for the methods being tested themselves, so it will show double the data it should
//on coverage percentage, and statements. It is possible this can be filtered out in OpenCover, but I am not sure
//how to do it without completel excluding the actual coverage data
feedback.NumberOfBranchesHit = coverageSession.Summary.VisitedBranchPoints;
feedback.NumberOfMissingStatements = (coverageSession.Summary.NumSequencePoints / 2) - coverageSession.Summary.VisitedSequencePoints;
feedback.NumberOfStatements = coverageSession.Summary.NumSequencePoints / 2;
feedback.CoveragePercentage = (double)coverageSession.Summary.BranchCoverage * 2;
}
