public bool Preprocessing(SubmissionDto submissionDto, string directory, out EngineWorkingDirectories workingDirectories,
out FeedbackDto feedback)
{
workingDirectories = new EngineWorkingDirectories();
if (!ValidateSubmission(submissionDto, out feedback))
{
return(false);
}
var handler = Factory.FileHandler();
workingDirectories.ParentDirectory = directory;
workingDirectories.ReferenceTestSuit = Path.Combine(directory, "ReferenceTestSuit");
workingDirectories.Solution = Path.Combine(directory, Path.GetFileNameWithoutExtension(submissionDto.SolutionFolderName));
workingDirectories.StudentTestSuit = Path.Combine(directory, "StudentTestSuit");
handler.UnzipByteArray(submissionDto.ReferenceTestSolution, directory,
workingDirectories.ReferenceTestSuit);
handler.UnzipByteArray(submissionDto.AssignmentSolution, directory,
workingDirectories.Solution);
handler.UnzipByteArray(submissionDto.TestCaseSolution, directory,
workingDirectories.StudentTestSuit);
var exitCode = Factory.Pytest().Run(workingDirectories.StudentTestSuit, "$null", workingDirectories.StudentTestSuit);
if (exitCode != 0)
{
feedback.Message = "Student's solution didn't pass their own Test Suite";
return(false);
}
return(true);
}
public CoverageSession GetTotalCoverage(EngineWorkingDirectories workingDirectories)
{
var dir = Directory.GetCurrentDirectory();
var path = Directory.CreateDirectory(dir + "\\results");
OpenTrace openTrace = new OpenTrace(dir);
var testPath = Path.Combine(dir + "\\results" + "\\TotalCoverage.xml");
var file = System.IO.File.Create(testPath);
file.Close();
var name = workingDirectories.StudentDllPath.Split('\\');
openTrace.Run(null, path.FullName, "results", workingDirectories.StudentDllPath + "\\bin\\Debug\\" + name.LastOrDefault() + ".dll");
XmlSerializer serializer = new XmlSerializer(typeof(CoverageSession));
FileStream fs = new FileStream(path.FullName + "\\results.xml", FileMode.Open);
CoverageSession coverageSession;
coverageSession = (CoverageSession)serializer.Deserialize(fs);
fs.Close();
Directory.Delete(path.FullName, true);
return(coverageSession);
}
public bool Preprocessing(SubmissionDto submissionDto, string directory, out EngineWorkingDirectories workingDirectories,
out FeedbackDto feedback)
{
workingDirectories = new EngineWorkingDirectories();
if (!ValidateSubmission(submissionDto, out feedback))
{
return(false);
}
var handler = Factory.FileHandler();
workingDirectories.ParentDirectory = directory;
workingDirectories.ReferenceTestSuit = Path.Combine(directory, "ReferenceTestSuit");
workingDirectories.Solution = Path.Combine(directory, submissionDto.SolutionFolderName);
workingDirectories.StudentTestSuit = Path.Combine(directory, "StudentTestSuit");
handler.UnzipByteArray(submissionDto.ReferenceTestSolution, directory,
workingDirectories.ReferenceTestSuit);
handler.UnzipByteArray(submissionDto.ReferenceSolution, directory,
workingDirectories.Solution);
handler.UnzipByteArray(submissionDto.TestCaseSolution, directory,
workingDirectories.StudentTestSuit);
return(true);
}
public List <AnnotatedTest> Setup(string arguments, string output, EngineWorkingDirectories workingDirectory)
{
string dllPath = arguments;
string[] argString = { "", "", "" };
//Can configure to use .csproj if given the correct path
arguments = GetBuildFile(arguments, ".csproj");
argString[0] = arguments;
//Can configure for .sln
arguments = GetBuildFile(dllPath, ".sln");
argString[1] = arguments;
string addPath = @"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\Common7\IDE\PublicAssemblies";
var process = new System.Diagnostics.Process();
var startInfo =
new ProcessStartInfo()
{
FileName = @"C:\Windows\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe",
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
Arguments = argString[1] + " /p:" + '"' + "ReferencePath=" + addPath + '"',
WorkingDirectory = workingDirectory.ParentDirectory
};
process.StartInfo = startInfo;
process.Start();
var buildSummary = process.StandardOutput.ReadToEnd();
process.WaitForExit();
var exitCode = process.ExitCode;
process.Close();
process.Dispose();
if (exitCode == 1 || !buildSummary.Contains("Build succeeded."))
{
throw new Exception(".csproj file Build failed. Path: " + arguments + " could not be executed properly.");
}
DirectoryInfo di = new DirectoryInfo(dllPath);
var newPath = Path.Combine(dllPath + "\\bin\\Debug\\" + di.Name + ".dll");
PluginLoader loader = PluginLoader.CreateFromAssemblyFile(newPath);
var annotations = GetAnnotations(loader.LoadDefaultAssembly());
return(annotations);
}
public string GetDirectoryPaths(string testDirectory, string solution, ref EngineWorkingDirectories workingDirectories)
{
//Creates new solutin folder for combination of solution and test
var newPath = Directory.CreateDirectory(testDirectory + "//CombineSolution");
//Copy content to new folder
DirectoryCopy(solution, newPath.FullName, true);
var solutionPath = newPath.GetDirectories();
string returnString = "";
if (solutionPath[0] != null)
{
var path = Directory.GetDirectories(testDirectory);
if (path[1] != null)
{
DirectoryCopy(path[1], solutionPath[0].FullName, true);
var dirs = Directory.GetDirectories(solutionPath[0].FullName);
foreach (var d in dirs)
{
if (d.Contains("Test"))
{
returnString = d;
}
}
}
else
{
throw new Exception("Directory not found!");
}
}
else
{
throw new Exception("Directory not found!");
}
if (testDirectory.Contains("Student"))
{
workingDirectories.StudentDllPath = returnString;
}
else
{
workingDirectories.InstructorDllPath = returnString;
}
return(returnString);
}
public void Run(SubmissionDto submissionDto, ref EngineWorkingDirectories workingDirectories, ref FeedbackDto feedback)
{
var instructorTestSuit = CollectTestCases(workingDirectories.ReferenceTestSuit, workingDirectories.Solution, ref workingDirectories);
var studentTestSuit = CollectTestCases(workingDirectories.StudentTestSuit, workingDirectories.Solution, ref workingDirectories);
var coverDirectory = Path.Combine(workingDirectories.ParentDirectory, "CoverDirectory");
Directory.CreateDirectory(coverDirectory);
var tracer = new Tracer(EngineFactory);
var instructorTestCoverage = tracer.GetTestCoverage(workingDirectories.InstructorDllPath,
coverDirectory, instructorTestSuit);
var studentTestCoverage = tracer.GetTestCoverage(workingDirectories.StudentDllPath,
coverDirectory, studentTestSuit);
var comparator = EngineFactory.Comparator();
comparator.Compare(ref annotatedTests, ref instructorTestCoverage, ref studentTestCoverage,
ref feedback, workingDirectories);
}
public void GetStats(SubmissionDto submission, ref EngineWorkingDirectories workingDirectories, ref FeedbackDto feedback)
{
var pytestcov = _factory.PytestCov();
var filepath = Path.Combine(workingDirectories.ParentDirectory, "stats");
pytestcov.Run($"--cov-branch --cov={workingDirectories.Solution} .", filepath, workingDirectories.StudentTestSuit);
var file = File.ReadAllLines(filepath);
var found = false;
foreach (var line in file)
{
if (found)
{
if (!line.StartsWith('-') && !line.StartsWith('=') && !string.IsNullOrWhiteSpace(line))
{
var match = Regex.Match(line, @"(.+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(%)(\s*)");
if (match.Groups.Count == 14)
{
feedback.NumberOfStatements = int.Parse(match.Groups[3].Value);
feedback.NumberOfMissingStatements = int.Parse(match.Groups[5].Value);
feedback.NumberOfBranchesHit = int.Parse(match.Groups[7].Value);
feedback.CoveragePercentage = Convert.ToDouble(int.Parse(match.Groups[11].Value)) / 100.00f;
}
}
}
else
{
if (_coverageHeaderRegex.IsMatch(line))
{
found = true;
}
}
}
}
public IList <IndividualTest> CollectTestCases(string testDirectory, string solution, ref EngineWorkingDirectories workingDirectories)
{
AnnotatedTestParser testParser = new AnnotatedTestParser();
string path = GetDirectoryPaths(testDirectory, solution, ref workingDirectories);
var annotations = testParser.Setup(path, null, workingDirectories);
List <IndividualTest> individualTests = new List <IndividualTest>();
foreach (AnnotatedTest t in annotations)
{
individualTests.Add(t.IndividualTest);
}
annotatedTests = annotations;
return(individualTests);
}
public void Compare(ref IList <AnnotatedTest> annotatedTests, ref TraceTests instructor,
ref TraceTests student, ref FeedbackDto feedback, EngineWorkingDirectories workingDirectories)
{
studentDirectory = workingDirectories.StudentDllPath;
instructorDirectory = workingDirectories.InstructorDllPath;
var InstructorMethodData = GetMethodMetaDatas(instructor.TestCoverages);
var instCov = GetInstructorCoverage(InstructorMethodData, ref instructor);
var StudentMethodData = GetMethodMetaDatas(student.TestCoverages);
foreach (var coverage in instructor.TestCoverages)
{
var annotation = FindAnnotation(ref annotatedTests, coverage);
var studentTests = new List <StudentTestDto>();
TestStatusEnum testStatus = TestStatusEnum.Redundant;
if (!coverage.Pass)
{
testStatus = TestStatusEnum.Failed;
}
else
{
var testData = StudentMethodData
.SelectMany(x => x
.Where(y => y.TestName == coverage.Test.TestName));
var studentCoverage = FindCoverage(testData);
int count = 0;
foreach (var test in instCov)
{
if (test.Item1.Count() == studentCoverage.Count())
{
var areEq = test.Item1.SequenceEqual(studentCoverage);
var instDto = feedback.InstructorTests.FirstOrDefault(x => x.Name == test.Item2);
if (instDto != null)
{
if (instDto.TestStatus == TestStatusEnum.Covered)
{
studentTests.Add(
new StudentTestDto()
{
TestStatus = TestStatusEnum.Redundant,
Name = coverage.Test.TestName,
Passed = coverage.Pass
});
testStatus = TestStatusEnum.Redundant;
}
}
else if (areEq && count == 0)
{
studentTests.Add(
new StudentTestDto()
{
TestStatus = TestStatusEnum.Covered,
Name = coverage.Test.TestName,
Passed = coverage.Pass
});
count++;
testStatus = TestStatusEnum.Covered;
}
else if (!areEq)
{
studentTests.Add(
new StudentTestDto()
{
TestStatus = TestStatusEnum.Uncovered,
Name = coverage.Test.TestName,
Passed = coverage.Pass
});
testStatus = TestStatusEnum.Uncovered;
}
}
}
}
foreach (var test in feedback.InstructorTests)
{
if (test.StudentTests.Count() == 0)
{
test.TestStatus = TestStatusEnum.Uncovered;
}
if (test.StudentTests.Count() > 1)
{
test.TestStatus = TestStatusEnum.Redundant;
var t = test.StudentTests.ToArray().Where(x => x.Name == x.Name);
if (t.Count() == test.StudentTests.Count())
{
test.TestStatus = TestStatusEnum.Covered;
}
}
}
feedback.InstructorTests.Add(
new InstructorTestDto
{
Name = coverage.Test.TestName,
Concepts = (annotation == null) ? new string[] { } : annotation.Concepts.ToArray(),
EquivalenceClass = (annotation == null) ? "NONE" : annotation.EquivalanceClass,
TestStatus = testStatus,
StudentTests = studentTests
});
}
var coverageSession = GetTotalCoverage(workingDirectories);
//OpenCover generates reports for the methods being tested themselves, so it will show double the data it should
//on coverage percentage, and statements. It is possible this can be filtered out in OpenCover, but I am not sure
//how to do it without completel excluding the actual coverage data
feedback.NumberOfBranchesHit = coverageSession.Summary.VisitedBranchPoints;
feedback.NumberOfMissingStatements = (coverageSession.Summary.NumSequencePoints / 2) - coverageSession.Summary.VisitedSequencePoints;
feedback.NumberOfStatements = coverageSession.Summary.NumSequencePoints / 2;
feedback.CoveragePercentage = (double)coverageSession.Summary.BranchCoverage * 2;
}
public void GetStats(SubmissionDto submission, ref EngineWorkingDirectories workingDirectories, ref FeedbackDto feedback)
{
}
