public bool Preprocessing(SubmissionDto submissionDto, string directory, out EngineWorkingDirectories workingDirectories, out FeedbackDto feedback) { workingDirectories = new EngineWorkingDirectories(); if (!ValidateSubmission(submissionDto, out feedback)) { return(false); } var handler = Factory.FileHandler(); workingDirectories.ParentDirectory = directory; workingDirectories.ReferenceTestSuit = Path.Combine(directory, "ReferenceTestSuit"); workingDirectories.Solution = Path.Combine(directory, Path.GetFileNameWithoutExtension(submissionDto.SolutionFolderName)); workingDirectories.StudentTestSuit = Path.Combine(directory, "StudentTestSuit"); handler.UnzipByteArray(submissionDto.ReferenceTestSolution, directory, workingDirectories.ReferenceTestSuit); handler.UnzipByteArray(submissionDto.AssignmentSolution, directory, workingDirectories.Solution); handler.UnzipByteArray(submissionDto.TestCaseSolution, directory, workingDirectories.StudentTestSuit); var exitCode = Factory.Pytest().Run(workingDirectories.StudentTestSuit, "$null", workingDirectories.StudentTestSuit); if (exitCode != 0) { feedback.Message = "Student's solution didn't pass their own Test Suite"; return(false); } return(true); }
public CoverageSession GetTotalCoverage(EngineWorkingDirectories workingDirectories) { var dir = Directory.GetCurrentDirectory(); var path = Directory.CreateDirectory(dir + "\\results"); OpenTrace openTrace = new OpenTrace(dir); var testPath = Path.Combine(dir + "\\results" + "\\TotalCoverage.xml"); var file = System.IO.File.Create(testPath); file.Close(); var name = workingDirectories.StudentDllPath.Split('\\'); openTrace.Run(null, path.FullName, "results", workingDirectories.StudentDllPath + "\\bin\\Debug\\" + name.LastOrDefault() + ".dll"); XmlSerializer serializer = new XmlSerializer(typeof(CoverageSession)); FileStream fs = new FileStream(path.FullName + "\\results.xml", FileMode.Open); CoverageSession coverageSession; coverageSession = (CoverageSession)serializer.Deserialize(fs); fs.Close(); Directory.Delete(path.FullName, true); return(coverageSession); }
public bool Preprocessing(SubmissionDto submissionDto, string directory, out EngineWorkingDirectories workingDirectories, out FeedbackDto feedback) { workingDirectories = new EngineWorkingDirectories(); if (!ValidateSubmission(submissionDto, out feedback)) { return(false); } var handler = Factory.FileHandler(); workingDirectories.ParentDirectory = directory; workingDirectories.ReferenceTestSuit = Path.Combine(directory, "ReferenceTestSuit"); workingDirectories.Solution = Path.Combine(directory, submissionDto.SolutionFolderName); workingDirectories.StudentTestSuit = Path.Combine(directory, "StudentTestSuit"); handler.UnzipByteArray(submissionDto.ReferenceTestSolution, directory, workingDirectories.ReferenceTestSuit); handler.UnzipByteArray(submissionDto.ReferenceSolution, directory, workingDirectories.Solution); handler.UnzipByteArray(submissionDto.TestCaseSolution, directory, workingDirectories.StudentTestSuit); return(true); }
public List <AnnotatedTest> Setup(string arguments, string output, EngineWorkingDirectories workingDirectory) { string dllPath = arguments; string[] argString = { "", "", "" }; //Can configure to use .csproj if given the correct path arguments = GetBuildFile(arguments, ".csproj"); argString[0] = arguments; //Can configure for .sln arguments = GetBuildFile(dllPath, ".sln"); argString[1] = arguments; string addPath = @"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\Common7\IDE\PublicAssemblies"; var process = new System.Diagnostics.Process(); var startInfo = new ProcessStartInfo() { FileName = @"C:\Windows\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe", RedirectStandardInput = true, RedirectStandardOutput = true, UseShellExecute = false, Arguments = argString[1] + " /p:" + '"' + "ReferencePath=" + addPath + '"', WorkingDirectory = workingDirectory.ParentDirectory }; process.StartInfo = startInfo; process.Start(); var buildSummary = process.StandardOutput.ReadToEnd(); process.WaitForExit(); var exitCode = process.ExitCode; process.Close(); process.Dispose(); if (exitCode == 1 || !buildSummary.Contains("Build succeeded.")) { throw new Exception(".csproj file Build failed. Path: " + arguments + " could not be executed properly."); } DirectoryInfo di = new DirectoryInfo(dllPath); var newPath = Path.Combine(dllPath + "\\bin\\Debug\\" + di.Name + ".dll"); PluginLoader loader = PluginLoader.CreateFromAssemblyFile(newPath); var annotations = GetAnnotations(loader.LoadDefaultAssembly()); return(annotations); }
public string GetDirectoryPaths(string testDirectory, string solution, ref EngineWorkingDirectories workingDirectories) { //Creates new solutin folder for combination of solution and test var newPath = Directory.CreateDirectory(testDirectory + "//CombineSolution"); //Copy content to new folder DirectoryCopy(solution, newPath.FullName, true); var solutionPath = newPath.GetDirectories(); string returnString = ""; if (solutionPath[0] != null) { var path = Directory.GetDirectories(testDirectory); if (path[1] != null) { DirectoryCopy(path[1], solutionPath[0].FullName, true); var dirs = Directory.GetDirectories(solutionPath[0].FullName); foreach (var d in dirs) { if (d.Contains("Test")) { returnString = d; } } } else { throw new Exception("Directory not found!"); } } else { throw new Exception("Directory not found!"); } if (testDirectory.Contains("Student")) { workingDirectories.StudentDllPath = returnString; } else { workingDirectories.InstructorDllPath = returnString; } return(returnString); }
public void Run(SubmissionDto submissionDto, ref EngineWorkingDirectories workingDirectories, ref FeedbackDto feedback) { var instructorTestSuit = CollectTestCases(workingDirectories.ReferenceTestSuit, workingDirectories.Solution, ref workingDirectories); var studentTestSuit = CollectTestCases(workingDirectories.StudentTestSuit, workingDirectories.Solution, ref workingDirectories); var coverDirectory = Path.Combine(workingDirectories.ParentDirectory, "CoverDirectory"); Directory.CreateDirectory(coverDirectory); var tracer = new Tracer(EngineFactory); var instructorTestCoverage = tracer.GetTestCoverage(workingDirectories.InstructorDllPath, coverDirectory, instructorTestSuit); var studentTestCoverage = tracer.GetTestCoverage(workingDirectories.StudentDllPath, coverDirectory, studentTestSuit); var comparator = EngineFactory.Comparator(); comparator.Compare(ref annotatedTests, ref instructorTestCoverage, ref studentTestCoverage, ref feedback, workingDirectories); }
public void GetStats(SubmissionDto submission, ref EngineWorkingDirectories workingDirectories, ref FeedbackDto feedback) { var pytestcov = _factory.PytestCov(); var filepath = Path.Combine(workingDirectories.ParentDirectory, "stats"); pytestcov.Run($"--cov-branch --cov={workingDirectories.Solution} .", filepath, workingDirectories.StudentTestSuit); var file = File.ReadAllLines(filepath); var found = false; foreach (var line in file) { if (found) { if (!line.StartsWith('-') && !line.StartsWith('=') && !string.IsNullOrWhiteSpace(line)) { var match = Regex.Match(line, @"(.+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(%)(\s*)"); if (match.Groups.Count == 14) { feedback.NumberOfStatements = int.Parse(match.Groups[3].Value); feedback.NumberOfMissingStatements = int.Parse(match.Groups[5].Value); feedback.NumberOfBranchesHit = int.Parse(match.Groups[7].Value); feedback.CoveragePercentage = Convert.ToDouble(int.Parse(match.Groups[11].Value)) / 100.00f; } } } else { if (_coverageHeaderRegex.IsMatch(line)) { found = true; } } } }
public IList <IndividualTest> CollectTestCases(string testDirectory, string solution, ref EngineWorkingDirectories workingDirectories) { AnnotatedTestParser testParser = new AnnotatedTestParser(); string path = GetDirectoryPaths(testDirectory, solution, ref workingDirectories); var annotations = testParser.Setup(path, null, workingDirectories); List <IndividualTest> individualTests = new List <IndividualTest>(); foreach (AnnotatedTest t in annotations) { individualTests.Add(t.IndividualTest); } annotatedTests = annotations; return(individualTests); }
public void Compare(ref IList <AnnotatedTest> annotatedTests, ref TraceTests instructor, ref TraceTests student, ref FeedbackDto feedback, EngineWorkingDirectories workingDirectories) { studentDirectory = workingDirectories.StudentDllPath; instructorDirectory = workingDirectories.InstructorDllPath; var InstructorMethodData = GetMethodMetaDatas(instructor.TestCoverages); var instCov = GetInstructorCoverage(InstructorMethodData, ref instructor); var StudentMethodData = GetMethodMetaDatas(student.TestCoverages); foreach (var coverage in instructor.TestCoverages) { var annotation = FindAnnotation(ref annotatedTests, coverage); var studentTests = new List <StudentTestDto>(); TestStatusEnum testStatus = TestStatusEnum.Redundant; if (!coverage.Pass) { testStatus = TestStatusEnum.Failed; } else { var testData = StudentMethodData .SelectMany(x => x .Where(y => y.TestName == coverage.Test.TestName)); var studentCoverage = FindCoverage(testData); int count = 0; foreach (var test in instCov) { if (test.Item1.Count() == studentCoverage.Count()) { var areEq = test.Item1.SequenceEqual(studentCoverage); var instDto = feedback.InstructorTests.FirstOrDefault(x => x.Name == test.Item2); if (instDto != null) { if (instDto.TestStatus == TestStatusEnum.Covered) { studentTests.Add( new StudentTestDto() { TestStatus = TestStatusEnum.Redundant, Name = coverage.Test.TestName, Passed = coverage.Pass }); testStatus = TestStatusEnum.Redundant; } } else if (areEq && count == 0) { studentTests.Add( new StudentTestDto() { TestStatus = TestStatusEnum.Covered, Name = coverage.Test.TestName, Passed = coverage.Pass }); count++; testStatus = TestStatusEnum.Covered; } else if (!areEq) { studentTests.Add( new StudentTestDto() { TestStatus = TestStatusEnum.Uncovered, Name = coverage.Test.TestName, Passed = coverage.Pass }); testStatus = TestStatusEnum.Uncovered; } } } } foreach (var test in feedback.InstructorTests) { if (test.StudentTests.Count() == 0) { test.TestStatus = TestStatusEnum.Uncovered; } if (test.StudentTests.Count() > 1) { test.TestStatus = TestStatusEnum.Redundant; var t = test.StudentTests.ToArray().Where(x => x.Name == x.Name); if (t.Count() == test.StudentTests.Count()) { test.TestStatus = TestStatusEnum.Covered; } } } feedback.InstructorTests.Add( new InstructorTestDto { Name = coverage.Test.TestName, Concepts = (annotation == null) ? new string[] { } : annotation.Concepts.ToArray(), EquivalenceClass = (annotation == null) ? "NONE" : annotation.EquivalanceClass, TestStatus = testStatus, StudentTests = studentTests }); } var coverageSession = GetTotalCoverage(workingDirectories); //OpenCover generates reports for the methods being tested themselves, so it will show double the data it should //on coverage percentage, and statements. It is possible this can be filtered out in OpenCover, but I am not sure //how to do it without completel excluding the actual coverage data feedback.NumberOfBranchesHit = coverageSession.Summary.VisitedBranchPoints; feedback.NumberOfMissingStatements = (coverageSession.Summary.NumSequencePoints / 2) - coverageSession.Summary.VisitedSequencePoints; feedback.NumberOfStatements = coverageSession.Summary.NumSequencePoints / 2; feedback.CoveragePercentage = (double)coverageSession.Summary.BranchCoverage * 2; }
public void GetStats(SubmissionDto submission, ref EngineWorkingDirectories workingDirectories, ref FeedbackDto feedback) { }