/// <summary> /// Builds a submission archive containing the submissions of /// all students. /// </summary> public async Task<Stream> BuildSubmissionArchiveAsync( Project project, IArchive templateContents, IList<StudentSubmission> submissions) { var stream = _fileSystem.CreateNewTempFile(); using (ZipArchive archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) { foreach (var result in submissions) { await WriteSubmissionToArchiveAsync ( archive, project, result.Student, templateContents, result.Contents ); result.Contents.Dispose(); } } stream.Position = 0; return stream; }
/// <summary> /// Writes the contents of a submission to an archive. /// </summary> private async Task WriteSubmissionToArchiveAsync( ZipArchive archive, Project project, ClassroomMembership student, IArchive templateContents, IArchive submissionContents) { var studentFolder = $"EclipseProjects\\{student.GitHubTeam}"; // The project will contain all non-immutable submission files, // plus all immutable and private files from the template project. var projectContents = submissionContents.Files .Where ( entry => project.GetFileType(entry) == FileType.Public ) .Concat ( templateContents.Files.Where ( entry => project.GetFileType(entry) != FileType.Public ) ) .ToList(); foreach (var entry in projectContents) { if (ExcludeEntry(project, entry)) continue; var contents = _transformer.GetFileContents(project, student, entry); var archiveFilePath = entry.FullPath; var archiveFileFolder = archiveFilePath.Contains("/") ? archiveFilePath.Substring(0, archiveFilePath.LastIndexOf("/")) : archiveFilePath; var localFileFolder = $"{studentFolder}\\{archiveFileFolder}"; var fileName = archiveFilePath.Substring(archiveFilePath.LastIndexOf("/") + 1); var localFilePath = $"{localFileFolder}\\{fileName}"; // Add the file to the student project folder. var projectFolderEntry = archive.CreateEntry(localFilePath); using (Stream stream = projectFolderEntry.Open()) { await stream.WriteAsync(contents, offset: 0, count: contents.Length); } // Add the file to the folder containing all files, if applicable. if (fileName.EndsWith(".java") && project.GetFileType(entry) == FileType.Public) { var allFilesEntry = archive.CreateEntry($"AllFiles\\{student.GitHubTeam}-{fileName}"); using (Stream stream = allFilesEntry.Open()) { await stream.WriteAsync(contents, offset: 0, count: contents.Length); } } } }
/// <summary> /// Retrieves a list of student repositories for a given project. /// </summary> public async Task<IDictionary<ClassroomMembership, GitHubRepository>> GetStudentRepositoriesAsync( Project project, IList<ClassroomMembership> students) { var orgName = project.Classroom.GitHubOrganization; var repoList = await _repoClient.GetAllRepositoriesAsync(orgName); var repoDictionary = repoList.ToDictionary ( repo => repo.Name, repo => repo ); return students .Where ( student => repoDictionary.ContainsKey ( GetRepoName(project, student) ) ) .ToDictionary ( student => student, student => repoDictionary[GetRepoName(project, student)] ); }
/// <summary> /// Executes before the action is executed. /// </summary> protected override async Task InitializeAsync() { await base.InitializeAsync(); Project = await ProjectService.GetProjectAsync(ClassroomName, ProjectName); ViewBag.Project = Project; }
/// <summary> /// Downloads the contents of the project template. /// </summary> public async Task<IArchive> DownloadTemplateContentsAsync(Project project) { return await _repoClient.GetRepositoryContentsAsync ( project.Classroom.GitHubOrganization, $"{project.Name}_Template", null /*branchName*/, ArchiveStore.Memory ); }
public void GetRepoName_ReturnsCorrectName() { var repoMetadataRetriever = new RepositoryMetadataRetriever(repoClient: null); var project = new Project() { Name = "Project1" }; var student = new ClassroomMembership() { GitHubTeam = "LastNameFirstName" }; var result = repoMetadataRetriever.GetRepoName(project, student); Assert.Equal("Project1_LastNameFirstName", result); }
/// <summary> /// Returns the file's contents, with any applicable /// transformations applied. /// </summary> public byte[] GetFileContents( Project project, ClassroomMembership student, IArchiveFile entry) { if (entry.FullPath.EndsWith(".project")) { var newContents = entry.GetEncodedData().Replace ( $"<name>{project.Name}</name>", $"<name>{project.Name}_{student.GitHubTeam}</name>" ); using (var memoryStream = new MemoryStream()) using (var streamWriter = new StreamWriter(memoryStream)) { streamWriter.Write(newContents); streamWriter.Flush(); return memoryStream.ToArray(); } } else if (entry.FullPath.EndsWith(".classpath")) { var jUnitPath = "org.eclipse.jdt.junit.JUNIT_CONTAINER/4"; using (var stream = new MemoryStream(entry.GetRawData())) { var projectNode = XElement.Load(stream); var hasJUnit = projectNode .Elements(XName.Get("classpathentry")) .Any ( elt => elt.Attribute(XName.Get("path")) != null && elt.Attribute(XName.Get("path")).Value == jUnitPath ); if (!hasJUnit) projectNode.Add(XElement.Parse($"<classpathentry kind=\"con\" path=\"{jUnitPath}\"/>")); using (var newStream = new MemoryStream()) { projectNode.Save(newStream); newStream.Flush(); return newStream.ToArray(); } } } else { return entry.GetRawData(); } }
/// <summary> /// Returns a list of all valid submission commit SHAs for the given user/project. /// </summary> public async Task<ICollection<string>> GetSubmissionCandidatesAsync( Project project, User user) { var allCommits = await GetAllCommitsAsync ( project, GetStudent(project, user) ); return new HashSet<string> ( allCommits .Where(c => c.Parents.Count > 0) .Select(commit => commit.Sha) ); }
/// <summary> /// Returns a list of push events for the given project. /// </summary> public async Task<IList<StudentRepoPushEvents>> GetAllPushEventsAsync( Project project, IList<ClassroomMembership> students) { var studentRepos = await _repoMetadataRetriever.GetStudentRepositoriesAsync ( project, students ); return await _operationRunner.RunOperationsAsync ( studentRepos.Keys, student => GetAllPushEventsAsync ( student, studentRepos[student] ) ); }
public async Task GetStudentRepositoriesAsync_ReturnsCorrectRepositories() { var project = new Project() { Name = "Project1", Classroom = new Classroom() { GitHubOrganization = "GitHubOrg" } }; var students = Collections.CreateList ( new ClassroomMembership() { GitHubTeam = "Last1First1" }, new ClassroomMembership() { GitHubTeam = "Last2First2" } ); var reposInOrganization = Collections.CreateList ( new GitHubRepository(0, "GitHubOrg", "Project1_Last1First1"), new GitHubRepository(1, "GitHubOrg", "Project1_Last2First2"), new GitHubRepository(2, "GitHubOrg", "SomeOtherProject_Last1First1"), new GitHubRepository(3, "GitHubOrg", "SomeOtherProject_Last3First3") ); var repoClient = new Mock<IGitHubRepositoryClient>(); repoClient .Setup(rc => rc.GetAllRepositoriesAsync("GitHubOrg")) .ReturnsAsync(reposInOrganization); var repoMetadataRetriever = new RepositoryMetadataRetriever(repoClient.Object); var results = await repoMetadataRetriever.GetStudentRepositoriesAsync ( project, students ); Assert.Equal(2, results.Count); Assert.Equal(0, results[students[0]].Id); Assert.Equal(1, results[students[1]].Id); }
public async Task GetAllPushEventsAsync_ReturnsPushEvents() { var project = new Project(); var students = Collections.CreateList(new ClassroomMembership()); var pushEvents = Collections.CreateList<GitHubPushEvent>(); var repoMetadataRetriever = new Mock<IRepositoryMetadataRetriever>(); repoMetadataRetriever .Setup(rmr => rmr.GetStudentRepositoriesAsync(project, students)) .ReturnsAsync ( new Dictionary<ClassroomMembership, GitHubRepository>() { [students[0]] = new GitHubRepository(1, "GitHubOrg", "GitHubRepoName") } ); var repoClient = new Mock<IGitHubRepositoryClient>(); repoClient .Setup(rc => rc.GetPushEventsAsync("GitHubOrg", "GitHubRepoName")) .ReturnsAsync(pushEvents); var operationRunner = new MockOperationRunner(); var pushEventRetriever = new PushEventRetriever ( repoMetadataRetriever.Object, repoClient.Object, operationRunner ); var results = await pushEventRetriever.GetAllPushEventsAsync(project, students); Assert.Equal(results.Count, 1); Assert.Equal(students[0], results[0].Student); Assert.Equal(pushEvents, results[0].PushEvents); }
/// <summary> /// Retunrs a list of files in a project repository. /// </summary> public async Task<IList<ProjectRepositoryFile>> GetRepoFileListAsync( Project project) { using ( var repoFiles = await _repoClient.GetRepositoryContentsAsync ( project.Classroom.GitHubOrganization, project.TemplateRepoName, null /*branchName*/, ArchiveStore.Memory ) ) { return repoFiles.Files.Select ( entry => new ProjectRepositoryFile ( project.GetFileType(entry), entry.FullPath ) ).ToList(); } }
/// <summary> /// Return the estimated build duration. /// </summary> private async Task<TimeSpan> GetEstimatedBuildDurationAsync(Project project, int userId) { var duration = await GetBuildsDescending(project) .Where(build => build.Commit.UserId == userId) .Where(build => build.Status == BuildStatus.Completed) .Select(build => build.DateCompleted - build.DateStarted) .FirstOrDefaultAsync(); if (duration == default(TimeSpan)) { return TimeSpan.FromSeconds(c_defaultEstimate); } return duration; }
/// <summary> /// Returns whether or not the given build is the latest build. /// </summary> private async Task<bool> IsLatestBuildAsync(Project project, Build build) { var latestBuildId = await GetBuildsDescending(project) .Where(b => b.Commit.UserId == build.Commit.UserId) .Select(b => b.Id) .FirstAsync(); return build.Id == latestBuildId; }
/// <summary> /// Returns a list of commits, in descending order. /// </summary> private IQueryable<Commit> GetCommitsDescending(Project project, int userId) { return _dbContext.Commits .Where(commit => commit.ProjectId == project.Id) .Where(commit => commit.UserId == userId) .Include(commit => commit.Build) .Include(commit => commit.Build.TestResults) .Include(commit => commit.User) .Include(commit => commit.User.ClassroomMemberships) .Include(commit => commit.Project.Classroom) .OrderByDescending(commit => commit.PushDate) .ThenByDescending(commit => commit.CommitDate); }
/// <summary> /// Returns the student classroom membership for the user. /// </summary> private ClassroomMembership GetStudent(Project project, User user) { return user.ClassroomMemberships.Single ( cm => cm.Classroom == project.Classroom ); }
/// <summary> /// Creates a repository for the given student, and pushes the non-test files /// from the source project to the new repository. /// </summary> private async Task<CreateAndPushResult> CreateAndPushAsync( Project project, ClassroomMembership student, string webhookUrl, bool overwriteIfSafe, ICollection<GitHubTeam> teams, ICollection<GitHubRepository> repositories, IArchive templateContents) { string orgName = project.Classroom.GitHubOrganization; string repoName = $"{project.Name}_{student.GitHubTeam}"; try { var repository = repositories.SingleOrDefault(repo => repo.Name == repoName); var team = teams.First(teamCandidate => teamCandidate.Name == student.GitHubTeam); bool repositoryAlreadyExisted = (repository != null); if (repositoryAlreadyExisted) { if (!overwriteIfSafe) return CreateAndPushResult.Exists; var commits = await _repoClient.GetAllCommitsAsync(orgName, repoName); if (commits.Count > c_numInitialCommits) return CreateAndPushResult.Exists; } else { repository = await _repoClient.CreateRepositoryAsync ( orgName, repoName, team, overwrite: false ); var staffTeam = GetStaffTeam(project.Classroom, teams); if (staffTeam != null) { await _teamClient.AddRepositoryAsync(orgName, repoName, staffTeam); } } await _repoClient.OverwriteRepositoryAsync ( repository, c_starterCommitMessage, templateContents, entry => project.GetFileType(entry) != FileType.Private, entry => project.GetFileType(entry) == FileType.Immutable ); await _repoClient.EnsurePushWebhookAsync(repository, webhookUrl); return repositoryAlreadyExisted ? CreateAndPushResult.Overwritten : CreateAndPushResult.Created; } catch (Exception ex) { _logger.LogError ( (EventId)0, ex, "Failed to create repository {RepoName} in organization {Org}.", repoName, orgName ); return CreateAndPushResult.Failed; } }
/// <summary> /// Creates a build job for a new commit received by a push event. /// Returns the job ID for the build job. /// </summary> public async Task<string> CreateBuildJobAsync( Project project, PushEventCommit newCommit, string buildResultCallbackUrl) { var projectJob = new ProjectJob ( newCommit.Commit.BuildRequestToken, newCommit.PushEvent.Repository.Owner.Name, project.Name, newCommit.PushEvent.Repository.Name, $"{project.Name}_Template", newCommit.Commit.Sha, project.PrivateFilePaths .Select(p => p.Path) .Concat(project.ImmutableFilePaths.Select(p => p.Path)) .ToList(), project.TestClasses .Select(tc => tc.ClassName) .ToList(), buildResultCallbackUrl ); var jobId = await _jobQueueClient.EnqueueAsync<IProjectRunnerService> ( service => service.ExecuteProjectJobAsync ( projectJob, _operationIdProvider.OperationId ) ); return jobId; }
/// <summary> /// Returns a checkpoint. /// </summary> private Checkpoint GetCheckpoint(Project project) { return new Checkpoint() { Name = "Checkpoint1", DisplayName = "Checkpoint 1", Project = project, ProjectId = project.Id }; }
/// <summary> /// Adds a project to the database. /// </summary> public TestDatabaseBuilder AddProject( string classroomName, string projectName, bool explicitSubmissions = true) { var classroom = _buildContext.Classrooms .Single(c => c.Name == classroomName); var project = new Project() { Name = projectName, ExplicitSubmissionRequired = explicitSubmissions, ClassroomId = classroom.Id }; _buildContext.Projects.Add(project); _buildContext.SaveChanges(); return this; }
/// <summary> /// Returns all commits for the given user/project. /// </summary> private async Task<ICollection<GitHubCommit>> GetAllCommitsAsync( Project project, ClassroomMembership student) { var orgName = student.Classroom.GitHubOrganization; var repoName = project.GetStudentRepoName(student); return await _repoClient.GetAllCommitsAsync(orgName, repoName); }
/// <summary> /// Creates repositories for the given students, and pushes the non-private /// files from the source project to the new repository. /// </summary> public async Task<IList<CreateStudentRepoResult>> CreateReposAsync( Project project, IList<ClassroomMembership> students, string webhookUrl, bool overwriteIfSafe) { var orgName = project.Classroom.GitHubOrganization; var templateRepoName = project.TemplateRepoName; var teams = await _teamClient.GetAllTeamsAsync(orgName); var repositories = await _repoClient.GetAllRepositoriesAsync(orgName); using ( var templateContents = await _repoClient.GetRepositoryContentsAsync ( orgName, templateRepoName, null /*branchName*/, ArchiveStore.Memory ) ) { return await _operationRunner.RunOperationsAsync ( students, async student => new CreateStudentRepoResult ( student.User, await CreateAndPushAsync ( project, student, webhookUrl, overwriteIfSafe, teams, repositories, templateContents ) ) ); } }
/// <summary> /// Ensures that webhooks are present in all student repositories. /// </summary> public async Task EnsureWebHooksPresentAsync( Project project, IList<ClassroomMembership> students, string webhookUrl) { var studentRepos = await _repoMetadataRetriever.GetStudentRepositoriesAsync ( project, students ); await _operationRunner.RunOperationsAsync ( studentRepos.Values, repo => EnsureWebHookPresentAsync(repo, webhookUrl) ); }
/// <summary> /// Should we exclude this entry? /// </summary> private bool ExcludeEntry(Project project, IArchiveFile entry) { if (entry.FullPath.EndsWith(".project") && !entry.FullPath.EndsWith($"{project.Name}/.project")) { return true; } return false; }
/// <summary> /// Returns a list of the number of pass/failed tests for each build, /// in ascending order of push date. /// </summary> public async Task<IList<BuildTestCount>> GetBuildTestCountsAsync(Project project, int userId) { return await _dbContext.Builds .Where(build => build.Status == BuildStatus.Completed) .Where(build => build.Commit.ProjectId == project.Id) .Where(build => build.Commit.UserId == userId) .OrderBy(build => build.Commit.PushDate) .ThenBy(build => build.Commit.CommitDate) .Select ( build => new BuildTestCount ( build.Id, build.Commit.PushDate, build.TestResults.Count(tr => tr.Succeeded), build.TestResults.Count(tr => !tr.Succeeded) ) ).ToListAsync(); }
/// <summary> /// Returns the latest commit for the given user. /// </summary> public async Task<Commit> GetLatestCommitAsync(Project project, int userId) { return await GetCommitsDescending(project, userId) .FirstOrDefaultAsync(); }
/// <summary> /// Returns a commit. /// </summary> private Commit GetCommit(User user, Project project) { return new Commit() { Sha = "Commit3", User = user, UserId = user.Id, Project = project, ProjectId = project.Id }; }
public async Task<IActionResult> Edit(string projectName, Project project) { if (ModelState.IsValid) { await ProjectService.UpdateProjectAsync(ClassroomName, project); return RedirectToAction("Index"); } else { return View("CreateEdit", project); } }
/// <summary> /// Given a list of push events, along with a list of /// existing recorded commits, returns a list of commits /// that need to be processed. /// <param name="project">The project whose commits we are processing.</param> /// <param name="repoEventLists">The push events to process if needed.</param> /// <param name="existingCommits">The commits that have already been processed.</param> /// </summary> public IList<PushEventCommit> GetNewCommitsToProcess( Project project, ICollection<CommitDescriptor> existingCommits, IList<StudentRepoPushEvents> repoEventLists) { var allPushEvents = repoEventLists .SelectMany ( repoEventList => repoEventList.PushEvents, (repoEventList, pushEvent) => new { repoEventList.Student, PushEvent = pushEvent } ).ToList(); var allPushEventCommits = allPushEvents .SelectMany ( studentPushEvent => studentPushEvent.PushEvent.Commits, (studentPushEvent, commit) => new { studentPushEvent.Student, studentPushEvent.PushEvent, Commit = commit } ).ToList(); var allPushEventCommitDescriptors = allPushEventCommits .Select ( studentPushEventCommit => new { studentPushEventCommit.Student, studentPushEventCommit.PushEvent, studentPushEventCommit.Commit, CommitDescriptor = new CommitDescriptor ( studentPushEventCommit.Commit.Id, project.Id, studentPushEventCommit.Student.UserId ) } ).ToList(); var allNewPushEventCommitDescriptors = allPushEventCommitDescriptors .Where ( newStudentCommit => !existingCommits.Contains ( newStudentCommit.CommitDescriptor ) ).ToList(); var commitsToAdd = allNewPushEventCommitDescriptors .Select ( newStudentCommit => new PushEventCommit ( newStudentCommit.PushEvent, new Commit() { Sha = newStudentCommit.Commit.Id, ProjectId = project.Id, UserId = newStudentCommit.Student.UserId, PushDate = newStudentCommit.PushEvent.CreatedAt.UtcDateTime, CommitDate = newStudentCommit.Commit.Timestamp.UtcDateTime, Message = newStudentCommit.Commit.Message, BuildRequestToken = project.ExplicitSubmissionRequired ? Guid.NewGuid().ToString() : null } ) ).ToList(); var uniqueCommitsToAdd = commitsToAdd .GroupBy ( commitToAdd => new { commitToAdd.Commit.UserId, commitToAdd.Commit.Sha } ) .Select ( group => group.OrderBy(c => c.Commit.PushDate).Last() ).ToList(); return uniqueCommitsToAdd; }
/// <summary> /// Returns a query for the builds of a given user, in ascending order. /// </summary> private IOrderedQueryable<Build> GetBuildsDescending(Project project) { return _dbContext.Builds .Where(build => build.Commit.ProjectId == project.Id) .Include(build => build.Commit) .Include(build => build.Commit.User.ClassroomMemberships) .Include(build => build.TestResults) .OrderByDescending(build => build.Commit.PushDate) .ThenByDescending(build => build.Commit.CommitDate); }