/// <summary>
		/// Builds a submission archive containing the submissions of
		/// all students.
		/// </summary>
		public async Task<Stream> BuildSubmissionArchiveAsync(
			Project project,
			IArchive templateContents,
			IList<StudentSubmission> submissions)
		{
			var stream = _fileSystem.CreateNewTempFile();

			using (ZipArchive archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true))
			{
				foreach (var result in submissions)
				{
					await WriteSubmissionToArchiveAsync
					(
						archive,
						project,
						result.Student,
						templateContents,
						result.Contents
					);

					result.Contents.Dispose();
				}
			}

			stream.Position = 0;
			return stream;
		}
		/// <summary>
		/// Writes the contents of a submission to an archive.
		/// </summary>
		private async Task WriteSubmissionToArchiveAsync(
			ZipArchive archive,
			Project project,
			ClassroomMembership student,
			IArchive templateContents,
			IArchive submissionContents)
		{
			var studentFolder = $"EclipseProjects\\{student.GitHubTeam}";

			// The project will contain all non-immutable submission files, 
			// plus all immutable and private files from the template project.

			var projectContents = submissionContents.Files
				.Where
				(
					entry => project.GetFileType(entry) == FileType.Public
				)
				.Concat
				(
					templateContents.Files.Where
					(
						entry => project.GetFileType(entry) != FileType.Public
					)
				)
				.ToList();

			foreach (var entry in projectContents)
			{
				if (ExcludeEntry(project, entry))
					continue;

				var contents = _transformer.GetFileContents(project, student, entry);

				var archiveFilePath = entry.FullPath;
				var archiveFileFolder = archiveFilePath.Contains("/")
					? archiveFilePath.Substring(0, archiveFilePath.LastIndexOf("/"))
					: archiveFilePath;

				var localFileFolder = $"{studentFolder}\\{archiveFileFolder}";
				var fileName = archiveFilePath.Substring(archiveFilePath.LastIndexOf("/") + 1);
				var localFilePath = $"{localFileFolder}\\{fileName}";

				// Add the file to the student project folder.
				var projectFolderEntry = archive.CreateEntry(localFilePath);
				using (Stream stream = projectFolderEntry.Open())
				{
					await stream.WriteAsync(contents, offset: 0, count: contents.Length);
				}

				// Add the file to the folder containing all files, if applicable.
				if (fileName.EndsWith(".java") && project.GetFileType(entry) == FileType.Public)
				{
					var allFilesEntry = archive.CreateEntry($"AllFiles\\{student.GitHubTeam}-{fileName}");
					using (Stream stream = allFilesEntry.Open())
					{
						await stream.WriteAsync(contents, offset: 0, count: contents.Length);
					}
				}
			}
		}
		/// <summary>
		/// Retrieves a list of student repositories for a given project.
		/// </summary>
		public async Task<IDictionary<ClassroomMembership, GitHubRepository>> GetStudentRepositoriesAsync(
			Project project,
			IList<ClassroomMembership> students)
		{
			var orgName = project.Classroom.GitHubOrganization;
			var repoList = await _repoClient.GetAllRepositoriesAsync(orgName);
			var repoDictionary = repoList.ToDictionary
			(
				repo => repo.Name,
				repo => repo
			);

			return students
				.Where
				(
					student => repoDictionary.ContainsKey
					(
						GetRepoName(project, student)
					)
				)
				.ToDictionary
				(
					student => student,
					student => repoDictionary[GetRepoName(project, student)]
				);
		}
		/// <summary>
		/// Executes before the action is executed.
		/// </summary>
		protected override async Task InitializeAsync()
		{
			await base.InitializeAsync();

			Project = await ProjectService.GetProjectAsync(ClassroomName, ProjectName);

			ViewBag.Project = Project;
		}
		/// <summary>
		/// Downloads the contents of the project template.
		/// </summary>
		public async Task<IArchive> DownloadTemplateContentsAsync(Project project)
		{
			return await _repoClient.GetRepositoryContentsAsync
			(
				project.Classroom.GitHubOrganization,
				$"{project.Name}_Template",
				null /*branchName*/,
				ArchiveStore.Memory
			);
		}
		public void GetRepoName_ReturnsCorrectName()
		{
			var repoMetadataRetriever = new RepositoryMetadataRetriever(repoClient: null);
			
			var project = new Project() { Name = "Project1" };
			var student = new ClassroomMembership() { GitHubTeam = "LastNameFirstName" };

			var result = repoMetadataRetriever.GetRepoName(project, student);

			Assert.Equal("Project1_LastNameFirstName", result);
		}
        /// <summary>
        /// Returns the file's contents, with any applicable 
        /// transformations applied.
        /// </summary>
        public byte[] GetFileContents(
            Project project,
            ClassroomMembership student,
            IArchiveFile entry)
        {
            if (entry.FullPath.EndsWith(".project"))
            {
                var newContents = entry.GetEncodedData().Replace
                (
                    $"<name>{project.Name}</name>",
                    $"<name>{project.Name}_{student.GitHubTeam}</name>"
                );

                using (var memoryStream = new MemoryStream())
                using (var streamWriter = new StreamWriter(memoryStream))
                {
                    streamWriter.Write(newContents);
                    streamWriter.Flush();

                    return memoryStream.ToArray();
                }
            }
            else if (entry.FullPath.EndsWith(".classpath"))
            {
                var jUnitPath = "org.eclipse.jdt.junit.JUNIT_CONTAINER/4";
                using (var stream = new MemoryStream(entry.GetRawData()))
                {
                    var projectNode = XElement.Load(stream);
                    var hasJUnit = projectNode
                        .Elements(XName.Get("classpathentry"))
                        .Any
                        (
                               elt => elt.Attribute(XName.Get("path")) != null
                            && elt.Attribute(XName.Get("path")).Value == jUnitPath
                        );

                    if (!hasJUnit)
                        projectNode.Add(XElement.Parse($"<classpathentry kind=\"con\" path=\"{jUnitPath}\"/>"));

                    using (var newStream = new MemoryStream())
                    {
                        projectNode.Save(newStream);
                        newStream.Flush();

                        return newStream.ToArray();
                    }
                }
            }
            else
            {
                return entry.GetRawData();
            }
        }
		/// <summary>
		/// Returns a list of all valid submission commit SHAs for the given user/project.
		/// </summary>
		public async Task<ICollection<string>> GetSubmissionCandidatesAsync(
			Project project,
			User user)
		{
			var allCommits = await GetAllCommitsAsync
			(
				project,
				GetStudent(project, user)	
			);

			return new HashSet<string>
			(
				allCommits
					.Where(c => c.Parents.Count > 0)
					.Select(commit => commit.Sha)
			);
		}
		/// <summary>
		/// Returns a list of push events for the given project.
		/// </summary>
		public async Task<IList<StudentRepoPushEvents>> GetAllPushEventsAsync(
			Project project,
			IList<ClassroomMembership> students)
		{
			var studentRepos = await _repoMetadataRetriever.GetStudentRepositoriesAsync
			(
				project, 
				students
			);

			return await _operationRunner.RunOperationsAsync
			(
				studentRepos.Keys,
				student => GetAllPushEventsAsync
				(
					student,
					studentRepos[student]
				)
			);
		}
		public async Task GetStudentRepositoriesAsync_ReturnsCorrectRepositories()
		{
			var project = new Project()
			{
				Name = "Project1",
				Classroom = new Classroom() { GitHubOrganization = "GitHubOrg" }
			};

			var students = Collections.CreateList
			(
				new ClassroomMembership() { GitHubTeam = "Last1First1" },
				new ClassroomMembership() { GitHubTeam = "Last2First2" }
			);

			var reposInOrganization = Collections.CreateList
			(
				new GitHubRepository(0, "GitHubOrg", "Project1_Last1First1"),
				new GitHubRepository(1, "GitHubOrg", "Project1_Last2First2"),
				new GitHubRepository(2, "GitHubOrg", "SomeOtherProject_Last1First1"),
				new GitHubRepository(3, "GitHubOrg", "SomeOtherProject_Last3First3")
			);

			var repoClient = new Mock<IGitHubRepositoryClient>();
			repoClient
				.Setup(rc => rc.GetAllRepositoriesAsync("GitHubOrg"))
				.ReturnsAsync(reposInOrganization);

			var repoMetadataRetriever = new RepositoryMetadataRetriever(repoClient.Object);

			var results = await repoMetadataRetriever.GetStudentRepositoriesAsync
			(
				project,
				students
			);

			Assert.Equal(2, results.Count);
			Assert.Equal(0, results[students[0]].Id);
			Assert.Equal(1, results[students[1]].Id);
		}
		public async Task GetAllPushEventsAsync_ReturnsPushEvents()
		{
			var project = new Project();
			var students = Collections.CreateList(new ClassroomMembership());
			var pushEvents = Collections.CreateList<GitHubPushEvent>();

			var repoMetadataRetriever = new Mock<IRepositoryMetadataRetriever>();
			repoMetadataRetriever
				.Setup(rmr => rmr.GetStudentRepositoriesAsync(project, students))
				.ReturnsAsync
				(
					new Dictionary<ClassroomMembership, GitHubRepository>()
					{
						[students[0]] = new GitHubRepository(1, "GitHubOrg", "GitHubRepoName")
					}
				);

			var repoClient = new Mock<IGitHubRepositoryClient>();
			repoClient
				.Setup(rc => rc.GetPushEventsAsync("GitHubOrg", "GitHubRepoName"))
				.ReturnsAsync(pushEvents);

			var operationRunner = new MockOperationRunner();

			var pushEventRetriever = new PushEventRetriever
			(
				repoMetadataRetriever.Object,
				repoClient.Object,
				operationRunner
			);

			var results = await pushEventRetriever.GetAllPushEventsAsync(project, students);

			Assert.Equal(results.Count, 1);
			Assert.Equal(students[0], results[0].Student);
			Assert.Equal(pushEvents, results[0].PushEvents);
		}
		/// <summary>
		/// Retunrs a list of files in a project repository.
		/// </summary>
		public async Task<IList<ProjectRepositoryFile>> GetRepoFileListAsync(
			Project project)
		{
			using
			(
				var repoFiles = await _repoClient.GetRepositoryContentsAsync
				(
					project.Classroom.GitHubOrganization,
					project.TemplateRepoName,
					null /*branchName*/,
					ArchiveStore.Memory
				)
			)
			{
				return repoFiles.Files.Select
				(
					entry => new ProjectRepositoryFile
					(
						project.GetFileType(entry),
						entry.FullPath
					)
				).ToList();
			}		
		}
Example #13
0
		/// <summary>
		/// Return the estimated build duration.
		/// </summary>
		private async Task<TimeSpan> GetEstimatedBuildDurationAsync(Project project, int userId)
		{
			var duration = await GetBuildsDescending(project)
				.Where(build => build.Commit.UserId == userId)
				.Where(build => build.Status == BuildStatus.Completed)
				.Select(build => build.DateCompleted - build.DateStarted)
				.FirstOrDefaultAsync();

			if (duration == default(TimeSpan))
			{
				return TimeSpan.FromSeconds(c_defaultEstimate);
			}

			return duration;
		}
Example #14
0
		/// <summary>
		/// Returns whether or not the given build is the latest build.
		/// </summary>
		private async Task<bool> IsLatestBuildAsync(Project project, Build build)
		{
			var latestBuildId = await GetBuildsDescending(project)
				.Where(b => b.Commit.UserId == build.Commit.UserId)
				.Select(b => b.Id)
				.FirstAsync();

			return build.Id == latestBuildId;
		}
Example #15
0
		/// <summary>
		/// Returns a list of commits, in descending order.
		/// </summary>
		private IQueryable<Commit> GetCommitsDescending(Project project, int userId)
		{
			return _dbContext.Commits
				.Where(commit => commit.ProjectId == project.Id)
				.Where(commit => commit.UserId == userId)
				.Include(commit => commit.Build)
				.Include(commit => commit.Build.TestResults)
				.Include(commit => commit.User)
				.Include(commit => commit.User.ClassroomMemberships)
				.Include(commit => commit.Project.Classroom)
				.OrderByDescending(commit => commit.PushDate)
				.ThenByDescending(commit => commit.CommitDate);
		}
		/// <summary>
		/// Returns the student classroom membership for the user.
		/// </summary>
		private ClassroomMembership GetStudent(Project project, User user)
		{
			return user.ClassroomMemberships.Single
			(
				cm => cm.Classroom == project.Classroom
			);
		}
		/// <summary>
		/// Creates a repository for the given student, and pushes the non-test files
		/// from the source project to the new repository.
		/// </summary>
		private async Task<CreateAndPushResult> CreateAndPushAsync(
			Project project,
			ClassroomMembership student,
			string webhookUrl,
			bool overwriteIfSafe,
			ICollection<GitHubTeam> teams,
			ICollection<GitHubRepository> repositories,
			IArchive templateContents)
		{
			string orgName = project.Classroom.GitHubOrganization;
			string repoName = $"{project.Name}_{student.GitHubTeam}";

			try
			{
				var repository = repositories.SingleOrDefault(repo => repo.Name == repoName);
				var team = teams.First(teamCandidate => teamCandidate.Name == student.GitHubTeam);
				bool repositoryAlreadyExisted = (repository != null);

				if (repositoryAlreadyExisted)
				{
					if (!overwriteIfSafe)
						return CreateAndPushResult.Exists;

					var commits = await _repoClient.GetAllCommitsAsync(orgName, repoName);
					if (commits.Count > c_numInitialCommits)
						return CreateAndPushResult.Exists;
				}
				else
				{
					repository = await _repoClient.CreateRepositoryAsync
					(
						orgName,
						repoName,
						team,
						overwrite: false
					);

					var staffTeam = GetStaffTeam(project.Classroom, teams);
					if (staffTeam != null)
					{
						await _teamClient.AddRepositoryAsync(orgName, repoName, staffTeam);
					}
				}

				await _repoClient.OverwriteRepositoryAsync
				(
					repository,
					c_starterCommitMessage,
					templateContents,
					entry => project.GetFileType(entry) != FileType.Private,
					entry => project.GetFileType(entry) == FileType.Immutable
				);

				await _repoClient.EnsurePushWebhookAsync(repository, webhookUrl);

				return repositoryAlreadyExisted
					? CreateAndPushResult.Overwritten
					: CreateAndPushResult.Created;
			}
			catch (Exception ex)
			{
				_logger.LogError
				(
					(EventId)0,
					ex,
					"Failed to create repository {RepoName} in organization {Org}.", repoName, orgName
				);

				return CreateAndPushResult.Failed;
			}
		}
		/// <summary>
		/// Creates a build job for a new commit received by a push event.
		/// Returns the job ID for the build job.
		/// </summary>
		public async Task<string> CreateBuildJobAsync(
			Project project,
			PushEventCommit newCommit,
			string buildResultCallbackUrl)
		{
			var projectJob = new ProjectJob
			(
				newCommit.Commit.BuildRequestToken,
				newCommit.PushEvent.Repository.Owner.Name,
				project.Name,
				newCommit.PushEvent.Repository.Name,
				$"{project.Name}_Template",
				newCommit.Commit.Sha,
				project.PrivateFilePaths
					.Select(p => p.Path)
					.Concat(project.ImmutableFilePaths.Select(p => p.Path))
					.ToList(),
				project.TestClasses
					.Select(tc => tc.ClassName)
					.ToList(),
				buildResultCallbackUrl
			);

			var jobId = await _jobQueueClient.EnqueueAsync<IProjectRunnerService>
			(
				service => service.ExecuteProjectJobAsync
				(
					projectJob,
					_operationIdProvider.OperationId
				)
			);

			return jobId;
		}
		/// <summary>
		/// Returns a checkpoint.
		/// </summary>
		private Checkpoint GetCheckpoint(Project project)
		{
			return new Checkpoint()
			{
				Name = "Checkpoint1",
				DisplayName = "Checkpoint 1",
				Project = project,
				ProjectId = project.Id
			};		    
		}
        /// <summary>
        /// Adds a project to the database.
        /// </summary>
        public TestDatabaseBuilder AddProject(
            string classroomName,
            string projectName,
            bool explicitSubmissions = true)
        {
            var classroom = _buildContext.Classrooms
                .Single(c => c.Name == classroomName);

            var project = new Project()
            {
                Name = projectName,
                ExplicitSubmissionRequired = explicitSubmissions,
                ClassroomId = classroom.Id
            };

            _buildContext.Projects.Add(project);
            _buildContext.SaveChanges();

            return this;
        }
		/// <summary>
		/// Returns all commits for the given user/project.
		/// </summary>
		private async Task<ICollection<GitHubCommit>> GetAllCommitsAsync(
			Project project, 
			ClassroomMembership student)
		{
			var orgName = student.Classroom.GitHubOrganization;
			var repoName = project.GetStudentRepoName(student);

			return await _repoClient.GetAllCommitsAsync(orgName, repoName);
		}
		/// <summary>
		/// Creates repositories for the given students, and pushes the non-private
		/// files from the source project to the new repository.
		/// </summary>
		public async Task<IList<CreateStudentRepoResult>> CreateReposAsync(
			Project project,
			IList<ClassroomMembership> students,
			string webhookUrl,
			bool overwriteIfSafe)
		{
			var orgName = project.Classroom.GitHubOrganization;
			var templateRepoName = project.TemplateRepoName;
			var teams = await _teamClient.GetAllTeamsAsync(orgName);
			var repositories = await _repoClient.GetAllRepositoriesAsync(orgName);
			using
			(
				var templateContents = await _repoClient.GetRepositoryContentsAsync
				(
					orgName,
					templateRepoName,
					null /*branchName*/,
					ArchiveStore.Memory
				)
			)
			{
				return await _operationRunner.RunOperationsAsync
				(
					students,
					async student => new CreateStudentRepoResult
					(
						student.User,
						await CreateAndPushAsync
						(
							project,
							student,
							webhookUrl,
							overwriteIfSafe,
							teams,
							repositories,
							templateContents
						)
					)
				);
			}
		}
		/// <summary>
		/// Ensures that webhooks are present in all student repositories.
		/// </summary>
		public async Task EnsureWebHooksPresentAsync(
			Project project,
			IList<ClassroomMembership> students,
			string webhookUrl)
		{
			var studentRepos = await _repoMetadataRetriever.GetStudentRepositoriesAsync
			(
				project, 
				students
			);

			await _operationRunner.RunOperationsAsync
			(
				studentRepos.Values,
				repo => EnsureWebHookPresentAsync(repo, webhookUrl)
			);
		}
		/// <summary>
		/// Should we exclude this entry?
		/// </summary>
		private bool ExcludeEntry(Project project, IArchiveFile entry)
		{
			if (entry.FullPath.EndsWith(".project")
				&& !entry.FullPath.EndsWith($"{project.Name}/.project"))
			{
				return true;
			}

			return false;
		}
Example #25
0
		/// <summary>
		/// Returns a list of the number of pass/failed tests for each build,
		/// in ascending order of push date.
		/// </summary>
		public async Task<IList<BuildTestCount>> GetBuildTestCountsAsync(Project project, int userId)
		{
			return await _dbContext.Builds
				.Where(build => build.Status == BuildStatus.Completed)
				.Where(build => build.Commit.ProjectId == project.Id)
				.Where(build => build.Commit.UserId == userId)
				.OrderBy(build => build.Commit.PushDate)
				.ThenBy(build => build.Commit.CommitDate)
				.Select
				(
					build => new BuildTestCount
					(
						build.Id,
						build.Commit.PushDate,
						build.TestResults.Count(tr => tr.Succeeded),
						build.TestResults.Count(tr => !tr.Succeeded)
					)
				).ToListAsync();
		}
Example #26
0
		/// <summary>
		/// Returns the latest commit for the given user.
		/// </summary>
		public async Task<Commit> GetLatestCommitAsync(Project project, int userId)
		{
			return await GetCommitsDescending(project, userId)
				.FirstOrDefaultAsync();
		}
		/// <summary>
		/// Returns a commit.
		/// </summary>
		private Commit GetCommit(User user, Project project)
		{
			return new Commit()
			{
				Sha = "Commit3",
				User = user,
				UserId = user.Id,
				Project = project,
				ProjectId = project.Id
			};
		}
		public async Task<IActionResult> Edit(string projectName, Project project)
		{
			if (ModelState.IsValid)
			{
				await ProjectService.UpdateProjectAsync(ClassroomName, project);

				return RedirectToAction("Index");
			}
			else
			{
				return View("CreateEdit", project);
			}
		}
		/// <summary>
		/// Given a list of push events, along with a list of 
		/// existing recorded commits, returns a list of commits
		/// that need to be processed.
		/// <param name="project">The project whose commits we are processing.</param>
		/// <param name="repoEventLists">The push events to process if needed.</param>
		/// <param name="existingCommits">The commits that have already been processed.</param>
		/// </summary>
		public IList<PushEventCommit> GetNewCommitsToProcess(
			Project project,
			ICollection<CommitDescriptor> existingCommits,
			IList<StudentRepoPushEvents> repoEventLists)
		{
			var allPushEvents = repoEventLists
				.SelectMany
				(
					repoEventList => repoEventList.PushEvents,
					(repoEventList, pushEvent) => new
					{
						repoEventList.Student,
						PushEvent = pushEvent
					}
				).ToList();

			var allPushEventCommits = allPushEvents
				.SelectMany
				(
					studentPushEvent => studentPushEvent.PushEvent.Commits,
					(studentPushEvent, commit) => new
					{
						studentPushEvent.Student,
						studentPushEvent.PushEvent,
						Commit = commit
					}
				).ToList();

			var allPushEventCommitDescriptors = allPushEventCommits
				.Select
				(
					studentPushEventCommit => new
					{
						studentPushEventCommit.Student,
						studentPushEventCommit.PushEvent,
						studentPushEventCommit.Commit,
						CommitDescriptor = new CommitDescriptor
						(
							studentPushEventCommit.Commit.Id,
							project.Id,
							studentPushEventCommit.Student.UserId
						)
					}
				).ToList();

			var allNewPushEventCommitDescriptors = allPushEventCommitDescriptors
				.Where
				(
					newStudentCommit => !existingCommits.Contains
					(
						newStudentCommit.CommitDescriptor
					)
				).ToList();

			var commitsToAdd = allNewPushEventCommitDescriptors
				.Select
				(
					newStudentCommit => new PushEventCommit
					(
						newStudentCommit.PushEvent,
						new Commit()
						{
							Sha = newStudentCommit.Commit.Id,
							ProjectId = project.Id,
							UserId = newStudentCommit.Student.UserId,
							PushDate = newStudentCommit.PushEvent.CreatedAt.UtcDateTime,
							CommitDate = newStudentCommit.Commit.Timestamp.UtcDateTime,
							Message = newStudentCommit.Commit.Message,
							BuildRequestToken = project.ExplicitSubmissionRequired
								? Guid.NewGuid().ToString()
								: null
						}
					)
				).ToList();

			var uniqueCommitsToAdd = commitsToAdd
				.GroupBy
				(
					commitToAdd => new
					{
						commitToAdd.Commit.UserId,
						commitToAdd.Commit.Sha
					}
				)
				.Select
				(
					group => group.OrderBy(c => c.Commit.PushDate).Last()
				).ToList();

			return uniqueCommitsToAdd;
		}
Example #30
0
		/// <summary>
		/// Returns a query for the builds of a given user, in ascending order.
		/// </summary>
		private IOrderedQueryable<Build> GetBuildsDescending(Project project)
		{
			return _dbContext.Builds
				.Where(build => build.Commit.ProjectId == project.Id)
				.Include(build => build.Commit)
				.Include(build => build.Commit.User.ClassroomMemberships)
				.Include(build => build.TestResults)
				.OrderByDescending(build => build.Commit.PushDate)
				.ThenByDescending(build => build.Commit.CommitDate);
		}