protected override IList <PullRequest> GetPullRequests(Repository repository) { var limit = 20; var skip = 0; var pullRequests = new List <PullRequest>(); DateTime past2Days = DateTime.Now.Date.AddDays(-2); do { Logger.LogDebug($"Fetching the top {limit} (skipping {skip}) pull requests for {repository.Name}."); List <PullRequest> updatedPullRequests = GetAsync.FetchResponseList <PullRequest>(RequestData, repository.PullRequestUrl + $"?status=Completed&$top={limit}&$skip={skip}", Logger).Result; if (updatedPullRequests == null) { return(pullRequests); } // Only show the completed pull requests that have been completed in the past two days List <PullRequest> filteredPullRequests = updatedPullRequests.Where(x => x.creationDate >= past2Days).ToList(); if (!filteredPullRequests.Any()) { Logger.LogDebug($"Finished fetching {pullRequests.Count} pull requests for repository {repository.Name}."); return(pullRequests); } pullRequests.AddRange(filteredPullRequests); skip += limit; } while (true); }
protected override void Update() { ConcurrentBag <Repository> populatedRepositories = new ConcurrentBag <Repository>(); Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, project => { IList <TFSAdvanced.Updater.Models.Repositories.Repository> repositories = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories?api=1.0").Result; if (repositories == null) { return; } Parallel.ForEach(repositories, new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, repo => { try { var populatedRepository = GetAsync.Fetch <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories/{repo.name}?api=1.0").Result; var repositoryDto = new Repository { Id = populatedRepository.id, Name = populatedRepository.name, PullRequestUrl = populatedRepository._links.pullRequests.href, Url = populatedRepository.remoteUrl, Project = new Project { Id = populatedRepository.project.id, Name = populatedRepository.project.name, Url = populatedRepository.project.url } }; var policyConfigurations = GetAsync.FetchResponseList <PolicyConfiguration>(requestData, $"{requestData.BaseAddress}/defaultcollection/{project.Id}/_apis/policy/configurations?api-version=2.0-preview.1").Result; foreach (var configuration in policyConfigurations) { if (configuration.type.displayName == "Minimum number of reviewers") { repositoryDto.MinimumApproverCount = configuration.settings.minimumApproverCount; } } populatedRepositories.Add(repositoryDto); } catch (Exception) { } }); }); var repositoryList = populatedRepositories.ToList(); repositoryRepository.Update(repositoryList); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = repositoryList.Count, UpdaterName = nameof(RepositoryUpdater) }); }
protected override void Update() { var pools = GetAsync.FetchResponseList <Pool>(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools?api-version=1.0").Result; if (pools != null) { poolRepository.Update(pools); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = pools.Count, UpdaterName = nameof(PoolUpdater) }); } }
protected override void Update() { List <Project> projects = GetAsync.FetchResponseList <Project>(requestData, $"{requestData.BaseAddress}/_apis/projects?api-version=1.0", Logger).Result; if (projects != null) { projectRepository.Update(projects.Select(x => new TFSAdvanced.Models.DTO.Project { Id = x.id, Name = x.name, Url = x.remoteUrl })); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = projects.Count, UpdaterName = nameof(ProjectUpdater) }); } }
protected override void Update() { DateTime yesterday = DateTime.Now.Date.AddDays(-1); var builds = new ConcurrentStack <TFSAdvanced.Updater.Models.Builds.Build>(); Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, project => { // Finished PR builds var projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&reasonFilter=validateShelveset&minFinishTime={yesterday:O}").Result; if (projectBuilds != null && projectBuilds.Any()) { builds.PushRange(projectBuilds.ToArray()); } // Current active builds projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&statusFilter=inProgress&statusFilter=notStarted").Result; if (projectBuilds != null && projectBuilds.Any()) { builds.PushRange(projectBuilds.ToArray()); } DateTime twoHoursAgo = DateTime.Now.AddHours(-2); // Because we want to capture the final state of any build that was running and just finished we are getting those too // Finished builds within the last 2 hours projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&minFinishTime={twoHoursAgo:O}").Result; if (projectBuilds != null && projectBuilds.Any()) { builds.PushRange(projectBuilds.ToArray()); } }); // The builds must be requested without the filter because the only filter available is minFinishTime, which will filter out those that haven't finished yet var buildLists = builds.ToList(); buildRepository.Update(buildLists.Select(CreateBuild)); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = buildLists.Count, UpdaterName = nameof(BuildUpdater) }); }
protected override void Update() { DateTime startTime = DateTime.Now; Logger.LogDebug($"Fetching Build Updates Since {lastRequest}"); var builds = new ConcurrentStack <TFSAdvanced.Updater.Models.Builds.Build>(); Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, project => { Logger.LogDebug($"Fetching finished build updates for project {project.Name}"); // Finished PR builds List <TFSAdvanced.Updater.Models.Builds.Build> projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&minFinishTime={lastRequest:O}", Logger).Result; if (projectBuilds != null && projectBuilds.Any()) { builds.PushRange(projectBuilds.ToArray()); } Logger.LogDebug($"Fetching active build updates for project {project.Name}"); // Current active builds projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&statusFilter=inProgress&statusFilter=notStarted", Logger).Result; if (projectBuilds != null && projectBuilds.Any()) { builds.PushRange(projectBuilds.ToArray()); } }); // The builds must be requested without the filter because the only filter available is minFinishTime, which will filter out those that haven't // finished yet List <TFSAdvanced.Updater.Models.Builds.Build> buildLists = builds.ToList(); buildRepository.Update(buildLists.Select(CreateBuild)); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = buildLists.Count, UpdaterName = nameof(BuildUpdater) }); // Use the start time so that there is a small amount of overlap lastRequest = startTime; }
protected override void Update() { var buildDefinitions = new ConcurrentBag <BuildDefinition>(); Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, project => { var definitions = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.BuildDefinition>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/definitions?api=2.2").Result; if (definitions == null) { logger.LogInformation($"Unable to get the definitiosn for the project {project.Name}"); return; } Parallel.ForEach(definitions, new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, definition => { var populatedDefinition = GetAsync.Fetch <TFSAdvanced.Updater.Models.Builds.BuildDefinition>(requestData, definition.url).Result; var repository = repositoryRepository.GetById(populatedDefinition.repository.id); buildDefinitions.Add(new BuildDefinition { DefaultBranch = populatedDefinition.repository.defaultBranch, Folder = populatedDefinition.path, Id = populatedDefinition.id, Name = populatedDefinition.name, Url = populatedDefinition._links.web.href, Repository = repository }); }); }); buildDefinitionRepository.Update(buildDefinitions); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = buildDefinitions.Count, UpdaterName = nameof(BuildDefinitionUpdater) }); }
protected override void Update() { var releases = new ConcurrentStack <ReleaseDefinition>(); Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, project => { var releaseDefinitions = GetAsync.FetchResponseList <Models.Releases.ReleaseDefinition>(requestData, $"{requestData.BaseReleaseManagerAddress}/{project.Id}/_apis/Release/definitions?api-version=3.0-preview.1").Result; if (releaseDefinitions != null) { Parallel.ForEach(releaseDefinitions, new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, releaseDefinition => { var populatedReleaseDefinition = GetAsync.Fetch <Models.Releases.ReleaseDefinition>(requestData, releaseDefinition.url).Result; releases.Push(CreateReleaseDefinition(populatedReleaseDefinition)); }); } }); releaseDefinitionRepository.Update(releases); }
protected override void Update() { ConcurrentBag <QueueJob> jobRequests = new ConcurrentBag <QueueJob>(); Parallel.ForEach(poolRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, pool => { var poolJobRequests = GetAsync.FetchResponseList <JobRequest>(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools/{pool.id}/jobrequests?api-version=1.0").Result; if (poolJobRequests != null) { foreach (var poolJobRequest in poolJobRequests) { QueueJob queueJob = new QueueJob { RequestId = poolJobRequest.requestId, QueuedTime = poolJobRequest.queueTime, AssignedTime = poolJobRequest.assignTime, FinishedTime = poolJobRequest.finishTime, Name = poolJobRequest.definition.name, Url = poolJobRequest.definition._links.web.href }; if (poolJobRequest.planType == PlanTypes.Build) { var build = buildRepository.GetBuild(poolJobRequest.owner.id); if (build != null) { queueJob.LaunchedBy = build.Creator; queueJob.StartedTime = build.StartedDate; queueJob.FinishedTime = build.FinishedDate; queueJob.BuildFolder = build.Folder; switch (build.BuildStatus) { case BuildStatus.NotStarted: queueJob.QueueJobStatus = QueueJobStatus.Queued; break; case BuildStatus.Abandonded: queueJob.QueueJobStatus = QueueJobStatus.Abandonded; break; case BuildStatus.Building: queueJob.QueueJobStatus = QueueJobStatus.Building; break; case BuildStatus.Cancelled: queueJob.QueueJobStatus = QueueJobStatus.Cancelled; break; case BuildStatus.Expired: case BuildStatus.Failed: queueJob.QueueJobStatus = QueueJobStatus.Failed; break; case BuildStatus.Succeeded: queueJob.QueueJobStatus = QueueJobStatus.Succeeded; break; } } var buildDefinition = buildDefinitionRepository.GetBuildDefinition(poolJobRequest.definition.id); if (buildDefinition != null && buildDefinition.Repository != null) { var project = projectRepository.GetProject(buildDefinition.Repository.Project.Id); if (project != null) { queueJob.Project = new Project { Id = project.Id, Name = project.Name, Url = project.Url }; } else { queueJob.Project = new Project { Name = "Unknown Project" }; } } else { queueJob.Project = new Project { Name = "Unknown Build Definition" }; } } else if (poolJobRequest.planType == PlanTypes.Release) { if (poolJobRequest.finishTime.HasValue) { switch (poolJobRequest.result) { case BuildResult.succeeded: queueJob.QueueJobStatus = QueueJobStatus.Succeeded; break; case BuildResult.abandoned: queueJob.QueueJobStatus = QueueJobStatus.Abandonded; break; case BuildResult.canceled: queueJob.QueueJobStatus = QueueJobStatus.Cancelled; break; case BuildResult.failed: case BuildResult.partiallySucceeded: queueJob.QueueJobStatus = QueueJobStatus.Failed; break; } } var releaseDefinition = releaseDefinitionRepository.GetReleaseDefinition(poolJobRequest.definition.id); if (releaseDefinition != null) { queueJob.Project = releaseDefinition.Project; } else { queueJob.Project = new Project { Name = "Unknown Release Definition" }; } } jobRequests.Add(queueJob); } } }); jobRequestRepository.Update(jobRequests); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = jobRequests.Count, UpdaterName = nameof(JobRequestUpdater) }); }
protected override void Update() { ConcurrentBag <PullRequest> allPullRequests = new ConcurrentBag <PullRequest>(); Parallel.ForEach(repositoryRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, repository => { if (string.IsNullOrEmpty(repository.PullRequestUrl)) { return; } var pullRequests = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.PullRequests.PullRequest>(requestData, repository.PullRequestUrl).Result; if (pullRequests == null) { return; } Parallel.ForEach(pullRequests, new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, pullRequest => { try { CommitLink commitId = null; if (pullRequest.lastMergeCommit != null) { commitId = pullRequest.lastMergeCommit; } else if (pullRequest.lastMergeSourceCommit != null) { commitId = pullRequest.lastMergeSourceCommit; } if (commitId == null) { logger.LogWarning($"Unable to get last merge commit for the pullrequest ({pullRequest.pullRequestId}) {pullRequest.description}"); return; } if (string.IsNullOrEmpty(commitId.commitId)) { logger.LogWarning($"Unable to get the last commitID for the pull request ({pullRequest.pullRequestId}) {pullRequest.description}"); return; } var build = buildRepository.GetBuildBySourceVersion(repository, commitId.commitId); var pullRequestDto = BuildPullRequest(pullRequest, build); pullRequestDto.Repository = repository; pullRequestDto.Url = BuildPullRequestUrl(pullRequest, requestData.BaseAddress); pullRequestDto.RequiredReviewers = repository.MinimumApproverCount; foreach (var reviewer in pullRequest.reviewers) { // Container reviewers do not count if (reviewer.isContainer) { continue; } if (reviewer.vote == (int)Vote.Approved) { pullRequestDto.AcceptedReviewers++; } } allPullRequests.Add(pullRequestDto); } catch (Exception e) { logger.LogError("Error parsing pull request", e); } }); }); var pullRequestsList = allPullRequests.ToList(); pullRequestRepository.Update(pullRequestsList); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = pullRequestsList.Count, UpdaterName = nameof(PullRequestUpdater) }); }
protected virtual IList <TFSAdvanced.Updater.Models.PullRequests.PullRequest> GetPullRequests(Repository repository) { return(GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.PullRequests.PullRequest>(RequestData, repository.PullRequestUrl, Logger).Result); }
protected override void Update() { var populatedRepositories = new ConcurrentBag <Repository>(); Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, project => { IList <TFSAdvanced.Updater.Models.Repositories.Repository> repositories = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories?api=1.0", Logger).Result; if (repositories == null) { return; } // policies are project scoped, so we only need to request once per project List <PolicyConfiguration> policyConfigurations = GetAsync.FetchResponseList <PolicyConfiguration>(requestData, $"{requestData.BaseAddress}/defaultcollection/{project.Id}/_apis/policy/configurations?api-version=2.0-preview.1", Logger).Result; Parallel.ForEach(repositories, new ParallelOptions { MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM }, repo => { try { TFSAdvanced.Updater.Models.Repositories.Repository populatedRepository = GetAsync.Fetch <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories/{repo.name}?api=1.0").Result; var repositoryDto = new Repository { Id = populatedRepository.id, Name = populatedRepository.name, PullRequestUrl = populatedRepository._links.pullRequests.href, Url = populatedRepository.remoteUrl, Project = new Project { Id = populatedRepository.project.id, Name = populatedRepository.project.name, Url = populatedRepository.project.url } }; foreach (PolicyConfiguration configuration in policyConfigurations) { if (configuration.type.id == MinimumReviewerPolicyId) { foreach (PolicyScope scope in configuration.settings.scope) { if (scope.repositoryId == repositoryDto.Id) { // NOTE: there could be multiple reviewer policies in a repo (per branch) so this may not be the correct level to store // this setting repositoryDto.MinimumApproverCount = configuration.settings.minimumApproverCount; break; } } } } populatedRepositories.Add(repositoryDto); } catch (Exception) { // ignored } }); }); List <Repository> repositoryList = populatedRepositories.ToList(); repositoryRepository.Update(repositoryList); updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = repositoryList.Count, UpdaterName = nameof(RepositoryUpdater) }); }