protected override IList <PullRequest> GetPullRequests(Repository repository)
{
var limit = 20;
var skip = 0;
var pullRequests = new List <PullRequest>();
DateTime past2Days = DateTime.Now.Date.AddDays(-2);
do
{
Logger.LogDebug($"Fetching the top {limit} (skipping {skip}) pull requests for {repository.Name}.");
List <PullRequest> updatedPullRequests = GetAsync.FetchResponseList <PullRequest>(RequestData, repository.PullRequestUrl + $"?status=Completed&$top={limit}&$skip={skip}", Logger).Result;
if (updatedPullRequests == null)
{
return(pullRequests);
}
// Only show the completed pull requests that have been completed in the past two days
List <PullRequest> filteredPullRequests = updatedPullRequests.Where(x => x.creationDate >= past2Days).ToList();
if (!filteredPullRequests.Any())
{
Logger.LogDebug($"Finished fetching {pullRequests.Count} pull requests for repository {repository.Name}.");
return(pullRequests);
}
pullRequests.AddRange(filteredPullRequests);
skip += limit;
} while (true);
}
protected override void Update()
{
ConcurrentBag <Repository> populatedRepositories = new ConcurrentBag <Repository>();
Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, project =>
{
IList <TFSAdvanced.Updater.Models.Repositories.Repository> repositories = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories?api=1.0").Result;
if (repositories == null)
{
return;
}
Parallel.ForEach(repositories, new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, repo =>
{
try
{
var populatedRepository = GetAsync.Fetch <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories/{repo.name}?api=1.0").Result;
var repositoryDto = new Repository
{
Id = populatedRepository.id,
Name = populatedRepository.name,
PullRequestUrl = populatedRepository._links.pullRequests.href,
Url = populatedRepository.remoteUrl,
Project = new Project
{
Id = populatedRepository.project.id,
Name = populatedRepository.project.name,
Url = populatedRepository.project.url
}
};
var policyConfigurations = GetAsync.FetchResponseList <PolicyConfiguration>(requestData, $"{requestData.BaseAddress}/defaultcollection/{project.Id}/_apis/policy/configurations?api-version=2.0-preview.1").Result;
foreach (var configuration in policyConfigurations)
{
if (configuration.type.displayName == "Minimum number of reviewers")
{
repositoryDto.MinimumApproverCount = configuration.settings.minimumApproverCount;
}
}
populatedRepositories.Add(repositoryDto);
}
catch (Exception)
{
}
});
});
var repositoryList = populatedRepositories.ToList();
repositoryRepository.Update(repositoryList);
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = repositoryList.Count, UpdaterName = nameof(RepositoryUpdater)
});
}
protected override void Update()
{
var pools = GetAsync.FetchResponseList <Pool>(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools?api-version=1.0").Result;
if (pools != null)
{
poolRepository.Update(pools);
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = pools.Count, UpdaterName = nameof(PoolUpdater)
});
}
}
protected override void Update()
{
List <Project> projects = GetAsync.FetchResponseList <Project>(requestData, $"{requestData.BaseAddress}/_apis/projects?api-version=1.0", Logger).Result;
if (projects != null)
{
projectRepository.Update(projects.Select(x => new TFSAdvanced.Models.DTO.Project
{
Id = x.id,
Name = x.name,
Url = x.remoteUrl
}));
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = projects.Count, UpdaterName = nameof(ProjectUpdater)
});
}
}
protected override void Update()
{
DateTime yesterday = DateTime.Now.Date.AddDays(-1);
var builds = new ConcurrentStack <TFSAdvanced.Updater.Models.Builds.Build>();
Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, project =>
{
// Finished PR builds
var projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&reasonFilter=validateShelveset&minFinishTime={yesterday:O}").Result;
if (projectBuilds != null && projectBuilds.Any())
{
builds.PushRange(projectBuilds.ToArray());
}
// Current active builds
projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&statusFilter=inProgress&statusFilter=notStarted").Result;
if (projectBuilds != null && projectBuilds.Any())
{
builds.PushRange(projectBuilds.ToArray());
}
DateTime twoHoursAgo = DateTime.Now.AddHours(-2);
// Because we want to capture the final state of any build that was running and just finished we are getting those too
// Finished builds within the last 2 hours
projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&minFinishTime={twoHoursAgo:O}").Result;
if (projectBuilds != null && projectBuilds.Any())
{
builds.PushRange(projectBuilds.ToArray());
}
});
// The builds must be requested without the filter because the only filter available is minFinishTime, which will filter out those that haven't finished yet
var buildLists = builds.ToList();
buildRepository.Update(buildLists.Select(CreateBuild));
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = buildLists.Count, UpdaterName = nameof(BuildUpdater)
});
}
protected override void Update()
{
DateTime startTime = DateTime.Now;
Logger.LogDebug($"Fetching Build Updates Since {lastRequest}");
var builds = new ConcurrentStack <TFSAdvanced.Updater.Models.Builds.Build>();
Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, project =>
{
Logger.LogDebug($"Fetching finished build updates for project {project.Name}");
// Finished PR builds
List <TFSAdvanced.Updater.Models.Builds.Build> projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&minFinishTime={lastRequest:O}", Logger).Result;
if (projectBuilds != null && projectBuilds.Any())
{
builds.PushRange(projectBuilds.ToArray());
}
Logger.LogDebug($"Fetching active build updates for project {project.Name}");
// Current active builds
projectBuilds = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.Build>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&statusFilter=inProgress&statusFilter=notStarted", Logger).Result;
if (projectBuilds != null && projectBuilds.Any())
{
builds.PushRange(projectBuilds.ToArray());
}
});
// The builds must be requested without the filter because the only filter available is minFinishTime, which will filter out those that haven't
// finished yet
List <TFSAdvanced.Updater.Models.Builds.Build> buildLists = builds.ToList();
buildRepository.Update(buildLists.Select(CreateBuild));
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = buildLists.Count, UpdaterName = nameof(BuildUpdater)
});
// Use the start time so that there is a small amount of overlap
lastRequest = startTime;
}
protected override void Update()
{
var buildDefinitions = new ConcurrentBag <BuildDefinition>();
Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, project =>
{
var definitions = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Builds.BuildDefinition>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/definitions?api=2.2").Result;
if (definitions == null)
{
logger.LogInformation($"Unable to get the definitiosn for the project {project.Name}");
return;
}
Parallel.ForEach(definitions, new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, definition =>
{
var populatedDefinition = GetAsync.Fetch <TFSAdvanced.Updater.Models.Builds.BuildDefinition>(requestData, definition.url).Result;
var repository = repositoryRepository.GetById(populatedDefinition.repository.id);
buildDefinitions.Add(new BuildDefinition
{
DefaultBranch = populatedDefinition.repository.defaultBranch,
Folder = populatedDefinition.path,
Id = populatedDefinition.id,
Name = populatedDefinition.name,
Url = populatedDefinition._links.web.href,
Repository = repository
});
});
});
buildDefinitionRepository.Update(buildDefinitions);
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = buildDefinitions.Count, UpdaterName = nameof(BuildDefinitionUpdater)
});
}
protected override void Update()
{
var releases = new ConcurrentStack <ReleaseDefinition>();
Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, project =>
{
var releaseDefinitions = GetAsync.FetchResponseList <Models.Releases.ReleaseDefinition>(requestData, $"{requestData.BaseReleaseManagerAddress}/{project.Id}/_apis/Release/definitions?api-version=3.0-preview.1").Result;
if (releaseDefinitions != null)
{
Parallel.ForEach(releaseDefinitions, new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, releaseDefinition =>
{
var populatedReleaseDefinition = GetAsync.Fetch <Models.Releases.ReleaseDefinition>(requestData, releaseDefinition.url).Result;
releases.Push(CreateReleaseDefinition(populatedReleaseDefinition));
});
}
});
releaseDefinitionRepository.Update(releases);
}
protected override void Update()
{
ConcurrentBag <QueueJob> jobRequests = new ConcurrentBag <QueueJob>();
Parallel.ForEach(poolRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, pool =>
{
var poolJobRequests = GetAsync.FetchResponseList <JobRequest>(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools/{pool.id}/jobrequests?api-version=1.0").Result;
if (poolJobRequests != null)
{
foreach (var poolJobRequest in poolJobRequests)
{
QueueJob queueJob = new QueueJob
{
RequestId = poolJobRequest.requestId,
QueuedTime = poolJobRequest.queueTime,
AssignedTime = poolJobRequest.assignTime,
FinishedTime = poolJobRequest.finishTime,
Name = poolJobRequest.definition.name,
Url = poolJobRequest.definition._links.web.href
};
if (poolJobRequest.planType == PlanTypes.Build)
{
var build = buildRepository.GetBuild(poolJobRequest.owner.id);
if (build != null)
{
queueJob.LaunchedBy = build.Creator;
queueJob.StartedTime = build.StartedDate;
queueJob.FinishedTime = build.FinishedDate;
queueJob.BuildFolder = build.Folder;
switch (build.BuildStatus)
{
case BuildStatus.NotStarted:
queueJob.QueueJobStatus = QueueJobStatus.Queued;
break;
case BuildStatus.Abandonded:
queueJob.QueueJobStatus = QueueJobStatus.Abandonded;
break;
case BuildStatus.Building:
queueJob.QueueJobStatus = QueueJobStatus.Building;
break;
case BuildStatus.Cancelled:
queueJob.QueueJobStatus = QueueJobStatus.Cancelled;
break;
case BuildStatus.Expired:
case BuildStatus.Failed:
queueJob.QueueJobStatus = QueueJobStatus.Failed;
break;
case BuildStatus.Succeeded:
queueJob.QueueJobStatus = QueueJobStatus.Succeeded;
break;
}
}
var buildDefinition = buildDefinitionRepository.GetBuildDefinition(poolJobRequest.definition.id);
if (buildDefinition != null && buildDefinition.Repository != null)
{
var project = projectRepository.GetProject(buildDefinition.Repository.Project.Id);
if (project != null)
{
queueJob.Project = new Project
{
Id = project.Id,
Name = project.Name,
Url = project.Url
};
}
else
{
queueJob.Project = new Project
{
Name = "Unknown Project"
};
}
}
else
{
queueJob.Project = new Project
{
Name = "Unknown Build Definition"
};
}
}
else if (poolJobRequest.planType == PlanTypes.Release)
{
if (poolJobRequest.finishTime.HasValue)
{
switch (poolJobRequest.result)
{
case BuildResult.succeeded:
queueJob.QueueJobStatus = QueueJobStatus.Succeeded;
break;
case BuildResult.abandoned:
queueJob.QueueJobStatus = QueueJobStatus.Abandonded;
break;
case BuildResult.canceled:
queueJob.QueueJobStatus = QueueJobStatus.Cancelled;
break;
case BuildResult.failed:
case BuildResult.partiallySucceeded:
queueJob.QueueJobStatus = QueueJobStatus.Failed;
break;
}
}
var releaseDefinition = releaseDefinitionRepository.GetReleaseDefinition(poolJobRequest.definition.id);
if (releaseDefinition != null)
{
queueJob.Project = releaseDefinition.Project;
}
else
{
queueJob.Project = new Project
{
Name = "Unknown Release Definition"
};
}
}
jobRequests.Add(queueJob);
}
}
});
jobRequestRepository.Update(jobRequests);
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = jobRequests.Count, UpdaterName = nameof(JobRequestUpdater)
});
}
protected override void Update()
{
ConcurrentBag <PullRequest> allPullRequests = new ConcurrentBag <PullRequest>();
Parallel.ForEach(repositoryRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, repository =>
{
if (string.IsNullOrEmpty(repository.PullRequestUrl))
{
return;
}
var pullRequests = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.PullRequests.PullRequest>(requestData, repository.PullRequestUrl).Result;
if (pullRequests == null)
{
return;
}
Parallel.ForEach(pullRequests, new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, pullRequest =>
{
try
{
CommitLink commitId = null;
if (pullRequest.lastMergeCommit != null)
{
commitId = pullRequest.lastMergeCommit;
}
else if (pullRequest.lastMergeSourceCommit != null)
{
commitId = pullRequest.lastMergeSourceCommit;
}
if (commitId == null)
{
logger.LogWarning($"Unable to get last merge commit for the pullrequest ({pullRequest.pullRequestId}) {pullRequest.description}");
return;
}
if (string.IsNullOrEmpty(commitId.commitId))
{
logger.LogWarning($"Unable to get the last commitID for the pull request ({pullRequest.pullRequestId}) {pullRequest.description}");
return;
}
var build = buildRepository.GetBuildBySourceVersion(repository, commitId.commitId);
var pullRequestDto = BuildPullRequest(pullRequest, build);
pullRequestDto.Repository = repository;
pullRequestDto.Url = BuildPullRequestUrl(pullRequest, requestData.BaseAddress);
pullRequestDto.RequiredReviewers = repository.MinimumApproverCount;
foreach (var reviewer in pullRequest.reviewers)
{
// Container reviewers do not count
if (reviewer.isContainer)
{
continue;
}
if (reviewer.vote == (int)Vote.Approved)
{
pullRequestDto.AcceptedReviewers++;
}
}
allPullRequests.Add(pullRequestDto);
}
catch (Exception e)
{
logger.LogError("Error parsing pull request", e);
}
});
});
var pullRequestsList = allPullRequests.ToList();
pullRequestRepository.Update(pullRequestsList);
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = pullRequestsList.Count, UpdaterName = nameof(PullRequestUpdater)
});
}
protected virtual IList <TFSAdvanced.Updater.Models.PullRequests.PullRequest> GetPullRequests(Repository repository)
{
return(GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.PullRequests.PullRequest>(RequestData, repository.PullRequestUrl, Logger).Result);
}
protected override void Update()
{
var populatedRepositories = new ConcurrentBag <Repository>();
Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, project =>
{
IList <TFSAdvanced.Updater.Models.Repositories.Repository> repositories = GetAsync.FetchResponseList <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories?api=1.0", Logger).Result;
if (repositories == null)
{
return;
}
// policies are project scoped, so we only need to request once per project
List <PolicyConfiguration> policyConfigurations = GetAsync.FetchResponseList <PolicyConfiguration>(requestData, $"{requestData.BaseAddress}/defaultcollection/{project.Id}/_apis/policy/configurations?api-version=2.0-preview.1", Logger).Result;
Parallel.ForEach(repositories, new ParallelOptions {
MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM
}, repo =>
{
try
{
TFSAdvanced.Updater.Models.Repositories.Repository populatedRepository = GetAsync.Fetch <TFSAdvanced.Updater.Models.Repositories.Repository>(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories/{repo.name}?api=1.0").Result;
var repositoryDto = new Repository
{
Id = populatedRepository.id,
Name = populatedRepository.name,
PullRequestUrl = populatedRepository._links.pullRequests.href,
Url = populatedRepository.remoteUrl,
Project = new Project
{
Id = populatedRepository.project.id,
Name = populatedRepository.project.name,
Url = populatedRepository.project.url
}
};
foreach (PolicyConfiguration configuration in policyConfigurations)
{
if (configuration.type.id == MinimumReviewerPolicyId)
{
foreach (PolicyScope scope in configuration.settings.scope)
{
if (scope.repositoryId == repositoryDto.Id)
{
// NOTE: there could be multiple reviewer policies in a repo (per branch) so this may not be the correct level to store
// this setting
repositoryDto.MinimumApproverCount = configuration.settings.minimumApproverCount;
break;
}
}
}
}
populatedRepositories.Add(repositoryDto);
}
catch (Exception)
{
// ignored
}
});
});
List <Repository> repositoryList = populatedRepositories.ToList();
repositoryRepository.Update(repositoryList);
updateStatusRepository.UpdateStatus(new UpdateStatus {
LastUpdate = DateTime.Now, UpdatedRecords = repositoryList.Count, UpdaterName = nameof(RepositoryUpdater)
});
}