private async Task <List <GitRepoTopicPublishRecord> > GetPublishHistory(GitHubRepository repo) { var github = new GitHubClient(new ProductHeaderValue("OPSMetrics")); var token = new Credentials(repo.AuthToken); github.Credentials = token; var repository = await github.Repository.Get(repo.Owner, repo.RepositoryName); var pullRequestRequest = new PullRequestRequest() { State = ItemState.Closed, Base = "live" }; var pullRequests = await github.Repository.PullRequest.GetAllForRepository(repository.Owner.Login, repository.Name, pullRequestRequest); List <GitRepoTopicPublishRecord> history = new List <GitRepoTopicPublishRecord>(); foreach (var pullRequest in pullRequests) { var merged = await github.PullRequest.Merged(repository.Owner.Login, repository.Name, pullRequest.Number); if (merged) { List <GitRepoTopicPublishRecord> records = new List <GitRepoTopicPublishRecord>(); var files = await github.Repository.PullRequest.Files(repository.Owner.Login, repository.Name, pullRequest.Number); var fileNames = files.Where(file => file.FileName.EndsWith(".md")).Select(file => file.FileName).ToList(); var commitInfos = await github.PullRequest.Commits(repository.Owner.Login, repository.Name, pullRequest.Number); foreach (var commitInfo in commitInfos) { var commit = await github.Repository.Commits.Get(repository.Owner.Login, repository.Name, commitInfo.Sha); if (commit.Parents.Count < 2) // Ignore merge commits { var updatedFiles = commit.Files; var updatedTopics = updatedFiles.Where(v => fileNames.Contains(v.Filename)).ToList(); foreach (var updatedTopic in updatedTopics) { AddToRecords(records, repo.PartitionKey, pullRequest.Number, updatedTopic.Filename, pullRequest.MergedAt.Value.DateTime, commit.Author == null ? null : commit.Author.Id.ToString()); // Merged time of PR as publish time instead of commit time if (updatedTopic.PreviousFileName != null) // If the topic is renamed { AddToRecords(records, repo.PartitionKey, pullRequest.Number, updatedTopic.PreviousFileName, pullRequest.MergedAt.Value.DateTime, commit.Author == null ? null : commit.Author.Id.ToString()); } } } } history.AddRange(records); } } return(history); }
async Task <List <string> > GetFileCountForExtension(GitHubRepository repo, string branchName, string extension) { var github = new GitHubClient(new ProductHeaderValue("OPSMetrics")); var token = new Credentials(repo.AuthToken); github.Credentials = token; //var repository = await github.Repository.Get(repo.Owner, repo.RepositoryName); //var tree = await github.GitDatabase.Tree.GetRecursive(repository.Owner.Login, repository.Name, branchName); var tree = await github.GitDatabase.Tree.GetRecursive(repo.Owner, repo.RepositoryName, branchName); int count = 0; List <string> topics = new List <string>(); if (tree.Truncated == false) { var items = tree.Tree; foreach (var item in items) { if (item.Type == TreeType.Blob) { var itemExtension = System.IO.Path.GetExtension(item.Path); if (string.Equals(itemExtension, "." + extension, StringComparison.OrdinalIgnoreCase)) { count++; topics.Add(item.Path); } } } } else { // If 'truncated' is true, the number of items in the tree array exceeded GitHub's maximum limit. // We need to use the non-recursive method of fetching trees, and fetch one sub-tree at a time. //tree = await github.GitDatabase.Tree.Get(repository.Owner.Login, repository.Name, branchName); //topics = await TraverseTreeManually(tree, github, repository, extension); tree = await github.GitDatabase.Tree.Get(repo.Owner, repo.RepositoryName, branchName); topics = await TraverseTreeManually(tree, github, repo.Owner, repo.RepositoryName, extension); } return(topics); }
async Task <int> GetRecentUpdatedTopicCountOnLiveBranch(GitHubRepository repo, int day) { var github = new GitHubClient(new ProductHeaderValue("OPSMetrics")); var token = new Credentials(repo.AuthToken); github.Credentials = token; var repository = await github.Repository.Get(repo.Owner, repo.RepositoryName); var pullRequestRequest = new PullRequestRequest() { State = ItemState.Closed, Base = "live" }; var pullRequests = await github.Repository.PullRequest.GetAllForRepository(repository.Owner.Login, repository.Name, pullRequestRequest); HashSet <string> recentUpdatedTopicList = new HashSet <string>(); var timeThreshold = new DateTimeOffset(DateTime.UtcNow.AddDays(0 - day)); foreach (var pullRequest in pullRequests) { if (pullRequest.MergedAt.HasValue && DateTimeOffset.Compare(timeThreshold, pullRequest.MergedAt.Value) <= 0) { var merged = await github.PullRequest.Merged(repository.Owner.Login, repository.Name, pullRequest.Number); if (merged) { var updatedFiles = await github.Repository.PullRequest.Files(repository.Owner.Login, repository.Name, pullRequest.Number); foreach (var updatedFile in updatedFiles) { if (!(string.Equals(updatedFile.Status, "removed")) && string.Equals(System.IO.Path.GetExtension(updatedFile.FileName), ".md")) { recentUpdatedTopicList.Add(updatedFile.FileName); } } } } } return(recentUpdatedTopicList.Count); }
protected override object Extract() { if (SharedObject_Prod_All == null) { return(null); } List <GitRepoTopicPublishRecord2> ret = new List <GitRepoTopicPublishRecord2>(); List <GitHubRepository> repos = SharedObject_Prod_All as List <GitHubRepository>; DateTime start = DateTime.Parse("2010-01-01"); InsightDBHelper.InsightDBHelper.ConnectDBWithConnectString(OPSDataSyncConnStr); var dataRow = InsightDBHelper.InsightDBHelper.ExecuteQuery("SELECT MAX(StartTime) FROM OPS_RepoTopicPublishRecords2 WITH (NOLOCK)"); if (dataRow != null && dataRow.Length > 0 && dataRow[0] != null && dataRow[0].ItemArray != null && dataRow[0].ItemArray.Length > 0 && dataRow[0].ItemArray[0] != null && !string.IsNullOrEmpty(dataRow[0].ItemArray[0].ToString())) { start = DateTime.Parse(dataRow[0].ItemArray[0].ToString()).AddSeconds(1); } List <OPSBuildInfo> opsBuildInfo = new List <OPSBuildInfo>(); string utcNowStr = DateTime.UtcNow.ToString("yyyyMMddHHmmss"); foreach (var r in repos) { GetOPSBuildInfo(start.ToString("yyyyMMddHHmmss"), utcNowStr, r.PartitionKey, null, opsBuildInfo); } // Get publish records for a test repo that is used by PM for testing GetOPSBuildInfo(start.ToString("yyyyMMddHHmmss"), utcNowStr, "1f799a29-9d37-b5e7-3c64-8283809f281d", null, opsBuildInfo); foreach (var buildItem in opsBuildInfo) { GitHubRepository repo = null; foreach (var r in repos) { if (r.Owner.Equals(buildItem.repository_account, StringComparison.OrdinalIgnoreCase) && r.RepositoryName.Equals(buildItem.repository_name, StringComparison.OrdinalIgnoreCase)) { repo = r; break; } } // Get publish records for a test repo that is used by PM for testing if (repo == null && !buildItem.repository_id.Equals("1f799a29-9d37-b5e7-3c64-8283809f281d", StringComparison.OrdinalIgnoreCase)) { continue; } if (buildItem.repository_id.Equals("1f799a29-9d37-b5e7-3c64-8283809f281d", StringComparison.OrdinalIgnoreCase)) { repo = new GitHubRepository() { PartitionKey = "1f799a29-9d37-b5e7-3c64-8283809f281d", IsLocalization = false, GitRepositoryType = "Vso" }; } if (buildItem.branch_name.Equals("live", StringComparison.OrdinalIgnoreCase) || buildItem.branch_name.Equals("master", StringComparison.OrdinalIgnoreCase)) { if (string.IsNullOrEmpty(buildItem.change_log_url)) { continue; } JavaScriptSerializer jsonSerializer = new JavaScriptSerializer(); string contentInJson = Util.GetJsonContent(buildItem.change_log_url); OPSBuildInfo_ChangeLog change_log = jsonSerializer.Deserialize <OPSBuildInfo_ChangeLog>(contentInJson); Dictionary <string, int[]> wordChangeCountPerFile = null; if (repo.GitRepositoryType.Equals("GitHub", StringComparison.OrdinalIgnoreCase) && !repo.IsLocalization) { //Get redirect_url by calling change_log_url string redirect_url = change_log.redirect_url; wordChangeCountPerFile = GetWordChangeCountPerFile(redirect_url, repo.AuthToken); } var authorLogins = new HashSet <string>(from commit in change_log.commits select commit.committer_login_name); var authorNames = new HashSet <string>(from commit in change_log.commits select commit.committer_name); List <GitRepoTopicPublishContentUpdate> topics = new List <GitRepoTopicPublishContentUpdate>(); foreach (var file in change_log.files) { if (file.file_name.EndsWith(".md")) { int?deleteCount = null, insertCount = null; if (wordChangeCountPerFile != null && wordChangeCountPerFile.Count != 0 && repo.GitRepositoryType.Equals("GitHub", StringComparison.OrdinalIgnoreCase) && !repo.IsLocalization) { if (wordChangeCountPerFile.ContainsKey(file.file_name)) { deleteCount = wordChangeCountPerFile[file.file_name][0]; insertCount = wordChangeCountPerFile[file.file_name][1]; } } GitRepoTopicPublishContentUpdate contentUpdate = new GitRepoTopicPublishContentUpdate() { TopicPath = file.file_name, Insertions = file.insertions, Deletions = file.deletions, Status = file.status, DeletionsOfWord = deleteCount, InsertionsOfWord = insertCount }; topics.Add(contentUpdate); } } ret.Add(new GitRepoTopicPublishRecord2() { PublishId = buildItem.id, PartitionKey = repo.PartitionKey, TopicPaths = topics, Branch = buildItem.branch_name, StartTime = buildItem.started_at == null ? (DateTime?)null : DateTime.Parse(buildItem.started_at), EndTime = buildItem.ended_at == null ? (DateTime?)null : DateTime.Parse(buildItem.ended_at), AuthorLogins = authorLogins, AuthorNames = authorNames, Status = buildItem.status }); } else { var record = new GitRepoTopicPublishRecord2() { PublishId = buildItem.id, PartitionKey = repo.PartitionKey, TopicPaths = null, Branch = buildItem.branch_name, StartTime = buildItem.started_at == null ? (DateTime?)null : DateTime.Parse(buildItem.started_at), EndTime = buildItem.ended_at == null ? (DateTime?)null : DateTime.Parse(buildItem.ended_at), AuthorLogins = new HashSet <string>(), AuthorNames = new HashSet <string>(), Status = buildItem.status }; ret.Add(record); } } return(ret); }
private int GetCommitNumber(int pullNumber, Dictionary <int, int> dic_PrevCommitNOInPull, GitHubRepository repo, bool isUpdate = false) { if (!isUpdate) { if (dic_PrevCommitNOInPull.ContainsKey(pullNumber)) { return(dic_PrevCommitNOInPull[pullNumber]); } } string commitNOUrl = string.Format("https://api.github.com/repos/{0}/{1}/pulls/{2}", repo.Owner, repo.RepositoryName, pullNumber); GitCommitNOInPull gitCommitNOInPull = Util.CallGitHubAPI <GitCommitNOInPull>(commitNOUrl, repo.AuthToken) as GitCommitNOInPull; int commitsNOInPull = gitCommitNOInPull == null ? 0 : gitCommitNOInPull.Commits; if (commitsNOInPull == 0) { return(0); } string commitInPullUrl = string.Format("https://api.github.com/repos/{0}/{1}/pulls/{2}/commits", repo.Owner, repo.RepositoryName, pullNumber); List <GitCommit> commitsInPull = Util.CallGitHubAPI <List <GitCommit> >(commitInPullUrl, repo.AuthToken) as List <GitCommit>; foreach (GitCommit commit in commitsInPull) { GitInnerCommit innerCommit = commit.Commit; if (innerCommit == null || innerCommit.Committer == null || string.IsNullOrEmpty(innerCommit.Committer.Date)) { continue; } //This commit is another PullRequest if (innerCommit != null && !string.IsNullOrEmpty(innerCommit.Message) && innerCommit.Message.StartsWith("Merge pull request #", StringComparison.OrdinalIgnoreCase)) { int mergedPullNO = -1, prevCommitNO; string message = innerCommit.Message; string[] tmp = message.Split(' '); int.TryParse(tmp[3].Split(new char[] { '#' }, StringSplitOptions.RemoveEmptyEntries)[0], out mergedPullNO); if (dic_PrevCommitNOInPull.ContainsKey(mergedPullNO)) { prevCommitNO = dic_PrevCommitNOInPull[mergedPullNO]; //Already exists } else { prevCommitNO = GetCommitNumber(mergedPullNO, dic_PrevCommitNOInPull, repo); } commitsNOInPull = commitsNOInPull - 1 + prevCommitNO; } } if (isUpdate) { dic_PrevCommitNOInPull.Remove(pullNumber); } dic_PrevCommitNOInPull.Add(pullNumber, commitsNOInPull); //Add new item return(commitsNOInPull); }