private async Task <List <GitRepoTopicPublishRecord> > GetPublishHistory(GitHubRepository repo)
{
var github = new GitHubClient(new ProductHeaderValue("OPSMetrics"));
var token = new Credentials(repo.AuthToken);
github.Credentials = token;
var repository = await github.Repository.Get(repo.Owner, repo.RepositoryName);
var pullRequestRequest = new PullRequestRequest()
{
State = ItemState.Closed, Base = "live"
};
var pullRequests = await github.Repository.PullRequest.GetAllForRepository(repository.Owner.Login, repository.Name, pullRequestRequest);
List <GitRepoTopicPublishRecord> history = new List <GitRepoTopicPublishRecord>();
foreach (var pullRequest in pullRequests)
{
var merged = await github.PullRequest.Merged(repository.Owner.Login, repository.Name, pullRequest.Number);
if (merged)
{
List <GitRepoTopicPublishRecord> records = new List <GitRepoTopicPublishRecord>();
var files = await github.Repository.PullRequest.Files(repository.Owner.Login, repository.Name, pullRequest.Number);
var fileNames = files.Where(file => file.FileName.EndsWith(".md")).Select(file => file.FileName).ToList();
var commitInfos = await github.PullRequest.Commits(repository.Owner.Login, repository.Name, pullRequest.Number);
foreach (var commitInfo in commitInfos)
{
var commit = await github.Repository.Commits.Get(repository.Owner.Login, repository.Name, commitInfo.Sha);
if (commit.Parents.Count < 2) // Ignore merge commits
{
var updatedFiles = commit.Files;
var updatedTopics = updatedFiles.Where(v => fileNames.Contains(v.Filename)).ToList();
foreach (var updatedTopic in updatedTopics)
{
AddToRecords(records, repo.PartitionKey, pullRequest.Number, updatedTopic.Filename,
pullRequest.MergedAt.Value.DateTime, commit.Author == null ? null : commit.Author.Id.ToString()); // Merged time of PR as publish time instead of commit time
if (updatedTopic.PreviousFileName != null) // If the topic is renamed
{
AddToRecords(records, repo.PartitionKey, pullRequest.Number, updatedTopic.PreviousFileName,
pullRequest.MergedAt.Value.DateTime, commit.Author == null ? null : commit.Author.Id.ToString());
}
}
}
}
history.AddRange(records);
}
}
return(history);
}
async Task <List <string> > GetFileCountForExtension(GitHubRepository repo, string branchName, string extension)
{
var github = new GitHubClient(new ProductHeaderValue("OPSMetrics"));
var token = new Credentials(repo.AuthToken);
github.Credentials = token;
//var repository = await github.Repository.Get(repo.Owner, repo.RepositoryName);
//var tree = await github.GitDatabase.Tree.GetRecursive(repository.Owner.Login, repository.Name, branchName);
var tree = await github.GitDatabase.Tree.GetRecursive(repo.Owner, repo.RepositoryName, branchName);
int count = 0;
List <string> topics = new List <string>();
if (tree.Truncated == false)
{
var items = tree.Tree;
foreach (var item in items)
{
if (item.Type == TreeType.Blob)
{
var itemExtension = System.IO.Path.GetExtension(item.Path);
if (string.Equals(itemExtension, "." + extension, StringComparison.OrdinalIgnoreCase))
{
count++;
topics.Add(item.Path);
}
}
}
}
else
{
// If 'truncated' is true, the number of items in the tree array exceeded GitHub's maximum limit.
// We need to use the non-recursive method of fetching trees, and fetch one sub-tree at a time.
//tree = await github.GitDatabase.Tree.Get(repository.Owner.Login, repository.Name, branchName);
//topics = await TraverseTreeManually(tree, github, repository, extension);
tree = await github.GitDatabase.Tree.Get(repo.Owner, repo.RepositoryName, branchName);
topics = await TraverseTreeManually(tree, github, repo.Owner, repo.RepositoryName, extension);
}
return(topics);
}
async Task <int> GetRecentUpdatedTopicCountOnLiveBranch(GitHubRepository repo, int day)
{
var github = new GitHubClient(new ProductHeaderValue("OPSMetrics"));
var token = new Credentials(repo.AuthToken);
github.Credentials = token;
var repository = await github.Repository.Get(repo.Owner, repo.RepositoryName);
var pullRequestRequest = new PullRequestRequest()
{
State = ItemState.Closed, Base = "live"
};
var pullRequests = await github.Repository.PullRequest.GetAllForRepository(repository.Owner.Login, repository.Name, pullRequestRequest);
HashSet <string> recentUpdatedTopicList = new HashSet <string>();
var timeThreshold = new DateTimeOffset(DateTime.UtcNow.AddDays(0 - day));
foreach (var pullRequest in pullRequests)
{
if (pullRequest.MergedAt.HasValue && DateTimeOffset.Compare(timeThreshold, pullRequest.MergedAt.Value) <= 0)
{
var merged = await github.PullRequest.Merged(repository.Owner.Login, repository.Name, pullRequest.Number);
if (merged)
{
var updatedFiles = await github.Repository.PullRequest.Files(repository.Owner.Login, repository.Name, pullRequest.Number);
foreach (var updatedFile in updatedFiles)
{
if (!(string.Equals(updatedFile.Status, "removed")) &&
string.Equals(System.IO.Path.GetExtension(updatedFile.FileName), ".md"))
{
recentUpdatedTopicList.Add(updatedFile.FileName);
}
}
}
}
}
return(recentUpdatedTopicList.Count);
}
protected override object Extract()
{
if (SharedObject_Prod_All == null)
{
return(null);
}
List <GitRepoTopicPublishRecord2> ret = new List <GitRepoTopicPublishRecord2>();
List <GitHubRepository> repos = SharedObject_Prod_All as List <GitHubRepository>;
DateTime start = DateTime.Parse("2010-01-01");
InsightDBHelper.InsightDBHelper.ConnectDBWithConnectString(OPSDataSyncConnStr);
var dataRow = InsightDBHelper.InsightDBHelper.ExecuteQuery("SELECT MAX(StartTime) FROM OPS_RepoTopicPublishRecords2 WITH (NOLOCK)");
if (dataRow != null &&
dataRow.Length > 0 &&
dataRow[0] != null &&
dataRow[0].ItemArray != null &&
dataRow[0].ItemArray.Length > 0 &&
dataRow[0].ItemArray[0] != null &&
!string.IsNullOrEmpty(dataRow[0].ItemArray[0].ToString()))
{
start = DateTime.Parse(dataRow[0].ItemArray[0].ToString()).AddSeconds(1);
}
List <OPSBuildInfo> opsBuildInfo = new List <OPSBuildInfo>();
string utcNowStr = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
foreach (var r in repos)
{
GetOPSBuildInfo(start.ToString("yyyyMMddHHmmss"), utcNowStr, r.PartitionKey, null, opsBuildInfo);
}
// Get publish records for a test repo that is used by PM for testing
GetOPSBuildInfo(start.ToString("yyyyMMddHHmmss"), utcNowStr, "1f799a29-9d37-b5e7-3c64-8283809f281d", null, opsBuildInfo);
foreach (var buildItem in opsBuildInfo)
{
GitHubRepository repo = null;
foreach (var r in repos)
{
if (r.Owner.Equals(buildItem.repository_account, StringComparison.OrdinalIgnoreCase) &&
r.RepositoryName.Equals(buildItem.repository_name, StringComparison.OrdinalIgnoreCase))
{
repo = r;
break;
}
}
// Get publish records for a test repo that is used by PM for testing
if (repo == null && !buildItem.repository_id.Equals("1f799a29-9d37-b5e7-3c64-8283809f281d", StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (buildItem.repository_id.Equals("1f799a29-9d37-b5e7-3c64-8283809f281d", StringComparison.OrdinalIgnoreCase))
{
repo = new GitHubRepository()
{
PartitionKey = "1f799a29-9d37-b5e7-3c64-8283809f281d", IsLocalization = false, GitRepositoryType = "Vso"
};
}
if (buildItem.branch_name.Equals("live", StringComparison.OrdinalIgnoreCase) ||
buildItem.branch_name.Equals("master", StringComparison.OrdinalIgnoreCase))
{
if (string.IsNullOrEmpty(buildItem.change_log_url))
{
continue;
}
JavaScriptSerializer jsonSerializer = new JavaScriptSerializer();
string contentInJson = Util.GetJsonContent(buildItem.change_log_url);
OPSBuildInfo_ChangeLog change_log = jsonSerializer.Deserialize <OPSBuildInfo_ChangeLog>(contentInJson);
Dictionary <string, int[]> wordChangeCountPerFile = null;
if (repo.GitRepositoryType.Equals("GitHub", StringComparison.OrdinalIgnoreCase) && !repo.IsLocalization)
{
//Get redirect_url by calling change_log_url
string redirect_url = change_log.redirect_url;
wordChangeCountPerFile = GetWordChangeCountPerFile(redirect_url, repo.AuthToken);
}
var authorLogins = new HashSet <string>(from commit in change_log.commits select commit.committer_login_name);
var authorNames = new HashSet <string>(from commit in change_log.commits select commit.committer_name);
List <GitRepoTopicPublishContentUpdate> topics = new List <GitRepoTopicPublishContentUpdate>();
foreach (var file in change_log.files)
{
if (file.file_name.EndsWith(".md"))
{
int?deleteCount = null, insertCount = null;
if (wordChangeCountPerFile != null && wordChangeCountPerFile.Count != 0 &&
repo.GitRepositoryType.Equals("GitHub", StringComparison.OrdinalIgnoreCase) && !repo.IsLocalization)
{
if (wordChangeCountPerFile.ContainsKey(file.file_name))
{
deleteCount = wordChangeCountPerFile[file.file_name][0];
insertCount = wordChangeCountPerFile[file.file_name][1];
}
}
GitRepoTopicPublishContentUpdate contentUpdate = new GitRepoTopicPublishContentUpdate()
{
TopicPath = file.file_name,
Insertions = file.insertions,
Deletions = file.deletions,
Status = file.status,
DeletionsOfWord = deleteCount,
InsertionsOfWord = insertCount
};
topics.Add(contentUpdate);
}
}
ret.Add(new GitRepoTopicPublishRecord2()
{
PublishId = buildItem.id,
PartitionKey = repo.PartitionKey,
TopicPaths = topics,
Branch = buildItem.branch_name,
StartTime = buildItem.started_at == null ? (DateTime?)null : DateTime.Parse(buildItem.started_at),
EndTime = buildItem.ended_at == null ? (DateTime?)null : DateTime.Parse(buildItem.ended_at),
AuthorLogins = authorLogins,
AuthorNames = authorNames,
Status = buildItem.status
});
}
else
{
var record = new GitRepoTopicPublishRecord2()
{
PublishId = buildItem.id,
PartitionKey = repo.PartitionKey,
TopicPaths = null,
Branch = buildItem.branch_name,
StartTime = buildItem.started_at == null ? (DateTime?)null : DateTime.Parse(buildItem.started_at),
EndTime = buildItem.ended_at == null ? (DateTime?)null : DateTime.Parse(buildItem.ended_at),
AuthorLogins = new HashSet <string>(),
AuthorNames = new HashSet <string>(),
Status = buildItem.status
};
ret.Add(record);
}
}
return(ret);
}
private int GetCommitNumber(int pullNumber, Dictionary <int, int> dic_PrevCommitNOInPull, GitHubRepository repo, bool isUpdate = false)
{
if (!isUpdate)
{
if (dic_PrevCommitNOInPull.ContainsKey(pullNumber))
{
return(dic_PrevCommitNOInPull[pullNumber]);
}
}
string commitNOUrl = string.Format("https://api.github.com/repos/{0}/{1}/pulls/{2}", repo.Owner, repo.RepositoryName, pullNumber);
GitCommitNOInPull gitCommitNOInPull = Util.CallGitHubAPI <GitCommitNOInPull>(commitNOUrl, repo.AuthToken) as GitCommitNOInPull;
int commitsNOInPull = gitCommitNOInPull == null ? 0 : gitCommitNOInPull.Commits;
if (commitsNOInPull == 0)
{
return(0);
}
string commitInPullUrl = string.Format("https://api.github.com/repos/{0}/{1}/pulls/{2}/commits", repo.Owner, repo.RepositoryName, pullNumber);
List <GitCommit> commitsInPull = Util.CallGitHubAPI <List <GitCommit> >(commitInPullUrl, repo.AuthToken) as List <GitCommit>;
foreach (GitCommit commit in commitsInPull)
{
GitInnerCommit innerCommit = commit.Commit;
if (innerCommit == null || innerCommit.Committer == null || string.IsNullOrEmpty(innerCommit.Committer.Date))
{
continue;
}
//This commit is another PullRequest
if (innerCommit != null && !string.IsNullOrEmpty(innerCommit.Message) && innerCommit.Message.StartsWith("Merge pull request #", StringComparison.OrdinalIgnoreCase))
{
int mergedPullNO = -1, prevCommitNO;
string message = innerCommit.Message;
string[] tmp = message.Split(' ');
int.TryParse(tmp[3].Split(new char[] { '#' }, StringSplitOptions.RemoveEmptyEntries)[0], out mergedPullNO);
if (dic_PrevCommitNOInPull.ContainsKey(mergedPullNO))
{
prevCommitNO = dic_PrevCommitNOInPull[mergedPullNO]; //Already exists
}
else
{
prevCommitNO = GetCommitNumber(mergedPullNO, dic_PrevCommitNOInPull, repo);
}
commitsNOInPull = commitsNOInPull - 1 + prevCommitNO;
}
}
if (isUpdate)
{
dic_PrevCommitNOInPull.Remove(pullNumber);
}
dic_PrevCommitNOInPull.Add(pullNumber, commitsNOInPull); //Add new item
return(commitsNOInPull);
}