// pagenation is ?page=2&per_page=100
/// <summary>
/// the url is like this: https://github.com/andyliuliming/WALinuxAgent.git
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public List <User> Extract(ExtractRequest extractRequest, List <string> user_logins)
{
GithubFeed githubFee = this.Parse(extractRequest.URL);
List <User> users = new List <User>();
RestApiCaller <User> userInfoCaller = new RestApiCaller <User>(ApiFormats.BaseUri);
// 3. extract the commit info ("login")
int leftUserLength = user_logins.Count - extractRequest.StartIndex;
int numberToRetrieve = leftUserLength > extractRequest.Left ? extractRequest.Left : leftUserLength;
for (int i = 0; i < numberToRetrieve; i++)
{
Console.WriteLine("getting the user's info: " + user_logins[extractRequest.StartIndex + i]);
string urlParameters = string.Format(ApiFormats.UserApi, user_logins[extractRequest.StartIndex + i]);
User user = null;
Retry(
() =>
{
user = userInfoCaller.CallApi("get", extractRequest.AccessToken, urlParameters, null);
},
() =>
{
extractRequest.Left--;
}
);
users.Add(user);
}
extractRequest.StartIndex += numberToRetrieve;
return(users);
}
public HashSet <string> ExtractUserLogin(ExtractRequest extractRequest)
{
GithubFeed githubFee = this.Parse(extractRequest.URL);
HashSet <string> user_logins = new HashSet <string>();
// 1. first get the branches https://api.github.com/repos/torvalds/linux
// https://api.github.com/repos/apache/hadoop
// 2. get the commits from the branches. https://api.github.com/repos/torvalds/linux/commits?page=3&per_page=100
RestApiCaller <List <CommitDetail> > commitInfoCaller = new RestApiCaller <List <CommitDetail> >(ApiFormats.BaseUri);
string repoBaseUri = string.Format(ApiFormats.CommitRelativePathPattern, githubFee.owner, githubFee.repo);
while (extractRequest.Left > 0)
{
Console.WriteLine("getting the page: " + extractRequest.StartPage);
string urlParameters = repoBaseUri + "?page=" + extractRequest.StartPage + "&per_page=" + extractRequest.PerPage;
List <CommitDetail> pagedDetails = null;
Retry(
() =>
{
pagedDetails = commitInfoCaller.CallApi("get", extractRequest.AccessToken, urlParameters, null);
},
() =>
{
extractRequest.Left--;
}
);
if (pagedDetails == null || pagedDetails.Count == 0)
{
break;
}
else
{
for (int i = 0; i < pagedDetails.Count; i++)
{
if (pagedDetails[i].author != null)
{
user_logins.Add(pagedDetails[i].author.login);
}
}
extractRequest.StartPage++;
}
}
return(user_logins);
}
private void RealWork()
{
commitInfoCaller.AccountResource = accountResource;
userInfoCaller.AccountResource = accountResource;
this.InterateChange <TalentCandidate>(db.TalentCandidates.OrderBy(tc => tc.Id), new Action <TalentCandidate>(tc =>
{
string urlParameters = string.Format(ApiFormats.UserApi, tc.Login);
Trace.TraceWarning(string.Format("getting the user {0}", tc.Login));
User user = userInfoCaller.CallApi("get", urlParameters);
if (user != null)
{
tc.Company = getEmptyOrValue(user.company);
tc.Email = getEmptyOrValue(user.email);
tc.Followers = getEmptyOrValue(user.followers);
tc.FollowersUrl = getEmptyOrValue(user.followers_url);
tc.Location = getEmptyOrValue(user.location);
tc.Name = getEmptyOrValue(user.name);
tc.ReposUrl = getEmptyOrValue(user.repos_url);
tc.Timestamp = DateTime.UtcNow;
}
else
{
Trace.TraceError("user result is null for: " + urlParameters);
}
}));
this.ResetTalentCandidateCommits();
foreach (GithubRepo githubRepo in db.GithubRepoes.ToList())
{
// clear the calculating number of the repository
this.ResetContributersToRepo(githubRepo);
int startPage = 0;
int perPage = 100;
GithubFeed githubFee = this.Parse(githubRepo.Url);
string repoBaseUri = string.Format(ApiFormats.CommitRelativePathPattern, githubFee.owner, githubFee.repo);
while (true)
{
string urlParameters = repoBaseUri + "?page=" + startPage + "&per_page=" + perPage;
List <CommitDetail> pagedCommitDetails = commitInfoCaller.CallApi("get", urlParameters);
if (pagedCommitDetails == null || pagedCommitDetails.Count == 0)
{
break;
}
else
{
for (int i = 0; i < pagedCommitDetails.Count; i++)
{
if (pagedCommitDetails[i].author != null)
{
string loginString = pagedCommitDetails[i].author.login;
TalentCandidate candidate = db.TalentCandidates.Where(
(tc) => tc.Login == loginString).FirstOrDefault();
if (candidate == null)
{
using (var transaction = db.Database.BeginTransaction())
{
candidate = new TalentCandidate();
candidate.Company = string.Empty;
candidate.Email = string.Empty;
candidate.Followers = string.Empty;
candidate.FollowersUrl = string.Empty;
candidate.Location = string.Empty;
candidate.Login = pagedCommitDetails[i].author.login;
candidate.Name = string.Empty;
candidate.ReposUrl = string.Empty;
candidate.Timestamp = DateTime.UtcNow;
db.TalentCandidates.Add(candidate);
db.SaveChanges();
transaction.Commit();
}
}
ContributerToRepo contributerInfo = null;
using (var transaction = db.Database.BeginTransaction())
{
contributerInfo = db.ContributerToRepoes.Where(ctp => ctp.RepoId == githubRepo.Id && ctp.TalentCandidateId == candidate.Id).FirstOrDefault();
if (contributerInfo == null)
{
contributerInfo = new ContributerToRepo();
contributerInfo.RepoId = githubRepo.Id;
contributerInfo.TalentCandidateId = candidate.Id;
contributerInfo.RepoUrl = githubRepo.Url;
contributerInfo.TalentCandidateName = (candidate.Name == null ? string.Empty : candidate.Name);
db.ContributerToRepoes.Add(contributerInfo);
db.SaveChanges();
}
contributerInfo.RepoUrl = githubRepo.Url;
contributerInfo.TalentCandidateName = (candidate.Name == null ? string.Empty : candidate.Name);
contributerInfo.CalculatingCommitNumber += 1;
candidate.CalculatingTotalCommits += 1;
db.SaveChanges();
transaction.Commit();
}
}
}
startPage++;
}
}
this.SetValueForContributerToRepo(githubRepo);
}
this.SetValueForTalentCandidateCommits();
}