public void SaveRenamedRepo(string oldOwner, string oldName, GitHubRepo newRepo)
{
string renameFile = Path.Combine(GetRepoFolder(oldOwner, oldName), "rename.txt");
Directory.CreateDirectory(Path.GetDirectoryName(renameFile));
File.WriteAllText(renameFile, newRepo.ToString());
}
IEnumerable <GitHubRepo> ReadRepos()
{
foreach (var line in File.ReadLines(_repoListFile))
{
var repo = GitHubRepo.Parse(line);
yield return(repo);
}
}
public static GitHubRepo Parse(string line)
{
var fields = line.Split('\t');
var ret = new GitHubRepo();
ret.ID = fields[0];
ret.Owner = fields[1];
ret.Name = fields[2];
ret.Language = fields[3];
ret.Stars = int.Parse(fields[4]);
return(ret);
}
// Returns true if the repo has been renamed
public GitHubRepo GetRenamedRepo(GitHubRepo repo)
{
string renameFile = Path.Combine(GetRepoFolder(repo.Owner, repo.Name), "rename.txt");
if (File.Exists(renameFile))
{
var line = File.ReadAllLines(renameFile).First();
var ret = GitHubRepo.Parse(line);
if (ret.Owner == repo.Owner && ret.Name == repo.Name)
{
throw new InvalidDataException($"Repo {ret.Owner}/{ret.Name} redirects to itself");
}
return(ret);
}
else
{
return(null);
}
}
async Task <IEnumerable <SearchResult> > SearchRepoAsync(GitHubRepo repo, bool handleRenamedRepos = true)
{
object operation = null;
try
{
List <SearchResult> ret = new List <SearchResult>();
if (_storage.IsNotFound(repo.Owner, repo.Name))
{
_logger.Verbose("{Repo} previously not found, skipping", repo.Owner + "/" + repo.Name);
return(Enumerable.Empty <SearchResult>());
}
if (_storage.HasRepoResults(repo.Owner, repo.Name))
{
_logger.Verbose("{Repo} already downloaded", repo.Owner + "/" + repo.Name);
ret = _storage.GetRepoResults(repo.Owner, repo.Name).ToList();
}
else
{
var request = new SearchCodeRequest()
{
FileName = "project.json",
};
request.Repos.Add(repo.Owner, repo.Name);
int totalResultsReturned = 0;
while (true)
{
if (_cancelToken.IsCancellationRequested)
{
return(Enumerable.Empty <SearchResult>());
}
operation = new { Operation = "Search", Repo = repo.Owner + "/" + repo.Name, Page = request.Page };
SearchCodeResult result;
ApiValidationException validationException = null;
ExceptionDispatchInfo validationExceptionDispatchInfo = null;
result = await
_searchThrottler.RunAsync <SearchCodeResult>(
async() =>
{
// Do a try/catch inside here so that renamed repos don't get logged as failures by the throttler
try
{
return(await _client.Search.SearchCode(request));
}
catch (ApiValidationException ex) when(handleRenamedRepos)
{
validationException = ex;
validationExceptionDispatchInfo = ExceptionDispatchInfo.Capture(ex);
return(null);
}
},
operation
);
if (result == null && validationException != null)
{
_logger.Debug(validationException, "Api validation exception for {Operation}, checking for renamed repo", operation);
var renameOperation = new { Operation = "RenameCheck", Repo = repo.Owner + "/" + repo.Name };
var potentiallyRenamedRepo = await _throttler.RunAsync <Repository>(
async() =>
{
try
{
return(await _client.Repository.Get(repo.Owner, repo.Name));
}
catch (NotFoundException)
{
return(null);
}
},
renameOperation
);
if (potentiallyRenamedRepo == null)
{
_logger.Information("Repo {Repo} not found", renameOperation.Repo);
_storage.SaveNotFound(repo.Owner, repo.Name, true);
return(Enumerable.Empty <SearchResult>());
}
if (potentiallyRenamedRepo.Owner.Login == repo.Owner && potentiallyRenamedRepo.Name == repo.Name)
{
_logger.Error("Repo was not renamed, Api validation must have failed for some other reason for {Operation}", operation);
validationExceptionDispatchInfo.Throw();
}
var newRepo = repo.Clone();
newRepo.Owner = potentiallyRenamedRepo.Owner.Login;
newRepo.Name = potentiallyRenamedRepo.Name;
_logger.Information("Repo {OldRepo} has been renamed to {Repo}", renameOperation.Repo, newRepo.Owner + "/" + newRepo.Name);
_storage.SaveRenamedRepo(repo.Owner, repo.Name, newRepo);
return(await SearchRepoAsync(newRepo, false));
}
foreach (var item in result.Items)
{
string destFile = _storage.GetFilePath(repo.Owner, repo.Name, item.Path);
if (Path.GetFileName(destFile).Equals("project.json", StringComparison.OrdinalIgnoreCase))
{
ret.Add(new SearchResult(item));
}
else
{
_logger.Information("{Path} was not a project.json file in {Repo}, ignoring", item.Path, repo.Owner + "/" + repo.Name);
}
}
if (result.IncompleteResults)
{
_logger.Error("Incomplete search results for {Repo}", repo.Owner + "/" + repo.Name);
break;
}
totalResultsReturned += result.Items.Count;
if (totalResultsReturned >= result.TotalCount)
{
break;
}
else
{
request.Page += 1;
}
}
_storage.RecordRepoResults(repo.Owner, repo.Name, ret);
_logger.Information("Completed searching repo {Repo}", repo.Owner + "/" + repo.Name);
}
return(ret);
}
catch (Exception ex)
{
_logger.Error(ex, "{Operation} failed", operation);
return(Enumerable.Empty <SearchResult>());
}
}
void RecordRepoCompleted(GitHubRepo repo, IEnumerable <SearchResult> results)
{
_storage.RecordRepoResults(repo.Owner, repo.Name, results);
_logger.Information("Completed repo {Repo}", repo.Owner + "/" + repo.Name);
}
