public async Task OrchestrateAsync(RunTimeConfiguration config)
{
bool moreRepostoriesToRead = false;
var sourceRepositoriesRead = 0;
var sourceRepositoriesAnalyzed = 0;
var repositoryAnalysisErrors = new List <(string repoName, string errorMessage, string errorStackTrace)>();
try
{
var userOrOranizationNameQueryStringKey = string.Empty;
var userOrOganziationNameQueryStringValue = string.Empty;
if (!string.IsNullOrWhiteSpace(config.User))
{
userOrOranizationNameQueryStringKey = "user";
userOrOganziationNameQueryStringValue = config.User;
}
else
{
userOrOranizationNameQueryStringKey = "organization";
userOrOganziationNameQueryStringValue = config.Organization;
}
restClient.BaseUrl = new Uri(config.Url);
string endCursor = null;
var stopWatch = Stopwatch.StartNew();
do
{
logger.Information($"Reading next batch of {config.BatchSize} repositories for login {config.User ?? config.Organization}");
CursorPagedResults <RepositorySummary> results = null;
if (endCursor != null)
{
var request = new RestRequest("/api/repositorysource/repositories");
request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue);
request.AddQueryParameter("take", config.BatchSize.ToString());
request.AddQueryParameter("endCursor", endCursor);
var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request);
if (!response.IsSuccessful)
{
throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}");
}
results = response.Data;
}
else
{
var request = new RestRequest("/api/repositorysource/repositories");
request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue);
request.AddQueryParameter("take", config.BatchSize.ToString());
var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request);
if (!response.IsSuccessful)
{
throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}");
}
results = response.Data;
}
sourceRepositoriesRead += results.Results.Count();
endCursor = results.EndCursor;
moreRepostoriesToRead = results.MoreToRead;
var repositoryAnalysisTasks = new List <Task>();
using (var semaphore = new SemaphoreSlim(config.Concurrency))
{
foreach (var result in results.Results)
{
// await here until there is a room for this task
await semaphore.WaitAsync();
repositoryAnalysisTasks.Add(SendAnalysisRequest(semaphore, result));
}
await Task.WhenAll(repositoryAnalysisTasks);
}
logger.Information($"Finished analyizing batch of {config.BatchSize} repositories. {sourceRepositoriesAnalyzed} respositories analyzed thus far");
} while (moreRepostoriesToRead);
stopWatch.Stop();
logger.Information($"\nAnalyized {sourceRepositoriesAnalyzed} out of {sourceRepositoriesRead} repositories in {stopWatch.Elapsed.TotalMinutes} minutes");
logger.Information($"\nThere were {repositoryAnalysisErrors.Count} analyisis errors");
foreach (var repositoryAnalysisError in repositoryAnalysisErrors)
{
logger.Error($"{repositoryAnalysisError.repoName} - {repositoryAnalysisError.errorMessage}");
}
Console.WriteLine("\nExecution complete!");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
}
catch (Exception ex)
{
logger.Error($"FATAL EXCEPTION OCCURRED! {ex.Message}");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
}
async Task SendAnalysisRequest(SemaphoreSlim semaphore, RepositorySummary repositorySummary)
{
try
{
if (config.AsOf.HasValue)
{
logger.Information($"Starting analysis of {repositorySummary.Url} as of {config.AsOf.Value.ToString("F")}");
}
else
{
logger.Information($"Starting analysis of {repositorySummary.Url}");
}
var repositoryAnalysis = new RepositoryAnalysis
{
ForceCompleteRefresh = config.RefreshAll,
RepositoryLastUpdatedOn = repositorySummary.UpdatedAt,
RepositoryId = repositorySummary.Url,
AsOf = config.AsOf
};
var request = new RestRequest("/api/repositoryanalysis/", Method.POST);
request.AddJsonBody(repositoryAnalysis);
if (sourceRepositoriesAnalyzed == 0)
{
// If no requests have been completed then set the timeout to be higher as if an organization
// is being targeted then the reading of all the team information can take a few minutes.
request.Timeout = config.FirstApiCallTimeout;
}
var response = await restClient.ExecuteTaskAsync(request);
if (!response.IsSuccessful)
{
if (response.ErrorMessage != null && response.ErrorMessage.StartsWith("No connection could be made because the target machine actively refused it"))
{
logger.Error($"UNABLE TO REACH API!!");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
return;
}
else
{
logger.Error(response?.ErrorMessage);
repositoryAnalysisErrors.Add((repositorySummary.Url, response.StatusDescription, null));
}
}
}
catch (Exception ex)
{
repositoryAnalysisErrors.Add((repositorySummary.Url, ex.Message, ex.StackTrace));
}
finally
{
semaphore.Release();
Interlocked.Increment(ref sourceRepositoriesAnalyzed);
}
}
}
public async Task Post(RepositoryAnalysis repositoyAnalysis)
{
await repositoryAnalysisManager.CreateAsync(repositoyAnalysis).ConfigureAwait(false);
}
public async Task CreateAsync(RepositoryAnalysis repositoryAnalysis)
{
var parsedRepoUrl = ParseRepositoryUrl();
// maybe change the name of this var to reflect how it's different than the repositoryAnalysis.repsoitoryId?
var repositoryId = $"{parsedRepoUrl.Host}|{parsedRepoUrl.Owner}|{parsedRepoUrl.Name}";
var repository = await repositoryManager.ReadAsync(repositoryId, repositoryAnalysis.AsOf).ConfigureAwait(false);
DateTime?repositoryLastUpdatedOn = null;
// If a last updated time for the repo was provided, use that to hopefully save an API call
if (repositoryAnalysis.RepositoryLastUpdatedOn.HasValue)
{
repositoryLastUpdatedOn = repositoryAnalysis.RepositoryLastUpdatedOn.Value;
}
else
{
var repositorySummary = await repositorySourceManager.ReadRepositorySummaryAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name).ConfigureAwait(false);
repositoryLastUpdatedOn = repositorySummary.UpdatedAt;
}
if (repositoryAnalysis.ForceCompleteRefresh || repository == null || repositoryLastUpdatedOn > repository.CurrentState.RepositoryLastUpdatedOn)
{
// Do repository summary call to get the commit Id of the latest commit and the date that commit was pushed for the snapshot
// populate the snapshot date with the corresponding manager calls (E.G. ScrapeDependenciesAsync)
// Do full repository read to get all the current state stuff (including calls to get derived data like devops integrations)
var sourceRepository = await repositorySourceManager.ReadRepositoryAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name).ConfigureAwait(false);
var repositoryCurrentState = new RepositoryCurrentState();
repositoryCurrentState.Id = repositoryId;
repositoryCurrentState.Name = sourceRepository.Name;
repositoryCurrentState.Owner = parsedRepoUrl.Owner;
repositoryCurrentState.DefaultBranch = sourceRepository.DefaultBranchName;
repositoryCurrentState.HasIssues = sourceRepository.IssueCount > 0;
repositoryCurrentState.HasProjects = sourceRepository.ProjectCount > 0;
repositoryCurrentState.HasPullRequests = sourceRepository.PullRequestCount > 0;
repositoryCurrentState.RepositoryCreatedOn = sourceRepository.CreatedAt;
repositoryCurrentState.RepositoryLastUpdatedOn = sourceRepository.PushedAt;
repositoryCurrentState.Teams = sourceRepository.Teams;
repositoryCurrentState.Topics = sourceRepository.TopicNames?.Select(name => new RepositoryTopic {
Name = name
}).ToList();
repositoryCurrentState.DevOpsIntegrations = await ScrapeDevOpsIntegrations(repositoryCurrentState.Name).ConfigureAwait(false);
// Need to pick a branch for the snapshot stuff
string branchName = null;
if (sourceRepository.BranchNames.Contains("master"))
{
branchName = "master";
}
else if (sourceRepository.BranchNames.Contains("development"))
{
branchName = "development";
}
else if (!string.IsNullOrWhiteSpace(sourceRepository.DefaultBranchName))
{
branchName = sourceRepository.DefaultBranchName;
}
RepositorySnapshot repositorySnapshot = null;
repositorySnapshot = new RepositorySnapshot();
// Have to set the windows in the manager
repositorySnapshot.RepositoryCurrentStateRepositoryId = repositoryCurrentState.Id;
repositorySnapshot.TakenOn = DateTime.Now;
repositorySnapshot.BranchUsed = branchName;
if (branchName != null)
{
repositorySnapshot.Dependencies = await ScrapeDependenciesAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name, branchName, repositoryAnalysis.AsOf).ConfigureAwait(false);
repositorySnapshot.Files = await repositorySourceManager.ReadFilesAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name, branchName, repositoryAnalysis.AsOf).ConfigureAwait(false);
}
else
{
repositorySnapshot.Dependencies = new List <RepositoryDependency>();
repositorySnapshot.Files = new List <RepositoryFile>();
}
repositorySnapshot.TypesAndImplementations = await ScrapeRepositoryTypeAndImplementation(
parsedRepoUrl.Name,
repositorySnapshot.Files,
repositorySnapshot.Dependencies,
repositoryCurrentState.Topics?.Select(topic => topic.Name),
new BacklogInfo { HasIssues = repositoryCurrentState.HasIssues ?? false },
repositoryAnalysis.AsOf).ConfigureAwait(false);
var updatedRepository = new Repository
{
CurrentState = repositoryCurrentState,
Snapshot = repositorySnapshot
};
await repositoryManager.UpsertAsync(updatedRepository, repositoryAnalysis.AsOf).ConfigureAwait(false);
}
(string Owner, string Name, string Host) ParseRepositoryUrl()
{
var repositoryUri = new Uri(repositoryAnalysis.RepositoryId);
var owner = repositoryUri.Segments[1].TrimEnd('/');
var name = repositoryUri.Segments[2].TrimEnd('/');
var host = repositoryUri.Host;
return(owner, name, host);
}
}
