public async Task OrchestrateAsync(RunTimeConfiguration config) { bool moreRepostoriesToRead = false; var sourceRepositoriesRead = 0; var sourceRepositoriesAnalyzed = 0; var repositoryAnalysisErrors = new List <(string repoName, string errorMessage, string errorStackTrace)>(); try { var userOrOranizationNameQueryStringKey = string.Empty; var userOrOganziationNameQueryStringValue = string.Empty; if (!string.IsNullOrWhiteSpace(config.User)) { userOrOranizationNameQueryStringKey = "user"; userOrOganziationNameQueryStringValue = config.User; } else { userOrOranizationNameQueryStringKey = "organization"; userOrOganziationNameQueryStringValue = config.Organization; } restClient.BaseUrl = new Uri(config.Url); string endCursor = null; var stopWatch = Stopwatch.StartNew(); do { logger.Information($"Reading next batch of {config.BatchSize} repositories for login {config.User ?? config.Organization}"); CursorPagedResults <RepositorySummary> results = null; if (endCursor != null) { var request = new RestRequest("/api/repositorysource/repositories"); request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue); request.AddQueryParameter("take", config.BatchSize.ToString()); request.AddQueryParameter("endCursor", endCursor); var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request); if (!response.IsSuccessful) { throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}"); } results = response.Data; } else { var request = new RestRequest("/api/repositorysource/repositories"); request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue); request.AddQueryParameter("take", config.BatchSize.ToString()); var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request); if (!response.IsSuccessful) { throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}"); } results = response.Data; } sourceRepositoriesRead += results.Results.Count(); endCursor = results.EndCursor; moreRepostoriesToRead = results.MoreToRead; var repositoryAnalysisTasks = new List <Task>(); using (var semaphore = new SemaphoreSlim(config.Concurrency)) { foreach (var result in results.Results) { // await here until there is a room for this task await semaphore.WaitAsync(); repositoryAnalysisTasks.Add(SendAnalysisRequest(semaphore, result)); } await Task.WhenAll(repositoryAnalysisTasks); } logger.Information($"Finished analyizing batch of {config.BatchSize} repositories. {sourceRepositoriesAnalyzed} respositories analyzed thus far"); } while (moreRepostoriesToRead); stopWatch.Stop(); logger.Information($"\nAnalyized {sourceRepositoriesAnalyzed} out of {sourceRepositoriesRead} repositories in {stopWatch.Elapsed.TotalMinutes} minutes"); logger.Information($"\nThere were {repositoryAnalysisErrors.Count} analyisis errors"); foreach (var repositoryAnalysisError in repositoryAnalysisErrors) { logger.Error($"{repositoryAnalysisError.repoName} - {repositoryAnalysisError.errorMessage}"); } Console.WriteLine("\nExecution complete!"); if (!runningInContainer) { Console.WriteLine("\nPress any key to exit"); Console.ReadKey(); } } catch (Exception ex) { logger.Error($"FATAL EXCEPTION OCCURRED! {ex.Message}"); if (!runningInContainer) { Console.WriteLine("\nPress any key to exit"); Console.ReadKey(); } } async Task SendAnalysisRequest(SemaphoreSlim semaphore, RepositorySummary repositorySummary) { try { if (config.AsOf.HasValue) { logger.Information($"Starting analysis of {repositorySummary.Url} as of {config.AsOf.Value.ToString("F")}"); } else { logger.Information($"Starting analysis of {repositorySummary.Url}"); } var repositoryAnalysis = new RepositoryAnalysis { ForceCompleteRefresh = config.RefreshAll, RepositoryLastUpdatedOn = repositorySummary.UpdatedAt, RepositoryId = repositorySummary.Url, AsOf = config.AsOf }; var request = new RestRequest("/api/repositoryanalysis/", Method.POST); request.AddJsonBody(repositoryAnalysis); if (sourceRepositoriesAnalyzed == 0) { // If no requests have been completed then set the timeout to be higher as if an organization // is being targeted then the reading of all the team information can take a few minutes. request.Timeout = config.FirstApiCallTimeout; } var response = await restClient.ExecuteTaskAsync(request); if (!response.IsSuccessful) { if (response.ErrorMessage != null && response.ErrorMessage.StartsWith("No connection could be made because the target machine actively refused it")) { logger.Error($"UNABLE TO REACH API!!"); if (!runningInContainer) { Console.WriteLine("\nPress any key to exit"); Console.ReadKey(); } return; } else { logger.Error(response?.ErrorMessage); repositoryAnalysisErrors.Add((repositorySummary.Url, response.StatusDescription, null)); } } } catch (Exception ex) { repositoryAnalysisErrors.Add((repositorySummary.Url, ex.Message, ex.StackTrace)); } finally { semaphore.Release(); Interlocked.Increment(ref sourceRepositoriesAnalyzed); } } }
public async Task Post(RepositoryAnalysis repositoyAnalysis) { await repositoryAnalysisManager.CreateAsync(repositoyAnalysis).ConfigureAwait(false); }
public async Task CreateAsync(RepositoryAnalysis repositoryAnalysis) { var parsedRepoUrl = ParseRepositoryUrl(); // maybe change the name of this var to reflect how it's different than the repositoryAnalysis.repsoitoryId? var repositoryId = $"{parsedRepoUrl.Host}|{parsedRepoUrl.Owner}|{parsedRepoUrl.Name}"; var repository = await repositoryManager.ReadAsync(repositoryId, repositoryAnalysis.AsOf).ConfigureAwait(false); DateTime?repositoryLastUpdatedOn = null; // If a last updated time for the repo was provided, use that to hopefully save an API call if (repositoryAnalysis.RepositoryLastUpdatedOn.HasValue) { repositoryLastUpdatedOn = repositoryAnalysis.RepositoryLastUpdatedOn.Value; } else { var repositorySummary = await repositorySourceManager.ReadRepositorySummaryAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name).ConfigureAwait(false); repositoryLastUpdatedOn = repositorySummary.UpdatedAt; } if (repositoryAnalysis.ForceCompleteRefresh || repository == null || repositoryLastUpdatedOn > repository.CurrentState.RepositoryLastUpdatedOn) { // Do repository summary call to get the commit Id of the latest commit and the date that commit was pushed for the snapshot // populate the snapshot date with the corresponding manager calls (E.G. ScrapeDependenciesAsync) // Do full repository read to get all the current state stuff (including calls to get derived data like devops integrations) var sourceRepository = await repositorySourceManager.ReadRepositoryAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name).ConfigureAwait(false); var repositoryCurrentState = new RepositoryCurrentState(); repositoryCurrentState.Id = repositoryId; repositoryCurrentState.Name = sourceRepository.Name; repositoryCurrentState.Owner = parsedRepoUrl.Owner; repositoryCurrentState.DefaultBranch = sourceRepository.DefaultBranchName; repositoryCurrentState.HasIssues = sourceRepository.IssueCount > 0; repositoryCurrentState.HasProjects = sourceRepository.ProjectCount > 0; repositoryCurrentState.HasPullRequests = sourceRepository.PullRequestCount > 0; repositoryCurrentState.RepositoryCreatedOn = sourceRepository.CreatedAt; repositoryCurrentState.RepositoryLastUpdatedOn = sourceRepository.PushedAt; repositoryCurrentState.Teams = sourceRepository.Teams; repositoryCurrentState.Topics = sourceRepository.TopicNames?.Select(name => new RepositoryTopic { Name = name }).ToList(); repositoryCurrentState.DevOpsIntegrations = await ScrapeDevOpsIntegrations(repositoryCurrentState.Name).ConfigureAwait(false); // Need to pick a branch for the snapshot stuff string branchName = null; if (sourceRepository.BranchNames.Contains("master")) { branchName = "master"; } else if (sourceRepository.BranchNames.Contains("development")) { branchName = "development"; } else if (!string.IsNullOrWhiteSpace(sourceRepository.DefaultBranchName)) { branchName = sourceRepository.DefaultBranchName; } RepositorySnapshot repositorySnapshot = null; repositorySnapshot = new RepositorySnapshot(); // Have to set the windows in the manager repositorySnapshot.RepositoryCurrentStateRepositoryId = repositoryCurrentState.Id; repositorySnapshot.TakenOn = DateTime.Now; repositorySnapshot.BranchUsed = branchName; if (branchName != null) { repositorySnapshot.Dependencies = await ScrapeDependenciesAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name, branchName, repositoryAnalysis.AsOf).ConfigureAwait(false); repositorySnapshot.Files = await repositorySourceManager.ReadFilesAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name, branchName, repositoryAnalysis.AsOf).ConfigureAwait(false); } else { repositorySnapshot.Dependencies = new List <RepositoryDependency>(); repositorySnapshot.Files = new List <RepositoryFile>(); } repositorySnapshot.TypesAndImplementations = await ScrapeRepositoryTypeAndImplementation( parsedRepoUrl.Name, repositorySnapshot.Files, repositorySnapshot.Dependencies, repositoryCurrentState.Topics?.Select(topic => topic.Name), new BacklogInfo { HasIssues = repositoryCurrentState.HasIssues ?? false }, repositoryAnalysis.AsOf).ConfigureAwait(false); var updatedRepository = new Repository { CurrentState = repositoryCurrentState, Snapshot = repositorySnapshot }; await repositoryManager.UpsertAsync(updatedRepository, repositoryAnalysis.AsOf).ConfigureAwait(false); } (string Owner, string Name, string Host) ParseRepositoryUrl() { var repositoryUri = new Uri(repositoryAnalysis.RepositoryId); var owner = repositoryUri.Segments[1].TrimEnd('/'); var name = repositoryUri.Segments[2].TrimEnd('/'); var host = repositoryUri.Host; return(owner, name, host); } }