public override async Task <int> ExecuteAsync() { try { EnsureOptionsCompatibility(_options); // use a set to accumulate dependencies as we go HashSet <StrippedDependency> accumulatedDependencies = new HashSet <StrippedDependency>(); // at the end of each depth level, these are added to the queue to clone Queue <StrippedDependency> dependenciesToClone = new Queue <StrippedDependency>(); RemoteFactory remoteFactory = new RemoteFactory(_options); if (string.IsNullOrWhiteSpace(_options.RepoUri)) { Local local = new Local(Logger); IEnumerable <DependencyDetail> rootDependencies = await local.GetDependenciesAsync(); IEnumerable <StrippedDependency> stripped = rootDependencies.Select(d => StrippedDependency.GetDependency(d)); foreach (StrippedDependency d in stripped) { accumulatedDependencies.Add(d); } Logger.LogInformation($"Found {rootDependencies.Count()} local dependencies. Starting deep clone..."); } else { // Start with the root repo we were asked to clone StrippedDependency rootDep = StrippedDependency.GetDependency(_options.RepoUri, _options.Version); accumulatedDependencies.Add(rootDep); Logger.LogInformation($"Starting deep clone of {rootDep.RepoUri}@{rootDep.Commit}"); } while (accumulatedDependencies.Any()) { // add this level's dependencies to the queue and clear it for the next level foreach (StrippedDependency d in accumulatedDependencies) { dependenciesToClone.Enqueue(d); } accumulatedDependencies.Clear(); // this will do one level of clones at a time while (dependenciesToClone.Any()) { StrippedDependency repo = dependenciesToClone.Dequeue(); // the folder for the specific repo-hash we are cloning. these will be orphaned from the .gitdir. string repoPath = GetRepoDirectory(_options.ReposFolder, repo.RepoUri, repo.Commit); // the "master" folder, which continues to be linked to the .git directory string masterGitRepoPath = GetMasterGitRepoPath(_options.ReposFolder, repo.RepoUri); // the .gitdir that is shared among all repo-hashes (temporarily, before they are orphaned) string masterRepoGitDirPath = GetMasterGitDirPath(_options.GitDirFolder, repo.RepoUri); // used for the specific-commit version of the repo Local local; // Scenarios we handle: no/existing/orphaned master folder cross no/existing .gitdir await HandleMasterCopy(remoteFactory, repo.RepoUri, masterGitRepoPath, masterRepoGitDirPath, Logger); // if using the default .gitdir path, get that for use in the specific clone. if (masterRepoGitDirPath == null) { masterRepoGitDirPath = GetDefaultMasterGitDirPath(_options.ReposFolder, repo.RepoUri); } local = HandleRepoAtSpecificHash(repoPath, repo.Commit, masterRepoGitDirPath, Logger); Logger.LogDebug($"Starting to look for dependencies in {repoPath}"); try { IEnumerable <DependencyDetail> deps = await local.GetDependenciesAsync(); IEnumerable <DependencyDetail> filteredDeps = FilterToolsetDependencies(deps, _options.IncludeToolset, Logger); Logger.LogDebug($"Got {deps.Count()} dependencies and filtered to {filteredDeps.Count()} dependencies"); foreach (DependencyDetail d in filteredDeps) { StrippedDependency dep = StrippedDependency.GetDependency(d); // e.g. arcade depends on previous versions of itself to build, so this would go on forever if (d.RepoUri == repo.RepoUri) { Logger.LogDebug($"Skipping self-dependency in {repo.RepoUri} ({repo.Commit} => {d.Commit})"); } // circular dependencies that have different hashes, e.g. DotNet-Trusted -> core-setup -> DotNet-Trusted -> ... else if (dep.HasDependencyOn(repo)) { Logger.LogDebug($"Skipping already-seen circular dependency from {repo.RepoUri} to {d.RepoUri}"); } else if (_options.IgnoredRepos.Any(r => r.Equals(d.RepoUri, StringComparison.OrdinalIgnoreCase))) { Logger.LogDebug($"Skipping ignored repo {d.RepoUri} (at {d.Commit})"); } else if (string.IsNullOrWhiteSpace(d.Commit)) { Logger.LogWarning($"Skipping dependency from {repo.RepoUri}@{repo.Commit} to {d.RepoUri}: Missing commit."); } else { StrippedDependency stripped = StrippedDependency.GetDependency(d); Logger.LogDebug($"Adding new dependency {stripped.RepoUri}@{stripped.Commit}"); repo.AddDependency(dep); accumulatedDependencies.Add(stripped); } } Logger.LogDebug($"done looking for dependencies in {repoPath} at {repo.Commit}"); } catch (DirectoryNotFoundException) { Logger.LogWarning($"Repo {repoPath} appears to have no '/eng' directory at commit {repo.Commit}. Dependency chain is broken here."); } catch (FileNotFoundException) { Logger.LogWarning($"Repo {repoPath} appears to have no '/eng/Version.Details.xml' file at commit {repo.Commit}. Dependency chain is broken here."); } finally { // delete the .gitdir redirect to orphan the repo. // we want to do this because otherwise all of these folder will show as dirty in Git, // and any operations on them will affect the master copy and all the others, which // could be confusing. string repoGitRedirectPath = Path.Combine(repoPath, ".git"); if (File.Exists(repoGitRedirectPath)) { Logger.LogDebug($"Deleting .gitdir redirect {repoGitRedirectPath}"); File.Delete(repoGitRedirectPath); } else { Logger.LogDebug($"No .gitdir redirect found at {repoGitRedirectPath}"); } } } // end inner while(dependenciesToClone.Any()) if (_options.CloneDepth == 0 && accumulatedDependencies.Any()) { Logger.LogInformation($"Reached clone depth limit, aborting with {accumulatedDependencies.Count} dependencies remaining"); foreach (StrippedDependency d in accumulatedDependencies) { Logger.LogDebug($"Abandoning dependency {d.RepoUri}@{d.Commit}"); } break; } else { _options.CloneDepth--; Logger.LogDebug($"Clone depth remaining: {_options.CloneDepth}"); Logger.LogDebug($"Dependencies remaining: {accumulatedDependencies.Count}"); } } // end outer while(accumulatedDependencies.Any()) return(Constants.SuccessCode); } catch (Exception exc) { Logger.LogError(exc, "Something failed while cloning."); return(Constants.ErrorCode); } }