public Remote(DarcSettings settings, ILogger logger) { ValidateSettings(settings); _logger = logger; if (settings.GitType == GitRepoType.GitHub) { _gitClient = new GitHubClient(settings.PersonalAccessToken, _logger); } else if (settings.GitType == GitRepoType.AzureDevOps) { _gitClient = new AzureDevOpsClient(settings.PersonalAccessToken, _logger); } // Only initialize the file manager if we have a git client, which excludes "None" if (_gitClient != null) { _fileManager = new GitFileManager(_gitClient, _logger); } // Initialize the bar client if there is a password if (!string.IsNullOrEmpty(settings.BuildAssetRegistryPassword)) { if (!string.IsNullOrEmpty(settings.BuildAssetRegistryBaseUri)) { _barClient = ApiFactory.GetAuthenticated(settings.BuildAssetRegistryBaseUri, settings.BuildAssetRegistryPassword); } else { _barClient = ApiFactory.GetAuthenticated(settings.BuildAssetRegistryPassword); } } }
public Local(string gitPath, ILogger logger) { _repo = Directory.GetParent(gitPath).FullName; _logger = logger; _gitClient = new LocalGitClient(_logger); _fileManager = new GitFileManager(_gitClient, _logger); }
public Local(ILogger logger, string overrideRootPath = null) { _repo = overrideRootPath ?? LocalHelpers.GetRootDir(GitExecutable, logger); _logger = logger; _gitClient = new LocalGitClient(GitExecutable, _logger); _fileManager = new GitFileManager(_gitClient, _logger); }
public static async Task UpdateGlobalJson( GitFileManager fileManager, string repository, string branch, DependencyDetail dependency) { var dependencyMapping = new Dictionary <string, string> { { "Microsoft.DotNet.Arcade.Sdk", "msbuild-sdks" }, { "Microsoft.DotNet.Build.Tasks.SharedFramework.Sdk", "msbuild-sdks" }, { "Microsoft.DotNet.Helix.Sdk", "msbuild-sdks" }, { "Microsoft.DotNet.SharedFramework.Sdk", "msbuild-sdks" }, { "Microsoft.NET.SharedFramework.Sdk", "msbuild-sdks" }, { "dotnet", "tools" } }; if (!dependencyMapping.ContainsKey(dependency.Name)) { throw new Exception($"Dependency '{dependency.Name}' has no parent mapping defined."); } string parent = dependencyMapping[dependency.Name]; await fileManager.AddDependencyToGlobalJson( repository, branch, parent, dependency.Name, dependency.Version); await fileManager.AddDependencyToVersionDetailsAsync( repository, branch, dependency); }
public Remote(IGitRepo gitClient, IBarClient barClient, ILogger logger) { _logger = logger; _barClient = barClient; _gitClient = gitClient; if (_gitClient != null) { _fileManager = new GitFileManager(_gitClient, _logger); } }
public static async Task UpdateGlobalJson(GitFileManager fileManager, string repository, DependencyDetail dependency) { Dictionary <string, string> dependencyMapping = new Dictionary <string, string> { { "Microsoft.DotNet.Arcade.Sdk", "msbuild-sdks" }, { "dotnet", "tools" } }; if (!dependencyMapping.ContainsKey(dependency.Name)) { throw new Exception($"Dependency '{dependency.Name}' has no parent mapping defined."); } string parent = dependencyMapping[dependency.Name]; await fileManager.AddDependencyToGlobalJson(Path.Combine(repository, VersionFilePath.GlobalJson), parent, dependency.Name, dependency.Version); await fileManager.AddDependencyToVersionDetails(Path.Combine(repository, VersionFilePath.VersionDetailsXml), dependency, DependencyType.Toolset); }
/// <summary> /// Cloning big repos takes a considerable amount of time when checking out the files. When /// working on batched subscription, the operation could take more than an hour causing the /// GitHub token to expire. By doing sparse and shallow checkout, we only deal with the files /// we need avoiding to check the complete repo shaving time from the overall push process /// </summary> /// <param name="filesToCommit">Collection of files to update.</param> /// <param name="repoUri">The repository to push the files to.</param> /// <param name="branch">The branch to push the files to.</param> /// <param name="commitMessage">The commmit message.</param> /// <returns></returns> protected async Task CommitFilesAsync( List <GitFile> filesToCommit, string repoUri, string branch, string commitMessage, ILogger logger, string pat, string dotnetMaestroName, string dotnetMaestroEmail) { logger.LogInformation("Pushing files to {branch}", branch); string tempRepoFolder = Path.Combine(TemporaryRepositoryPath, Path.GetRandomFileName()); string remote = "origin"; try { string clonedRepo = null; logger.LogInformation("Sparse and shallow checkout of branch {branch} in {repoUri}...", branch, repoUri); clonedRepo = LocalHelpers.SparseAndShallowCheckout(GitExecutable, repoUri, branch, tempRepoFolder, logger, remote, dotnetMaestroName, dotnetMaestroEmail, pat); foreach (GitFile file in filesToCommit) { string filePath = Path.Combine(clonedRepo, file.FilePath); if (file.Operation == GitFileOperation.Add) { if (!File.Exists(filePath)) { string parentFolder = Directory.GetParent(filePath).FullName; Directory.CreateDirectory(parentFolder); } using (FileStream stream = File.Create(filePath)) { byte[] contentBytes = GetUtf8ContentBytes(file.Content, file.ContentEncoding); await stream.WriteAsync(contentBytes, 0, contentBytes.Length); } } else if (file.Operation == GitFileOperation.Delete) { File.Delete(filePath); } LocalHelpers.ExecuteCommand(GitExecutable, $"add {filePath}", logger, clonedRepo); } LocalHelpers.ExecuteCommand(GitExecutable, $"commit -m \"{commitMessage}\"", logger, clonedRepo); LocalHelpers.ExecuteCommand(GitExecutable, $"-c core.askpass= -c credential.helper= push {remote} {branch}", logger, clonedRepo); } catch (Exception exc) { // This was originally a DarcException. Making it an actual Exception so we get to see in AppInsights if something failed while // commiting the changes throw new Exception($"Something went wrong when pushing the files to repo {repoUri} in branch {branch}", exc); } finally { try { // .git/objects hierarchy are marked as read-only so we need to unset the read-only attribute otherwise an UnauthorizedAccessException is thrown. GitFileManager.NormalizeAttributes(tempRepoFolder); Directory.Delete(tempRepoFolder, true); } catch (DirectoryNotFoundException) { // If the directory wasn't found, that means that the clone operation above failed // but this error isn't interesting at all. } catch (Exception exc) { throw new Exception($"Something went wrong while trying to delete the folder {tempRepoFolder}", exc); } } }
/// <summary> /// We used to group commits in a tree object so there would be only one commit per /// change but this doesn't work for trees that end up being too big (around 20K files). /// By using LibGit2Sharp we still group changes in one and we don't need to create a new /// tree. Everything happens locally in the host executing the push. /// </summary> /// <param name="filesToCommit">Collection of files to update.</param> /// <param name="repoUri">The repository to push the files to.</param> /// <param name="branch">The branch to push the files to.</param> /// <param name="commitMessage">The commmit message.</param> /// <returns></returns> protected async Task CommitFilesAsync( List <GitFile> filesToCommit, string repoUri, string branch, string commitMessage, ILogger _logger, string pat) { string dotnetMaestro = "dotnet-maestro"; using (_logger.BeginScope("Pushing files to {branch}", branch)) { string tempRepoFolder = Path.Combine(TemporaryRepositoryPath, Path.GetRandomFileName()); try { string repoPath = LibGit2Sharp.Repository.Clone( repoUri, tempRepoFolder, new LibGit2Sharp.CloneOptions { BranchName = branch, Checkout = true, CredentialsProvider = (url, user, cred) => new LibGit2Sharp.UsernamePasswordCredentials { // The PAT is actually the only thing that matters here, the username // will be ignored. Username = dotnetMaestro, Password = pat } }); using (LibGit2Sharp.Repository localRepo = new LibGit2Sharp.Repository(repoPath)) { foreach (GitFile file in filesToCommit) { string filePath = Path.Combine(tempRepoFolder, file.FilePath); if (file.Operation == GitFileOperation.Add) { if (!File.Exists(filePath)) { string parentFolder = Directory.GetParent(filePath).FullName; Directory.CreateDirectory(parentFolder); } using (FileStream stream = File.Create(filePath)) { byte[] contentBytes = GetUtf8ContentBytes(file.Content, file.ContentEncoding); await stream.WriteAsync(contentBytes, 0, contentBytes.Length); } } else { File.Delete(Path.Combine(tempRepoFolder, file.FilePath)); } } LibGit2Sharp.Commands.Stage(localRepo, "*"); LibGit2Sharp.Signature author = new LibGit2Sharp.Signature(dotnetMaestro, $"@{dotnetMaestro}", DateTime.Now); LibGit2Sharp.Signature commiter = author; localRepo.Commit(commitMessage, author, commiter, new LibGit2Sharp.CommitOptions { AllowEmptyCommit = false, PrettifyMessage = true }); localRepo.Network.Push(localRepo.Branches[branch], new LibGit2Sharp.PushOptions { CredentialsProvider = (url, user, cred) => new LibGit2Sharp.UsernamePasswordCredentials { // The PAT is actually the only thing that matters here, the username // will be ignored. Username = dotnetMaestro, Password = pat } }); } } catch (LibGit2Sharp.EmptyCommitException) { _logger.LogInformation("There was nothing to commit..."); } catch (Exception exc) { // This was originally a DarcException. Making it an actual Exception so we get to see in AppInsights if something failed while // commiting the changes throw new Exception($"Something went wrong when pushing the files to repo {repoUri} in branch {branch}", exc); } finally { try { // Libgit2Sharp behaves similarly to git and marks files under the .git/objects hierarchy as read-only, // thus if the read-only attribute is not unset an UnauthorizedAccessException is thrown. GitFileManager.NormalizeAttributes(tempRepoFolder); Directory.Delete(tempRepoFolder, true); } catch (DirectoryNotFoundException) { // If the directory wasn't found, that means that the clone operation above failed // but this error isn't interesting at all. } } } }
/// <summary> /// We used to group commits in a tree object so there would be only one commit per /// change but this doesn't work for trees that end up being too big (around 20K files). /// By using LibGit2Sharp we still group changes in one and we don't need to create a new /// tree. Everything happens locally in the host executing the push. /// </summary> /// <param name="filesToCommit">Collection of files to update.</param> /// <param name="repoUri">The repository to push the files to.</param> /// <param name="branch">The branch to push the files to.</param> /// <param name="commitMessage">The commmit message.</param> /// <returns></returns> public async Task PushFilesAsync( List <GitFile> filesToCommit, string repoUri, string branch, string commitMessage) { string dotnetMaestro = "dotnet-maestro"; using (_logger.BeginScope("Pushing files to {branch}", branch)) { (string owner, string repo) = ParseRepoUri(repoUri); string tempRepoFolder = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); try { string repoPath = LibGit2Sharp.Repository.Clone( repoUri, tempRepoFolder, new LibGit2Sharp.CloneOptions { BranchName = branch, Checkout = true }); using (LibGit2Sharp.Repository localRepo = new LibGit2Sharp.Repository(repoPath)) { foreach (GitFile file in filesToCommit) { string filePath = Path.Combine(tempRepoFolder, file.FilePath); if (file.Operation == GitFileOperation.Add) { if (!File.Exists(filePath)) { string parentFolder = Directory.GetParent(filePath).FullName; Directory.CreateDirectory(parentFolder); } using (FileStream stream = File.Create(filePath)) { byte[] contentBytes = this.GetContentBytes(file.Content); await stream.WriteAsync(contentBytes, 0, contentBytes.Length); } } else { File.Delete(Path.Combine(tempRepoFolder, file.FilePath)); } } LibGit2Sharp.Commands.Stage(localRepo, "*"); LibGit2Sharp.Signature author = new LibGit2Sharp.Signature(dotnetMaestro, $"@{dotnetMaestro}", DateTime.Now); LibGit2Sharp.Signature commiter = author; localRepo.Commit(commitMessage, author, commiter, new LibGit2Sharp.CommitOptions { AllowEmptyCommit = false, PrettifyMessage = true }); localRepo.Network.Push(localRepo.Branches[branch], new LibGit2Sharp.PushOptions { CredentialsProvider = (url, user, cred) => new LibGit2Sharp.UsernamePasswordCredentials { Username = dotnetMaestro, Password = Client.Credentials.Password } }); } } catch (LibGit2Sharp.EmptyCommitException) { _logger.LogInformation("There was nothing to commit..."); } catch (Exception exc) { throw new DarcException($"Something went wrong when pushing the files to repo {repo} in branch {branch}", exc); } finally { // Libgit2Sharp behaves similarly to git and marks files under the .git/objects hierarchy as read-only, // thus if the read-only attribute is not unset an UnauthorizedAccessException is thrown. GitFileManager.NormalizeAttributes(tempRepoFolder); Directory.Delete(tempRepoFolder, true); } } }