/// <summary> /// We used to group commits in a tree object so there would be only one commit per /// change but this doesn't work for trees that end up being too big (around 20K files). /// By using LibGit2Sharp we still group changes in one and we don't need to create a new /// tree. Everything happens locally in the host executing the push. /// </summary> /// <param name="filesToCommit">Collection of files to update.</param> /// <param name="repoUri">The repository to push the files to.</param> /// <param name="branch">The branch to push the files to.</param> /// <param name="commitMessage">The commmit message.</param> /// <returns></returns> protected async Task CommitFilesAsync( List <GitFile> filesToCommit, string repoUri, string branch, string commitMessage, ILogger _logger, string pat) { string dotnetMaestro = "dotnet-maestro"; using (_logger.BeginScope("Pushing files to {branch}", branch)) { string tempRepoFolder = Path.Combine(TemporaryRepositoryPath, Path.GetRandomFileName()); try { string repoPath = LibGit2Sharp.Repository.Clone( repoUri, tempRepoFolder, new LibGit2Sharp.CloneOptions { BranchName = branch, Checkout = true, CredentialsProvider = (url, user, cred) => new LibGit2Sharp.UsernamePasswordCredentials { // The PAT is actually the only thing that matters here, the username // will be ignored. Username = dotnetMaestro, Password = pat } }); using (LibGit2Sharp.Repository localRepo = new LibGit2Sharp.Repository(repoPath)) { foreach (GitFile file in filesToCommit) { string filePath = Path.Combine(tempRepoFolder, file.FilePath); if (file.Operation == GitFileOperation.Add) { if (!File.Exists(filePath)) { string parentFolder = Directory.GetParent(filePath).FullName; Directory.CreateDirectory(parentFolder); } using (FileStream stream = File.Create(filePath)) { byte[] contentBytes = GetUtf8ContentBytes(file.Content, file.ContentEncoding); await stream.WriteAsync(contentBytes, 0, contentBytes.Length); } } else { File.Delete(Path.Combine(tempRepoFolder, file.FilePath)); } } LibGit2Sharp.Commands.Stage(localRepo, "*"); LibGit2Sharp.Signature author = new LibGit2Sharp.Signature(dotnetMaestro, $"@{dotnetMaestro}", DateTime.Now); LibGit2Sharp.Signature commiter = author; localRepo.Commit(commitMessage, author, commiter, new LibGit2Sharp.CommitOptions { AllowEmptyCommit = false, PrettifyMessage = true }); localRepo.Network.Push(localRepo.Branches[branch], new LibGit2Sharp.PushOptions { CredentialsProvider = (url, user, cred) => new LibGit2Sharp.UsernamePasswordCredentials { // The PAT is actually the only thing that matters here, the username // will be ignored. Username = dotnetMaestro, Password = pat } }); } } catch (LibGit2Sharp.EmptyCommitException) { _logger.LogInformation("There was nothing to commit..."); } catch (Exception exc) { // This was originally a DarcException. Making it an actual Exception so we get to see in AppInsights if something failed while // commiting the changes throw new Exception($"Something went wrong when pushing the files to repo {repoUri} in branch {branch}", exc); } finally { try { // Libgit2Sharp behaves similarly to git and marks files under the .git/objects hierarchy as read-only, // thus if the read-only attribute is not unset an UnauthorizedAccessException is thrown. GitFileManager.NormalizeAttributes(tempRepoFolder); Directory.Delete(tempRepoFolder, true); } catch (DirectoryNotFoundException) { // If the directory wasn't found, that means that the clone operation above failed // but this error isn't interesting at all. } } } }
/// <summary> /// Cloning big repos takes a considerable amount of time when checking out the files. When /// working on batched subscription, the operation could take more than an hour causing the /// GitHub token to expire. By doing sparse and shallow checkout, we only deal with the files /// we need avoiding to check the complete repo shaving time from the overall push process /// </summary> /// <param name="filesToCommit">Collection of files to update.</param> /// <param name="repoUri">The repository to push the files to.</param> /// <param name="branch">The branch to push the files to.</param> /// <param name="commitMessage">The commmit message.</param> /// <returns></returns> protected async Task CommitFilesAsync( List <GitFile> filesToCommit, string repoUri, string branch, string commitMessage, ILogger logger, string pat, string dotnetMaestroName, string dotnetMaestroEmail) { logger.LogInformation("Pushing files to {branch}", branch); string tempRepoFolder = Path.Combine(TemporaryRepositoryPath, Path.GetRandomFileName()); string remote = "origin"; try { string clonedRepo = null; logger.LogInformation("Sparse and shallow checkout of branch {branch} in {repoUri}...", branch, repoUri); clonedRepo = LocalHelpers.SparseAndShallowCheckout(GitExecutable, repoUri, branch, tempRepoFolder, logger, remote, dotnetMaestroName, dotnetMaestroEmail, pat); foreach (GitFile file in filesToCommit) { string filePath = Path.Combine(clonedRepo, file.FilePath); if (file.Operation == GitFileOperation.Add) { if (!File.Exists(filePath)) { string parentFolder = Directory.GetParent(filePath).FullName; Directory.CreateDirectory(parentFolder); } using (FileStream stream = File.Create(filePath)) { byte[] contentBytes = GetUtf8ContentBytes(file.Content, file.ContentEncoding); await stream.WriteAsync(contentBytes, 0, contentBytes.Length); } } else if (file.Operation == GitFileOperation.Delete) { File.Delete(filePath); } LocalHelpers.ExecuteCommand(GitExecutable, $"add {filePath}", logger, clonedRepo); } LocalHelpers.ExecuteCommand(GitExecutable, $"commit -m \"{commitMessage}\"", logger, clonedRepo); LocalHelpers.ExecuteCommand(GitExecutable, $"-c core.askpass= -c credential.helper= push {remote} {branch}", logger, clonedRepo); } catch (Exception exc) { // This was originally a DarcException. Making it an actual Exception so we get to see in AppInsights if something failed while // commiting the changes throw new Exception($"Something went wrong when pushing the files to repo {repoUri} in branch {branch}", exc); } finally { try { // .git/objects hierarchy are marked as read-only so we need to unset the read-only attribute otherwise an UnauthorizedAccessException is thrown. GitFileManager.NormalizeAttributes(tempRepoFolder); Directory.Delete(tempRepoFolder, true); } catch (DirectoryNotFoundException) { // If the directory wasn't found, that means that the clone operation above failed // but this error isn't interesting at all. } catch (Exception exc) { throw new Exception($"Something went wrong while trying to delete the folder {tempRepoFolder}", exc); } } }
/// <summary> /// We used to group commits in a tree object so there would be only one commit per /// change but this doesn't work for trees that end up being too big (around 20K files). /// By using LibGit2Sharp we still group changes in one and we don't need to create a new /// tree. Everything happens locally in the host executing the push. /// </summary> /// <param name="filesToCommit">Collection of files to update.</param> /// <param name="repoUri">The repository to push the files to.</param> /// <param name="branch">The branch to push the files to.</param> /// <param name="commitMessage">The commmit message.</param> /// <returns></returns> public async Task PushFilesAsync( List <GitFile> filesToCommit, string repoUri, string branch, string commitMessage) { string dotnetMaestro = "dotnet-maestro"; using (_logger.BeginScope("Pushing files to {branch}", branch)) { (string owner, string repo) = ParseRepoUri(repoUri); string tempRepoFolder = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); try { string repoPath = LibGit2Sharp.Repository.Clone( repoUri, tempRepoFolder, new LibGit2Sharp.CloneOptions { BranchName = branch, Checkout = true }); using (LibGit2Sharp.Repository localRepo = new LibGit2Sharp.Repository(repoPath)) { foreach (GitFile file in filesToCommit) { string filePath = Path.Combine(tempRepoFolder, file.FilePath); if (file.Operation == GitFileOperation.Add) { if (!File.Exists(filePath)) { string parentFolder = Directory.GetParent(filePath).FullName; Directory.CreateDirectory(parentFolder); } using (FileStream stream = File.Create(filePath)) { byte[] contentBytes = this.GetContentBytes(file.Content); await stream.WriteAsync(contentBytes, 0, contentBytes.Length); } } else { File.Delete(Path.Combine(tempRepoFolder, file.FilePath)); } } LibGit2Sharp.Commands.Stage(localRepo, "*"); LibGit2Sharp.Signature author = new LibGit2Sharp.Signature(dotnetMaestro, $"@{dotnetMaestro}", DateTime.Now); LibGit2Sharp.Signature commiter = author; localRepo.Commit(commitMessage, author, commiter, new LibGit2Sharp.CommitOptions { AllowEmptyCommit = false, PrettifyMessage = true }); localRepo.Network.Push(localRepo.Branches[branch], new LibGit2Sharp.PushOptions { CredentialsProvider = (url, user, cred) => new LibGit2Sharp.UsernamePasswordCredentials { Username = dotnetMaestro, Password = Client.Credentials.Password } }); } } catch (LibGit2Sharp.EmptyCommitException) { _logger.LogInformation("There was nothing to commit..."); } catch (Exception exc) { throw new DarcException($"Something went wrong when pushing the files to repo {repo} in branch {branch}", exc); } finally { // Libgit2Sharp behaves similarly to git and marks files under the .git/objects hierarchy as read-only, // thus if the read-only attribute is not unset an UnauthorizedAccessException is thrown. GitFileManager.NormalizeAttributes(tempRepoFolder); Directory.Delete(tempRepoFolder, true); } } }