public Remote(DarcSettings settings, ILogger logger)
{
ValidateSettings(settings);
_logger = logger;
if (settings.GitType == GitRepoType.GitHub)
{
_gitClient = new GitHubClient(settings.PersonalAccessToken, _logger);
}
else if (settings.GitType == GitRepoType.AzureDevOps)
{
_gitClient = new AzureDevOpsClient(settings.PersonalAccessToken, _logger);
}
// Only initialize the file manager if we have a git client, which excludes "None"
if (_gitClient != null)
{
_fileManager = new GitFileManager(_gitClient, _logger);
}
// Initialize the bar client if there is a password
if (!string.IsNullOrEmpty(settings.BuildAssetRegistryPassword))
{
if (!string.IsNullOrEmpty(settings.BuildAssetRegistryBaseUri))
{
_barClient = ApiFactory.GetAuthenticated(settings.BuildAssetRegistryBaseUri, settings.BuildAssetRegistryPassword);
}
else
{
_barClient = ApiFactory.GetAuthenticated(settings.BuildAssetRegistryPassword);
}
}
}
public Local(string gitPath, ILogger logger)
{
_repo = Directory.GetParent(gitPath).FullName;
_logger = logger;
_gitClient = new LocalGitClient(_logger);
_fileManager = new GitFileManager(_gitClient, _logger);
}
public Local(ILogger logger, string overrideRootPath = null)
{
_repo = overrideRootPath ?? LocalHelpers.GetRootDir(GitExecutable, logger);
_logger = logger;
_gitClient = new LocalGitClient(GitExecutable, _logger);
_fileManager = new GitFileManager(_gitClient, _logger);
}
public static async Task UpdateGlobalJson(
GitFileManager fileManager,
string repository,
string branch,
DependencyDetail dependency)
{
var dependencyMapping = new Dictionary <string, string>
{
{ "Microsoft.DotNet.Arcade.Sdk", "msbuild-sdks" },
{ "Microsoft.DotNet.Build.Tasks.SharedFramework.Sdk", "msbuild-sdks" },
{ "Microsoft.DotNet.Helix.Sdk", "msbuild-sdks" },
{ "Microsoft.DotNet.SharedFramework.Sdk", "msbuild-sdks" },
{ "Microsoft.NET.SharedFramework.Sdk", "msbuild-sdks" },
{ "dotnet", "tools" }
};
if (!dependencyMapping.ContainsKey(dependency.Name))
{
throw new Exception($"Dependency '{dependency.Name}' has no parent mapping defined.");
}
string parent = dependencyMapping[dependency.Name];
await fileManager.AddDependencyToGlobalJson(
repository,
branch,
parent,
dependency.Name,
dependency.Version);
await fileManager.AddDependencyToVersionDetailsAsync(
repository,
branch,
dependency);
}
public Remote(IGitRepo gitClient, IBarClient barClient, ILogger logger)
{
_logger = logger;
_barClient = barClient;
_gitClient = gitClient;
if (_gitClient != null)
{
_fileManager = new GitFileManager(_gitClient, _logger);
}
}
public static async Task UpdateGlobalJson(GitFileManager fileManager, string repository, DependencyDetail dependency)
{
Dictionary <string, string> dependencyMapping = new Dictionary <string, string>
{
{ "Microsoft.DotNet.Arcade.Sdk", "msbuild-sdks" },
{ "dotnet", "tools" }
};
if (!dependencyMapping.ContainsKey(dependency.Name))
{
throw new Exception($"Dependency '{dependency.Name}' has no parent mapping defined.");
}
string parent = dependencyMapping[dependency.Name];
await fileManager.AddDependencyToGlobalJson(Path.Combine(repository, VersionFilePath.GlobalJson), parent, dependency.Name, dependency.Version);
await fileManager.AddDependencyToVersionDetails(Path.Combine(repository, VersionFilePath.VersionDetailsXml), dependency, DependencyType.Toolset);
}
/// <summary>
/// Cloning big repos takes a considerable amount of time when checking out the files. When
/// working on batched subscription, the operation could take more than an hour causing the
/// GitHub token to expire. By doing sparse and shallow checkout, we only deal with the files
/// we need avoiding to check the complete repo shaving time from the overall push process
/// </summary>
/// <param name="filesToCommit">Collection of files to update.</param>
/// <param name="repoUri">The repository to push the files to.</param>
/// <param name="branch">The branch to push the files to.</param>
/// <param name="commitMessage">The commmit message.</param>
/// <returns></returns>
protected async Task CommitFilesAsync(
List <GitFile> filesToCommit,
string repoUri,
string branch,
string commitMessage,
ILogger logger,
string pat,
string dotnetMaestroName,
string dotnetMaestroEmail)
{
logger.LogInformation("Pushing files to {branch}", branch);
string tempRepoFolder = Path.Combine(TemporaryRepositoryPath, Path.GetRandomFileName());
string remote = "origin";
try
{
string clonedRepo = null;
logger.LogInformation("Sparse and shallow checkout of branch {branch} in {repoUri}...", branch, repoUri);
clonedRepo = LocalHelpers.SparseAndShallowCheckout(GitExecutable, repoUri, branch, tempRepoFolder, logger, remote, dotnetMaestroName, dotnetMaestroEmail, pat);
foreach (GitFile file in filesToCommit)
{
string filePath = Path.Combine(clonedRepo, file.FilePath);
if (file.Operation == GitFileOperation.Add)
{
if (!File.Exists(filePath))
{
string parentFolder = Directory.GetParent(filePath).FullName;
Directory.CreateDirectory(parentFolder);
}
using (FileStream stream = File.Create(filePath))
{
byte[] contentBytes = GetUtf8ContentBytes(file.Content, file.ContentEncoding);
await stream.WriteAsync(contentBytes, 0, contentBytes.Length);
}
}
else if (file.Operation == GitFileOperation.Delete)
{
File.Delete(filePath);
}
LocalHelpers.ExecuteCommand(GitExecutable, $"add {filePath}", logger, clonedRepo);
}
LocalHelpers.ExecuteCommand(GitExecutable, $"commit -m \"{commitMessage}\"", logger, clonedRepo);
LocalHelpers.ExecuteCommand(GitExecutable, $"-c core.askpass= -c credential.helper= push {remote} {branch}", logger, clonedRepo);
}
catch (Exception exc)
{
// This was originally a DarcException. Making it an actual Exception so we get to see in AppInsights if something failed while
// commiting the changes
throw new Exception($"Something went wrong when pushing the files to repo {repoUri} in branch {branch}", exc);
}
finally
{
try
{
// .git/objects hierarchy are marked as read-only so we need to unset the read-only attribute otherwise an UnauthorizedAccessException is thrown.
GitFileManager.NormalizeAttributes(tempRepoFolder);
Directory.Delete(tempRepoFolder, true);
}
catch (DirectoryNotFoundException)
{
// If the directory wasn't found, that means that the clone operation above failed
// but this error isn't interesting at all.
}
catch (Exception exc)
{
throw new Exception($"Something went wrong while trying to delete the folder {tempRepoFolder}", exc);
}
}
}
/// <summary>
/// We used to group commits in a tree object so there would be only one commit per
/// change but this doesn't work for trees that end up being too big (around 20K files).
/// By using LibGit2Sharp we still group changes in one and we don't need to create a new
/// tree. Everything happens locally in the host executing the push.
/// </summary>
/// <param name="filesToCommit">Collection of files to update.</param>
/// <param name="repoUri">The repository to push the files to.</param>
/// <param name="branch">The branch to push the files to.</param>
/// <param name="commitMessage">The commmit message.</param>
/// <returns></returns>
protected async Task CommitFilesAsync(
List <GitFile> filesToCommit,
string repoUri,
string branch,
string commitMessage,
ILogger _logger,
string pat)
{
string dotnetMaestro = "dotnet-maestro";
using (_logger.BeginScope("Pushing files to {branch}", branch))
{
string tempRepoFolder = Path.Combine(TemporaryRepositoryPath, Path.GetRandomFileName());
try
{
string repoPath = LibGit2Sharp.Repository.Clone(
repoUri,
tempRepoFolder,
new LibGit2Sharp.CloneOptions
{
BranchName = branch,
Checkout = true,
CredentialsProvider = (url, user, cred) =>
new LibGit2Sharp.UsernamePasswordCredentials
{
// The PAT is actually the only thing that matters here, the username
// will be ignored.
Username = dotnetMaestro,
Password = pat
}
});
using (LibGit2Sharp.Repository localRepo = new LibGit2Sharp.Repository(repoPath))
{
foreach (GitFile file in filesToCommit)
{
string filePath = Path.Combine(tempRepoFolder, file.FilePath);
if (file.Operation == GitFileOperation.Add)
{
if (!File.Exists(filePath))
{
string parentFolder = Directory.GetParent(filePath).FullName;
Directory.CreateDirectory(parentFolder);
}
using (FileStream stream = File.Create(filePath))
{
byte[] contentBytes = GetUtf8ContentBytes(file.Content, file.ContentEncoding);
await stream.WriteAsync(contentBytes, 0, contentBytes.Length);
}
}
else
{
File.Delete(Path.Combine(tempRepoFolder, file.FilePath));
}
}
LibGit2Sharp.Commands.Stage(localRepo, "*");
LibGit2Sharp.Signature author = new LibGit2Sharp.Signature(dotnetMaestro, $"@{dotnetMaestro}", DateTime.Now);
LibGit2Sharp.Signature commiter = author;
localRepo.Commit(commitMessage, author, commiter, new LibGit2Sharp.CommitOptions
{
AllowEmptyCommit = false,
PrettifyMessage = true
});
localRepo.Network.Push(localRepo.Branches[branch], new LibGit2Sharp.PushOptions
{
CredentialsProvider = (url, user, cred) =>
new LibGit2Sharp.UsernamePasswordCredentials
{
// The PAT is actually the only thing that matters here, the username
// will be ignored.
Username = dotnetMaestro,
Password = pat
}
});
}
}
catch (LibGit2Sharp.EmptyCommitException)
{
_logger.LogInformation("There was nothing to commit...");
}
catch (Exception exc)
{
// This was originally a DarcException. Making it an actual Exception so we get to see in AppInsights if something failed while
// commiting the changes
throw new Exception($"Something went wrong when pushing the files to repo {repoUri} in branch {branch}", exc);
}
finally
{
try
{
// Libgit2Sharp behaves similarly to git and marks files under the .git/objects hierarchy as read-only,
// thus if the read-only attribute is not unset an UnauthorizedAccessException is thrown.
GitFileManager.NormalizeAttributes(tempRepoFolder);
Directory.Delete(tempRepoFolder, true);
}
catch (DirectoryNotFoundException)
{
// If the directory wasn't found, that means that the clone operation above failed
// but this error isn't interesting at all.
}
}
}
}
/// <summary>
/// We used to group commits in a tree object so there would be only one commit per
/// change but this doesn't work for trees that end up being too big (around 20K files).
/// By using LibGit2Sharp we still group changes in one and we don't need to create a new
/// tree. Everything happens locally in the host executing the push.
/// </summary>
/// <param name="filesToCommit">Collection of files to update.</param>
/// <param name="repoUri">The repository to push the files to.</param>
/// <param name="branch">The branch to push the files to.</param>
/// <param name="commitMessage">The commmit message.</param>
/// <returns></returns>
public async Task PushFilesAsync(
List <GitFile> filesToCommit,
string repoUri,
string branch,
string commitMessage)
{
string dotnetMaestro = "dotnet-maestro";
using (_logger.BeginScope("Pushing files to {branch}", branch))
{
(string owner, string repo) = ParseRepoUri(repoUri);
string tempRepoFolder = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
try
{
string repoPath = LibGit2Sharp.Repository.Clone(
repoUri,
tempRepoFolder,
new LibGit2Sharp.CloneOptions
{
BranchName = branch,
Checkout = true
});
using (LibGit2Sharp.Repository localRepo = new LibGit2Sharp.Repository(repoPath))
{
foreach (GitFile file in filesToCommit)
{
string filePath = Path.Combine(tempRepoFolder, file.FilePath);
if (file.Operation == GitFileOperation.Add)
{
if (!File.Exists(filePath))
{
string parentFolder = Directory.GetParent(filePath).FullName;
Directory.CreateDirectory(parentFolder);
}
using (FileStream stream = File.Create(filePath))
{
byte[] contentBytes = this.GetContentBytes(file.Content);
await stream.WriteAsync(contentBytes, 0, contentBytes.Length);
}
}
else
{
File.Delete(Path.Combine(tempRepoFolder, file.FilePath));
}
}
LibGit2Sharp.Commands.Stage(localRepo, "*");
LibGit2Sharp.Signature author = new LibGit2Sharp.Signature(dotnetMaestro, $"@{dotnetMaestro}", DateTime.Now);
LibGit2Sharp.Signature commiter = author;
localRepo.Commit(commitMessage, author, commiter, new LibGit2Sharp.CommitOptions
{
AllowEmptyCommit = false,
PrettifyMessage = true
});
localRepo.Network.Push(localRepo.Branches[branch], new LibGit2Sharp.PushOptions
{
CredentialsProvider = (url, user, cred) =>
new LibGit2Sharp.UsernamePasswordCredentials
{
Username = dotnetMaestro,
Password = Client.Credentials.Password
}
});
}
}
catch (LibGit2Sharp.EmptyCommitException)
{
_logger.LogInformation("There was nothing to commit...");
}
catch (Exception exc)
{
throw new DarcException($"Something went wrong when pushing the files to repo {repo} in branch {branch}", exc);
}
finally
{
// Libgit2Sharp behaves similarly to git and marks files under the .git/objects hierarchy as read-only,
// thus if the read-only attribute is not unset an UnauthorizedAccessException is thrown.
GitFileManager.NormalizeAttributes(tempRepoFolder);
Directory.Delete(tempRepoFolder, true);
}
}
}