public async Task <IRemote> GetRemoteAsync(string repoUrl, ILogger logger)
{
using (logger.BeginScope($"Getting remote for repo {repoUrl}."))
{
// Normalize the url with the AzDO client prior to attempting to
// get a token. When we do coherency updates we build a repo graph and
// may end up traversing links to classic azdo uris.
string normalizedUrl = AzureDevOpsClient.NormalizeUrl(repoUrl);
Uri normalizedRepoUri = new Uri(normalizedUrl);
// Look up the setting for where the repo root should be held. Default to empty,
// which will use the temp directory.
string temporaryRepositoryRoot = Configuration.GetValue <string>("DarcTemporaryRepoRoot", null);
if (string.IsNullOrEmpty(temporaryRepositoryRoot))
{
temporaryRepositoryRoot = _tempFiles.GetFilePath("repos");
}
IGitRepo gitClient;
long installationId = await Context.GetInstallationId(normalizedUrl);
await ExponentialRetry.RetryAsync(
async() => await EnsureLocalGit(logger),
ex => logger.LogError(ex, $"Failed to install git to local temporary directory."),
ex => true);
switch (normalizedRepoUri.Host)
{
case "github.com":
if (installationId == default)
{
throw new GithubApplicationInstallationException($"No installation is avaliable for repository '{normalizedUrl}'");
}
gitClient = new GitHubClient(_gitExecutable, await GitHubTokenProvider.GetTokenForInstallationAsync(installationId),
logger, temporaryRepositoryRoot, Cache.Cache);
break;
case "dev.azure.com":
gitClient = new AzureDevOpsClient(_gitExecutable, await AzureDevOpsTokenProvider.GetTokenForRepository(normalizedUrl),
logger, temporaryRepositoryRoot);
break;
default:
throw new NotImplementedException($"Unknown repo url type {normalizedUrl}");
}
;
return(new Remote(gitClient, new MaestroBarClient(Context), logger));
}
}
public void SecondCleanupFixesBrokenFirstCleanup()
{
StreamWriter writer = null;
string parent;
try
{
using (var tempFiles = new TemporaryFiles(MockBuilder.StatelessServiceContext(),
NullLogger <TemporaryFiles> .Instance))
{
tempFiles.Initialize();
string testPath = tempFiles.GetFilePath("asdfpoiu");
writer = File.CreateText(testPath);
}
}
finally
{
writer?.Dispose();
}
using (var tempFiles = new TemporaryFiles(MockBuilder.StatelessServiceContext(),
NullLogger <TemporaryFiles> .Instance))
{
tempFiles.Initialize();
string testPath = tempFiles.GetFilePath("asdfpoiu");
parent = Path.GetDirectoryName(testPath);
}
Directory.Exists(parent).Should().BeFalse();
}
public void PreInitializeIsNoop()
{
using (var tempFiles = new TemporaryFiles(MockBuilder.StatelessServiceContext(),
NullLogger <TemporaryFiles> .Instance))
{
string testPath = tempFiles.GetFilePath("asdfpoiu");
string parent = Path.GetDirectoryName(testPath);
Directory.Exists(parent).Should().BeFalse();
}
}
public void InitializeCreatedRoot()
{
using (var tempFiles = new TemporaryFiles(MockBuilder.StatelessServiceContext(),
NullLogger <TemporaryFiles> .Instance))
{
tempFiles.Initialize();
string testPath = tempFiles.GetFilePath("asdfpoiu");
string parent = Path.GetDirectoryName(testPath);
Assert.True(Directory.Exists(parent));
}
}
/// <summary>
/// Download and install git to the a temporary location.
/// Git is used by DarcLib, and the Service Fabric nodes do not have it installed natively.
///
/// The file is assumed to be on a public endpoint.
/// We return the git client executable so that this call may be easily wrapped in RetryAsync
/// </summary>
public async Task <string> GetPathToLocalGitAsync()
{
// Determine whether we need to do any downloading at all.
if (!string.IsNullOrEmpty(_gitExecutable))
{
return(_gitExecutable);
}
await _semaphoreSlim.WaitAsync();
try
{
// Determine whether another thread ended up getting the lock and downloaded git
// in the meantime.
if (string.IsNullOrEmpty(_gitExecutable))
{
using (_operations.BeginOperation($"Installing a local copy of git"))
{
string gitLocation = _configuration.GetValue <string>("GitDownloadLocation", null);
string[] pathSegments = new Uri(gitLocation, UriKind.Absolute).Segments;
string remoteFileName = pathSegments[pathSegments.Length - 1];
string gitRoot = _tempFiles.GetFilePath("git-portable");
string targetPath = Path.Combine(gitRoot, Path.GetFileNameWithoutExtension(remoteFileName));
string gitZipFile = Path.Combine(gitRoot, remoteFileName);
_logger.LogInformation($"Downloading git from '{gitLocation}' to '{gitZipFile}'");
Directory.CreateDirectory(targetPath);
using (HttpClient client = new HttpClient())
using (FileStream outStream = new FileStream(gitZipFile, FileMode.Create, FileAccess.Write))
using (var inStream = await client.GetStreamAsync(gitLocation))
{
await inStream.CopyToAsync(outStream);
}
_logger.LogInformation($"Extracting '{gitZipFile}' to '{targetPath}'");
ZipFile.ExtractToDirectory(gitZipFile, targetPath, overwriteFiles: true);
_gitExecutable = Path.Combine(targetPath, "bin", "git.exe");
}
}
}
finally
{
_semaphoreSlim.Release();
}
return(_gitExecutable);
}
private static void CleanTempFolderForTest()
{
using (var tempFiles = new TemporaryFiles(MockBuilder.StatelessServiceContext(),
NullLogger <TemporaryFiles> .Instance))
{
tempFiles.Initialize();
string testPath = tempFiles.GetFilePath("asdfpoiu");
string parent = Path.GetDirectoryName(testPath);
if (Directory.Exists(parent))
{
Directory.Delete(parent);
}
}
}
public void DisposeCleansUp()
{
string parent;
using (var tempFiles = new TemporaryFiles(MockBuilder.StatelessServiceContext(),
NullLogger <TemporaryFiles> .Instance))
{
tempFiles.Initialize();
string testPath = tempFiles.GetFilePath("asdfpoiu");
File.WriteAllText(testPath, "Test content");
parent = Path.GetDirectoryName(testPath);
}
Directory.Exists(parent).Should().BeFalse();
}
public void CleanupResilientToOpenHandles()
{
StreamWriter writer = null;
try
{
string parent;
using (var tempFiles = new TemporaryFiles(MockBuilder.StatelessServiceContext(),
NullLogger <TemporaryFiles> .Instance))
{
tempFiles.Initialize();
string testPath = tempFiles.GetFilePath("asdfpoiu");
writer = File.CreateText(testPath);
parent = Path.GetDirectoryName(testPath);
}
Directory.Exists(parent).Should().BeTrue();
}
finally
{
writer?.Dispose();
}
}
/// <summary>
/// Download and install git to the a temporary location.
/// Git is used by DarcLib, and the Service Fabric nodes do not have it installed natively.
///
/// The file is assumed to be on a public endpoint.
/// We return the git client executable so that this call may be easily wrapped in RetryAsync
/// </summary>
public async Task <string> GetPathToLocalGitAsync()
{
// Determine whether we need to do any downloading at all.
if (!string.IsNullOrEmpty(_gitExecutable) && File.Exists(_gitExecutable))
{
// We should also mke sure that the git executable that exists runs properly
try
{
LocalHelpers.CheckGitInstallation(_gitExecutable, _logger);
return(_gitExecutable);
}
catch
{
_logger.LogWarning($"Something went wrong with validating git executable at {_gitExecutable}. Downloading new version.");
}
}
await _semaphoreSlim.WaitAsync();
try
{
// Determine whether another thread ended up getting the lock and downloaded git
// in the meantime.
if (string.IsNullOrEmpty(_gitExecutable) || !File.Exists(_gitExecutable))
{
using (_operations.BeginOperation($"Installing a local copy of git"))
{
string gitLocation = _configuration.GetValue <string>("GitDownloadLocation", null);
string[] pathSegments = new Uri(gitLocation, UriKind.Absolute).Segments;
string remoteFileName = pathSegments[pathSegments.Length - 1];
string gitRoot = _tempFiles.GetFilePath("git-portable");
string targetPath = Path.Combine(gitRoot, Path.GetFileNameWithoutExtension(remoteFileName));
string gitZipFile = Path.Combine(gitRoot, remoteFileName);
_logger.LogInformation($"Downloading git from '{gitLocation}' to '{gitZipFile}'");
if (Directory.Exists(targetPath))
{
Directory.Delete(targetPath, true);
}
Directory.CreateDirectory(targetPath);
using (HttpClient client = new HttpClient())
using (FileStream outStream = new FileStream(gitZipFile, FileMode.Create, FileAccess.Write))
using (var inStream = await client.GetStreamAsync(gitLocation))
{
await inStream.CopyToAsync(outStream);
}
_logger.LogInformation($"Extracting '{gitZipFile}' to '{targetPath}'");
ZipFile.ExtractToDirectory(gitZipFile, targetPath, overwriteFiles: true);
_gitExecutable = Path.Combine(targetPath, "bin", "git.exe");
}
}
}
finally
{
_semaphoreSlim.Release();
}
// Will throw if something is wrong with the git executable, forcing a retry
LocalHelpers.CheckGitInstallation(_gitExecutable, _logger);
return(_gitExecutable);
}