public void GetCloneDirectory_REPO_should_return_proper_value_when_called()
{
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
var repo = new RepositoryResource()
{
Alias = "alias",
Id = "repo1",
Type = "git",
Url = null,
};
// If name is not set and url is not set, then it should use alias
Assert.Equal("alias", RepositoryUtil.GetCloneDirectory(repo));
// If url is set, it should choose url over alias
repo.Url = new Uri("https://[email protected]/jpricket/MyFirstProject/_git/repo1_url");
Assert.Equal("repo1_url", RepositoryUtil.GetCloneDirectory(repo));
// If name is set, it should choose name over alias or url
repo.Properties.Set(RepositoryPropertyNames.Name, "MyFirstProject/repo1_name");
Assert.Equal("repo1_name", RepositoryUtil.GetCloneDirectory(repo));
}
}
private TrackingConfig LoadIfExists(
IExecutionContext executionContext,
string file)
{
Trace.Entering();
Trace.Verbose($"Loading {file}");
// The tracking config will not exist for a new definition.
if (!File.Exists(file))
{
Trace.Verbose($"Tracking file does not exist: {file}");
return(null);
}
TrackingConfig result = null;
// Load the content and distinguish between tracking config file
// version 1 and file version 2.
string content = File.ReadAllText(file);
string fileFormatVersionJsonProperty = StringUtil.Format(
@"""{0}""",
TrackingConfig.FileFormatVersionJsonProperty);
if (content.Contains(fileFormatVersionJsonProperty))
{
// The config is the new format.
Trace.Verbose("Parsing new tracking config format.");
result = JsonConvert.DeserializeObject <TrackingConfig>(content);
}
else
{
// Attempt to parse the legacy format.
Trace.Verbose("Parsing legacy tracking config format.");
var legacyTrackingConfig = LegacyTrackingConfig.TryParse(content);
if (legacyTrackingConfig == null)
{
executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content));
}
else
{
// Convert legacy format to the new format.
result = ConvertToNewFormat(
executionContext,
legacyTrackingConfig,
RepositoryUtil.GetCloneDirectory(legacyTrackingConfig.RepositoryUrl),
RepositoryUtil.GuessRepositoryType(legacyTrackingConfig.RepositoryUrl));
}
}
if (result != null)
{
result.FileLocation = file;
}
return(result);
}
public void GetCloneDirectory_STRING_should_return_proper_value_when_called()
{
// These test cases were inspired by the test cases that git.exe uses
// see https://github.com/git/git/blob/53f9a3e157dbbc901a02ac2c73346d375e24978c/t/t5603-clone-dirname.sh#L21
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
// basic syntax with bare and non-bare variants
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo.git"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git"));
// similar, but using ssh URL rather than host:path syntax
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo.git"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/.git"));
// we should remove trailing slashes and .git suffixes
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo///"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/.git/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo.git/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo.git///"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo///.git/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/.git///"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo///"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo.git/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo.git///"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo///.git/"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git///"));
Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git///"));
Assert.Equal("repo", RepositoryUtil.GetCloneDirectory("host:foo/repo"));
// omitting the path should default to the hostname
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://host/"));
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/"));
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("host:/"));
// auth materials should be redacted
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/"));
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host:1234/"));
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@rd@host:1234/"));
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("user@host:/"));
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("user:password@host:/"));
Assert.Equal("host", RepositoryUtil.GetCloneDirectory("user:passw@rd@host:/"));
// trailing port-like numbers should not be stripped for paths
Assert.Equal("1234", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/test:1234"));
Assert.Equal("1234", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/test:1234.git"));
}
}
public void GetCloneDirectory_STRING_should_throw_on_null()
{
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
Assert.Throws <ArgumentNullException>(() => RepositoryUtil.GetCloneDirectory((string)null));
}
}
public RepositoryTrackingInfo(RepositoryResource repositoryResource, string sourcesDirectoryRoot)
{
if (repositoryResource != null)
{
Identifier = repositoryResource.Alias;
RepositoryType = repositoryResource.Type;
RepositoryUrl = repositoryResource.Url.AbsoluteUri;
SourcesDirectory = Path.Combine(sourcesDirectoryRoot, RepositoryUtil.GetCloneDirectory(repositoryResource));
}
}
private bool IsCheckoutToCustomPath(TrackingConfig trackingConfig, RepositoryInfo repoInfo, TaskStep selfCheckoutTask)
{
string path;
string selfRepoName = RepositoryUtil.GetCloneDirectory(repoInfo.PrimaryRepository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Name));
string defaultRepoCheckoutPath = Path.GetFullPath(Path.Combine(trackingConfig.SourcesDirectory, selfRepoName));
return selfCheckoutTask != null
&& selfCheckoutTask.Inputs.TryGetValue(PipelineConstants.CheckoutTaskInputs.Path, out path)
&& !string.Equals(Path.GetFullPath(Path.Combine(trackingConfig.BuildDirectory, path)),
defaultRepoCheckoutPath,
IOUtil.FilePathStringComparison);
}
public TrackingConfig(
IExecutionContext executionContext,
IList <RepositoryResource> repositories,
int buildDirectory)
: this()
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(repositories, nameof(repositories));
// Get the repo that we are going to checkout first to create the tracking info from.
var primaryRepository = RepositoryUtil.GetPrimaryRepository(repositories);
// Set the directories.
BuildDirectory = buildDirectory.ToString(CultureInfo.InvariantCulture);
ArtifactsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.ArtifactsDirectory);
SourcesDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.SourcesDirectory);
TestResultsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.TestResultsDirectory);
// Set the other properties.
CollectionId = executionContext.Variables.System_CollectionId;
DefinitionId = executionContext.Variables.System_DefinitionId;
RepositoryUrl = primaryRepository?.Url.AbsoluteUri;
RepositoryType = primaryRepository?.Type;
System = BuildSystem;
UpdateJobRunProperties(executionContext);
foreach (var repo in repositories)
{
RepositoryTrackingInfo.Add(new Build.RepositoryTrackingInfo
{
Identifier = repo.Alias,
RepositoryType = repo.Type,
RepositoryUrl = repo.Url.AbsoluteUri,
SourcesDirectory = Path.Combine(SourcesDirectory, RepositoryUtil.GetCloneDirectory(repo)),
});
}
// Now that we have all the repositories set up, we can compute the config hash
HashKey = TrackingConfigHashAlgorithm.ComputeHash(CollectionId, DefinitionId, RepositoryTrackingInfo);
}
private TrackingConfig LoadIfExists(
IExecutionContext executionContext,
string file)
{
Trace.Entering();
Trace.Verbose($"Loading {file}");
// The tracking config will not exist for a new definition.
if (!File.Exists(file))
{
Trace.Verbose($"Tracking file does not exist: {file}");
return(null);
}
TrackingConfig result = null;
// Load the content and distinguish between tracking config file
// version 1 and file version 2.
string content = File.ReadAllText(file);
string fileFormatVersionJsonProperty = StringUtil.Format(
@"""{0}""",
TrackingConfig.FileFormatVersionJsonProperty);
if (content.Contains(fileFormatVersionJsonProperty))
{
// The config is the new format.
Trace.Verbose("Parsing new tracking config format.");
result = JsonConvert.DeserializeObject <TrackingConfig>(content);
if (result != null)
{
// if RepositoryTrackingInfo is empty, then we should create an entry so the rest
// of the logic after this will act correctly
if (result.RepositoryTrackingInfo.Count == 0)
{
result.RepositoryTrackingInfo.Add(new Build.RepositoryTrackingInfo
{
Identifier = RepositoryUtil.DefaultPrimaryRepositoryName,
RepositoryType = result.RepositoryType,
RepositoryUrl = result.RepositoryUrl,
SourcesDirectory = result.SourcesDirectory,
});
}
}
}
else
{
// Attempt to parse the legacy format.
Trace.Verbose("Parsing legacy tracking config format.");
var legacyTrackingConfig = LegacyTrackingConfig.TryParse(content);
if (legacyTrackingConfig == null)
{
executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content));
}
else
{
// Convert legacy format to the new format.
result = ConvertToNewFormat(
executionContext,
legacyTrackingConfig,
RepositoryUtil.GetCloneDirectory(legacyTrackingConfig.RepositoryUrl),
RepositoryUtil.GuessRepositoryType(legacyTrackingConfig.RepositoryUrl));
}
}
if (result != null)
{
result.FileLocation = file;
}
return(result);
}
public TrackingConfig PrepareDirectory(
IExecutionContext executionContext,
IList <RepositoryResource> repositories,
WorkspaceOptions workspace)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repositories, nameof(repositories));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Create the tracking config for this execution of the pipeline
var agentSettings = HostContext.GetService <IConfigurationStore>().GetSettings();
var newConfig = trackingManager.Create(executionContext, repositories, ShouldOverrideBuildDirectory(repositories, agentSettings));
// Load the tracking config from the last execution of the pipeline
var existingConfig = trackingManager.LoadExistingTrackingConfig(executionContext);
// If there aren't any major changes, merge the configurations and use the same workspace
if (trackingManager.AreTrackingConfigsCompatible(executionContext, newConfig, existingConfig))
{
newConfig = trackingManager.MergeTrackingConfigs(executionContext, newConfig, existingConfig);
}
else if (existingConfig != null)
{
// If the previous config had different repos, get a new workspace folder and mark the old one for clean up
trackingManager.MarkForGarbageCollection(executionContext, existingConfig);
// If the config file was updated to a new config, we need to delete the legacy artifact/staging directories.
// DeleteDirectory will check for the existence of the folders first.
DeleteDirectory(
executionContext,
description: "legacy artifacts directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory));
DeleteDirectory(
executionContext,
description: "legacy staging directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory));
}
// Save any changes to the config file
trackingManager.UpdateTrackingConfig(executionContext, newConfig);
// Prepare the build directory.
// There are 2 ways to provide build directory clean policy.
// 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016)
// 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI)
BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, workspace);
CreateDirectory(
executionContext,
description: "build directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory),
deleteExisting: cleanOption == BuildCleanOption.All);
CreateDirectory(
executionContext,
description: "artifacts directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.ArtifactsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "test results directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.TestResultsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "binaries directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Binary);
var defaultSourceDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory);
CreateDirectory(
executionContext,
description: "source directory",
path: defaultSourceDirectory,
deleteExisting: cleanOption == BuildCleanOption.Source);
// Set the default clone path for each repository (the Checkout task may override this later)
foreach (var repository in repositories)
{
String repoPath;
if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1)
var repoSourceDirectory = newConfig?.RepositoryTrackingInfo.Where(item => string.Equals(item.Identifier, repository.Alias, StringComparison.OrdinalIgnoreCase)).Select(item => item.SourcesDirectory).FirstOrDefault();
if (repoSourceDirectory != null)
{
repoPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), repoSourceDirectory);
}
else
{
repoPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory, RepositoryUtil.GetCloneDirectory(repository));
}
}
else
{
// For single checkouts, the repository is rooted to the sources folder (_work/1/s)
repoPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory);
}
if (!string.Equals(repoPath, defaultSourceDirectory, StringComparison.Ordinal))
{
CreateDirectory(
executionContext,
description: "repository source directory",
path: repoPath,
deleteExisting: cleanOption == BuildCleanOption.Source);
}
Trace.Info($"Set repository path for repository {repository.Alias} to '{repoPath}'");
repository.Properties.Set <string>(RepositoryPropertyNames.Path, repoPath);
}
return(newConfig);
}
public override async Task RunAsync(AgentTaskPluginExecutionContext executionContext, CancellationToken token)
{
var sourceSkipVar = StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("agent.source.skip")?.Value) ||
!StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("build.syncSources")?.Value ?? bool.TrueString);
if (sourceSkipVar)
{
executionContext.Output($"Skip sync source for repository.");
return;
}
var repoAlias = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, true);
var repo = executionContext.Repositories.Single(x => string.Equals(x.Alias, repoAlias, StringComparison.OrdinalIgnoreCase));
executionContext.PublishTelemetry(area: "AzurePipelinesAgent", feature: "Checkout", properties: new Dictionary <string, string>
{
{ "RepoType", $"{repo.Type}" },
{ "HostOS", $"{PlatformUtil.HostOS}" }
});
MergeCheckoutOptions(executionContext, repo);
var currentRepoPath = repo.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path);
var buildDirectory = executionContext.Variables.GetValueOrDefault("agent.builddirectory")?.Value;
var tempDirectory = executionContext.Variables.GetValueOrDefault("agent.tempdirectory")?.Value;
ArgUtil.NotNullOrEmpty(currentRepoPath, nameof(currentRepoPath));
ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory));
ArgUtil.NotNullOrEmpty(tempDirectory, nameof(tempDirectory));
// Determine the path that we should clone/move the repository into
const string sourcesDirectory = "s"; //Constants.Build.Path.SourcesDirectory
string expectRepoPath;
var path = executionContext.GetInput("path");
if (!string.IsNullOrEmpty(path))
{
// When the checkout task provides a path, always use that one
expectRepoPath = IOUtil.ResolvePath(buildDirectory, path);
if (!expectRepoPath.StartsWith(buildDirectory.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar))
{
throw new ArgumentException($"Input path '{path}' should resolve to a directory under '{buildDirectory}', current resolved path '{expectRepoPath}'.");
}
}
else if (HasMultipleCheckouts(executionContext))
{
// When there are multiple checkout tasks (and this one didn't set the path), default to directory 1/s/<repoName>
expectRepoPath = Path.Combine(buildDirectory, sourcesDirectory, RepositoryUtil.GetCloneDirectory(repo));
}
else
{
// When there's a single checkout task that doesn't have path set, default to sources directory 1/s
expectRepoPath = Path.Combine(buildDirectory, sourcesDirectory);
}
// Update the repository path in the worker process
executionContext.UpdateRepositoryPath(repoAlias, expectRepoPath);
executionContext.Debug($"Repository requires to be placed at '{expectRepoPath}', current location is '{currentRepoPath}'");
if (!string.Equals(currentRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), expectRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), IOUtil.FilePathStringComparison))
{
executionContext.Output($"Repository is current at '{currentRepoPath}', move to '{expectRepoPath}'.");
var count = 1;
var staging = Path.Combine(tempDirectory, $"_{count}");
while (Directory.Exists(staging))
{
count++;
staging = Path.Combine(tempDirectory, $"_{count}");
}
try
{
executionContext.Debug($"Move existing repository '{currentRepoPath}' to '{expectRepoPath}' via staging directory '{staging}'.");
IOUtil.MoveDirectory(currentRepoPath, expectRepoPath, staging, CancellationToken.None);
}
catch (Exception ex)
{
executionContext.Debug("Catch exception during repository move.");
executionContext.Debug(ex.ToString());
executionContext.Warning("Unable move and reuse existing repository to required location.");
IOUtil.DeleteDirectory(expectRepoPath, CancellationToken.None);
}
executionContext.Output($"Repository will be located at '{expectRepoPath}'.");
repo.Properties.Set <string>(Pipelines.RepositoryPropertyNames.Path, expectRepoPath);
}
ISourceProvider sourceProvider = SourceProviderFactory.GetSourceProvider(repo.Type);
await sourceProvider.GetSourceAsync(executionContext, repo, token);
}
private string GetDefaultRepositoryPath(
IExecutionContext executionContext,
RepositoryResource repository,
string defaultSourcesDirectory)
{
if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1)
return(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory, RepositoryUtil.GetCloneDirectory(repository)));
}
else
{
// For single checkouts, the repository is rooted to the sources folder (_work/1/s)
return(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory));
}
}
private string GetDefaultRepositoryPath(
IExecutionContext executionContext,
RepositoryResource repository,
TrackingConfig newConfig
)
{
string repoPath = String.Empty;
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1)
var repoSourceDirectory = newConfig?.RepositoryTrackingInfo.Where(item => string.Equals(item.Identifier, repository.Alias, StringComparison.OrdinalIgnoreCase)).Select(item => item.SourcesDirectory).FirstOrDefault();
if (repoSourceDirectory != null)
{
repoPath = Path.Combine(workDirectory, repoSourceDirectory);
}
else
{
repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory, RepositoryUtil.GetCloneDirectory(repository));
}
}
else
{
// For single checkouts, the repository is rooted to the sources folder (_work/1/s)
repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory);
}
return(repoPath);
}
