public override string GetLocalPath(IExecutionContext executionContext, RepositoryResource repository, string path)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repository, nameof(repository));
ArgUtil.NotNull(repository.Endpoint, nameof(repository.Endpoint));
path = path ?? string.Empty;
if (path.StartsWith("$/") || path.StartsWith(@"$\"))
{
// Create the tf command manager.
var tf = HostContext.CreateService <ITfsVCCommandManager>();
tf.CancellationToken = CancellationToken.None;
tf.Repository = repository;
tf.Endpoint = executionContext.Endpoints.Single(x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) || (repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name, StringComparison.OrdinalIgnoreCase)));
tf.ExecutionContext = executionContext;
// Attempt to resolve the path.
string localPath = tf.ResolvePath(serverPath: path);
if (!string.IsNullOrEmpty(localPath))
{
return(localPath);
}
}
// Return the original path.
return(path);
}
public override string GetLocalPath(IExecutionContext executionContext, RepositoryResource repository, string path)
{
Trace.Verbose("Entering SvnSourceProvider.GetLocalPath");
ISvnCommandManager svn = HostContext.CreateService <ISvnCommandManager>();
svn.Init(executionContext, repository, CancellationToken.None);
// We assume that this is a server path first.
string serverPath = svn.NormalizeRelativePath(path, '/', '\\').Trim();
string localPath;
if (serverPath.StartsWith("^/"))
{
// Convert the server path to the relative one using SVN work copy mappings
string sourcesDirectory = repository.Properties.Get <string>(RepositoryPropertyNames.Path);
localPath = svn.ResolveServerPath(serverPath, sourcesDirectory);
}
else
{
// normalize the path back to the local file system one.
localPath = svn.NormalizeRelativePath(serverPath, Path.DirectorySeparatorChar, '/');
}
Trace.Verbose("Leaving SvnSourceProvider.GetLocalPath");
return(localPath);
}
private string GetDefaultRepositoryPath(
IExecutionContext executionContext,
RepositoryResource repository,
TrackingConfig newConfig
)
{
string repoPath = String.Empty;
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1)
var repoSourceDirectory = newConfig?.RepositoryTrackingInfo.Where(item => string.Equals(item.Identifier, repository.Alias, StringComparison.OrdinalIgnoreCase)).Select(item => item.SourcesDirectory).FirstOrDefault();
if (repoSourceDirectory != null)
{
repoPath = Path.Combine(workDirectory, repoSourceDirectory);
}
else
{
repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory, RepositoryUtil.GetCloneDirectory(repository));
}
}
else
{
// For single checkouts, the repository is rooted to the sources folder (_work/1/s)
repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory);
}
return(repoPath);
}
public static string GetSourceDirectoryHashKey(this RepositoryResource repository, IExecutionContext executionContext)
{
// Validate parameters.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repository.Url, nameof(repository.Url));
// Calculate the hash key.
const string Format = "{{{{ \r\n \"system\" : \"build\", \r\n \"collectionId\" = \"{0}\", \r\n \"definitionId\" = \"{1}\", \r\n \"repositoryUrl\" = \"{2}\", \r\n \"sourceFolder\" = \"{{0}}\",\r\n \"hashKey\" = \"{{1}}\"\r\n}}}}";
string hashInput = string.Format(
CultureInfo.InvariantCulture,
Format,
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_DefinitionId,
repository.Url.AbsoluteUri);
using (SHA1 sha1Hash = SHA1.Create())
{
byte[] data = sha1Hash.ComputeHash(Encoding.UTF8.GetBytes(hashInput));
StringBuilder hexString = new StringBuilder();
for (int i = 0; i < data.Length; i++)
{
hexString.Append(data[i].ToString("x2"));
}
return(hexString.ToString());
}
}
public void TrackingConfig_ctor_should_fill_in_fields_correctly()
{
using (TestHostContext tc = Setup(out Mock <IExecutionContext> mockExecutionContext))
{
// Arrange.
var repository = new RepositoryResource()
{
Type = RepositoryTypes.Git, Url = new Uri(RepositoryUrl)
};
// Act.
var config = new TrackingConfig(mockExecutionContext.Object, new[] { repository }, DefinitionId);
// Assert.
Assert.Equal(Path.Combine("322", "a"), config.ArtifactsDirectory);
Assert.Equal("322", config.BuildDirectory);
Assert.Equal(CollectionId, config.CollectionId);
Assert.Equal(CollectionUrl, config.CollectionUrl);
Assert.Equal(DefinitionId.ToString(), config.DefinitionId);
Assert.Equal(DefinitionName, config.DefinitionName);
Assert.Equal(3, config.FileFormatVersion);
Assert.Equal(null, config.FileLocation);
Assert.Equal("ea7c71421cca06c927f73627b66d6b4f4c3a5f4a", config.HashKey);
Assert.Equal(RepositoryTypes.Git, config.RepositoryType);
Assert.Equal(RepositoryUrl, config.RepositoryUrl);
Assert.Equal(Path.Combine("322", "s"), config.SourcesDirectory);
Assert.Equal("build", config.System);
Assert.Equal(Path.Combine("322", "TestResults"), config.TestResultsDirectory);
Assert.NotNull(config.RepositoryTrackingInfo);
Assert.Equal(true, config.ShouldSerializeRepositoryTrackingInfo());
Assert.Equal(1, config.RepositoryTrackingInfo.Count);
Assert.Equal(RepositoryUrl, config.RepositoryTrackingInfo[0].RepositoryUrl);
}
}
public void GetRepository_should_return_correct_value_when_called()
{
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
var repo1 = new RepositoryResource
{
Alias = "repo1",
Id = "repo1",
Type = "git",
};
var repo2 = new RepositoryResource
{
Alias = "repo2",
Id = "repo2",
Type = "git",
};
var repoSelf = new RepositoryResource
{
Alias = "self",
Id = "repo3",
Type = "git",
};
Assert.Equal(repo1, RepositoryUtil.GetRepository(new[] { repo1, repo2 }, "repo1"));
Assert.Equal(repo2, RepositoryUtil.GetRepository(new[] { repo1, repo2 }, "repo2"));
Assert.Equal(repo1, RepositoryUtil.GetRepository(new[] { repoSelf, repo1, repo2 }, "repo1"));
Assert.Equal(repo2, RepositoryUtil.GetRepository(new[] { repoSelf, repo1, repo2 }, "repo2"));
Assert.Equal(repoSelf, RepositoryUtil.GetRepository(new[] { repoSelf, repo1, repo2 }, "self"));
Assert.Equal(null, RepositoryUtil.GetRepository(new[] { repoSelf, repo1, repo2 }, "unknown"));
}
}
public void GetCloneDirectory_REPO_should_return_proper_value_when_called()
{
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
var repo = new RepositoryResource()
{
Alias = "alias",
Id = "repo1",
Type = "git",
Url = null,
};
// If name is not set and url is not set, then it should use alias
Assert.Equal("alias", RepositoryUtil.GetCloneDirectory(repo));
// If url is set, it should choose url over alias
repo.Url = new Uri("https://[email protected]/jpricket/MyFirstProject/_git/repo1_url");
Assert.Equal("repo1_url", RepositoryUtil.GetCloneDirectory(repo));
// If name is set, it should choose name over alias or url
repo.Properties.Set(RepositoryPropertyNames.Name, "MyFirstProject/repo1_name");
Assert.Equal("repo1_name", RepositoryUtil.GetCloneDirectory(repo));
}
}
public TrackingConfig UpdateDirectory(
IExecutionContext executionContext,
RepositoryResource repository)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repository, nameof(repository));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Defer to the source provider to calculate the hash key.
Trace.Verbose("Calculating build directory hash key.");
string hashKey = repository.GetSourceDirectoryHashKey(executionContext);
Trace.Verbose($"Hash key: {hashKey}");
// Load the existing tracking file.
string trackingFile = Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
Constants.Build.Path.SourceRootMappingDirectory,
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_DefinitionId,
Constants.Build.Path.TrackingConfigFile);
Trace.Verbose($"Loading tracking config if exists: {trackingFile}");
TrackingConfigBase existingConfig = trackingManager.LoadIfExists(executionContext, trackingFile);
ArgUtil.NotNull(existingConfig, nameof(existingConfig));
TrackingConfig newConfig = ConvertToNewFormat(executionContext, repository, existingConfig);
ArgUtil.NotNull(newConfig, nameof(newConfig));
var repoPath = repository.Properties.Get <string>(RepositoryPropertyNames.Path);
ArgUtil.NotNullOrEmpty(repoPath, nameof(repoPath));
Trace.Info($"Update repository path for repository {repository.Alias} to '{repoPath}'");
string buildDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory);
if (repoPath.StartsWith(buildDirectory + Path.DirectorySeparatorChar) || repoPath.StartsWith(buildDirectory + Path.AltDirectorySeparatorChar))
{
// The sourcesDirectory in tracking file is a relative path to agent's work folder.
newConfig.SourcesDirectory = repoPath.Substring(HostContext.GetDirectory(WellKnownDirectory.Work).Length + 1).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
else
{
throw new ArgumentException($"Repository path '{repoPath}' should be located under agent's work directory '{buildDirectory}'.");
}
// Update the tracking config files.
Trace.Verbose("Updating job run properties.");
trackingManager.UpdateJobRunProperties(executionContext, newConfig, trackingFile);
return(newConfig);
}
public RepositoryTrackingInfo(RepositoryResource repositoryResource, string sourcesDirectoryRoot)
{
if (repositoryResource != null)
{
Identifier = repositoryResource.Alias;
RepositoryType = repositoryResource.Type;
RepositoryUrl = repositoryResource.Url.AbsoluteUri;
SourcesDirectory = Path.Combine(sourcesDirectoryRoot, RepositoryUtil.GetCloneDirectory(repositoryResource));
}
}
private TrackingConfig ConvertToNewFormat(
IExecutionContext executionContext,
RepositoryResource repository,
TrackingConfigBase config)
{
Trace.Entering();
// If it's already in the new format, return it.
TrackingConfig newConfig = config as TrackingConfig;
if (newConfig != null)
{
return(newConfig);
}
// Delete the legacy artifact/staging directories.
LegacyTrackingConfig legacyConfig = config as LegacyTrackingConfig;
DeleteDirectory(
executionContext,
description: "legacy artifacts directory",
path: Path.Combine(legacyConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory));
DeleteDirectory(
executionContext,
description: "legacy staging directory",
path: Path.Combine(legacyConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory));
// Determine the source directory name. Check if the directory is named "s" already.
// Convert the source directory to be named "s" if there is a problem with the old name.
string sourcesDirectoryNameOnly = Constants.Build.Path.SourcesDirectory;
string repositoryName = repository.Properties.Get <string>(RepositoryPropertyNames.Name);
if (!Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, sourcesDirectoryNameOnly)) &&
!String.Equals(repositoryName, Constants.Build.Path.ArtifactsDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(repositoryName, Constants.Build.Path.LegacyArtifactsDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(repositoryName, Constants.Build.Path.LegacyStagingDirectory, StringComparison.OrdinalIgnoreCase) &&
!String.Equals(repositoryName, Constants.Build.Path.TestResultsDirectory, StringComparison.OrdinalIgnoreCase) &&
!repositoryName.Contains("\\") &&
!repositoryName.Contains("/") &&
Directory.Exists(Path.Combine(legacyConfig.BuildDirectory, repositoryName)))
{
sourcesDirectoryNameOnly = repositoryName;
}
// Convert to the new format.
newConfig = new TrackingConfig(
executionContext,
legacyConfig,
sourcesDirectoryNameOnly,
repository.Type,
// The legacy artifacts directory has been deleted at this point - see above - so
// switch the configuration to using the new naming scheme.
useNewArtifactsDirectoryName: true);
return(newConfig);
}
/// <summary>
/// Returns the folder name that would be created by calling 'git.exe clone'.
/// This is just the relative folder name not a full path.
/// The repo name is used if provided, then repo url, and finally repo alias.
/// </summary>
public static string GetCloneDirectory(RepositoryResource repository)
{
ArgUtil.NotNull(repository, nameof(repository));
string repoName =
repository.Properties.Get <string>(RepositoryPropertyNames.Name) ??
repository.Url?.AbsoluteUri ??
repository.Alias;
return(GetCloneDirectory(repoName));
}
public static Boolean TestOverrideBuildDirectory(this RepositoryResource repository, AgentSettings settings)
{
if (repository.Type == TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.Tfvc)
{
return(settings.IsHosted);
}
else
{
return(false);
}
}
public void GetRepositoryForLocalPath_should_return_correct_values()
{
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
var repo1 = new RepositoryResource
{
Alias = "repo1",
Id = "repo1",
Type = "git",
};
repo1.Properties.Set(RepositoryPropertyNames.Path, Path.Combine("root", "1", "s", "repo1"));
var repo2 = new RepositoryResource
{
Alias = "repo2",
Id = "repo2",
Type = "git",
};
repo2.Properties.Set(RepositoryPropertyNames.Path, Path.Combine("root", "1", "s", "repo2"));
var repo3 = new RepositoryResource
{
Alias = "repo3",
Id = "repo3",
Type = "git",
};
// repo3 has no path
// Make sure null is returned if nothing matches or inputs are invalid
Assert.Equal(null, RepositoryUtil.GetRepositoryForLocalPath(null, null));
Assert.Equal(null, RepositoryUtil.GetRepositoryForLocalPath(null, Path.Combine("root", "1", "s", "not_a_repo")));
Assert.Equal(null, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, null));
Assert.Equal(null, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, "not a path"));
Assert.Equal(null, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, Path.Combine("root", "1", "s", "not_a_repo")));
Assert.Equal(null, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, Path.Combine("root", "1", "s")));
Assert.Equal(null, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, Path.Combine("root", "1", "s", "repo3")));
// Make sure the first repo is returned if there is only one
Assert.Equal(repo1, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1 }, Path.Combine("root", "1", "s", "not_a_repo")));
Assert.Equal(repo2, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo2 }, "not a path"));
Assert.Equal(repo3, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo3 }, "not a path"));
// Make sure the matching repo is returned if there is more than one
Assert.Equal(repo1, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, Path.Combine("root", "1", "s", "repo1")));
Assert.Equal(repo1, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, Path.Combine("root", "1", "s", "repo1", "sub", "path", "file.txt")));
Assert.Equal(repo2, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, Path.Combine("root", "1", "s", "repo2")));
Assert.Equal(repo2, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo1, repo2, repo3 }, Path.Combine("root", "1", "s", "repo2", "sub", "path", "file.txt")));
Assert.Equal(repo2, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo3, repo1, repo2 }, Path.Combine("root", "1", "s", "repo2")));
Assert.Equal(repo2, RepositoryUtil.GetRepositoryForLocalPath(new[] { repo3, repo1, repo2 }, Path.Combine("root", "1", "s", "repo2", "sub", "path", "file.txt")));
}
}
private void InitializeExecutionContext(TestHostContext hc, bool isMultiCheckout = false)
{
List <string> warnings;
_variables = new Variables(
hostContext: hc,
copy: new Dictionary <string, VariableValue>(),
warnings: out warnings);
_repositories = new List <RepositoryResource>();
_selfRepo = new RepositoryResource()
{
Alias = "self",
Id = Guid.NewGuid().ToString(),
Name = "mainRepo",
Type = "git",
};
_selfRepo.Properties.Set(RepositoryPropertyNames.Path, "/1/s");
_otherRepo = new RepositoryResource()
{
Alias = "repo2",
Id = Guid.NewGuid().ToString(),
Name = "otherRepo",
Type = "git",
};
_otherRepo.Properties.Set(RepositoryPropertyNames.Path, "/1/otherRepo");
_repositories.Add(_selfRepo);
_repositories.Add(_otherRepo);
_ec = new Mock <IExecutionContext>();
_ec.SetupAllProperties();
_ec.Setup(x => x.Variables).Returns(_variables);
_ec.Setup(x => x.Repositories).Returns(_repositories);
_ec.Setup(x => x.GetHostContext()).Returns(hc);
_ec.Setup(x => x.SetVariable(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <bool>()))
.Callback <string, string, bool, bool, bool, bool>((name, value, secret, b2, b3, readOnly) => _variables.Set(name, value, secret, readOnly));
if (isMultiCheckout)
{
var jobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
jobSettings.Add(Agent.Sdk.WellKnownJobSettings.HasMultipleCheckouts, Boolean.TrueString);
_ec.Setup(x => x.JobSettings).Returns(jobSettings);
}
var directoryManager = new Mock <Microsoft.VisualStudio.Services.Agent.Worker.Build.IBuildDirectoryManager>();
directoryManager.Setup(x => x.GetRelativeRepositoryPath(It.IsAny <string>(), It.IsAny <string>()))
.Returns <string, string>((bd, path) => GetLastPathPart(path));
hc.SetSingleton(directoryManager.Object);
}
public TrackingConfig UpdateDirectory(
IExecutionContext executionContext,
RepositoryResource repository)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repository, nameof(repository));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Defer to the source provider to calculate the hash key.
Trace.Verbose("Calculating build directory hash key.");
string hashKey = repository.GetSourceDirectoryHashKey(executionContext);
Trace.Verbose($"Hash key: {hashKey}");
// Load the existing tracking file.
string trackingFile = Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
Constants.Build.Path.SourceRootMappingDirectory,
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_DefinitionId,
Constants.Build.Path.TrackingConfigFile);
Trace.Verbose($"Loading tracking config if exists: {trackingFile}");
TrackingConfigBase existingConfig = trackingManager.LoadIfExists(executionContext, trackingFile);
ArgUtil.NotNull(existingConfig, nameof(existingConfig));
TrackingConfig newConfig = ConvertToNewFormat(executionContext, repository, existingConfig);
ArgUtil.NotNull(newConfig, nameof(newConfig));
var repoPath = repository.Properties.Get <string>(RepositoryPropertyNames.Path);
ArgUtil.NotNullOrEmpty(repoPath, nameof(repoPath));
Trace.Info($"Update repository path for repository {repository.Alias} to '{repoPath}'");
string buildDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory);
newConfig.SourcesDirectory = GetRelativeRepositoryPath(buildDirectory, repoPath);
// Update the tracking config files.
Trace.Verbose("Updating job run properties.");
trackingManager.UpdateJobRunProperties(executionContext, newConfig, trackingFile);
return(newConfig);
}
public void Init(
IExecutionContext context,
RepositoryResource repository,
CancellationToken cancellationToken)
{
// Validation.
ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(repository, nameof(repository));
ArgUtil.NotNull(cancellationToken, nameof(cancellationToken));
ArgUtil.NotNull(repository.Url, nameof(repository.Url));
ArgUtil.Equal(true, repository.Url.IsAbsoluteUri, nameof(repository.Url.IsAbsoluteUri));
ArgUtil.NotNull(repository.Endpoint, nameof(repository.Endpoint));
ServiceEndpoint endpoint = context.Endpoints.Single(
x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) ||
(repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase)));
ArgUtil.NotNull(endpoint.Data, nameof(endpoint.Data));
ArgUtil.NotNull(endpoint.Authorization, nameof(endpoint.Authorization));
ArgUtil.NotNull(endpoint.Authorization.Parameters, nameof(endpoint.Authorization.Parameters));
ArgUtil.Equal(EndpointAuthorizationSchemes.UsernamePassword, endpoint.Authorization.Scheme, nameof(endpoint.Authorization.Scheme));
_context = context;
_repository = repository;
_endpoint = endpoint;
_cancellationToken = cancellationToken;
// Find svn in %Path%
string svnPath = WhichUtil.Which("svn", trace: Trace);
if (string.IsNullOrEmpty(svnPath))
{
throw new Exception(StringUtil.Loc("SvnNotInstalled"));
}
else
{
_context.Debug($"Found svn installation path: {svnPath}.");
_svn = svnPath;
}
// External providers may need basic auth or tokens
endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out _username);
endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out _password);
_acceptUntrusted = endpoint.Data.ContainsKey(EndpointData.SvnAcceptUntrustedCertificates) &&
StringUtil.ConvertToBoolean(endpoint.Data[EndpointData.SvnAcceptUntrustedCertificates], defaultValue: false);
}
private string GetDefaultRepositoryPath(
IExecutionContext executionContext,
RepositoryResource repository,
string defaultSourcesDirectory)
{
if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1)
return(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory, RepositoryUtil.GetCloneDirectory(repository)));
}
else
{
// For single checkouts, the repository is rooted to the sources folder (_work/1/s)
return(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory));
}
}
public void GetTriggeringRepository_should_return_correct_value_when_called()
{
using (TestHostContext hc = new TestHostContext(this))
{
Tracing trace = hc.GetTrace();
var repo1 = new RepositoryResource
{
Alias = "repo1",
Id = "repo1",
Type = "git",
};
var repo2 = new RepositoryResource
{
Alias = "repo2",
Id = "repo2",
Type = "git",
};
var repoSelf = new RepositoryResource
{
Alias = "self",
Id = "repo3",
Type = "git",
};
// No properties set
Assert.Equal(null, RepositoryUtil.GetTriggeringRepository(null));
Assert.Equal(repo1, RepositoryUtil.GetTriggeringRepository(new[] { repo1 }));
Assert.Equal(repo2, RepositoryUtil.GetTriggeringRepository(new[] { repo2 }));
Assert.Equal(repoSelf, RepositoryUtil.GetTriggeringRepository(new[] { repoSelf }));
Assert.Equal(null, RepositoryUtil.GetTriggeringRepository(new[] { repo1, repo2 }));
Assert.Equal(repoSelf, RepositoryUtil.GetTriggeringRepository(new[] { repoSelf, repo1, repo2 }));
Assert.Equal(repoSelf, RepositoryUtil.GetTriggeringRepository(new[] { repo1, repoSelf, repo2 }));
Assert.Equal(repoSelf, RepositoryUtil.GetTriggeringRepository(new[] { repo1, repo2, repoSelf }));
// With IsPrimaryRepository set
repo2.Properties.Set(RepositoryUtil.IsTriggeringRepository, Boolean.TrueString);
Assert.Equal(repo2, RepositoryUtil.GetTriggeringRepository(new[] { repo1, repo2, repoSelf }));
repo2.Properties.Set(RepositoryUtil.IsTriggeringRepository, Boolean.FalseString);
Assert.Equal(repoSelf, RepositoryUtil.GetTriggeringRepository(new[] { repo1, repo2, repoSelf }));
}
}
public TrackingConfig UpdateDirectory(
IExecutionContext executionContext,
RepositoryResource updatedRepository)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(updatedRepository, nameof(updatedRepository));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Determine new repository path
var repoPath = updatedRepository.Properties.Get <string>(RepositoryPropertyNames.Path);
ArgUtil.NotNullOrEmpty(repoPath, nameof(repoPath));
Trace.Info($"Update repository path for repository {updatedRepository.Alias} to '{repoPath}'");
// Get the config
var trackingConfig = trackingManager.LoadExistingTrackingConfig(executionContext);
// Update the repositoryInfo on the config
string buildDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), trackingConfig.BuildDirectory);
string relativeRepoPath = GetRelativeRepositoryPath(buildDirectory, repoPath);
var effectedRepo = trackingConfig.RepositoryTrackingInfo.FirstOrDefault(r => string.Equals(r.Identifier, updatedRepository.Alias, StringComparison.OrdinalIgnoreCase));
if (effectedRepo != null)
{
Trace.Info($"Found repository {updatedRepository.Alias}'");
effectedRepo.SourcesDirectory = relativeRepoPath;
}
// Also update the SourcesDirectory on the tracking info if there is only one repo.
if (trackingConfig.RepositoryTrackingInfo.Count == 1)
{
Trace.Info($"Updating SourcesDirectory to {updatedRepository.Alias}'");
trackingConfig.SourcesDirectory = relativeRepoPath;
}
// Update the tracking config files.
Trace.Verbose("Updating job run properties.");
trackingManager.UpdateTrackingConfig(executionContext, trackingConfig);
return(trackingConfig);
}
public TrackingConfig(
IExecutionContext executionContext,
RepositoryResource repository,
int buildDirectory,
string hashKey)
{
// Set the directories.
BuildDirectory = buildDirectory.ToString(CultureInfo.InvariantCulture);
ArtifactsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.ArtifactsDirectory);
SourcesDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.SourcesDirectory);
TestResultsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.TestResultsDirectory);
// Set the other properties.
CollectionId = executionContext.Variables.System_CollectionId;
DefinitionId = executionContext.Variables.System_DefinitionId;
HashKey = hashKey;
RepositoryUrl = repository.Url.AbsoluteUri;
RepositoryType = repository.Type;
System = BuildSystem;
UpdateJobRunProperties(executionContext);
}
public Task <RepositoryResource> Update(RepositoryResource resource, string currentSlug = null)
=> ApiClientWrapper.Update <RepositoryResource>($"{RootUri}/{currentSlug ?? resource.Slug}", resource);
public Task Remove(RepositoryResource resource)
=> ApiClientWrapper.Remove($"{RootUri}/{resource.Slug}");
public TrackingConfig Create(
IExecutionContext executionContext,
RepositoryResource repository,
string hashKey,
string file,
bool overrideBuildDirectory)
{
Trace.Entering();
// Get or create the top-level tracking config.
TopLevelTrackingConfig topLevelConfig;
string topLevelFile = Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
Constants.Build.Path.SourceRootMappingDirectory,
Constants.Build.Path.TopLevelTrackingConfigFile);
Trace.Verbose($"Loading top-level tracking config if exists: {topLevelFile}");
if (!File.Exists(topLevelFile))
{
topLevelConfig = new TopLevelTrackingConfig();
}
else
{
topLevelConfig = JsonConvert.DeserializeObject <TopLevelTrackingConfig>(File.ReadAllText(topLevelFile));
if (topLevelConfig == null)
{
executionContext.Warning($"Rebuild corruptted top-level tracking configure file {topLevelFile}.");
// save the corruptted file in case we need to investigate more.
File.Copy(topLevelFile, $"{topLevelFile}.corruptted", true);
topLevelConfig = new TopLevelTrackingConfig();
DirectoryInfo workDir = new DirectoryInfo(HostContext.GetDirectory(WellKnownDirectory.Work));
foreach (var dir in workDir.EnumerateDirectories())
{
// we scan the entire _work directory and find the directory with the highest integer number.
if (int.TryParse(dir.Name, NumberStyles.Integer, CultureInfo.InvariantCulture, out int lastBuildNumber) &&
lastBuildNumber > topLevelConfig.LastBuildDirectoryNumber)
{
topLevelConfig.LastBuildDirectoryNumber = lastBuildNumber;
}
}
}
}
// Determine the build directory.
if (overrideBuildDirectory)
{
// This should only occur during hosted builds. This was added due to TFVC.
// TFVC does not allow a local path for a single machine to be mapped in multiple
// workspaces. The machine name for a hosted images is not unique.
//
// So if a customer is running two hosted builds at the same time, they could run
// into the local mapping conflict.
//
// The workaround is to force the build directory to be different across all concurrent
// hosted builds (for TFVC). The agent ID will be unique across all concurrent hosted
// builds so that can safely be used as the build directory.
ArgUtil.Equal(default(int), topLevelConfig.LastBuildDirectoryNumber, nameof(topLevelConfig.LastBuildDirectoryNumber));
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
topLevelConfig.LastBuildDirectoryNumber = settings.AgentId;
}
else
{
topLevelConfig.LastBuildDirectoryNumber++;
}
// Update the top-level tracking config.
topLevelConfig.LastBuildDirectoryCreatedOn = DateTimeOffset.Now;
WriteToFile(topLevelFile, topLevelConfig);
// Create the new tracking config.
TrackingConfig config = new TrackingConfig(
executionContext,
repository,
topLevelConfig.LastBuildDirectoryNumber,
hashKey);
WriteToFile(file, config);
return(config);
}
public RepositoryScope ScopeToRepository(RepositoryResource repository) => new RepositoryScope(this, repository.Slug, _client);
public TrackingConfig PrepareDirectory(
IExecutionContext executionContext,
RepositoryResource repository,
WorkspaceOptions workspace)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repository, nameof(repository));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Defer to the source provider to calculate the hash key.
Trace.Verbose("Calculating build directory hash key.");
string hashKey = repository.GetSourceDirectoryHashKey(executionContext);
Trace.Verbose($"Hash key: {hashKey}");
// Load the existing tracking file if one already exists.
string trackingFile = Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
Constants.Build.Path.SourceRootMappingDirectory,
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_DefinitionId,
Constants.Build.Path.TrackingConfigFile);
Trace.Verbose($"Loading tracking config if exists: {trackingFile}");
TrackingConfigBase existingConfig = trackingManager.LoadIfExists(executionContext, trackingFile);
// Check if the build needs to be garbage collected. If the hash key
// has changed, then the existing build directory cannot be reused.
TrackingConfigBase garbageConfig = null;
if (existingConfig != null &&
!string.Equals(existingConfig.HashKey, hashKey, StringComparison.OrdinalIgnoreCase))
{
// Just store a reference to the config for now. It can safely be
// marked for garbage collection only after the new build directory
// config has been created.
Trace.Verbose($"Hash key from existing tracking config does not match. Existing key: {existingConfig.HashKey}");
garbageConfig = existingConfig;
existingConfig = null;
}
// Create a new tracking config if required.
TrackingConfig newConfig;
if (existingConfig == null)
{
Trace.Verbose("Creating a new tracking config file.");
var agentSetting = HostContext.GetService <IConfigurationStore>().GetSettings();
newConfig = trackingManager.Create(
executionContext,
repository,
hashKey,
trackingFile,
repository.TestOverrideBuildDirectory(agentSetting));
ArgUtil.NotNull(newConfig, nameof(newConfig));
}
else
{
// Convert legacy format to the new format if required.
newConfig = ConvertToNewFormat(executionContext, repository, existingConfig);
// Fill out repository type if it's not there.
// repository type is a new property introduced for maintenance job
if (string.IsNullOrEmpty(newConfig.RepositoryType))
{
newConfig.RepositoryType = repository.Type;
}
// For existing tracking config files, update the job run properties.
Trace.Verbose("Updating job run properties.");
trackingManager.UpdateJobRunProperties(executionContext, newConfig, trackingFile);
}
// Mark the old configuration for garbage collection.
if (garbageConfig != null)
{
Trace.Verbose("Marking existing config for garbage collection.");
trackingManager.MarkForGarbageCollection(executionContext, garbageConfig);
}
// Prepare the build directory.
// There are 2 ways to provide build directory clean policy.
// 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016)
// 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI)
BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, workspace);
CreateDirectory(
executionContext,
description: "build directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory),
deleteExisting: cleanOption == BuildCleanOption.All);
CreateDirectory(
executionContext,
description: "artifacts directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.ArtifactsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "test results directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.TestResultsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "binaries directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Binary);
CreateDirectory(
executionContext,
description: "source directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.SourcesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Source);
return(newConfig);
}
public void Init()
{
sharpBucket = TestHelpers.GetV2ClientAuthenticatedWithOAuth();
repositoriesEndPoint = sharpBucket.RepositoriesEndPoint();
repositoryResource = repositoriesEndPoint.RepositoryResource(ACCOUNT_NAME, REPOSITORY_NAME);
}
public virtual string GetLocalPath(IExecutionContext executionContext, RepositoryResource repository, string path)
{
return(path);
}
public static async Task <(bool success, string error)> InsertRepositoryEntity(RepositoryResource resource, CloudTable table, ILogger log)
{
log.LogInformation($"");
try
{
var newEntity = new RepositoryEntity(resource.Owner, resource.Name);
newEntity.Subscribers.Add(resource.Subscriber);
var insertOperation = TableOperation.Insert(newEntity);
await table.ExecuteAsync(insertOperation);
}
catch (StorageException sex)
{
return(false, sex.RequestInformation.ExtendedErrorInformation.ErrorMessage);
}
catch (Exception ex)
{
return(false, ex.Message);
}
return(true, "");
}
private ISourceProvider GetSourceProvider(IExecutionContext executionContext, RepositoryResource repository)
{
if (repository != null)
{
var extensionManager = HostContext.GetService <IExtensionManager>();
List <ISourceProvider> sourceProviders = extensionManager.GetExtensions <ISourceProvider>();
var sourceProvider = sourceProviders.FirstOrDefault(x => string.Equals(x.RepositoryType, repository.Type, StringComparison.OrdinalIgnoreCase));
return(sourceProvider);
}
return(null);
}
public static async Task <(bool success, string error)> UpdateRepositoryEntity(RepositoryEntity existing, RepositoryResource resource, CloudTable table, ILogger log)
{
if (!existing.Subscribers.Contains(resource.Subscriber))
{
log.LogInformation($"");
try
{
existing.Subscribers.Add(resource.Subscriber);
var updateOperation = TableOperation.Replace(existing);
await table.ExecuteAsync(updateOperation);
}
catch (StorageException sex)
{
return(false, sex.RequestInformation.ExtendedErrorInformation.ErrorMessage);
}
catch (Exception ex)
{
return(false, ex.Message);
}
}
return(true, "");
}
