public void GetCloneDirectory_REPO_should_return_proper_value_when_called() { using (TestHostContext hc = new TestHostContext(this)) { Tracing trace = hc.GetTrace(); var repo = new RepositoryResource() { Alias = "alias", Id = "repo1", Type = "git", Url = null, }; // If name is not set and url is not set, then it should use alias Assert.Equal("alias", RepositoryUtil.GetCloneDirectory(repo)); // If url is set, it should choose url over alias repo.Url = new Uri("https://[email protected]/jpricket/MyFirstProject/_git/repo1_url"); Assert.Equal("repo1_url", RepositoryUtil.GetCloneDirectory(repo)); // If name is set, it should choose name over alias or url repo.Properties.Set(RepositoryPropertyNames.Name, "MyFirstProject/repo1_name"); Assert.Equal("repo1_name", RepositoryUtil.GetCloneDirectory(repo)); } }
private TrackingConfig LoadIfExists( IExecutionContext executionContext, string file) { Trace.Entering(); Trace.Verbose($"Loading {file}"); // The tracking config will not exist for a new definition. if (!File.Exists(file)) { Trace.Verbose($"Tracking file does not exist: {file}"); return(null); } TrackingConfig result = null; // Load the content and distinguish between tracking config file // version 1 and file version 2. string content = File.ReadAllText(file); string fileFormatVersionJsonProperty = StringUtil.Format( @"""{0}""", TrackingConfig.FileFormatVersionJsonProperty); if (content.Contains(fileFormatVersionJsonProperty)) { // The config is the new format. Trace.Verbose("Parsing new tracking config format."); result = JsonConvert.DeserializeObject <TrackingConfig>(content); } else { // Attempt to parse the legacy format. Trace.Verbose("Parsing legacy tracking config format."); var legacyTrackingConfig = LegacyTrackingConfig.TryParse(content); if (legacyTrackingConfig == null) { executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content)); } else { // Convert legacy format to the new format. result = ConvertToNewFormat( executionContext, legacyTrackingConfig, RepositoryUtil.GetCloneDirectory(legacyTrackingConfig.RepositoryUrl), RepositoryUtil.GuessRepositoryType(legacyTrackingConfig.RepositoryUrl)); } } if (result != null) { result.FileLocation = file; } return(result); }
public void GetCloneDirectory_STRING_should_return_proper_value_when_called() { // These test cases were inspired by the test cases that git.exe uses // see https://github.com/git/git/blob/53f9a3e157dbbc901a02ac2c73346d375e24978c/t/t5603-clone-dirname.sh#L21 using (TestHostContext hc = new TestHostContext(this)) { Tracing trace = hc.GetTrace(); // basic syntax with bare and non-bare variants Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo.git")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git")); // similar, but using ssh URL rather than host:path syntax Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo.git")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/.git")); // we should remove trailing slashes and .git suffixes Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo///")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/.git/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo.git/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo.git///")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo///.git/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("ssh://host/foo/.git///")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo///")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo.git/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo.git///")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo///.git/")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git///")); Assert.Equal("foo", RepositoryUtil.GetCloneDirectory("host:foo/.git///")); Assert.Equal("repo", RepositoryUtil.GetCloneDirectory("host:foo/repo")); // omitting the path should default to the hostname Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://host/")); Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/")); Assert.Equal("host", RepositoryUtil.GetCloneDirectory("host:/")); // auth materials should be redacted Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/")); Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host:1234/")); Assert.Equal("host", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@rd@host:1234/")); Assert.Equal("host", RepositoryUtil.GetCloneDirectory("user@host:/")); Assert.Equal("host", RepositoryUtil.GetCloneDirectory("user:password@host:/")); Assert.Equal("host", RepositoryUtil.GetCloneDirectory("user:passw@rd@host:/")); // trailing port-like numbers should not be stripped for paths Assert.Equal("1234", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/test:1234")); Assert.Equal("1234", RepositoryUtil.GetCloneDirectory("ssh://*****:*****@host/test:1234.git")); } }
public void GetCloneDirectory_STRING_should_throw_on_null() { using (TestHostContext hc = new TestHostContext(this)) { Tracing trace = hc.GetTrace(); Assert.Throws <ArgumentNullException>(() => RepositoryUtil.GetCloneDirectory((string)null)); } }
public RepositoryTrackingInfo(RepositoryResource repositoryResource, string sourcesDirectoryRoot) { if (repositoryResource != null) { Identifier = repositoryResource.Alias; RepositoryType = repositoryResource.Type; RepositoryUrl = repositoryResource.Url.AbsoluteUri; SourcesDirectory = Path.Combine(sourcesDirectoryRoot, RepositoryUtil.GetCloneDirectory(repositoryResource)); } }
private bool IsCheckoutToCustomPath(TrackingConfig trackingConfig, RepositoryInfo repoInfo, TaskStep selfCheckoutTask) { string path; string selfRepoName = RepositoryUtil.GetCloneDirectory(repoInfo.PrimaryRepository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Name)); string defaultRepoCheckoutPath = Path.GetFullPath(Path.Combine(trackingConfig.SourcesDirectory, selfRepoName)); return selfCheckoutTask != null && selfCheckoutTask.Inputs.TryGetValue(PipelineConstants.CheckoutTaskInputs.Path, out path) && !string.Equals(Path.GetFullPath(Path.Combine(trackingConfig.BuildDirectory, path)), defaultRepoCheckoutPath, IOUtil.FilePathStringComparison); }
public TrackingConfig( IExecutionContext executionContext, IList <RepositoryResource> repositories, int buildDirectory) : this() { ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(repositories, nameof(repositories)); // Get the repo that we are going to checkout first to create the tracking info from. var primaryRepository = RepositoryUtil.GetPrimaryRepository(repositories); // Set the directories. BuildDirectory = buildDirectory.ToString(CultureInfo.InvariantCulture); ArtifactsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.ArtifactsDirectory); SourcesDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.SourcesDirectory); TestResultsDirectory = Path.Combine(BuildDirectory, Constants.Build.Path.TestResultsDirectory); // Set the other properties. CollectionId = executionContext.Variables.System_CollectionId; DefinitionId = executionContext.Variables.System_DefinitionId; RepositoryUrl = primaryRepository?.Url.AbsoluteUri; RepositoryType = primaryRepository?.Type; System = BuildSystem; UpdateJobRunProperties(executionContext); foreach (var repo in repositories) { RepositoryTrackingInfo.Add(new Build.RepositoryTrackingInfo { Identifier = repo.Alias, RepositoryType = repo.Type, RepositoryUrl = repo.Url.AbsoluteUri, SourcesDirectory = Path.Combine(SourcesDirectory, RepositoryUtil.GetCloneDirectory(repo)), }); } // Now that we have all the repositories set up, we can compute the config hash HashKey = TrackingConfigHashAlgorithm.ComputeHash(CollectionId, DefinitionId, RepositoryTrackingInfo); }
private TrackingConfig LoadIfExists( IExecutionContext executionContext, string file) { Trace.Entering(); Trace.Verbose($"Loading {file}"); // The tracking config will not exist for a new definition. if (!File.Exists(file)) { Trace.Verbose($"Tracking file does not exist: {file}"); return(null); } TrackingConfig result = null; // Load the content and distinguish between tracking config file // version 1 and file version 2. string content = File.ReadAllText(file); string fileFormatVersionJsonProperty = StringUtil.Format( @"""{0}""", TrackingConfig.FileFormatVersionJsonProperty); if (content.Contains(fileFormatVersionJsonProperty)) { // The config is the new format. Trace.Verbose("Parsing new tracking config format."); result = JsonConvert.DeserializeObject <TrackingConfig>(content); if (result != null) { // if RepositoryTrackingInfo is empty, then we should create an entry so the rest // of the logic after this will act correctly if (result.RepositoryTrackingInfo.Count == 0) { result.RepositoryTrackingInfo.Add(new Build.RepositoryTrackingInfo { Identifier = RepositoryUtil.DefaultPrimaryRepositoryName, RepositoryType = result.RepositoryType, RepositoryUrl = result.RepositoryUrl, SourcesDirectory = result.SourcesDirectory, }); } } } else { // Attempt to parse the legacy format. Trace.Verbose("Parsing legacy tracking config format."); var legacyTrackingConfig = LegacyTrackingConfig.TryParse(content); if (legacyTrackingConfig == null) { executionContext.Warning(StringUtil.Loc("UnableToParseBuildTrackingConfig0", content)); } else { // Convert legacy format to the new format. result = ConvertToNewFormat( executionContext, legacyTrackingConfig, RepositoryUtil.GetCloneDirectory(legacyTrackingConfig.RepositoryUrl), RepositoryUtil.GuessRepositoryType(legacyTrackingConfig.RepositoryUrl)); } } if (result != null) { result.FileLocation = file; } return(result); }
public TrackingConfig PrepareDirectory( IExecutionContext executionContext, IList <RepositoryResource> repositories, WorkspaceOptions workspace) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables)); ArgUtil.NotNull(repositories, nameof(repositories)); var trackingManager = HostContext.GetService <ITrackingManager>(); // Create the tracking config for this execution of the pipeline var agentSettings = HostContext.GetService <IConfigurationStore>().GetSettings(); var newConfig = trackingManager.Create(executionContext, repositories, ShouldOverrideBuildDirectory(repositories, agentSettings)); // Load the tracking config from the last execution of the pipeline var existingConfig = trackingManager.LoadExistingTrackingConfig(executionContext); // If there aren't any major changes, merge the configurations and use the same workspace if (trackingManager.AreTrackingConfigsCompatible(executionContext, newConfig, existingConfig)) { newConfig = trackingManager.MergeTrackingConfigs(executionContext, newConfig, existingConfig); } else if (existingConfig != null) { // If the previous config had different repos, get a new workspace folder and mark the old one for clean up trackingManager.MarkForGarbageCollection(executionContext, existingConfig); // If the config file was updated to a new config, we need to delete the legacy artifact/staging directories. // DeleteDirectory will check for the existence of the folders first. DeleteDirectory( executionContext, description: "legacy artifacts directory", path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory)); DeleteDirectory( executionContext, description: "legacy staging directory", path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory)); } // Save any changes to the config file trackingManager.UpdateTrackingConfig(executionContext, newConfig); // Prepare the build directory. // There are 2 ways to provide build directory clean policy. // 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016) // 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI) BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, workspace); CreateDirectory( executionContext, description: "build directory", path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory), deleteExisting: cleanOption == BuildCleanOption.All); CreateDirectory( executionContext, description: "artifacts directory", path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.ArtifactsDirectory), deleteExisting: true); CreateDirectory( executionContext, description: "test results directory", path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.TestResultsDirectory), deleteExisting: true); CreateDirectory( executionContext, description: "binaries directory", path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory), deleteExisting: cleanOption == BuildCleanOption.Binary); var defaultSourceDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory); CreateDirectory( executionContext, description: "source directory", path: defaultSourceDirectory, deleteExisting: cleanOption == BuildCleanOption.Source); // Set the default clone path for each repository (the Checkout task may override this later) foreach (var repository in repositories) { String repoPath; if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings)) { // If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1) var repoSourceDirectory = newConfig?.RepositoryTrackingInfo.Where(item => string.Equals(item.Identifier, repository.Alias, StringComparison.OrdinalIgnoreCase)).Select(item => item.SourcesDirectory).FirstOrDefault(); if (repoSourceDirectory != null) { repoPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), repoSourceDirectory); } else { repoPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory, RepositoryUtil.GetCloneDirectory(repository)); } } else { // For single checkouts, the repository is rooted to the sources folder (_work/1/s) repoPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory); } if (!string.Equals(repoPath, defaultSourceDirectory, StringComparison.Ordinal)) { CreateDirectory( executionContext, description: "repository source directory", path: repoPath, deleteExisting: cleanOption == BuildCleanOption.Source); } Trace.Info($"Set repository path for repository {repository.Alias} to '{repoPath}'"); repository.Properties.Set <string>(RepositoryPropertyNames.Path, repoPath); } return(newConfig); }
public override async Task RunAsync(AgentTaskPluginExecutionContext executionContext, CancellationToken token) { var sourceSkipVar = StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("agent.source.skip")?.Value) || !StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("build.syncSources")?.Value ?? bool.TrueString); if (sourceSkipVar) { executionContext.Output($"Skip sync source for repository."); return; } var repoAlias = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, true); var repo = executionContext.Repositories.Single(x => string.Equals(x.Alias, repoAlias, StringComparison.OrdinalIgnoreCase)); executionContext.PublishTelemetry(area: "AzurePipelinesAgent", feature: "Checkout", properties: new Dictionary <string, string> { { "RepoType", $"{repo.Type}" }, { "HostOS", $"{PlatformUtil.HostOS}" } }); MergeCheckoutOptions(executionContext, repo); var currentRepoPath = repo.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path); var buildDirectory = executionContext.Variables.GetValueOrDefault("agent.builddirectory")?.Value; var tempDirectory = executionContext.Variables.GetValueOrDefault("agent.tempdirectory")?.Value; ArgUtil.NotNullOrEmpty(currentRepoPath, nameof(currentRepoPath)); ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory)); ArgUtil.NotNullOrEmpty(tempDirectory, nameof(tempDirectory)); // Determine the path that we should clone/move the repository into const string sourcesDirectory = "s"; //Constants.Build.Path.SourcesDirectory string expectRepoPath; var path = executionContext.GetInput("path"); if (!string.IsNullOrEmpty(path)) { // When the checkout task provides a path, always use that one expectRepoPath = IOUtil.ResolvePath(buildDirectory, path); if (!expectRepoPath.StartsWith(buildDirectory.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar)) { throw new ArgumentException($"Input path '{path}' should resolve to a directory under '{buildDirectory}', current resolved path '{expectRepoPath}'."); } } else if (HasMultipleCheckouts(executionContext)) { // When there are multiple checkout tasks (and this one didn't set the path), default to directory 1/s/<repoName> expectRepoPath = Path.Combine(buildDirectory, sourcesDirectory, RepositoryUtil.GetCloneDirectory(repo)); } else { // When there's a single checkout task that doesn't have path set, default to sources directory 1/s expectRepoPath = Path.Combine(buildDirectory, sourcesDirectory); } // Update the repository path in the worker process executionContext.UpdateRepositoryPath(repoAlias, expectRepoPath); executionContext.Debug($"Repository requires to be placed at '{expectRepoPath}', current location is '{currentRepoPath}'"); if (!string.Equals(currentRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), expectRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), IOUtil.FilePathStringComparison)) { executionContext.Output($"Repository is current at '{currentRepoPath}', move to '{expectRepoPath}'."); var count = 1; var staging = Path.Combine(tempDirectory, $"_{count}"); while (Directory.Exists(staging)) { count++; staging = Path.Combine(tempDirectory, $"_{count}"); } try { executionContext.Debug($"Move existing repository '{currentRepoPath}' to '{expectRepoPath}' via staging directory '{staging}'."); IOUtil.MoveDirectory(currentRepoPath, expectRepoPath, staging, CancellationToken.None); } catch (Exception ex) { executionContext.Debug("Catch exception during repository move."); executionContext.Debug(ex.ToString()); executionContext.Warning("Unable move and reuse existing repository to required location."); IOUtil.DeleteDirectory(expectRepoPath, CancellationToken.None); } executionContext.Output($"Repository will be located at '{expectRepoPath}'."); repo.Properties.Set <string>(Pipelines.RepositoryPropertyNames.Path, expectRepoPath); } ISourceProvider sourceProvider = SourceProviderFactory.GetSourceProvider(repo.Type); await sourceProvider.GetSourceAsync(executionContext, repo, token); }
private string GetDefaultRepositoryPath( IExecutionContext executionContext, RepositoryResource repository, string defaultSourcesDirectory) { if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings)) { // If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1) return(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory, RepositoryUtil.GetCloneDirectory(repository))); } else { // For single checkouts, the repository is rooted to the sources folder (_work/1/s) return(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory)); } }
private string GetDefaultRepositoryPath( IExecutionContext executionContext, RepositoryResource repository, TrackingConfig newConfig ) { string repoPath = String.Empty; string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings)) { // If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1) var repoSourceDirectory = newConfig?.RepositoryTrackingInfo.Where(item => string.Equals(item.Identifier, repository.Alias, StringComparison.OrdinalIgnoreCase)).Select(item => item.SourcesDirectory).FirstOrDefault(); if (repoSourceDirectory != null) { repoPath = Path.Combine(workDirectory, repoSourceDirectory); } else { repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory, RepositoryUtil.GetCloneDirectory(repository)); } } else { // For single checkouts, the repository is rooted to the sources folder (_work/1/s) repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory); } return(repoPath); }