public void ReturnsCorrectHashKey() { using (TestHostContext tc = new TestHostContext(this)) { // Arrange. var executionContext = new Mock <IExecutionContext>(); List <string> warnings; executionContext .Setup(x => x.Variables) .Returns(new Variables(tc, copy: new Dictionary <string, VariableValue>(), warnings: out warnings)); executionContext.Object.Variables.Set(Constants.Variables.System.CollectionId, "7aee6dde-6381-4098-93e7-50a8264cf066"); executionContext.Object.Variables.Set(Constants.Variables.System.DefinitionId, "7"); Pipelines.RepositoryResource repository = new Pipelines.RepositoryResource() { Url = new Uri("http://contoso:8080/tfs/DefaultCollection/gitTest/_git/gitTest") }; // Act. string hashKey = repository.GetSourceDirectoryHashKey(executionContext.Object); // Assert. Assert.Equal("5c5c3d7ac33cca6604736eb3af977f23f1cf1146", hashKey); } }
public TestHostContext Setup([CallerMemberName] string name = "") { // Setup the host context. TestHostContext hc = new TestHostContext(this, name); // Create a random work path. _workFolder = hc.GetDirectory(WellKnownDirectory.Work); // Setup the execution context. _ec = new Mock <IExecutionContext>(); List <string> warnings; _variables = new Variables(hc, new Dictionary <string, VariableValue>(), out warnings); _variables.Set(Constants.Variables.System.CollectionId, CollectionId); _variables.Set(WellKnownDistributedTaskVariables.TFCollectionUrl, CollectionUrl); _variables.Set(Constants.Variables.System.DefinitionId, DefinitionId); _variables.Set(Constants.Variables.Build.DefinitionName, DefinitionName); _ec.Setup(x => x.Variables).Returns(_variables); // Setup the endpoint. _repository = new Pipelines.RepositoryResource() { Url = new Uri(RepositoryUrl) }; // Setup the tracking manager. _trackingManager = new TrackingManager(); _trackingManager.Initialize(hc); return(hc); }
public void InitializeJob_should_mark_primary_repository_in_multicheckout() { // Note: the primary repository is defined as the first repository that is checked out in the job using (TestHostContext hc = CreateTestContext()) using (var ec = new Agent.Worker.ExecutionContext()) { // Arrange: Create a job request message. TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = new TimelineReference(); JobEnvironment environment = new JobEnvironment(); environment.SystemConnection = new ServiceEndpoint(); List <TaskInstance> tasks = new List <TaskInstance>(); tasks.Add(new TaskInstance() { Id = Pipelines.PipelineConstants.CheckoutTask.Id, Version = Pipelines.PipelineConstants.CheckoutTask.Version, Inputs = { { Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, "repo2" } } }); tasks.Add(new TaskInstance() { Id = Pipelines.PipelineConstants.CheckoutTask.Id, Version = Pipelines.PipelineConstants.CheckoutTask.Version, Inputs = { { Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, "repo3" } } }); Guid JobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = Pipelines.AgentJobRequestMessageUtil.Convert(new AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, environment, tasks)); var repo1 = new Pipelines.RepositoryResource() { Alias = "self" }; var repo2 = new Pipelines.RepositoryResource() { Alias = "repo2" }; var repo3 = new Pipelines.RepositoryResource() { Alias = "repo3" }; jobRequest.Resources.Repositories.Add(repo1); jobRequest.Resources.Repositories.Add(repo2); jobRequest.Resources.Repositories.Add(repo3); // Arrange: Setup the paging logger. var pagingLogger = new Mock <IPagingLogger>(); hc.EnqueueInstance(pagingLogger.Object); ec.Initialize(hc); // Act. ec.InitializeJob(jobRequest, CancellationToken.None); // Assert. Assert.NotNull(ec.JobSettings); Assert.Equal(Boolean.TrueString, ec.JobSettings[WellKnownJobSettings.HasMultipleCheckouts]); Assert.Equal("repo2", ec.JobSettings[WellKnownJobSettings.FirstRepositoryCheckedOut]); Assert.Equal(Boolean.FalseString, repo1.Properties.Get <string>(RepositoryUtil.IsPrimaryRepository, Boolean.FalseString)); Assert.Equal(Boolean.TrueString, repo2.Properties.Get <string>(RepositoryUtil.IsPrimaryRepository, Boolean.FalseString)); Assert.Equal(Boolean.FalseString, repo3.Properties.Get <string>(RepositoryUtil.IsPrimaryRepository, Boolean.FalseString)); } }
protected void MergeInputs(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository) { string clean = executionContext.GetInput("clean"); if (!string.IsNullOrEmpty(clean)) { repository.Properties.Set <bool>("clean", PluginUtil.ConvertToBoolean(clean)); } // there is no addition inputs for TFVC and SVN if (repository.Type == RepositoryTypes.Bitbucket || repository.Type == RepositoryTypes.GitHub || repository.Type == RepositoryTypes.GitHubEnterprise || repository.Type == RepositoryTypes.Git || repository.Type == RepositoryTypes.TfsGit) { string checkoutSubmodules = executionContext.GetInput("checkoutSubmodules"); if (!string.IsNullOrEmpty(checkoutSubmodules)) { repository.Properties.Set <bool>("checkoutSubmodules", PluginUtil.ConvertToBoolean(checkoutSubmodules)); } string checkoutNestedSubmodules = executionContext.GetInput("checkoutNestedSubmodules"); if (!string.IsNullOrEmpty(checkoutNestedSubmodules)) { repository.Properties.Set <bool>("checkoutNestedSubmodules", PluginUtil.ConvertToBoolean(checkoutNestedSubmodules)); } string preserveCredential = executionContext.GetInput("preserveCredential"); if (!string.IsNullOrEmpty(preserveCredential)) { repository.Properties.Set <bool>("preserveCredential", PluginUtil.ConvertToBoolean(preserveCredential)); } string gitLfsSupport = executionContext.GetInput("gitLfsSupport"); if (!string.IsNullOrEmpty(gitLfsSupport)) { repository.Properties.Set <bool>("gitLfsSupport", PluginUtil.ConvertToBoolean(gitLfsSupport)); } string acceptUntrustedCerts = executionContext.GetInput("acceptUntrustedCerts"); if (!string.IsNullOrEmpty(acceptUntrustedCerts)) { repository.Properties.Set <bool>("acceptUntrustedCerts", PluginUtil.ConvertToBoolean(acceptUntrustedCerts)); } string fetchDepth = executionContext.GetInput("fetchDepth"); if (!string.IsNullOrEmpty(fetchDepth)) { repository.Properties.Set <string>("fetchDepth", fetchDepth); } } }
public void VerifyJobRequestMessagePiiDataIsScrubbed() { // Arrange Pipelines.AgentJobRequestMessage message = CreateJobRequestMessage("jobwithpiidata"); // Populate PII variables foreach (string piiVariable in Variables.PiiVariables) { message.Variables.Add(piiVariable, "MyPiiVariable"); } foreach (string piiVariableSuffix in Variables.PiiArtifactVariableSuffixes) { message.Variables.Add($"{Variables.PiiArtifactVariablePrefix}.MyArtifact.{piiVariableSuffix}", "MyPiiVariable"); } // Populate the repository PII data Pipelines.RepositoryResource repository = new Pipelines.RepositoryResource(); repository.Properties.Set( Pipelines.RepositoryPropertyNames.VersionInfo, new Pipelines.VersionInfo() { Author = "MyAuthor", Message = "MyMessage" }); message.Resources.Repositories.Add(repository); // Act Pipelines.AgentJobRequestMessage scrubbedMessage = WorkerUtilities.ScrubPiiData(message); // Assert foreach (string piiVariable in Variables.PiiVariables) { scrubbedMessage.Variables.TryGetValue(piiVariable, out VariableValue value); Assert.Equal("[PII]", value.Value); } foreach (string piiVariableSuffix in Variables.PiiArtifactVariableSuffixes) { scrubbedMessage.Variables.TryGetValue($"{Variables.PiiArtifactVariablePrefix}.MyArtifact.{piiVariableSuffix}", out VariableValue value); Assert.Equal("[PII]", value.Value); } Pipelines.RepositoryResource scrubbedRepo = scrubbedMessage.Resources.Repositories[0]; Pipelines.VersionInfo scrubbedInfo = scrubbedRepo.Properties.Get <Pipelines.VersionInfo>(Pipelines.RepositoryPropertyNames.VersionInfo); Assert.Equal("[PII]", scrubbedInfo.Author); }
private Pipelines.RepositoryResource GetRepository(TestHostContext hostContext, String alias, String relativePath) { var workFolder = hostContext.GetDirectory(WellKnownDirectory.Work); var repo = new Pipelines.RepositoryResource() { Alias = alias, Type = Pipelines.RepositoryTypes.Git, }; repo.Properties.Set <string>(Pipelines.RepositoryPropertyNames.Path, Path.Combine(workFolder, "1", relativePath)); return(repo); }
/// <summary> /// Initializes svn command path and execution environment /// </summary> /// <param name="context">The build commands' execution context</param> /// <param name="endpoint">The Subversion server endpoint providing URL, username/password, and untrasted certs acceptace information</param> /// <param name="cancellationToken">The cancellation token used to stop svn command execution</param> public void Init( AgentTaskPluginExecutionContext context, Pipelines.RepositoryResource repository, CancellationToken cancellationToken) { // Validation. ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(repository, nameof(repository)); ArgUtil.NotNull(cancellationToken, nameof(cancellationToken)); ArgUtil.NotNull(repository.Url, nameof(repository.Url)); ArgUtil.Equal(true, repository.Url.IsAbsoluteUri, nameof(repository.Url.IsAbsoluteUri)); ArgUtil.NotNull(repository.Endpoint, nameof(repository.Endpoint)); ServiceEndpoint endpoint = context.Endpoints.Single( x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) || (repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase))); ArgUtil.NotNull(endpoint.Data, nameof(endpoint.Data)); ArgUtil.NotNull(endpoint.Authorization, nameof(endpoint.Authorization)); ArgUtil.NotNull(endpoint.Authorization.Parameters, nameof(endpoint.Authorization.Parameters)); ArgUtil.Equal(EndpointAuthorizationSchemes.UsernamePassword, endpoint.Authorization.Scheme, nameof(endpoint.Authorization.Scheme)); _context = context; _repository = repository; _cancellationToken = cancellationToken; // Find svn in %Path% string svnPath = WhichUtil.Which("svn", trace: context); if (string.IsNullOrEmpty(svnPath)) { throw new Exception(StringUtil.Loc("SvnNotInstalled")); } else { _context.Debug($"Found svn installation path: {svnPath}."); _svn = svnPath; } // External providers may need basic auth or tokens endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out _username); endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out _password); if (endpoint.Data.TryGetValue(EndpointData.AcceptUntrustedCertificates, out string endpointAcceptUntrustedCerts)) { _acceptUntrusted = StringUtil.ConvertToBoolean(endpointAcceptUntrustedCerts); } _acceptUntrusted = _acceptUntrusted || (context.GetCertConfiguration()?.SkipServerCertificateValidation ?? false); }
private Pipelines.RepositoryResource GetRepository(TestHostContext hostContext, String alias, String relativePath, String Name) { var workFolder = hostContext.GetDirectory(WellKnownDirectory.Work); var repo = new Pipelines.RepositoryResource() { Alias = alias, Type = Pipelines.RepositoryTypes.ExternalGit, Id = alias, Url = new Uri($"http://contoso.visualstudio.com/{Name}"), Name = Name, }; repo.Properties.Set <string>(Pipelines.RepositoryPropertyNames.Path, Path.Combine(workFolder, "1", relativePath)); return(repo); }
private void Setup(TestHostContext hostContext) { var repo = new Pipelines.RepositoryResource() { Alias = "myRepo", Type = Pipelines.RepositoryTypes.Git, }; repo.Properties.Set <string>(Pipelines.RepositoryPropertyNames.Path, Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Work), "1", "s")); _executionContext = new AgentTaskPluginExecutionContext(hostContext.GetTrace()) { Endpoints = new List <ServiceEndpoint>(), Inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase) { { Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, "myRepo" }, }, Repositories = new List <Pipelines.RepositoryResource> { repo }, Variables = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase) { { "agent.builddirectory", Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Work), "1") }, { "agent.workfolder", hostContext.GetDirectory(WellKnownDirectory.Work) }, { "agent.tempdirectory", hostContext.GetDirectory(WellKnownDirectory.Temp) } }, }; _sourceProvider = new Mock <ISourceProvider>(); _sourceProviderFactory = new Mock <ISourceProviderFactory>(); _sourceProviderFactory .Setup(x => x.GetSourceProvider(It.IsAny <String>())) .Returns(_sourceProvider.Object); _checkoutTask = new CheckoutTask(_sourceProviderFactory.Object); }
/// <summary> /// Initializes svn command path and execution environment /// </summary> /// <param name="context">The build commands' execution context</param> /// <param name="endpoint">The Subversion server endpoint providing URL, username/password, and untrasted certs acceptace information</param> /// <param name="cancellationToken">The cancellation token used to stop svn command execution</param> public void Init( AgentTaskPluginExecutionContext context, Pipelines.RepositoryResource repository, CancellationToken cancellationToken) { // Validation. PluginUtil.NotNull(context, nameof(context)); PluginUtil.NotNull(repository, nameof(repository)); PluginUtil.NotNull(cancellationToken, nameof(cancellationToken)); PluginUtil.NotNull(repository.Url, nameof(repository.Url)); PluginUtil.Equal(true, repository.Url.IsAbsoluteUri, nameof(repository.Url.IsAbsoluteUri)); PluginUtil.NotNull(repository.Endpoint, nameof(repository.Endpoint)); ServiceEndpoint endpoint = context.Endpoints.Single(x => x.Id == repository.Endpoint.Id); PluginUtil.NotNull(endpoint.Data, nameof(endpoint.Data)); PluginUtil.NotNull(endpoint.Authorization, nameof(endpoint.Authorization)); PluginUtil.NotNull(endpoint.Authorization.Parameters, nameof(endpoint.Authorization.Parameters)); PluginUtil.Equal(EndpointAuthorizationSchemes.UsernamePassword, endpoint.Authorization.Scheme, nameof(endpoint.Authorization.Scheme)); _context = context; _repository = repository; _cancellationToken = cancellationToken; // Find svn in %Path% string svnPath = PluginUtil.Which("svn"); if (string.IsNullOrEmpty(svnPath)) { throw new Exception(PluginUtil.Loc("SvnNotInstalled")); } else { _context.Debug($"Found svn installation path: {svnPath}."); _svn = svnPath; } // External providers may need basic auth or tokens endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out _username); endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out _password); _acceptUntrusted = PluginUtil.ConvertToBoolean(repository.Properties.Get <string>(EndpointData.SvnAcceptUntrustedCertificates)); }
protected void MergeCheckoutOptions(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository) { // Merge the repository checkout options if ((!executionContext.Variables.TryGetValue("MERGE_CHECKOUT_OPTIONS", out VariableValue mergeCheckoutOptions) || !String.Equals(mergeCheckoutOptions.Value, "false", StringComparison.OrdinalIgnoreCase)) && repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions) is JObject checkoutOptions) { foreach (var pair in checkoutOptions) { var inputName = pair.Key; // Skip if unexpected checkout option if (!_checkoutOptions.Contains(inputName)) { executionContext.Debug($"Unexpected checkout option '{inputName}'"); continue; } // Skip if input defined if (executionContext.Inputs.TryGetValue(inputName, out string inputValue) && !string.IsNullOrEmpty(inputValue)) { continue; } try { executionContext.Inputs[inputName] = pair.Value.ToObject <String>(); } catch (Exception ex) { executionContext.Debug($"Error setting the checkout option '{inputName}': {ex.Message}"); } } } }
public Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository) { return(Task.CompletedTask); }
public static Pipelines.AgentJobRequestMessage ScrubPiiData(Pipelines.AgentJobRequestMessage message) { ArgUtil.NotNull(message, nameof(message)); var scrubbedVariables = new Dictionary <string, VariableValue>(); // Scrub the known PII variables foreach (var variable in message.Variables) { if (Variables.PiiVariables.Contains(variable.Key) || (variable.Key.StartsWith(Variables.PiiArtifactVariablePrefix, StringComparison.OrdinalIgnoreCase) && Variables.PiiArtifactVariableSuffixes.Any(varSuffix => variable.Key.EndsWith(varSuffix, StringComparison.OrdinalIgnoreCase)))) { scrubbedVariables[variable.Key] = "[PII]"; } else { scrubbedVariables[variable.Key] = variable.Value; } } var scrubbedRepositories = new List <Pipelines.RepositoryResource>(); // Scrub the repository resources foreach (var repository in message.Resources.Repositories) { Pipelines.RepositoryResource scrubbedRepository = repository.Clone(); var versionInfo = repository.Properties.Get <Pipelines.VersionInfo>(Pipelines.RepositoryPropertyNames.VersionInfo); if (versionInfo != null) { scrubbedRepository.Properties.Set( Pipelines.RepositoryPropertyNames.VersionInfo, new Pipelines.VersionInfo() { Author = "[PII]" }); } scrubbedRepositories.Add(scrubbedRepository); } var scrubbedJobResources = new Pipelines.JobResources(); scrubbedJobResources.Containers.AddRange(message.Resources.Containers); scrubbedJobResources.Endpoints.AddRange(message.Resources.Endpoints); scrubbedJobResources.Repositories.AddRange(scrubbedRepositories); scrubbedJobResources.SecureFiles.AddRange(message.Resources.SecureFiles); // Reconstitute a new agent job request message from the scrubbed parts return(new Pipelines.AgentJobRequestMessage( plan: message.Plan, timeline: message.Timeline, jobId: message.JobId, jobDisplayName: message.JobDisplayName, jobName: message.JobName, jobContainer: message.JobContainer, jobSidecarContainers: message.JobSidecarContainers, variables: scrubbedVariables, maskHints: message.MaskHints, jobResources: scrubbedJobResources, workspaceOptions: message.Workspace, steps: message.Steps)); }
public async Task GetSourceAsync( AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository, CancellationToken cancellationToken) { // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(repository, nameof(repository)); SvnCliManager svn = new SvnCliManager(); svn.Init(executionContext, repository, cancellationToken); // Determine the sources directory. string sourcesDirectory = repository.Properties.Get <string>("sourcedirecotry"); executionContext.Debug($"sourcesDirectory={sourcesDirectory}"); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); string sourceBranch = repository.Properties.Get <string>("sourcebranch"); executionContext.Debug($"sourceBranch={sourceBranch}"); string revision = repository.Version; if (string.IsNullOrWhiteSpace(revision)) { revision = "HEAD"; } executionContext.Debug($"revision={revision}"); bool clean = StringUtil.ConvertToBoolean(repository.Properties.Get <string>(EndpointData.Clean)); executionContext.Debug($"clean={clean}"); // Get the definition mappings. List <SvnMappingDetails> allMappings = JsonConvert.DeserializeObject <SvnWorkspace>(repository.Properties.Get <string>(EndpointData.SvnWorkspaceMapping)).Mappings; if (StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("system.debug")?.Value)) { allMappings.ForEach(m => executionContext.Debug($"ServerPath: {m.ServerPath}, LocalPath: {m.LocalPath}, Depth: {m.Depth}, Revision: {m.Revision}, IgnoreExternals: {m.IgnoreExternals}")); } Dictionary <string, SvnMappingDetails> normalizedMappings = svn.NormalizeMappings(allMappings); if (StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("system.debug")?.Value)) { executionContext.Debug($"Normalized mappings count: {normalizedMappings.Count}"); normalizedMappings.ToList().ForEach(p => executionContext.Debug($" [{p.Key}] ServerPath: {p.Value.ServerPath}, LocalPath: {p.Value.LocalPath}, Depth: {p.Value.Depth}, Revision: {p.Value.Revision}, IgnoreExternals: {p.Value.IgnoreExternals}")); } string normalizedBranch = svn.NormalizeRelativePath(sourceBranch, '/', '\\'); executionContext.Output(StringUtil.Loc("SvnSyncingRepo", repository.Properties.Get <string>("name"))); string effectiveRevision = await svn.UpdateWorkspace( sourcesDirectory, normalizedMappings, clean, normalizedBranch, revision); executionContext.Output(StringUtil.Loc("SvnBranchCheckedOut", normalizedBranch, repository.Properties.Get <string>("name"), effectiveRevision)); }
// TODO: Updates legacy config. private TestHostContext Setup( [CallerMemberName] string name = "", BuildCleanOption?cleanOption = null, ExistingConfigKind existingConfigKind = ExistingConfigKind.None) { // Setup the host context. TestHostContext hc = new TestHostContext(this, name); // Create a random work path. var configStore = new Mock <IConfigurationStore>(); _workFolder = hc.GetDirectory(WellKnownDirectory.Work); var settings = new AgentSettings() { WorkFolder = _workFolder }; configStore.Setup(x => x.GetSettings()).Returns(settings); hc.SetSingleton <IConfigurationStore>(configStore.Object); // Setup the execution context. _ec = new Mock <IExecutionContext>(); List <string> warnings; _variables = new Variables(hc, new Dictionary <string, VariableValue>(), out warnings); _variables.Set(Constants.Variables.System.CollectionId, CollectionId); _variables.Set(Constants.Variables.System.DefinitionId, DefinitionId); _variables.Set(Constants.Variables.Build.Clean, $"{cleanOption}"); _ec.Setup(x => x.Variables).Returns(_variables); // Store the expected tracking file path. _trackingFile = Path.Combine( _workFolder, Constants.Build.Path.SourceRootMappingDirectory, _ec.Object.Variables.System_CollectionId, _ec.Object.Variables.System_DefinitionId, Constants.Build.Path.TrackingConfigFile); // Setup the endpoint. _repository = new Pipelines.RepositoryResource() { Alias = "self", Type = Pipelines.RepositoryTypes.Git, Url = new Uri("http://contoso.visualstudio.com"), }; _repository.Properties.Set <String>(Pipelines.RepositoryPropertyNames.Name, "Some endpoint name"); _workspaceOptions = new Pipelines.WorkspaceOptions(); // // Setup the source provider. // _sourceProvider = new Mock<ISourceProvider>(); // _sourceProvider // .Setup(x => x.GetBuildDirectoryHashKey(_ec.Object, _repository)) // .Returns(HashKey); // hc.SetSingleton<ISourceProvider>(_sourceProvider.Object); // Store the existing config object. switch (existingConfigKind) { case ExistingConfigKind.Matching: _existingConfig = new TrackingConfig(_ec.Object, _repository, 1, HashKey); Assert.Equal("1", _existingConfig.BuildDirectory); break; case ExistingConfigKind.Nonmatching: _existingConfig = new TrackingConfig(_ec.Object, _repository, 2, NonmatchingHashKey); Assert.Equal("2", _existingConfig.BuildDirectory); break; case ExistingConfigKind.None: break; default: throw new NotSupportedException(); } // Store the new config object. if (existingConfigKind == ExistingConfigKind.Matching) { _newConfig = _existingConfig; } else { _newConfig = new TrackingConfig(_ec.Object, _repository, 3, HashKey); Assert.Equal("3", _newConfig.BuildDirectory); } // Setup the tracking manager. _trackingManager = new Mock <ITrackingManager>(); _trackingManager .Setup(x => x.LoadIfExists(_ec.Object, _trackingFile)) .Returns(_existingConfig); if (existingConfigKind == ExistingConfigKind.None || existingConfigKind == ExistingConfigKind.Nonmatching) { _trackingManager .Setup(x => x.Create(_ec.Object, _repository, HashKey, _trackingFile, false)) .Returns(_newConfig); if (existingConfigKind == ExistingConfigKind.Nonmatching) { _trackingManager .Setup(x => x.MarkForGarbageCollection(_ec.Object, _existingConfig)); } } else if (existingConfigKind == ExistingConfigKind.Matching) { _trackingManager .Setup(x => x.UpdateJobRunProperties(_ec.Object, _existingConfig, _trackingFile)); } else { throw new NotSupportedException(); } hc.SetSingleton <ITrackingManager>(_trackingManager.Object); // Setup the build directory manager. _buildDirectoryManager = new BuildDirectoryManager(); _buildDirectoryManager.Initialize(hc); return(hc); }
public async Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository) { if (_undoShelvesetPendingChanges) { string shelvesetName = repository.Properties.Get <string>("SourceTfvcShelveset"); executionContext.Debug($"Undo pending changes left by shelveset '{shelvesetName}'."); // Create the tf command manager. #if OS_WINDOWS var tf = new TFCliManager(); #else var tf = new TeeCliManager(); #endif tf.CancellationToken = CancellationToken.None; tf.Repository = repository; tf.ExecutionContext = executionContext; if (repository.Endpoint != null) { // the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection) // or a real service endpoint to external service which has a real id var endpoint = executionContext.Endpoints.Single(x => (x.Id == Guid.Empty && x.Name == repository.Endpoint.Name) || (x.Id != Guid.Empty && x.Id == repository.Endpoint.Id)); ArgUtil.NotNull(endpoint, nameof(endpoint)); tf.Endpoint = endpoint; } // Get the definition mappings. DefinitionWorkspaceMapping[] definitionMappings = JsonConvert.DeserializeObject <DefinitionWorkspaceMappings>(repository.Properties.Get <string>(EndpointData.TfvcWorkspaceMapping))?.Mappings; // Determine the sources directory. string sourcesDirectory = repository.Properties.Get <string>("sourcedirectory"); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); try { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } } } } catch (Exception ex) { // We can't undo pending changes, log a warning and continue. executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } }
public async Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository) { bool undoShelvesetPendingChanges = StringUtil.ConvertToBoolean(executionContext.TaskVariables.GetValueOrDefault("UndoShelvesetPendingChanges")?.Value); if (undoShelvesetPendingChanges) { string shelvesetName = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Shelveset); executionContext.Debug($"Undo pending changes left by shelveset '{shelvesetName}'."); // Create the tf command manager. ITfsVCCliManager tf; if (PlatformUtil.RunningOnWindows) { tf = new TFCliManager(); } else { tf = new TeeCliManager(); } tf.CancellationToken = CancellationToken.None; tf.Repository = repository; tf.ExecutionContext = executionContext; if (repository.Endpoint != null) { // the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection) // or a real service endpoint to external service which has a real id var endpoint = executionContext.Endpoints.Single( x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) || (repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase))); ArgUtil.NotNull(endpoint, nameof(endpoint)); tf.Endpoint = endpoint; } // Get the definition mappings. var workspaceMappings = repository.Properties.Get <IList <Pipelines.WorkspaceMapping> >(Pipelines.RepositoryPropertyNames.Mappings); DefinitionWorkspaceMapping[] definitionMappings = workspaceMappings.Select(x => new DefinitionWorkspaceMapping() { ServerPath = x.ServerPath, LocalPath = x.LocalPath, MappingType = x.Exclude ? DefinitionMappingType.Cloak : DefinitionMappingType.Map }).ToArray(); // Determine the sources directory. string sourcesDirectory = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); try { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } } } } catch (Exception ex) { // We can't undo pending changes, log a warning and continue. executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } }
public async Task GetSourceAsync( AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository, CancellationToken cancellationToken) { // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(repository, nameof(repository)); // Validate .NET Framework 4.6 or higher is installed. if (PlatformUtil.RunningOnWindows && !NetFrameworkUtil.Test(new Version(4, 6), executionContext)) { throw new Exception(StringUtil.Loc("MinimumNetFramework46")); } // determine if we've been asked to suppress some checkout step output bool reducedOutput = AgentKnobs.QuietCheckout.GetValue(executionContext).AsBoolean(); if (reducedOutput) { executionContext.Output(StringUtil.Loc("QuietCheckoutModeRequested")); executionContext.SetTaskVariable(AgentKnobs.QuietCheckoutRuntimeVarName, Boolean.TrueString); } // Create the tf command manager. ITfsVCCliManager tf; if (PlatformUtil.RunningOnWindows) { tf = new TFCliManager(); } else { tf = new TeeCliManager(); } tf.CancellationToken = cancellationToken; tf.Repository = repository; tf.ExecutionContext = executionContext; if (repository.Endpoint != null) { // the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection) // or a real service endpoint to external service which has a real id var endpoint = executionContext.Endpoints.Single( x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) || (repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase))); ArgUtil.NotNull(endpoint, nameof(endpoint)); tf.Endpoint = endpoint; } // Setup proxy. var agentProxy = executionContext.GetProxyConfiguration(); if (agentProxy != null && !string.IsNullOrEmpty(agentProxy.ProxyAddress) && !agentProxy.WebProxy.IsBypassed(repository.Url)) { executionContext.Debug($"Configure '{tf.FilePath}' to work through proxy server '{agentProxy.ProxyAddress}'."); tf.SetupProxy(agentProxy.ProxyAddress, agentProxy.ProxyUsername, agentProxy.ProxyPassword); } // Setup client certificate. var agentCertManager = executionContext.GetCertConfiguration(); if (agentCertManager != null && agentCertManager.SkipServerCertificateValidation) { executionContext.Debug("TF does not support ignoring SSL certificate validation error."); } // prepare client cert, if the repository's endpoint url match the TFS/VSTS url var systemConnection = executionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); if (!string.IsNullOrEmpty(agentCertManager?.ClientCertificateFile) && Uri.Compare(repository.Url, systemConnection.Url, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0) { executionContext.Debug($"Configure '{tf.FilePath}' to work with client cert '{agentCertManager.ClientCertificateFile}'."); tf.SetupClientCertificate(agentCertManager.ClientCertificateFile, agentCertManager.ClientCertificatePrivateKeyFile, agentCertManager.ClientCertificateArchiveFile, agentCertManager.ClientCertificatePassword); } // Add TF to the PATH. string tfPath = tf.FilePath; ArgUtil.File(tfPath, nameof(tfPath)); executionContext.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", PathUtil.PathVariable, Path.GetFileName(tfPath))); executionContext.PrependPath(Path.GetDirectoryName(tfPath)); executionContext.Debug($"PATH: '{Environment.GetEnvironmentVariable("PATH")}'"); if (PlatformUtil.RunningOnWindows) { // Set TFVC_BUILDAGENT_POLICYPATH string policyDllPath = Path.Combine(executionContext.Variables.GetValueOrDefault("Agent.HomeDirectory")?.Value, "externals", "tf", "Microsoft.TeamFoundation.VersionControl.Controls.dll"); ArgUtil.File(policyDllPath, nameof(policyDllPath)); const string policyPathEnvKey = "TFVC_BUILDAGENT_POLICYPATH"; executionContext.Output(StringUtil.Loc("SetEnvVar", policyPathEnvKey)); executionContext.SetVariable(policyPathEnvKey, policyDllPath); } // Check if the administrator accepted the license terms of the TEE EULA when configuring the agent. if (tf.Features.HasFlag(TfsVCFeatures.Eula) && StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("Agent.AcceptTeeEula")?.Value)) { // Check if the "tf eula -accept" command needs to be run for the current user. bool skipEula = false; try { skipEula = tf.TestEulaAccepted(); } catch (Exception ex) { executionContext.Debug("Unexpected exception while testing whether the TEE EULA has been accepted for the current user."); executionContext.Debug(ex.ToString()); } if (!skipEula) { // Run the command "tf eula -accept". try { await tf.EulaAsync(); } catch (Exception ex) { executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } } // Get the workspaces. executionContext.Output(StringUtil.Loc("QueryingWorkspaceInfo")); ITfsVCWorkspace[] tfWorkspaces = await tf.WorkspacesAsync(); // Determine the workspace name. string buildDirectory = executionContext.Variables.GetValueOrDefault("agent.builddirectory")?.Value; ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory)); string workspaceName = $"ws_{Path.GetFileName(buildDirectory)}_{executionContext.Variables.GetValueOrDefault("agent.id")?.Value}"; executionContext.SetVariable("build.repository.tfvc.workspace", workspaceName); // Get the definition mappings. var workspaceMappings = repository.Properties.Get <IList <Pipelines.WorkspaceMapping> >(Pipelines.RepositoryPropertyNames.Mappings); DefinitionWorkspaceMapping[] definitionMappings = workspaceMappings.Select(x => new DefinitionWorkspaceMapping() { ServerPath = x.ServerPath, LocalPath = x.LocalPath, MappingType = x.Exclude ? DefinitionMappingType.Cloak : DefinitionMappingType.Map }).ToArray(); // Determine the sources directory. string sourcesDirectory = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); // Attempt to re-use an existing workspace if the command manager supports scorch // or if clean is not specified. ITfsVCWorkspace existingTFWorkspace = null; bool clean = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean)); if (tf.Features.HasFlag(TfsVCFeatures.Scorch) || !clean) { existingTFWorkspace = WorkspaceUtil.MatchExactWorkspace( executionContext: executionContext, tfWorkspaces: tfWorkspaces, name: workspaceName, definitionMappings: definitionMappings, sourcesDirectory: sourcesDirectory); if (existingTFWorkspace != null) { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } } } // Scorch. if (clean) { // Try to scorch. try { await tf.ScorchAsync(); } catch (ProcessExitCodeException ex) { // Scorch failed. // Warn, drop the folder, and re-clone. executionContext.Warning(ex.Message); existingTFWorkspace = null; } } } } // Create a new workspace. if (existingTFWorkspace == null) { // Remove any conflicting workspaces. await RemoveConflictingWorkspacesAsync( tf : tf, tfWorkspaces : tfWorkspaces, name : workspaceName, directory : sourcesDirectory); // Remove any conflicting workspace from a different computer. // This is primarily a hosted scenario where a registered hosted // agent can land on a different computer each time. tfWorkspaces = await tf.WorkspacesAsync(matchWorkspaceNameOnAnyComputer : true); foreach (ITfsVCWorkspace tfWorkspace in tfWorkspaces ?? new ITfsVCWorkspace[0]) { await tf.TryWorkspaceDeleteAsync(tfWorkspace); } // Recreate the sources directory. executionContext.Debug($"Deleting: '{sourcesDirectory}'."); IOUtil.DeleteDirectory(sourcesDirectory, cancellationToken); Directory.CreateDirectory(sourcesDirectory); // Create the workspace. await tf.WorkspaceNewAsync(); // Remove the default mapping. if (tf.Features.HasFlag(TfsVCFeatures.DefaultWorkfoldMap)) { await tf.WorkfoldUnmapAsync("$/"); } // Sort the definition mappings. definitionMappings = (definitionMappings ?? new DefinitionWorkspaceMapping[0]) .OrderBy(x => x.NormalizedServerPath?.Length ?? 0) // By server path length. .ToArray() ?? new DefinitionWorkspaceMapping[0]; // Add the definition mappings to the workspace. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings) { switch (definitionMapping.MappingType) { case DefinitionMappingType.Cloak: // Add the cloak. await tf.WorkfoldCloakAsync(serverPath : definitionMapping.ServerPath); break; case DefinitionMappingType.Map: // Add the mapping. await tf.WorkfoldMapAsync( serverPath : definitionMapping.ServerPath, localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory)); break; default: throw new NotSupportedException(); } } } if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Get. await tf.GetAsync(localPath : sourcesDirectory, quiet : reducedOutput); } else { // Perform "get" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { await tf.GetAsync(localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory), quiet : reducedOutput); } } } // Steps for shelveset/gated. string shelvesetName = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Shelveset); if (!string.IsNullOrEmpty(shelvesetName)) { // Steps for gated. ITfsVCShelveset tfShelveset = null; string gatedShelvesetName = executionContext.Variables.GetValueOrDefault("build.gated.shelvesetname")?.Value; if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Clean the last-saved-checkin-metadata for existing workspaces. // // A better long term fix is to add a switch to "tf unshelve" that completely overwrites // the last-saved-checkin-metadata, instead of merging associated work items. // // The targeted workaround for now is to create a trivial change and "tf shelve /move", // which will delete the last-saved-checkin-metadata. if (existingTFWorkspace != null) { executionContext.Output("Cleaning last saved checkin metadata."); // Find a local mapped directory. string firstLocalDirectory = (definitionMappings ?? new DefinitionWorkspaceMapping[0]) .Where(x => x.MappingType == DefinitionMappingType.Map) .Select(x => x.GetRootedLocalPath(sourcesDirectory)) .FirstOrDefault(x => Directory.Exists(x)); if (firstLocalDirectory == null) { executionContext.Warning("No mapped folder found. Unable to clean last-saved-checkin-metadata."); } else { // Create a trival change and "tf shelve /move" to clear the // last-saved-checkin-metadata. string cleanName = "__tf_clean_wksp_metadata"; string tempCleanFile = Path.Combine(firstLocalDirectory, cleanName); try { File.WriteAllText(path: tempCleanFile, contents: "clean last-saved-checkin-metadata", encoding: Encoding.UTF8); await tf.AddAsync(tempCleanFile); await tf.ShelveAsync(shelveset : cleanName, commentFile : tempCleanFile, move : true); } catch (Exception ex) { executionContext.Warning($"Unable to clean last-saved-checkin-metadata. {ex.Message}"); try { await tf.UndoAsync(tempCleanFile); } catch (Exception ex2) { executionContext.Warning($"Unable to undo '{tempCleanFile}'. {ex2.Message}"); } } finally { IOUtil.DeleteFile(tempCleanFile); } } } // Get the shelveset metadata. tfShelveset = await tf.ShelvesetsAsync(shelveset : shelvesetName); // The above command throws if the shelveset is not found, // so the following assertion should never fail. ArgUtil.NotNull(tfShelveset, nameof(tfShelveset)); } // Unshelve. bool unshelveErrorsAllowed = AgentKnobs.AllowTfvcUnshelveErrors.GetValue(executionContext).AsBoolean(); await tf.UnshelveAsync(shelveset : shelvesetName, unshelveErrorsAllowed); // Ensure we undo pending changes for shelveset build at the end. executionContext.SetTaskVariable("UndoShelvesetPendingChanges", bool.TrueString); if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Create the comment file for reshelve. StringBuilder comment = new StringBuilder(tfShelveset.Comment ?? string.Empty); string runCi = executionContext.Variables.GetValueOrDefault("build.gated.runci")?.Value; bool gatedRunCi = StringUtil.ConvertToBoolean(runCi, true); if (!gatedRunCi) { if (comment.Length > 0) { comment.AppendLine(); } comment.Append("***NO_CI***"); } string commentFile = null; try { commentFile = Path.GetTempFileName(); File.WriteAllText(path: commentFile, contents: comment.ToString(), encoding: Encoding.UTF8); // Reshelve. await tf.ShelveAsync(shelveset : gatedShelvesetName, commentFile : commentFile, move : false); } finally { // Cleanup the comment file. if (File.Exists(commentFile)) { File.Delete(commentFile); } } } } // Cleanup proxy settings. if (agentProxy != null && !string.IsNullOrEmpty(agentProxy.ProxyAddress) && !agentProxy.WebProxy.IsBypassed(repository.Url)) { executionContext.Debug($"Remove proxy setting for '{tf.FilePath}' to work through proxy server '{agentProxy.ProxyAddress}'."); tf.CleanupProxySetting(); } // Set intra-task variable for post job cleanup executionContext.SetTaskVariable("repository", repository.Alias); }