public async Task PostJobCleanupAsync(IExecutionContext executionContext, ServiceEndpoint endpoint) { if (_undoShelvesetPendingChanges) { string shelvesetName = GetEndpointData(endpoint, Constants.EndpointData.SourceTfvcShelveset); executionContext.Debug($"Undo pending changes left by shelveset '{shelvesetName}'."); // Create the tf command manager. var tf = HostContext.CreateService <ITfsVCCommandManager>(); tf.CancellationToken = executionContext.CancellationToken; tf.Endpoint = endpoint; tf.ExecutionContext = executionContext; // Get the definition mappings. DefinitionWorkspaceMapping[] definitionMappings = JsonConvert.DeserializeObject <DefinitionWorkspaceMappings>(endpoint.Data[WellKnownEndpointData.TfvcWorkspaceMapping])?.Mappings; // Determine the sources directory. string sourcesDirectory = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); try { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, executionContext.CancellationToken); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, executionContext.CancellationToken); }); } } } } } catch (Exception ex) { // We can't undo pending changes, log a warning and continue. executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } }
public async Task GetSourceAsync( IExecutionContext executionContext, ServiceEndpoint endpoint, CancellationToken cancellationToken) { Trace.Entering(); // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(endpoint, nameof(endpoint)); #if OS_WINDOWS // Validate .NET Framework 4.6 or higher is installed. var netFrameworkUtil = HostContext.GetService <INetFrameworkUtil>(); if (!netFrameworkUtil.Test(new Version(4, 6))) { throw new Exception(StringUtil.Loc("MinimumNetFramework46")); } #endif // Create the tf command manager. var tf = HostContext.CreateService <ITfsVCCommandManager>(); tf.CancellationToken = cancellationToken; tf.Endpoint = endpoint; tf.ExecutionContext = executionContext; // Setup proxy. var agentProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(endpoint.Url)) { executionContext.Debug($"Configure '{tf.FilePath}' to work through proxy server '{executionContext.Variables.Agent_ProxyUrl}'."); tf.SetupProxy(executionContext.Variables.Agent_ProxyUrl, executionContext.Variables.Agent_ProxyUsername, executionContext.Variables.Agent_ProxyPassword); } // Add TF to the PATH. string tfPath = tf.FilePath; ArgUtil.File(tfPath, nameof(tfPath)); var varUtil = HostContext.GetService <IVarUtil>(); executionContext.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", Constants.PathVariable, Path.GetFileName(tfPath))); varUtil.PrependPath(Path.GetDirectoryName(tfPath)); executionContext.Debug($"{Constants.PathVariable}: '{Environment.GetEnvironmentVariable(Constants.PathVariable)}'"); #if OS_WINDOWS // Set TFVC_BUILDAGENT_POLICYPATH string policyDllPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.ServerOM), "Microsoft.TeamFoundation.VersionControl.Controls.dll"); ArgUtil.File(policyDllPath, nameof(policyDllPath)); const string policyPathEnvKey = "TFVC_BUILDAGENT_POLICYPATH"; executionContext.Output(StringUtil.Loc("SetEnvVar", policyPathEnvKey)); Environment.SetEnvironmentVariable(policyPathEnvKey, policyDllPath); #endif // Check if the administrator accepted the license terms of the TEE EULA when configuring the agent. AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings(); if (tf.Features.HasFlag(TfsVCFeatures.Eula) && settings.AcceptTeeEula) { // Check if the "tf eula -accept" command needs to be run for the current user. bool skipEula = false; try { skipEula = tf.TestEulaAccepted(); } catch (Exception ex) { executionContext.Debug("Unexpected exception while testing whether the TEE EULA has been accepted for the current user."); executionContext.Debug(ex.ToString()); } if (!skipEula) { // Run the command "tf eula -accept". try { await tf.EulaAsync(); } catch (Exception ex) { executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } } // Get the workspaces. executionContext.Output(StringUtil.Loc("QueryingWorkspaceInfo")); ITfsVCWorkspace[] tfWorkspaces = await tf.WorkspacesAsync(); // Determine the workspace name. string buildDirectory = executionContext.Variables.Agent_BuildDirectory; ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory)); string workspaceName = $"ws_{Path.GetFileName(buildDirectory)}_{settings.AgentId}"; executionContext.Variables.Set(Constants.Variables.Build.RepoTfvcWorkspace, workspaceName); // Get the definition mappings. DefinitionWorkspaceMapping[] definitionMappings = JsonConvert.DeserializeObject <DefinitionWorkspaceMappings>(endpoint.Data[WellKnownEndpointData.TfvcWorkspaceMapping])?.Mappings; // Determine the sources directory. string sourcesDirectory = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); // Attempt to re-use an existing workspace if the command manager supports scorch // or if clean is not specified. ITfsVCWorkspace existingTFWorkspace = null; bool clean = endpoint.Data.ContainsKey(WellKnownEndpointData.Clean) && StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean], defaultValue: false); if (tf.Features.HasFlag(TfsVCFeatures.Scorch) || !clean) { existingTFWorkspace = WorkspaceUtil.MatchExactWorkspace( executionContext: executionContext, tfWorkspaces: tfWorkspaces, name: workspaceName, definitionMappings: definitionMappings, sourcesDirectory: sourcesDirectory); if (existingTFWorkspace != null) { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } } } // Scorch. if (clean) { // Try to scorch. try { await tf.ScorchAsync(); } catch (ProcessExitCodeException ex) { // Scorch failed. // Warn, drop the folder, and re-clone. executionContext.Warning(ex.Message); existingTFWorkspace = null; } } } } // Create a new workspace. if (existingTFWorkspace == null) { // Remove any conflicting workspaces. await RemoveConflictingWorkspacesAsync( tf : tf, tfWorkspaces : tfWorkspaces, name : workspaceName, directory : sourcesDirectory); // Remove any conflicting workspace from a different computer. // This is primarily a hosted scenario where a registered hosted // agent can land on a different computer each time. tfWorkspaces = await tf.WorkspacesAsync(matchWorkspaceNameOnAnyComputer : true); foreach (ITfsVCWorkspace tfWorkspace in tfWorkspaces ?? new ITfsVCWorkspace[0]) { await tf.WorkspaceDeleteAsync(tfWorkspace); } // Recreate the sources directory. executionContext.Debug($"Deleting: '{sourcesDirectory}'."); IOUtil.DeleteDirectory(sourcesDirectory, cancellationToken); Directory.CreateDirectory(sourcesDirectory); // Create the workspace. await tf.WorkspaceNewAsync(); // Remove the default mapping. if (tf.Features.HasFlag(TfsVCFeatures.DefaultWorkfoldMap)) { await tf.WorkfoldUnmapAsync("$/"); } // Sort the definition mappings. definitionMappings = (definitionMappings ?? new DefinitionWorkspaceMapping[0]) .OrderBy(x => x.NormalizedServerPath?.Length ?? 0) // By server path length. .ToArray() ?? new DefinitionWorkspaceMapping[0]; // Add the definition mappings to the workspace. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings) { switch (definitionMapping.MappingType) { case DefinitionMappingType.Cloak: // Add the cloak. await tf.WorkfoldCloakAsync(serverPath : definitionMapping.ServerPath); break; case DefinitionMappingType.Map: // Add the mapping. await tf.WorkfoldMapAsync( serverPath : definitionMapping.ServerPath, localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory)); break; default: throw new NotSupportedException(); } } } if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Get. await tf.GetAsync(localPath : sourcesDirectory); } else { // Perform "get" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { await tf.GetAsync(localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory)); } } } // Steps for shelveset/gated. string shelvesetName = GetEndpointData(endpoint, Constants.EndpointData.SourceTfvcShelveset); if (!string.IsNullOrEmpty(shelvesetName)) { // Steps for gated. ITfsVCShelveset tfShelveset = null; string gatedShelvesetName = GetEndpointData(endpoint, Constants.EndpointData.GatedShelvesetName); if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Clean the last-saved-checkin-metadata for existing workspaces. // // A better long term fix is to add a switch to "tf unshelve" that completely overwrites // the last-saved-checkin-metadata, instead of merging associated work items. // // The targeted workaround for now is to create a trivial change and "tf shelve /move", // which will delete the last-saved-checkin-metadata. if (existingTFWorkspace != null) { executionContext.Output("Cleaning last saved checkin metadata."); // Find a local mapped directory. string firstLocalDirectory = (definitionMappings ?? new DefinitionWorkspaceMapping[0]) .Where(x => x.MappingType == DefinitionMappingType.Map) .Select(x => x.GetRootedLocalPath(sourcesDirectory)) .FirstOrDefault(x => Directory.Exists(x)); if (firstLocalDirectory == null) { executionContext.Warning("No mapped folder found. Unable to clean last-saved-checkin-metadata."); } else { // Create a trival change and "tf shelve /move" to clear the // last-saved-checkin-metadata. string cleanName = "__tf_clean_wksp_metadata"; string tempCleanFile = Path.Combine(firstLocalDirectory, cleanName); try { File.WriteAllText(path: tempCleanFile, contents: "clean last-saved-checkin-metadata", encoding: Encoding.UTF8); await tf.AddAsync(tempCleanFile); await tf.ShelveAsync(shelveset : cleanName, commentFile : tempCleanFile, move : true); } catch (Exception ex) { executionContext.Warning($"Unable to clean last-saved-checkin-metadata. {ex.Message}"); try { await tf.UndoAsync(tempCleanFile); } catch (Exception ex2) { executionContext.Warning($"Unable to undo '{tempCleanFile}'. {ex2.Message}"); } } finally { IOUtil.DeleteFile(tempCleanFile); } } } // Get the shelveset metadata. tfShelveset = await tf.ShelvesetsAsync(shelveset : shelvesetName); // The above command throws if the shelveset is not found, // so the following assertion should never fail. ArgUtil.NotNull(tfShelveset, nameof(tfShelveset)); } // Unshelve. await tf.UnshelveAsync(shelveset : shelvesetName); // Ensure we undo pending changes for shelveset build at the end. _undoShelvesetPendingChanges = true; if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Create the comment file for reshelve. StringBuilder comment = new StringBuilder(tfShelveset.Comment ?? string.Empty); string runCi = GetEndpointData(endpoint, Constants.EndpointData.GatedRunCI); bool gatedRunCi = StringUtil.ConvertToBoolean(runCi, true); if (!gatedRunCi) { if (comment.Length > 0) { comment.AppendLine(); } comment.Append(Constants.Build.NoCICheckInComment); } string commentFile = null; try { commentFile = Path.GetTempFileName(); File.WriteAllText(path: commentFile, contents: comment.ToString(), encoding: Encoding.UTF8); // Reshelve. await tf.ShelveAsync(shelveset : gatedShelvesetName, commentFile : commentFile, move : false); } finally { // Cleanup the comment file. if (File.Exists(commentFile)) { File.Delete(commentFile); } } } } // Cleanup proxy settings. if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(endpoint.Url)) { executionContext.Debug($"Remove proxy setting for '{tf.FilePath}' to work through proxy server '{executionContext.Variables.Agent_ProxyUrl}'."); tf.CleanupProxySetting(); } }
public async Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository) { bool undoShelvesetPendingChanges = StringUtil.ConvertToBoolean(executionContext.TaskVariables.GetValueOrDefault("UndoShelvesetPendingChanges")?.Value); if (undoShelvesetPendingChanges) { string shelvesetName = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Shelveset); executionContext.Debug($"Undo pending changes left by shelveset '{shelvesetName}'."); // Create the tf command manager. ITfsVCCliManager tf; if (PlatformUtil.RunningOnWindows) { tf = new TFCliManager(); } else { tf = new TeeCliManager(); } tf.CancellationToken = CancellationToken.None; tf.Repository = repository; tf.ExecutionContext = executionContext; if (repository.Endpoint != null) { // the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection) // or a real service endpoint to external service which has a real id var endpoint = executionContext.Endpoints.Single( x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) || (repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase))); ArgUtil.NotNull(endpoint, nameof(endpoint)); tf.Endpoint = endpoint; } // Get the definition mappings. var workspaceMappings = repository.Properties.Get <IList <Pipelines.WorkspaceMapping> >(Pipelines.RepositoryPropertyNames.Mappings); DefinitionWorkspaceMapping[] definitionMappings = workspaceMappings.Select(x => new DefinitionWorkspaceMapping() { ServerPath = x.ServerPath, LocalPath = x.LocalPath, MappingType = x.Exclude ? DefinitionMappingType.Cloak : DefinitionMappingType.Map }).ToArray(); // Determine the sources directory. string sourcesDirectory = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); try { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } } } } catch (Exception ex) { // We can't undo pending changes, log a warning and continue. executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } }
public async Task GetSourceAsync( AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository, CancellationToken cancellationToken) { // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(repository, nameof(repository)); // Validate .NET Framework 4.6 or higher is installed. if (PlatformUtil.RunningOnWindows && !NetFrameworkUtil.Test(new Version(4, 6), executionContext)) { throw new Exception(StringUtil.Loc("MinimumNetFramework46")); } // determine if we've been asked to suppress some checkout step output bool reducedOutput = AgentKnobs.QuietCheckout.GetValue(executionContext).AsBoolean(); if (reducedOutput) { executionContext.Output(StringUtil.Loc("QuietCheckoutModeRequested")); executionContext.SetTaskVariable(AgentKnobs.QuietCheckoutRuntimeVarName, Boolean.TrueString); } // Create the tf command manager. ITfsVCCliManager tf; if (PlatformUtil.RunningOnWindows) { tf = new TFCliManager(); } else { tf = new TeeCliManager(); } tf.CancellationToken = cancellationToken; tf.Repository = repository; tf.ExecutionContext = executionContext; if (repository.Endpoint != null) { // the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection) // or a real service endpoint to external service which has a real id var endpoint = executionContext.Endpoints.Single( x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) || (repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase))); ArgUtil.NotNull(endpoint, nameof(endpoint)); tf.Endpoint = endpoint; } // Setup proxy. var agentProxy = executionContext.GetProxyConfiguration(); if (agentProxy != null && !string.IsNullOrEmpty(agentProxy.ProxyAddress) && !agentProxy.WebProxy.IsBypassed(repository.Url)) { executionContext.Debug($"Configure '{tf.FilePath}' to work through proxy server '{agentProxy.ProxyAddress}'."); tf.SetupProxy(agentProxy.ProxyAddress, agentProxy.ProxyUsername, agentProxy.ProxyPassword); } // Setup client certificate. var agentCertManager = executionContext.GetCertConfiguration(); if (agentCertManager != null && agentCertManager.SkipServerCertificateValidation) { executionContext.Debug("TF does not support ignoring SSL certificate validation error."); } // prepare client cert, if the repository's endpoint url match the TFS/VSTS url var systemConnection = executionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); if (!string.IsNullOrEmpty(agentCertManager?.ClientCertificateFile) && Uri.Compare(repository.Url, systemConnection.Url, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0) { executionContext.Debug($"Configure '{tf.FilePath}' to work with client cert '{agentCertManager.ClientCertificateFile}'."); tf.SetupClientCertificate(agentCertManager.ClientCertificateFile, agentCertManager.ClientCertificatePrivateKeyFile, agentCertManager.ClientCertificateArchiveFile, agentCertManager.ClientCertificatePassword); } // Add TF to the PATH. string tfPath = tf.FilePath; ArgUtil.File(tfPath, nameof(tfPath)); executionContext.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", PathUtil.PathVariable, Path.GetFileName(tfPath))); executionContext.PrependPath(Path.GetDirectoryName(tfPath)); executionContext.Debug($"PATH: '{Environment.GetEnvironmentVariable("PATH")}'"); if (PlatformUtil.RunningOnWindows) { // Set TFVC_BUILDAGENT_POLICYPATH string policyDllPath = Path.Combine(executionContext.Variables.GetValueOrDefault("Agent.HomeDirectory")?.Value, "externals", "tf", "Microsoft.TeamFoundation.VersionControl.Controls.dll"); ArgUtil.File(policyDllPath, nameof(policyDllPath)); const string policyPathEnvKey = "TFVC_BUILDAGENT_POLICYPATH"; executionContext.Output(StringUtil.Loc("SetEnvVar", policyPathEnvKey)); executionContext.SetVariable(policyPathEnvKey, policyDllPath); } // Check if the administrator accepted the license terms of the TEE EULA when configuring the agent. if (tf.Features.HasFlag(TfsVCFeatures.Eula) && StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("Agent.AcceptTeeEula")?.Value)) { // Check if the "tf eula -accept" command needs to be run for the current user. bool skipEula = false; try { skipEula = tf.TestEulaAccepted(); } catch (Exception ex) { executionContext.Debug("Unexpected exception while testing whether the TEE EULA has been accepted for the current user."); executionContext.Debug(ex.ToString()); } if (!skipEula) { // Run the command "tf eula -accept". try { await tf.EulaAsync(); } catch (Exception ex) { executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } } // Get the workspaces. executionContext.Output(StringUtil.Loc("QueryingWorkspaceInfo")); ITfsVCWorkspace[] tfWorkspaces = await tf.WorkspacesAsync(); // Determine the workspace name. string buildDirectory = executionContext.Variables.GetValueOrDefault("agent.builddirectory")?.Value; ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory)); string workspaceName = $"ws_{Path.GetFileName(buildDirectory)}_{executionContext.Variables.GetValueOrDefault("agent.id")?.Value}"; executionContext.SetVariable("build.repository.tfvc.workspace", workspaceName); // Get the definition mappings. var workspaceMappings = repository.Properties.Get <IList <Pipelines.WorkspaceMapping> >(Pipelines.RepositoryPropertyNames.Mappings); DefinitionWorkspaceMapping[] definitionMappings = workspaceMappings.Select(x => new DefinitionWorkspaceMapping() { ServerPath = x.ServerPath, LocalPath = x.LocalPath, MappingType = x.Exclude ? DefinitionMappingType.Cloak : DefinitionMappingType.Map }).ToArray(); // Determine the sources directory. string sourcesDirectory = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); // Attempt to re-use an existing workspace if the command manager supports scorch // or if clean is not specified. ITfsVCWorkspace existingTFWorkspace = null; bool clean = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean)); if (tf.Features.HasFlag(TfsVCFeatures.Scorch) || !clean) { existingTFWorkspace = WorkspaceUtil.MatchExactWorkspace( executionContext: executionContext, tfWorkspaces: tfWorkspaces, name: workspaceName, definitionMappings: definitionMappings, sourcesDirectory: sourcesDirectory); if (existingTFWorkspace != null) { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } } } // Scorch. if (clean) { // Try to scorch. try { await tf.ScorchAsync(); } catch (ProcessExitCodeException ex) { // Scorch failed. // Warn, drop the folder, and re-clone. executionContext.Warning(ex.Message); existingTFWorkspace = null; } } } } // Create a new workspace. if (existingTFWorkspace == null) { // Remove any conflicting workspaces. await RemoveConflictingWorkspacesAsync( tf : tf, tfWorkspaces : tfWorkspaces, name : workspaceName, directory : sourcesDirectory); // Remove any conflicting workspace from a different computer. // This is primarily a hosted scenario where a registered hosted // agent can land on a different computer each time. tfWorkspaces = await tf.WorkspacesAsync(matchWorkspaceNameOnAnyComputer : true); foreach (ITfsVCWorkspace tfWorkspace in tfWorkspaces ?? new ITfsVCWorkspace[0]) { await tf.TryWorkspaceDeleteAsync(tfWorkspace); } // Recreate the sources directory. executionContext.Debug($"Deleting: '{sourcesDirectory}'."); IOUtil.DeleteDirectory(sourcesDirectory, cancellationToken); Directory.CreateDirectory(sourcesDirectory); // Create the workspace. await tf.WorkspaceNewAsync(); // Remove the default mapping. if (tf.Features.HasFlag(TfsVCFeatures.DefaultWorkfoldMap)) { await tf.WorkfoldUnmapAsync("$/"); } // Sort the definition mappings. definitionMappings = (definitionMappings ?? new DefinitionWorkspaceMapping[0]) .OrderBy(x => x.NormalizedServerPath?.Length ?? 0) // By server path length. .ToArray() ?? new DefinitionWorkspaceMapping[0]; // Add the definition mappings to the workspace. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings) { switch (definitionMapping.MappingType) { case DefinitionMappingType.Cloak: // Add the cloak. await tf.WorkfoldCloakAsync(serverPath : definitionMapping.ServerPath); break; case DefinitionMappingType.Map: // Add the mapping. await tf.WorkfoldMapAsync( serverPath : definitionMapping.ServerPath, localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory)); break; default: throw new NotSupportedException(); } } } if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Get. await tf.GetAsync(localPath : sourcesDirectory, quiet : reducedOutput); } else { // Perform "get" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { await tf.GetAsync(localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory), quiet : reducedOutput); } } } // Steps for shelveset/gated. string shelvesetName = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Shelveset); if (!string.IsNullOrEmpty(shelvesetName)) { // Steps for gated. ITfsVCShelveset tfShelveset = null; string gatedShelvesetName = executionContext.Variables.GetValueOrDefault("build.gated.shelvesetname")?.Value; if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Clean the last-saved-checkin-metadata for existing workspaces. // // A better long term fix is to add a switch to "tf unshelve" that completely overwrites // the last-saved-checkin-metadata, instead of merging associated work items. // // The targeted workaround for now is to create a trivial change and "tf shelve /move", // which will delete the last-saved-checkin-metadata. if (existingTFWorkspace != null) { executionContext.Output("Cleaning last saved checkin metadata."); // Find a local mapped directory. string firstLocalDirectory = (definitionMappings ?? new DefinitionWorkspaceMapping[0]) .Where(x => x.MappingType == DefinitionMappingType.Map) .Select(x => x.GetRootedLocalPath(sourcesDirectory)) .FirstOrDefault(x => Directory.Exists(x)); if (firstLocalDirectory == null) { executionContext.Warning("No mapped folder found. Unable to clean last-saved-checkin-metadata."); } else { // Create a trival change and "tf shelve /move" to clear the // last-saved-checkin-metadata. string cleanName = "__tf_clean_wksp_metadata"; string tempCleanFile = Path.Combine(firstLocalDirectory, cleanName); try { File.WriteAllText(path: tempCleanFile, contents: "clean last-saved-checkin-metadata", encoding: Encoding.UTF8); await tf.AddAsync(tempCleanFile); await tf.ShelveAsync(shelveset : cleanName, commentFile : tempCleanFile, move : true); } catch (Exception ex) { executionContext.Warning($"Unable to clean last-saved-checkin-metadata. {ex.Message}"); try { await tf.UndoAsync(tempCleanFile); } catch (Exception ex2) { executionContext.Warning($"Unable to undo '{tempCleanFile}'. {ex2.Message}"); } } finally { IOUtil.DeleteFile(tempCleanFile); } } } // Get the shelveset metadata. tfShelveset = await tf.ShelvesetsAsync(shelveset : shelvesetName); // The above command throws if the shelveset is not found, // so the following assertion should never fail. ArgUtil.NotNull(tfShelveset, nameof(tfShelveset)); } // Unshelve. bool unshelveErrorsAllowed = AgentKnobs.AllowTfvcUnshelveErrors.GetValue(executionContext).AsBoolean(); await tf.UnshelveAsync(shelveset : shelvesetName, unshelveErrorsAllowed); // Ensure we undo pending changes for shelveset build at the end. executionContext.SetTaskVariable("UndoShelvesetPendingChanges", bool.TrueString); if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Create the comment file for reshelve. StringBuilder comment = new StringBuilder(tfShelveset.Comment ?? string.Empty); string runCi = executionContext.Variables.GetValueOrDefault("build.gated.runci")?.Value; bool gatedRunCi = StringUtil.ConvertToBoolean(runCi, true); if (!gatedRunCi) { if (comment.Length > 0) { comment.AppendLine(); } comment.Append("***NO_CI***"); } string commentFile = null; try { commentFile = Path.GetTempFileName(); File.WriteAllText(path: commentFile, contents: comment.ToString(), encoding: Encoding.UTF8); // Reshelve. await tf.ShelveAsync(shelveset : gatedShelvesetName, commentFile : commentFile, move : false); } finally { // Cleanup the comment file. if (File.Exists(commentFile)) { File.Delete(commentFile); } } } } // Cleanup proxy settings. if (agentProxy != null && !string.IsNullOrEmpty(agentProxy.ProxyAddress) && !agentProxy.WebProxy.IsBypassed(repository.Url)) { executionContext.Debug($"Remove proxy setting for '{tf.FilePath}' to work through proxy server '{agentProxy.ProxyAddress}'."); tf.CleanupProxySetting(); } // Set intra-task variable for post job cleanup executionContext.SetTaskVariable("repository", repository.Alias); }
public async Task GetSourceAsync( IExecutionContext executionContext, ServiceEndpoint endpoint, CancellationToken cancellationToken) { Trace.Entering(); // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(endpoint, nameof(endpoint)); // Create the tf command manager. var tf = HostContext.CreateService <ITfsVCCommandManager>(); tf.CancellationToken = cancellationToken; tf.Endpoint = endpoint; tf.ExecutionContext = executionContext; // Add TF to the PATH. string tfPath = tf.FilePath; ArgUtil.File(tfPath, nameof(tfPath)); var varUtil = HostContext.GetService <IVarUtil>(); executionContext.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", Constants.PathVariable, Path.GetFileName(tfPath))); varUtil.PrependPath(Path.GetDirectoryName(tfPath)); executionContext.Debug($"{Constants.PathVariable}: '{Environment.GetEnvironmentVariable(Constants.PathVariable)}'"); // Check if the administrator accepted the license terms of the TEE EULA when configuring the agent. AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings(); if (tf.Features.HasFlag(TfsVCFeatures.Eula) && settings.AcceptTeeEula) { // Check if the "tf eula -accept" command needs to be run for the current user. bool skipEula = false; try { skipEula = tf.TestEulaAccepted(); } catch (Exception ex) { executionContext.Debug("Unexpected exception while testing whether the TEE EULA has been accepted for the current user."); executionContext.Debug(ex.ToString()); } if (!skipEula) { // Run the command "tf eula -accept". try { await tf.EulaAsync(); } catch (Exception ex) { executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } } // Get the workspaces. executionContext.Output(StringUtil.Loc("QueryingWorkspaceInfo")); ITfsVCWorkspace[] tfWorkspaces = await tf.WorkspacesAsync(); // Determine the workspace name. string buildDirectory = executionContext.Variables.Agent_BuildDirectory; ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory)); string workspaceName = $"ws_{Path.GetFileName(buildDirectory)}_{settings.AgentId}"; executionContext.Variables.Set(Constants.Variables.Build.RepoTfvcWorkspace, workspaceName); // Get the definition mappings. DefinitionWorkspaceMapping[] definitionMappings = JsonConvert.DeserializeObject <DefinitionWorkspaceMappings>(endpoint.Data[WellKnownEndpointData.TfvcWorkspaceMapping])?.Mappings; // Determine the sources directory. string sourcesDirectory = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); // Attempt to re-use an existing workspace if the command manager supports scorch // or if clean is not specified. ITfsVCWorkspace existingTFWorkspace = null; bool clean = endpoint.Data.ContainsKey(WellKnownEndpointData.Clean) && StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean], defaultValue: false); if (tf.Features.HasFlag(TfsVCFeatures.Scorch) || !clean) { existingTFWorkspace = WorkspaceUtil.MatchExactWorkspace( executionContext: executionContext, tfWorkspaces: tfWorkspaces, name: workspaceName, definitionMappings: definitionMappings, sourcesDirectory: sourcesDirectory); if (existingTFWorkspace != null) { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } } } // Scorch. if (clean) { // Try to scorch. try { await tf.ScorchAsync(); } catch (ProcessExitCodeException ex) { // Scorch failed. // Warn, drop the folder, and re-clone. executionContext.Warning(ex.Message); existingTFWorkspace = null; } } } } // Create a new workspace. if (existingTFWorkspace == null) { // Remove any conflicting workspaces. await RemoveConflictingWorkspacesAsync( tf : tf, tfWorkspaces : tfWorkspaces, name : workspaceName, directory : sourcesDirectory); // Remove any conflicting workspace from a different computer. // This is primarily a hosted scenario where a registered hosted // agent can land on a different computer each time. tfWorkspaces = await tf.WorkspacesAsync(matchWorkspaceNameOnAnyComputer : true); foreach (ITfsVCWorkspace tfWorkspace in tfWorkspaces ?? new ITfsVCWorkspace[0]) { await tf.WorkspaceDeleteAsync(tfWorkspace); } // Recreate the sources directory. executionContext.Debug($"Deleting: '{sourcesDirectory}'."); IOUtil.DeleteDirectory(sourcesDirectory, cancellationToken); Directory.CreateDirectory(sourcesDirectory); // Create the workspace. await tf.WorkspaceNewAsync(); // Remove the default mapping. if (tf.Features.HasFlag(TfsVCFeatures.DefaultWorkfoldMap)) { await tf.WorkfoldUnmapAsync("$/"); } // Sort the definition mappings. definitionMappings = (definitionMappings ?? new DefinitionWorkspaceMapping[0]) .OrderBy(x => x.NormalizedServerPath?.Length ?? 0) // By server path length. .ToArray() ?? new DefinitionWorkspaceMapping[0]; // Add the definition mappings to the workspace. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings) { switch (definitionMapping.MappingType) { case DefinitionMappingType.Cloak: // Add the cloak. await tf.WorkfoldCloakAsync(serverPath : definitionMapping.ServerPath); break; case DefinitionMappingType.Map: // Add the mapping. await tf.WorkfoldMapAsync( serverPath : definitionMapping.ServerPath, localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory)); break; default: throw new NotSupportedException(); } } } if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Get. await tf.GetAsync(localPath : sourcesDirectory); } else { // Perform "get" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { await tf.GetAsync(localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory)); } } } string shelvesetName = GetEndpointData(endpoint, Constants.EndpointData.SourceTfvcShelveset); if (!string.IsNullOrEmpty(shelvesetName)) { // Get the shelveset details. ITfsVCShelveset tfShelveset = null; string gatedShelvesetName = GetEndpointData(endpoint, Constants.EndpointData.GatedShelvesetName); if (!string.IsNullOrEmpty(gatedShelvesetName)) { tfShelveset = await tf.ShelvesetsAsync(shelveset : shelvesetName); // The command throws if the shelveset is not found. // This assertion should never fail. ArgUtil.NotNull(tfShelveset, nameof(tfShelveset)); } // Unshelve. await tf.UnshelveAsync(shelveset : shelvesetName); if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Create the comment file for reshelve. StringBuilder comment = new StringBuilder(tfShelveset.Comment ?? string.Empty); string runCi = GetEndpointData(endpoint, Constants.EndpointData.GatedRunCI); bool gatedRunCi = StringUtil.ConvertToBoolean(runCi, true); if (!gatedRunCi) { if (comment.Length > 0) { comment.AppendLine(); } comment.Append(Constants.Build.NoCICheckInComment); } string commentFile = null; try { commentFile = Path.GetTempFileName(); File.WriteAllText(path: commentFile, contents: comment.ToString(), encoding: Encoding.UTF8); // Reshelve. await tf.ShelveAsync(shelveset : gatedShelvesetName, commentFile : commentFile); } finally { // Cleanup the comment file. if (File.Exists(commentFile)) { File.Delete(commentFile); } } } } }
public async Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository) { if (_undoShelvesetPendingChanges) { string shelvesetName = repository.Properties.Get <string>("SourceTfvcShelveset"); executionContext.Debug($"Undo pending changes left by shelveset '{shelvesetName}'."); // Create the tf command manager. #if OS_WINDOWS var tf = new TFCliManager(); #else var tf = new TeeCliManager(); #endif tf.CancellationToken = CancellationToken.None; tf.Repository = repository; tf.ExecutionContext = executionContext; if (repository.Endpoint != null) { // the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection) // or a real service endpoint to external service which has a real id var endpoint = executionContext.Endpoints.Single(x => (x.Id == Guid.Empty && x.Name == repository.Endpoint.Name) || (x.Id != Guid.Empty && x.Id == repository.Endpoint.Id)); ArgUtil.NotNull(endpoint, nameof(endpoint)); tf.Endpoint = endpoint; } // Get the definition mappings. DefinitionWorkspaceMapping[] definitionMappings = JsonConvert.DeserializeObject <DefinitionWorkspaceMappings>(repository.Properties.Get <string>(EndpointData.TfvcWorkspaceMapping))?.Mappings; // Determine the sources directory. string sourcesDirectory = repository.Properties.Get <string>("sourcedirectory"); ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); try { if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot)) { // Undo pending changes. ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory); if (tfStatus?.HasPendingChanges ?? false) { await tf.UndoAsync(localPath : sourcesDirectory); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } else { // Perform "undo" for each map. foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0]) { if (definitionMapping.MappingType == DefinitionMappingType.Map) { // Check the status. string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory); ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath); if (tfStatus?.HasPendingChanges ?? false) { // Undo. await tf.UndoAsync(localPath : localPath); // Cleanup remaining files/directories from pend adds. tfStatus.AllAdds .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, CancellationToken.None); }); } } } } } catch (Exception ex) { // We can't undo pending changes, log a warning and continue. executionContext.Debug(ex.ToString()); executionContext.Warning(ex.Message); } } }