public void MarkExpiredForGarbageCollection(IExecutionContext executionContext, TimeSpan expiration)
{
Trace.Entering();
Trace.Info("Scan all SourceFolder tracking files.");
string searchRoot = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Build.Path.SourceRootMappingDirectory);
if (!Directory.Exists(searchRoot))
{
executionContext.Output(StringUtil.Loc("GCDirNotExist", searchRoot));
return;
}
var allTrackingFiles = Directory.EnumerateFiles(searchRoot, Constants.Build.Path.TrackingConfigFile, SearchOption.AllDirectories);
Trace.Verbose($"Find {allTrackingFiles.Count()} tracking files.");
executionContext.Output(StringUtil.Loc("DirExpireLimit", expiration.TotalDays));
executionContext.Output(StringUtil.Loc("CurrentUTC", DateTime.UtcNow.ToString("o")));
// scan all sourcefolder tracking file, find which folder has never been used since UTC-expiration
// the scan and garbage discovery should be best effort.
// if the tracking file is in old format, just delete the folder since the first time the folder been use we will convert the tracking file to new format.
foreach (var trackingFile in allTrackingFiles)
{
try
{
executionContext.Output(StringUtil.Loc("EvaluateTrackingFile", trackingFile));
TrackingConfigBase tracking = LoadIfExists(executionContext, trackingFile);
// detect whether the tracking file is in new format.
TrackingConfig newTracking = tracking as TrackingConfig;
if (newTracking == null)
{
LegacyTrackingConfig legacyConfig = tracking as LegacyTrackingConfig;
ArgUtil.NotNull(legacyConfig, nameof(LegacyTrackingConfig));
Trace.Verbose($"{trackingFile} is a old format tracking file.");
executionContext.Output(StringUtil.Loc("GCOldFormatTrackingFile", trackingFile));
MarkForGarbageCollection(executionContext, legacyConfig);
IOUtil.DeleteFile(trackingFile);
}
else
{
Trace.Verbose($"{trackingFile} is a new format tracking file.");
ArgUtil.NotNull(newTracking.LastRunOn, nameof(newTracking.LastRunOn));
executionContext.Output(StringUtil.Loc("BuildDirLastUseTIme", Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newTracking.BuildDirectory), newTracking.LastRunOnString));
if (DateTime.UtcNow - expiration > newTracking.LastRunOn)
{
executionContext.Output(StringUtil.Loc("GCUnusedTrackingFile", trackingFile, expiration.TotalDays));
MarkForGarbageCollection(executionContext, newTracking);
IOUtil.DeleteFile(trackingFile);
}
}
}
catch (Exception ex)
{
executionContext.Error(StringUtil.Loc("ErrorDuringBuildGC", trackingFile));
executionContext.Error(ex);
}
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
// Resolve the target script.
string target = GetTarget();
ArgUtil.NotNullOrEmpty(target, nameof(target));
string scriptFile = Path.Combine(TaskDirectory, target);
ArgUtil.File(scriptFile, nameof(scriptFile));
// Determine the working directory.
string workingDirectory = GetWorkingDirectory();
if (String.IsNullOrEmpty(workingDirectory))
{
workingDirectory = Path.GetDirectoryName(scriptFile);
}
else
{
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
}
// Copy the OM binaries into the legacy host folder.
ExecutionContext.Output(StringUtil.Loc("PrepareTaskExecutionHandler"));
IOUtil.CopyDirectory(
source: HostContext.GetDirectory(WellKnownDirectory.ServerOM),
target: HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost),
cancellationToken: ExecutionContext.CancellationToken);
Trace.Info("Finished copying files.");
// Add the legacy ps host environment variables.
AddLegacyHostEnvironmentVariables(scriptFile: scriptFile, workingDirectory: workingDirectory);
AddPrependPathToEnvironment();
// Add proxy setting to LegacyVSTSPowerShellHost.exe.config
var agentProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentProxy.ProxyAddress))
{
AddProxySetting(agentProxy);
}
// Invoke the process.
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OnDataReceived;
processInvoker.ErrorDataReceived += OnDataReceived;
try
{
String vstsPSHostExe = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), "LegacyVSTSPowerShellHost.exe");
Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : vstsPSHostExe,
arguments : "",
environment : Environment,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : null,
inheritConsoleHandler : !ExecutionContext.Variables.Retain_Default_Encoding,
cancellationToken : ExecutionContext.CancellationToken);
// the exit code from vstsPSHost.exe indicate how many error record we get during execution
// -1 exit code means infrastructure failure of Host itself.
// this is to match current handler's logic.
if (exitCode > 0)
{
if (ExecutionContext.Result != null)
{
ExecutionContext.Debug($"Task result already set. Not failing due to error count ({exitCode}).");
}
else
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode));
}
}
else if (exitCode < 0)
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode));
}
}
finally
{
processInvoker.OutputDataReceived -= OnDataReceived;
processInvoker.ErrorDataReceived -= OnDataReceived;
}
}
}
public async Task RunAsync(ActionRunStage stage)
{
if (stage == ActionRunStage.Post)
{
throw new NotSupportedException("Script action should not have 'Post' job action.");
}
// Validate args
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
ArgUtil.NotNull(githubContext, nameof(githubContext));
var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
Inputs.TryGetValue("script", out var contents);
contents = contents ?? string.Empty;
Inputs.TryGetValue("workingDirectory", out var workingDirectory);
var workspaceDir = githubContext["workspace"] as StringContextData;
workingDirectory = Path.Combine(workspaceDir, workingDirectory ?? string.Empty);
Inputs.TryGetValue("shell", out var shell);
var isContainerStepHost = StepHost is ContainerStepHost;
string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse <string>());
string commandPath, argFormat, shellCommand;
// Set up default command and arguments
if (string.IsNullOrEmpty(shell))
{
#if OS_WINDOWS
shellCommand = "pwsh";
commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
if (string.IsNullOrEmpty(commandPath))
{
shellCommand = "powershell";
Trace.Info($"Defaulting to {shellCommand}");
commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
}
ArgUtil.NotNullOrEmpty(commandPath, "Default Shell");
#else
shellCommand = "sh";
commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
#endif
argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
}
else
{
var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell);
shellCommand = parsed.shellCommand;
// For non-ContainerStepHost, the command must be located on the host by Which
commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
argFormat = $"{parsed.shellArgs}".TrimStart();
if (string.IsNullOrEmpty(argFormat))
{
argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
}
}
// No arg format was given, shell must be a built-in
if (string.IsNullOrEmpty(argFormat) || !argFormat.Contains("{0}"))
{
throw new ArgumentException("Invalid shell option. Shell must be a valid built-in (bash, sh, cmd, powershell, pwsh) or a format string containing '{0}'");
}
// We do not not the full path until we know what shell is being used, so that we can determine the file extension
var scriptFilePath = Path.Combine(tempDirectory, $"{Guid.NewGuid()}{ScriptHandlerHelpers.GetScriptFileExtension(shellCommand)}");
var resolvedScriptPath = $"{StepHost.ResolvePathForStepHost(scriptFilePath).Replace("\"", "\\\"")}";
// Format arg string with script path
var arguments = string.Format(argFormat, resolvedScriptPath);
// Fix up and write the script
contents = ScriptHandlerHelpers.FixUpScriptContents(shellCommand, contents);
#if OS_WINDOWS
// Normalize Windows line endings
contents = contents.Replace("\r\n", "\n").Replace("\n", "\r\n");
var encoding = ExecutionContext.Variables.Retain_Default_Encoding && Console.InputEncoding.CodePage != 65001
? Console.InputEncoding
: new UTF8Encoding(false);
#else
// Don't add a BOM. It causes the script to fail on some operating systems (e.g. on Ubuntu 14).
var encoding = new UTF8Encoding(false);
#endif
// Script is written to local path (ie host) but executed relative to the StepHost, which may be a container
File.WriteAllText(scriptFilePath, contents, encoding);
// Prepend PATH
AddPrependPathToEnvironment();
// expose context to environment
foreach (var context in ExecutionContext.ExpressionValues)
{
if (context.Value is IEnvironmentContextData runtimeContext && runtimeContext != null)
{
foreach (var env in runtimeContext.GetRuntimeEnvironmentVariables())
{
Environment[env.Key] = env.Value;
}
}
}
// dump out the command
var fileName = isContainerStepHost ? shellCommand : commandPath;
ExecutionContext.Debug($"{fileName} {arguments}");
using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager))
using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager))
{
StepHost.OutputDataReceived += stdoutManager.OnDataReceived;
StepHost.ErrorDataReceived += stderrManager.OnDataReceived;
// Execute
int exitCode = await StepHost.ExecuteAsync(workingDirectory : StepHost.ResolvePathForStepHost(workingDirectory),
fileName : fileName,
arguments : arguments,
environment : Environment,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
inheritConsoleHandler : !ExecutionContext.Variables.Retain_Default_Encoding,
cancellationToken : ExecutionContext.CancellationToken);
// Error
if (exitCode != 0)
{
ExecutionContext.Error($"Process completed with exit code {exitCode}.");
ExecutionContext.Result = TaskResult.Failed;
}
}
}
public async Task GetSourceAsync(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
CancellationToken cancellationToken)
{
Trace.Entering();
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(endpoint, nameof(endpoint));
#if OS_WINDOWS
// Validate .NET Framework 4.6 or higher is installed.
var netFrameworkUtil = HostContext.GetService <INetFrameworkUtil>();
if (!netFrameworkUtil.Test(new Version(4, 6)))
{
throw new Exception(StringUtil.Loc("MinimumNetFramework46"));
}
#endif
// Create the tf command manager.
var tf = HostContext.CreateService <ITfsVCCommandManager>();
tf.CancellationToken = cancellationToken;
tf.Endpoint = endpoint;
tf.ExecutionContext = executionContext;
// Setup proxy.
var agentProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(endpoint.Url))
{
executionContext.Debug($"Configure '{tf.FilePath}' to work through proxy server '{executionContext.Variables.Agent_ProxyUrl}'.");
tf.SetupProxy(executionContext.Variables.Agent_ProxyUrl, executionContext.Variables.Agent_ProxyUsername, executionContext.Variables.Agent_ProxyPassword);
}
// Setup client certificate.
var agentCertManager = HostContext.GetService <IAgentCertificateManager>();
if (agentCertManager.SkipServerCertificateValidation)
{
#if OS_WINDOWS
executionContext.Debug("TF.exe does not support ignore SSL certificate validation error.");
#else
executionContext.Debug("TF does not support ignore SSL certificate validation error.");
#endif
}
var configUrl = new Uri(HostContext.GetService <IConfigurationStore>().GetSettings().ServerUrl);
if (!string.IsNullOrEmpty(agentCertManager.ClientCertificateFile) &&
Uri.Compare(endpoint.Url, configUrl, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0)
{
executionContext.Debug($"Configure '{tf.FilePath}' to work with client cert '{agentCertManager.ClientCertificateFile}'.");
tf.SetupClientCertificate(agentCertManager.ClientCertificateFile, agentCertManager.ClientCertificatePrivateKeyFile, agentCertManager.ClientCertificateArchiveFile, agentCertManager.ClientCertificatePassword);
}
// Add TF to the PATH.
string tfPath = tf.FilePath;
ArgUtil.File(tfPath, nameof(tfPath));
var varUtil = HostContext.GetService <IVarUtil>();
executionContext.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", Constants.PathVariable, Path.GetFileName(tfPath)));
varUtil.PrependPath(Path.GetDirectoryName(tfPath));
executionContext.Debug($"{Constants.PathVariable}: '{Environment.GetEnvironmentVariable(Constants.PathVariable)}'");
#if OS_WINDOWS
// Set TFVC_BUILDAGENT_POLICYPATH
string policyDllPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.ServerOM), "Microsoft.TeamFoundation.VersionControl.Controls.dll");
ArgUtil.File(policyDllPath, nameof(policyDllPath));
const string policyPathEnvKey = "TFVC_BUILDAGENT_POLICYPATH";
executionContext.Output(StringUtil.Loc("SetEnvVar", policyPathEnvKey));
Environment.SetEnvironmentVariable(policyPathEnvKey, policyDllPath);
#endif
// Check if the administrator accepted the license terms of the TEE EULA when configuring the agent.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
if (tf.Features.HasFlag(TfsVCFeatures.Eula) && settings.AcceptTeeEula)
{
// Check if the "tf eula -accept" command needs to be run for the current user.
bool skipEula = false;
try
{
skipEula = tf.TestEulaAccepted();
}
catch (Exception ex)
{
executionContext.Debug("Unexpected exception while testing whether the TEE EULA has been accepted for the current user.");
executionContext.Debug(ex.ToString());
}
if (!skipEula)
{
// Run the command "tf eula -accept".
try
{
await tf.EulaAsync();
}
catch (Exception ex)
{
executionContext.Debug(ex.ToString());
executionContext.Warning(ex.Message);
}
}
}
// Get the workspaces.
executionContext.Output(StringUtil.Loc("QueryingWorkspaceInfo"));
ITfsVCWorkspace[] tfWorkspaces = await tf.WorkspacesAsync();
// Determine the workspace name.
string buildDirectory = executionContext.Variables.Agent_BuildDirectory;
ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory));
string workspaceName = $"ws_{Path.GetFileName(buildDirectory)}_{settings.AgentId}";
executionContext.Variables.Set(Constants.Variables.Build.RepoTfvcWorkspace, workspaceName);
// Get the definition mappings.
DefinitionWorkspaceMapping[] definitionMappings =
JsonConvert.DeserializeObject <DefinitionWorkspaceMappings>(endpoint.Data[EndpointData.TfvcWorkspaceMapping])?.Mappings;
// Determine the sources directory.
string sourcesDirectory = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory);
ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory));
// Attempt to re-use an existing workspace if the command manager supports scorch
// or if clean is not specified.
ITfsVCWorkspace existingTFWorkspace = null;
bool clean = false;
if (endpoint.Data.ContainsKey(EndpointData.Clean))
{
if (!GetConditionLiteralBool(endpoint.Data[EndpointData.Clean], out clean))
{
var expressionManager = HostContext.GetService <IExpressionManager>();
try
{
clean = expressionManager.Evaluate(
executionContext, expressionManager.Parse(executionContext, endpoint.Data[EndpointData.Clean])
).Value;
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
clean = false;
executionContext.Error(ex);
}
}
}
if (tf.Features.HasFlag(TfsVCFeatures.Scorch) || !clean)
{
existingTFWorkspace = WorkspaceUtil.MatchExactWorkspace(
executionContext: executionContext,
tfWorkspaces: tfWorkspaces,
name: workspaceName,
definitionMappings: definitionMappings,
sourcesDirectory: sourcesDirectory);
if (existingTFWorkspace != null)
{
if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot))
{
// Undo pending changes.
ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory);
if (tfStatus?.HasPendingChanges ?? false)
{
await tf.UndoAsync(localPath : sourcesDirectory);
// Cleanup remaining files/directories from pend adds.
tfStatus.AllAdds
.OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted.
.ToList()
.ForEach(x =>
{
executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem));
IOUtil.Delete(x.LocalItem, cancellationToken);
});
}
}
else
{
// Perform "undo" for each map.
foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0])
{
if (definitionMapping.MappingType == DefinitionMappingType.Map)
{
// Check the status.
string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory);
ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath);
if (tfStatus?.HasPendingChanges ?? false)
{
// Undo.
await tf.UndoAsync(localPath : localPath);
// Cleanup remaining files/directories from pend adds.
tfStatus.AllAdds
.OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted.
.ToList()
.ForEach(x =>
{
executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem));
IOUtil.Delete(x.LocalItem, cancellationToken);
});
}
}
}
}
// Scorch.
if (clean)
{
// Try to scorch.
try
{
await tf.ScorchAsync();
}
catch (ProcessExitCodeException ex)
{
// Scorch failed.
// Warn, drop the folder, and re-clone.
executionContext.Warning(ex.Message);
existingTFWorkspace = null;
}
}
}
}
// Create a new workspace.
if (existingTFWorkspace == null)
{
// Remove any conflicting workspaces.
await RemoveConflictingWorkspacesAsync(
tf : tf,
tfWorkspaces : tfWorkspaces,
name : workspaceName,
directory : sourcesDirectory);
// Remove any conflicting workspace from a different computer.
// This is primarily a hosted scenario where a registered hosted
// agent can land on a different computer each time.
tfWorkspaces = await tf.WorkspacesAsync(matchWorkspaceNameOnAnyComputer : true);
foreach (ITfsVCWorkspace tfWorkspace in tfWorkspaces ?? new ITfsVCWorkspace[0])
{
await tf.WorkspaceDeleteAsync(tfWorkspace);
}
// Recreate the sources directory.
executionContext.Debug($"Deleting: '{sourcesDirectory}'.");
IOUtil.DeleteDirectory(sourcesDirectory, cancellationToken);
Directory.CreateDirectory(sourcesDirectory);
// Create the workspace.
await tf.WorkspaceNewAsync();
// Remove the default mapping.
if (tf.Features.HasFlag(TfsVCFeatures.DefaultWorkfoldMap))
{
await tf.WorkfoldUnmapAsync("$/");
}
// Sort the definition mappings.
definitionMappings =
(definitionMappings ?? new DefinitionWorkspaceMapping[0])
.OrderBy(x => x.NormalizedServerPath?.Length ?? 0) // By server path length.
.ToArray() ?? new DefinitionWorkspaceMapping[0];
// Add the definition mappings to the workspace.
foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings)
{
switch (definitionMapping.MappingType)
{
case DefinitionMappingType.Cloak:
// Add the cloak.
await tf.WorkfoldCloakAsync(serverPath : definitionMapping.ServerPath);
break;
case DefinitionMappingType.Map:
// Add the mapping.
await tf.WorkfoldMapAsync(
serverPath : definitionMapping.ServerPath,
localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory));
break;
default:
throw new NotSupportedException();
}
}
}
if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot))
{
// Get.
await tf.GetAsync(localPath : sourcesDirectory);
}
else
{
// Perform "get" for each map.
foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0])
{
if (definitionMapping.MappingType == DefinitionMappingType.Map)
{
await tf.GetAsync(localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory));
}
}
}
// Steps for shelveset/gated.
string shelvesetName = GetEndpointData(endpoint, Constants.EndpointData.SourceTfvcShelveset);
if (!string.IsNullOrEmpty(shelvesetName))
{
// Steps for gated.
ITfsVCShelveset tfShelveset = null;
string gatedShelvesetName = GetEndpointData(endpoint, Constants.EndpointData.GatedShelvesetName);
if (!string.IsNullOrEmpty(gatedShelvesetName))
{
// Clean the last-saved-checkin-metadata for existing workspaces.
//
// A better long term fix is to add a switch to "tf unshelve" that completely overwrites
// the last-saved-checkin-metadata, instead of merging associated work items.
//
// The targeted workaround for now is to create a trivial change and "tf shelve /move",
// which will delete the last-saved-checkin-metadata.
if (existingTFWorkspace != null)
{
executionContext.Output("Cleaning last saved checkin metadata.");
// Find a local mapped directory.
string firstLocalDirectory =
(definitionMappings ?? new DefinitionWorkspaceMapping[0])
.Where(x => x.MappingType == DefinitionMappingType.Map)
.Select(x => x.GetRootedLocalPath(sourcesDirectory))
.FirstOrDefault(x => Directory.Exists(x));
if (firstLocalDirectory == null)
{
executionContext.Warning("No mapped folder found. Unable to clean last-saved-checkin-metadata.");
}
else
{
// Create a trival change and "tf shelve /move" to clear the
// last-saved-checkin-metadata.
string cleanName = "__tf_clean_wksp_metadata";
string tempCleanFile = Path.Combine(firstLocalDirectory, cleanName);
try
{
File.WriteAllText(path: tempCleanFile, contents: "clean last-saved-checkin-metadata", encoding: Encoding.UTF8);
await tf.AddAsync(tempCleanFile);
await tf.ShelveAsync(shelveset : cleanName, commentFile : tempCleanFile, move : true);
}
catch (Exception ex)
{
executionContext.Warning($"Unable to clean last-saved-checkin-metadata. {ex.Message}");
try
{
await tf.UndoAsync(tempCleanFile);
}
catch (Exception ex2)
{
executionContext.Warning($"Unable to undo '{tempCleanFile}'. {ex2.Message}");
}
}
finally
{
IOUtil.DeleteFile(tempCleanFile);
}
}
}
// Get the shelveset metadata.
tfShelveset = await tf.ShelvesetsAsync(shelveset : shelvesetName);
// The above command throws if the shelveset is not found,
// so the following assertion should never fail.
ArgUtil.NotNull(tfShelveset, nameof(tfShelveset));
}
// Unshelve.
await tf.UnshelveAsync(shelveset : shelvesetName);
// Ensure we undo pending changes for shelveset build at the end.
_undoShelvesetPendingChanges = true;
if (!string.IsNullOrEmpty(gatedShelvesetName))
{
// Create the comment file for reshelve.
StringBuilder comment = new StringBuilder(tfShelveset.Comment ?? string.Empty);
string runCi = GetEndpointData(endpoint, Constants.EndpointData.GatedRunCI);
bool gatedRunCi = StringUtil.ConvertToBoolean(runCi, true);
if (!gatedRunCi)
{
if (comment.Length > 0)
{
comment.AppendLine();
}
comment.Append(Constants.Build.NoCICheckInComment);
}
string commentFile = null;
try
{
commentFile = Path.GetTempFileName();
File.WriteAllText(path: commentFile, contents: comment.ToString(), encoding: Encoding.UTF8);
// Reshelve.
await tf.ShelveAsync(shelveset : gatedShelvesetName, commentFile : commentFile, move : false);
}
finally
{
// Cleanup the comment file.
if (File.Exists(commentFile))
{
File.Delete(commentFile);
}
}
}
}
// Cleanup proxy settings.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl) && !agentProxy.IsBypassed(endpoint.Url))
{
executionContext.Debug($"Remove proxy setting for '{tf.FilePath}' to work through proxy server '{executionContext.Variables.Agent_ProxyUrl}'.");
tf.CleanupProxySetting();
}
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
}
if (container.ImageOS != PlatformUtil.OS.Windows)
{
string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
if (string.IsNullOrEmpty(defaultWorkingDirectory))
{
throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir"));
}
string workingDirectory = IOUtil.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), container.ImageOS);
string mountWorkingDirectory = container.TranslateToHostPath(workingDirectory);
executionContext.Debug($"Default Working Directory {defaultWorkingDirectory}");
executionContext.Debug($"Working Directory {workingDirectory}");
executionContext.Debug($"Mount Working Directory {mountWorkingDirectory}");
container.MountVolumes.Add(new MountVolume(mountWorkingDirectory, workingDirectory));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
}
else
{
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
}
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
bool externalReadOnly = container.ImageOS != PlatformUtil.OS.Windows; // This code was refactored to use PlatformUtils. The previous implementation did not have the externals directory mounted read-only for Windows.
// That seems wrong, but to prevent any potential backwards compatibility issues, we are keeping the same logic
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), externalReadOnly));
if (container.ImageOS != PlatformUtil.OS.Windows)
{
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
}
if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
string node;
if (!string.IsNullOrEmpty(container.ContainerBringNodePath))
{
node = container.ContainerBringNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
// if on Mac OS X, require container to have node
if (PlatformUtil.RunningOnMacOS)
{
container.ContainerBringNodePath = "node";
node = container.ContainerBringNodePath;
}
// if running on Windows, and attempting to run linux container, require container to have node
else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux)
{
container.ContainerBringNodePath = "node";
node = container.ContainerBringNodePath;
}
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
if (container.IsJobContainer)
{
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
}
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Get port mappings of running container
if (executionContext.Container == null && !container.IsJobContainer)
{
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
executionContext.Variables.Set(
$"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}",
$"{port.HostPort}");
}
}
if (!PlatformUtil.RunningOnWindows)
{
if (container.IsJobContainer)
{
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_azpcontainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created sudo group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true);
if (setupDockerGroup)
{
executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName));
// Get docker.sock group id on Host
string statFormatOption = "-c %g";
if (PlatformUtil.RunningOnMacOS)
{
statFormatOption = "-f %g";
}
string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"{statFormatOption} /var/run/docker.sock")).FirstOrDefault();
// We need to find out whether there is a group with same GID inside the container
string existingGroupName = null;
List <string> groupsOutput = new List <string>();
int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput);
if (execGroupGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}");
}
if (groupsOutput.Count > 0)
{
// check all potential groups that might match the GID.
foreach (string groupOutput in groupsOutput)
{
if (!string.IsNullOrEmpty(groupOutput))
{
var groupSegments = groupOutput.Split(':');
if (groupSegments.Length != 4)
{
Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'");
}
else
{
// the output of /etc/group should looks like `group:x:gid:`
var groupName = groupSegments[0];
var groupId = groupSegments[2];
if (string.Equals(dockerSockGroupId, groupId))
{
existingGroupName = groupName;
break;
}
}
}
}
}
if (string.IsNullOrEmpty(existingGroupName))
{
// create a new group with same gid
existingGroupName = "azure_pipelines_docker";
int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker");
if (execDockerGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}");
}
}
// Add the new created user to the docker socket group.
int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}");
if (execGroupUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}");
}
// if path to node is just 'node', with no path, let's make sure it is actually there
if (string.Equals(container.ContainerBringNodePath, "node", StringComparison.OrdinalIgnoreCase))
{
List <string> nodeVersionOutput = new List <string>();
int execNodeVersionExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"node -v\"", nodeVersionOutput);
if (execNodeVersionExitCode != 0)
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. Got exit code {execNodeVersionExitCode} from docker exec");
}
if (nodeVersionOutput.Count > 0)
{
executionContext.Output($"Detected Node Version: {nodeVersionOutput[0]}");
Trace.Info($"Using node version {nodeVersionOutput[0]} in container {container.ContainerId}");
}
else
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. No output from node -v");
}
}
}
}
}
}
public async Task RunMaintenanceOperation(IExecutionContext executionContext)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
// this might be not accurate when the agent is configured for old TFS server
int totalAvailableTimeInMinutes = executionContext.Variables.GetInt("maintenance.jobtimeoutinminutes") ?? 60;
// start a timer to track how much time we used
Stopwatch totalTimeSpent = Stopwatch.StartNew();
var trackingManager = HostContext.GetService <ITrackingManager>();
int staleBuildDirThreshold = executionContext.Variables.GetInt("maintenance.deleteworkingdirectory.daysthreshold") ?? 0;
if (staleBuildDirThreshold > 0)
{
// scan unused build directories
executionContext.Output(StringUtil.Loc("DiscoverBuildDir", staleBuildDirThreshold));
trackingManager.MarkExpiredForGarbageCollection(executionContext, TimeSpan.FromDays(staleBuildDirThreshold));
}
else
{
executionContext.Output(StringUtil.Loc("GCBuildDirNotEnabled"));
return;
}
executionContext.Output(StringUtil.Loc("GCBuildDir"));
// delete unused build directories
trackingManager.DisposeCollectedGarbage(executionContext);
// give source provider a chance to run maintenance operation
Trace.Info("Scan all SourceFolder tracking files.");
string searchRoot = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Build.Path.SourceRootMappingDirectory);
if (!Directory.Exists(searchRoot))
{
executionContext.Output(StringUtil.Loc("GCDirNotExist", searchRoot));
return;
}
// <tracking config, tracking file path>
List <Tuple <TrackingConfig, string> > optimizeTrackingFiles = new List <Tuple <TrackingConfig, string> >();
var allTrackingFiles = Directory.EnumerateFiles(searchRoot, Constants.Build.Path.TrackingConfigFile, SearchOption.AllDirectories);
Trace.Verbose($"Find {allTrackingFiles.Count()} tracking files.");
foreach (var trackingFile in allTrackingFiles)
{
executionContext.Output(StringUtil.Loc("EvaluateTrackingFile", trackingFile));
TrackingConfigBase tracking = trackingManager.LoadIfExists(executionContext, trackingFile);
// detect whether the tracking file is in new format.
TrackingConfig newTracking = tracking as TrackingConfig;
if (newTracking == null)
{
executionContext.Output(StringUtil.Loc("GCOldFormatTrackingFile", trackingFile));
}
else if (string.IsNullOrEmpty(newTracking.RepositoryType))
{
// repository not been set.
executionContext.Output(StringUtil.Loc("SkipTrackingFileWithoutRepoType", trackingFile));
}
else
{
optimizeTrackingFiles.Add(new Tuple <TrackingConfig, string>(newTracking, trackingFile));
}
}
// Sort the all tracking file ASC by last maintenance attempted time
foreach (var trackingInfo in optimizeTrackingFiles.OrderBy(x => x.Item1.LastMaintenanceAttemptedOn))
{
// maintenance has been cancelled.
executionContext.CancellationToken.ThrowIfCancellationRequested();
bool runMainenance = false;
TrackingConfig trackingConfig = trackingInfo.Item1;
string trackingFile = trackingInfo.Item2;
if (trackingConfig.LastMaintenanceAttemptedOn == null)
{
// this folder never run maintenance before, we will do maintenance if there is more than half of the time remains.
if (totalTimeSpent.Elapsed.TotalMinutes < totalAvailableTimeInMinutes / 2) // 50% time left
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' has never run maintenance before. Skip since we may not have enough time.");
}
}
else if (trackingConfig.LastMaintenanceCompletedOn == null)
{
// this folder did finish maintenance last time, this might indicate we need more time for this working directory
if (totalTimeSpent.Elapsed.TotalMinutes < totalAvailableTimeInMinutes / 4) // 75% time left
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' didn't finish maintenance last time. Skip since we may not have enough time.");
}
}
else
{
// estimate time for running maintenance
TimeSpan estimateTime = trackingConfig.LastMaintenanceCompletedOn.Value - trackingConfig.LastMaintenanceAttemptedOn.Value;
// there is more than 10 mins left after we run maintenance on this repository directory
if (totalAvailableTimeInMinutes > totalTimeSpent.Elapsed.TotalMinutes + estimateTime.TotalMinutes + 10)
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' may take about '{estimateTime.TotalMinutes}' mins to finish maintenance. It's too risky since we only have '{totalAvailableTimeInMinutes - totalTimeSpent.Elapsed.TotalMinutes}' mins left for maintenance.");
}
}
if (runMainenance)
{
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = extensionManager.GetExtensions <ISourceProvider>().FirstOrDefault(x => string.Equals(x.RepositoryType, trackingConfig.RepositoryType, StringComparison.OrdinalIgnoreCase));
if (sourceProvider != null)
{
try
{
trackingManager.MaintenanceStarted(trackingConfig, trackingFile);
string repositoryPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), trackingConfig.SourcesDirectory);
await sourceProvider.RunMaintenanceOperations(executionContext, repositoryPath);
trackingManager.MaintenanceCompleted(trackingConfig, trackingFile);
}
catch (Exception ex)
{
executionContext.Error(StringUtil.Loc("ErrorDuringBuildGC", trackingFile));
executionContext.Error(ex);
}
}
}
}
}
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobDisplayName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
if (HostContext.RunMode == RunMode.Normal)
{
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
}
else if (HostContext.RunMode == RunMode.Local)
{
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
}
// Start the child process.
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
await notification.JobStarted(message.JobId);
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
await LogWorkerProcessUnhandledException(message, detailInfo);
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandle exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher prority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandle exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
public void ConfigureService(AgentSettings settings, CommandSettings command)
{
Trace.Entering();
if (!_windowsServiceHelper.IsRunningInElevatedMode())
{
Trace.Error("Needs Administrator privileges for configure agent as windows service.");
throw new SecurityException(StringUtil.Loc("NeedAdminForConfigAgentWinService"));
}
// TODO: Fix bug that exists in the legacy Windows agent where configuration using mirrored credentials causes an error, but the agent is still functional (after restarting). Mirrored credentials is a supported scenario and shouldn't manifest any errors.
// We use NetworkService as default account for build and release agent
// We use Local System as default account for deployment agent, deployment pool agent
bool isDeploymentGroupScenario = command.DeploymentGroup || command.DeploymentPool;
NTAccount defaultServiceAccount = isDeploymentGroupScenario ? _windowsServiceHelper.GetDefaultAdminServiceAccount() : _windowsServiceHelper.GetDefaultServiceAccount();
string logonAccount = command.GetWindowsLogonAccount(defaultValue: defaultServiceAccount.ToString(), descriptionMsg: StringUtil.Loc("WindowsLogonAccountNameDescription"));
string domainName;
string userName;
GetAccountSegments(logonAccount, out domainName, out userName);
if ((string.IsNullOrEmpty(domainName) || domainName.Equals(".", StringComparison.CurrentCultureIgnoreCase)) && !logonAccount.Contains('@'))
{
logonAccount = String.Format("{0}\\{1}", Environment.MachineName, userName);
domainName = Environment.MachineName;
}
Trace.Info("LogonAccount after transforming: {0}, user: {1}, domain: {2}", logonAccount, userName, domainName);
string logonPassword = string.Empty;
if (!defaultServiceAccount.Equals(new NTAccount(logonAccount)) && !NativeWindowsServiceHelper.IsWellKnownIdentity(logonAccount))
{
while (true)
{
logonPassword = command.GetWindowsLogonPassword(logonAccount);
if (_windowsServiceHelper.IsValidCredential(domainName, userName, logonPassword))
{
Trace.Info("Credential validation succeed");
break;
}
else
{
if (!command.Unattended)
{
Trace.Info("Invalid credential entered");
_term.WriteLine(StringUtil.Loc("InvalidWindowsCredential"));
}
else
{
throw new SecurityException(StringUtil.Loc("InvalidWindowsCredential"));
}
}
}
}
string serviceName;
string serviceDisplayName;
CalculateServiceName(settings, ServiceNamePattern, ServiceDisplayNamePattern, out serviceName, out serviceDisplayName);
if (_windowsServiceHelper.IsServiceExists(serviceName))
{
_term.WriteLine(StringUtil.Loc("ServiceAlreadyExists", serviceName));
_windowsServiceHelper.UninstallService(serviceName);
}
Trace.Info("Verifying if the account has LogonAsService permission");
if (_windowsServiceHelper.IsUserHasLogonAsServicePrivilege(domainName, userName))
{
Trace.Info($"Account: {logonAccount} already has Logon As Service Privilege.");
}
else
{
if (!_windowsServiceHelper.GrantUserLogonAsServicePrivilage(domainName, userName))
{
throw new InvalidOperationException(StringUtil.Loc("CanNotGrantPermission", logonAccount));
}
}
// grant permission for agent root folder and work folder
Trace.Info("Create local group and grant folder permission to service logon account.");
string agentRoot = HostContext.GetDirectory(WellKnownDirectory.Root);
string workFolder = HostContext.GetDirectory(WellKnownDirectory.Work);
Directory.CreateDirectory(workFolder);
_windowsServiceHelper.GrantDirectoryPermissionForAccount(logonAccount, new[] { agentRoot, workFolder });
_term.WriteLine(StringUtil.Loc("GrantingFilePermissions", logonAccount));
// install service.
_windowsServiceHelper.InstallService(serviceName, serviceDisplayName, logonAccount, logonPassword);
// create .service file with service name.
SaveServiceSettings(serviceName);
// Add registry key after installation
_windowsServiceHelper.CreateVstsAgentRegistryKey();
Trace.Info("Configuration was successful, trying to start the service");
_windowsServiceHelper.StartService(serviceName);
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Action, nameof(Action));
var taskManager = HostContext.GetService <IActionManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Load the task definition and choose the handler.
Definition definition = taskManager.LoadAction(ExecutionContext, Action);
ArgUtil.NotNull(definition, nameof(definition));
ActionExecutionData handlerData = definition.Data?.Execution;
ArgUtil.NotNull(handlerData, nameof(handlerData));
if (handlerData.HasPre &&
Action.Reference is Pipelines.RepositoryPathReference repoAction &&
string.Equals(repoAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
ExecutionContext.Warning($"`pre` execution is not supported for local action from '{repoAction.Path}'");
}
// The action has post cleanup defined.
// we need to create timeline record for them and add them to the step list that StepRunner is using
if (handlerData.HasPost && (Stage == ActionRunStage.Pre || Stage == ActionRunStage.Main))
{
string postDisplayName = $"Post {this.DisplayName}";
var repositoryReference = Action.Reference as RepositoryPathReference;
var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}";
var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" :
$"{repositoryReference.Name}{pathString}@{repositoryReference.Ref}";
ExecutionContext.Debug($"Register post job cleanup for action: {repoString}");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = Action;
actionRunner.Stage = ActionRunStage.Post;
actionRunner.Condition = handlerData.CleanupCondition;
actionRunner.DisplayName = postDisplayName;
ExecutionContext.RegisterPostJobStep(actionRunner);
}
IStepHost stepHost = HostContext.CreateService <IDefaultStepHost>();
// Makes directory for event_path data
var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
var workflowDirectory = Path.Combine(tempDirectory, "_github_workflow");
Directory.CreateDirectory(workflowDirectory);
var gitHubEvent = ExecutionContext.GetGitHubContext("event");
// adds the GitHub event path/file if the event exists
if (gitHubEvent != null)
{
var workflowFile = Path.Combine(workflowDirectory, "event.json");
Trace.Info($"Write event payload to {workflowFile}");
File.WriteAllText(workflowFile, gitHubEvent, new UTF8Encoding(false));
ExecutionContext.SetGitHubContext("event_path", workflowFile);
}
// Setup container stephost for running inside the container.
if (ExecutionContext.Container != null)
{
// Make sure required container is already created.
ArgUtil.NotNullOrEmpty(ExecutionContext.Container.ContainerId, nameof(ExecutionContext.Container.ContainerId));
var containerStepHost = HostContext.CreateService <IContainerStepHost>();
containerStepHost.Container = ExecutionContext.Container;
stepHost = containerStepHost;
}
// Load the inputs.
ExecutionContext.Debug("Loading inputs");
var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator();
var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions);
var userInputs = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair <string, string> input in inputs)
{
userInputs.Add(input.Key);
string message = "";
if (definition.Data?.Deprecated?.TryGetValue(input.Key, out message) == true)
{
ExecutionContext.Warning(String.Format("Input '{0}' has been deprecated with message: {1}", input.Key, message));
}
}
var validInputs = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
// Merge the default inputs from the definition
if (definition.Data?.Inputs != null)
{
var manifestManager = HostContext.GetService <IActionManifestManager>();
foreach (var input in definition.Data.Inputs)
{
string key = input.Key.AssertString("action input name").Value;
validInputs.Add(key);
if (!inputs.ContainsKey(key))
{
inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value);
}
}
}
// Validate inputs only for actions with action.yml
if (Action.Reference.Type == Pipelines.ActionSourceType.Repository)
{
foreach (var input in userInputs)
{
if (!validInputs.Contains(input))
{
ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']");
}
}
}
// Load the action environment.
ExecutionContext.Debug("Loading env");
var environment = new Dictionary <String, String>(VarUtil.EnvironmentVariableKeyComparer);
#if OS_WINDOWS
var envContext = ExecutionContext.ExpressionValues["env"] as DictionaryContextData;
#else
var envContext = ExecutionContext.ExpressionValues["env"] as CaseSensitiveDictionaryContextData;
#endif
// Apply environment from env context, env context contains job level env and action's evn block
foreach (var env in envContext)
{
environment[env.Key] = env.Value.ToString();
}
// Apply action's intra-action state at last
foreach (var state in ExecutionContext.IntraActionState)
{
environment[$"STATE_{state.Key}"] = state.Value ?? string.Empty;
}
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
Action.Reference,
stepHost,
handlerData,
inputs,
environment,
ExecutionContext.Variables,
actionDirectory: definition.Directory);
// Print out action details
handler.PrintActionDetails(Stage);
// Run the task.
await handler.RunAsync(Stage);
}
private async Task ProcessPluginCommandAsync(IAsyncCommandContext context, AgentCommandPluginExecutionContext pluginContext, string plugin, Command command, CancellationToken token)
{
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"command \"{plugin}\"";
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Any content coming from STDERR will indicate the command process failed.
// We can't use ## command for plugin to communicate, since we are already processing ## command
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
object stderrLock = new object();
List <string> stderr = new List <string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
context.Output(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
lock (stderrLock)
{
stderr.Add(e.Data);
};
};
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext));
int returnCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : file,
arguments : arguments,
environment : null,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : redirectStandardIn,
cancellationToken : token);
if (returnCode != 0)
{
context.Output(string.Join(Environment.NewLine, stderr));
throw new ProcessExitCodeException(returnCode, file, arguments);
}
else if (stderr.Count > 0)
{
throw new InvalidOperationException(string.Join(Environment.NewLine, stderr));
}
else
{
// Everything works fine.
// Return code is 0.
// No STDERR comes out.
}
}
}
private Assembly ResolveAssembly(AssemblyLoadContext context, AssemblyName assembly)
{
string assemblyFilename = assembly.Name + ".dll";
return(context.LoadFromAssemblyPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), assemblyFilename)));
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_logFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Diag), StringUtil.Format("{0}_{1:yyyyMMdd-HHmmss}-utc.log", nameof(InternetCheck), DateTime.UtcNow));
}
// Download all required actions.
// Make sure all condition inputs are valid.
// Build up three list of steps for jobrunner (pre-job, job, post-job).
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// Create a new timeline record for 'Set up job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug($"Starting: Set up job");
context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
var setting = HostContext.GetService <IConfigurationStore>().GetSettings();
var credFile = HostContext.GetConfigFile(WellKnownConfigFile.Credentials);
if (File.Exists(credFile))
{
var credData = IOUtil.LoadObject <CredentialData>(credFile);
if (credData != null &&
credData.Data.TryGetValue("clientId", out var clientId))
{
// print out HostName for self-hosted runner
context.Output($"Runner name: '{setting.AgentName}'");
context.Output($"Machine name: '{Environment.MachineName}'");
}
}
var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo);
if (File.Exists(setupInfoFile))
{
Trace.Info($"Load machine setup info from {setupInfoFile}");
try
{
var setupInfo = IOUtil.LoadObject <List <SetupInfo> >(setupInfoFile);
if (setupInfo?.Count > 0)
{
foreach (var info in setupInfo)
{
if (!string.IsNullOrEmpty(info?.Detail))
{
var groupName = info.Group;
if (string.IsNullOrEmpty(groupName))
{
groupName = "Machine Setup Info";
}
context.Output($"##[group]{groupName}");
var multiLines = info.Detail.Replace("\r\n", "\n").TrimEnd('\n').Split('\n');
foreach (var line in multiLines)
{
context.Output(line);
}
context.Output("##[endgroup]");
}
}
}
}
catch (Exception ex)
{
context.Output($"Fail to load and print machine setup info: {ex.Message}");
Trace.Error(ex);
}
}
var repoFullName = context.GetGitHubContext("repository");
ArgUtil.NotNull(repoFullName, nameof(repoFullName));
context.Debug($"Primary repository: {repoFullName}");
// Print proxy setting information for better diagnostic experience
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
}
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
}
// Prepare the workflow directory
context.Output("Prepare workflow directory");
var directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
context,
message.Workspace);
// Set the directory variables
context.Debug("Update context data");
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));
// Temporary hack for GHES alpha
var configurationStore = HostContext.GetService <IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings();
if (string.IsNullOrEmpty(context.GetGitHubContext("server_url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
{
var url = new Uri(runnerSettings.GitHubUrl);
var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
context.SetGitHubContext("server_url", $"{url.Scheme}://{url.Host}{portInfo}");
context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql");
}
// Evaluate the job-level environment variables
context.Debug("Evaluating job-level environment variables");
var templateEvaluator = context.ToPipelineTemplateEvaluator();
foreach (var token in message.EnvironmentVariables)
{
var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, jobContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
foreach (var pair in environmentVariables)
{
context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
}
}
// Evaluate the job container
context.Debug("Evaluating job container");
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
if (container != null)
{
jobContext.Container = new Container.ContainerInfo(HostContext, container);
}
// Evaluate the job service containers
context.Debug("Evaluating job service containers");
var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
if (serviceContainers?.Count > 0)
{
foreach (var pair in serviceContainers)
{
var networkAlias = pair.Key;
var serviceContainer = pair.Value;
jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
}
}
// Evaluate the job defaults
context.Debug("Evaluating job defaults");
foreach (var token in message.Defaults)
{
var defaults = token.AssertMapping("defaults");
if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)))
{
context.JobDefaults["run"] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase));
var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
foreach (var pair in jobDefaults)
{
if (!string.IsNullOrEmpty(pair.Value))
{
context.JobDefaults["run"][pair.Key] = pair.Value;
}
}
}
}
// Build up 2 lists of steps, pre-job, job
// Download actions not already in the cache
Trace.Info("Downloading actions");
var actionManager = HostContext.GetService <IActionManager>();
var prepareResult = await actionManager.PrepareActionsAsync(context, message.Steps);
preJobSteps.AddRange(prepareResult.ContainerSetupSteps);
// Add start-container steps, record and stop-container steps
if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (jobContext.Container != null)
{
containers.Add(jobContext.Container);
}
containers.AddRange(jobContext.ServiceContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: "Initialize containers",
data: (object)containers));
}
// Add action steps
foreach (var step in message.Steps)
{
if (step.Type == Pipelines.StepType.Action)
{
var action = step as Pipelines.ActionStep;
Trace.Info($"Adding {action.DisplayName}.");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = action;
actionRunner.Stage = ActionRunStage.Main;
actionRunner.Condition = step.Condition;
var contextData = new Pipelines.ContextData.DictionaryContextData();
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
contextData[pair.Key] = pair.Value;
}
}
actionRunner.TryEvaluateDisplayName(contextData, context);
jobSteps.Add(actionRunner);
if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep))
{
Trace.Info($"Adding pre-{action.DisplayName}.");
preStep.TryEvaluateDisplayName(contextData, context);
preStep.DisplayName = $"Pre {preStep.DisplayName}";
preJobSteps.Add(preStep);
}
}
}
var intraActionStates = new Dictionary <Guid, Dictionary <string, string> >();
foreach (var preStep in prepareResult.PreStepTracker)
{
intraActionStates[preStep.Key] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
}
// Create execution context for pre-job steps
foreach (var step in preJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
}
else if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
Guid stepId = Guid.NewGuid();
actionStep.ExecutionContext = jobContext.CreateChild(stepId, actionStep.DisplayName, stepId.ToString("N"), null, null, intraActionStates[actionStep.Action.Id]);
}
}
// Create execution context for job steps
foreach (var step in jobSteps)
{
if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName, intraActionState);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the RUNNER_TRACKING_ID env variable.
Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
context.Debug("Collect running processes for tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Debug("Finishing: Set up job");
context.Complete();
}
}
}
public TrackingConfig Create(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
string hashKey,
string file,
bool overrideBuildDirectory)
{
Trace.Entering();
// Get or create the top-level tracking config.
TopLevelTrackingConfig topLevelConfig;
string topLevelFile = Path.Combine(
IOUtil.GetWorkPath(HostContext),
Constants.Build.Path.SourceRootMappingDirectory,
Constants.Build.Path.TopLevelTrackingConfigFile);
Trace.Verbose($"Loading top-level tracking config if exists: {topLevelFile}");
if (!File.Exists(topLevelFile))
{
topLevelConfig = new TopLevelTrackingConfig();
}
else
{
topLevelConfig = JsonConvert.DeserializeObject <TopLevelTrackingConfig>(File.ReadAllText(topLevelFile));
if (topLevelConfig == null)
{
executionContext.Warning($"Rebuild corruptted top-level tracking configure file {topLevelFile}.");
// save the corruptted file in case we need to investigate more.
File.Copy(topLevelFile, $"{topLevelFile}.corruptted", true);
topLevelConfig = new TopLevelTrackingConfig();
DirectoryInfo workDir = new DirectoryInfo(HostContext.GetDirectory(WellKnownDirectory.Work));
foreach (var dir in workDir.EnumerateDirectories())
{
// we scan the entire _work directory and find the directory with the highest integer number.
if (int.TryParse(dir.Name, NumberStyles.Integer, CultureInfo.InvariantCulture, out int lastBuildNumber) &&
lastBuildNumber > topLevelConfig.LastBuildDirectoryNumber)
{
topLevelConfig.LastBuildDirectoryNumber = lastBuildNumber;
}
}
}
}
// Determine the build directory.
if (overrideBuildDirectory)
{
// This should only occur during hosted builds. This was added due to TFVC.
// TFVC does not allow a local path for a single machine to be mapped in multiple
// workspaces. The machine name for a hosted images is not unique.
//
// So if a customer is running two hosted builds at the same time, they could run
// into the local mapping conflict.
//
// The workaround is to force the build directory to be different across all concurrent
// hosted builds (for TFVC). The agent ID will be unique across all concurrent hosted
// builds so that can safely be used as the build directory.
ArgUtil.Equal(default(int), topLevelConfig.LastBuildDirectoryNumber, nameof(topLevelConfig.LastBuildDirectoryNumber));
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
topLevelConfig.LastBuildDirectoryNumber = settings.AgentId;
}
else
{
topLevelConfig.LastBuildDirectoryNumber++;
}
// Update the top-level tracking config.
topLevelConfig.LastBuildDirectoryCreatedOn = DateTimeOffset.Now;
WriteToFile(topLevelFile, topLevelConfig);
// Create the new tracking config.
TrackingConfig config = new TrackingConfig(
executionContext,
endpoint,
topLevelConfig.LastBuildDirectoryNumber,
hashKey);
WriteToFile(file, config);
return(config);
}
private void DeletePreviousVersionAgentBackup(CancellationToken token)
{
// delete previous backup agent (back compat, can be remove after serval sprints)
// bin.bak.2.99.0
// externals.bak.2.99.0
foreach (string existBackUp in Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "*.bak.*"))
{
Trace.Info($"Delete existing agent backup at {existBackUp}.");
try
{
IOUtil.DeleteDirectory(existBackUp, token);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.Error(ex);
Trace.Info($"Catch exception during delete backup folder {existBackUp}, ignore this error try delete the backup folder on next auto-update.");
}
}
// delete old bin.2.99.0 folder, only leave the current version and the latest download version
var allBinDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "bin.*");
if (allBinDirs.Length > 2)
{
// there are more than 2 bin.version folder.
// delete older bin.version folders.
foreach (var oldBinDir in allBinDirs)
{
if (string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin.{BuildConstants.AgentPackage.Version}"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin.{_targetPackage.Version}"), StringComparison.OrdinalIgnoreCase))
{
// skip for current agent version
continue;
}
Trace.Info($"Delete agent bin folder's backup at {oldBinDir}.");
try
{
IOUtil.DeleteDirectory(oldBinDir, token);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.Error(ex);
Trace.Info($"Catch exception during delete backup folder {oldBinDir}, ignore this error try delete the backup folder on next auto-update.");
}
}
}
// delete old externals.2.99.0 folder, only leave the current version and the latest download version
var allExternalsDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "externals.*");
if (allExternalsDirs.Length > 2)
{
// there are more than 2 externals.version folder.
// delete older externals.version folders.
foreach (var oldExternalDir in allExternalsDirs)
{
if (string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals.{BuildConstants.AgentPackage.Version}"), StringComparison.OrdinalIgnoreCase) ||
string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals.{_targetPackage.Version}"), StringComparison.OrdinalIgnoreCase))
{
// skip for current agent version
continue;
}
Trace.Info($"Delete agent externals folder's backup at {oldExternalDir}.");
try
{
IOUtil.DeleteDirectory(oldExternalDir, token);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Trace.Error(ex);
Trace.Info($"Catch exception during delete backup folder {oldExternalDir}, ignore this error try delete the backup folder on next auto-update.");
}
}
}
}
public ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile)
{
var templateContext = CreateTemplateContext(executionContext);
ActionDefinitionData actionDefinition = new ActionDefinitionData();
// Clean up file name real quick
// Instead of using Regex which can be computationally expensive,
// we can just remove the # of characters from the fileName according to the length of the basePath
string basePath = HostContext.GetDirectory(WellKnownDirectory.Actions);
string fileRelativePath = manifestFile;
if (manifestFile.Contains(basePath))
{
fileRelativePath = manifestFile.Remove(0, basePath.Length + 1);
}
try
{
var token = default(TemplateToken);
// Get the file ID
var fileId = templateContext.GetFileId(fileRelativePath);
// Add this file to the FileTable in executionContext if it hasn't been added already
// we use > since fileID is 1 indexed
if (fileId > executionContext.Global.FileTable.Count)
{
executionContext.Global.FileTable.Add(fileRelativePath);
}
// Read the file
var fileContent = File.ReadAllText(manifestFile);
using (var stringReader = new StringReader(fileContent))
{
var yamlObjectReader = new YamlObjectReader(fileId, stringReader);
token = TemplateReader.Read(templateContext, "action-root", yamlObjectReader, fileId, out _);
}
var actionMapping = token.AssertMapping("action manifest root");
var actionOutputs = default(MappingToken);
var actionRunValueToken = default(TemplateToken);
foreach (var actionPair in actionMapping)
{
var propertyName = actionPair.Key.AssertString($"action.yml property key");
switch (propertyName.Value)
{
case "name":
actionDefinition.Name = actionPair.Value.AssertString("name").Value;
break;
case "outputs":
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
actionOutputs = actionPair.Value.AssertMapping("outputs");
break;
}
Trace.Info($"Ignore action property outputs. Outputs for a whole action is not supported yet.");
break;
case "description":
actionDefinition.Description = actionPair.Value.AssertString("description").Value;
break;
case "inputs":
ConvertInputs(actionPair.Value, actionDefinition);
break;
case "runs":
// Defer runs token evaluation to after for loop to ensure that order of outputs doesn't matter.
actionRunValueToken = actionPair.Value;
break;
default:
Trace.Info($"Ignore action property {propertyName}.");
break;
}
}
// Evaluate Runs Last
if (actionRunValueToken != null)
{
actionDefinition.Execution = ConvertRuns(executionContext, templateContext, actionRunValueToken, fileRelativePath, actionOutputs);
}
}
catch (Exception ex)
{
Trace.Error(ex);
templateContext.Errors.Add(ex);
}
if (templateContext.Errors.Count > 0)
{
foreach (var error in templateContext.Errors)
{
Trace.Error($"Action.yml load error: {error.Message}");
executionContext.Error(error.Message);
}
throw new ArgumentException($"Fail to load {fileRelativePath}");
}
if (actionDefinition.Execution == null)
{
executionContext.Debug($"Loaded action.yml file: {StringUtil.ConvertToJson(actionDefinition)}");
throw new ArgumentException($"Top level 'runs:' section is required for {fileRelativePath}");
}
else
{
Trace.Info($"Loaded action.yml file: {StringUtil.ConvertToJson(actionDefinition)}");
}
return(actionDefinition);
}
public Task StartAsync(IExecutionContext context, List <IStep> steps, CancellationToken token)
{
Trace.Entering();
ArgUtil.NotNull(context, nameof(context));
List <PluginInfo> enabledPlugins = new List <PluginInfo>();
if (context.Variables.GetBoolean("agent.disablelogplugin") ?? false)
{
// all log plugs are disabled
context.Debug("All log plugins are disabled.");
}
else
{
foreach (var plugin in _logPlugins)
{
if (context.Variables.GetBoolean($"agent.disablelogplugin.{plugin.Key}") ?? false)
{
// skip plugin
context.Debug($"Log plugin '{plugin.Key}' is disabled.");
continue;
}
else
{
enabledPlugins.Add(plugin.Value);
}
}
}
if (enabledPlugins.Count > 0)
{
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
ArgUtil.File(file, $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"log \"{_instanceId.ToString("D")}\"";
var processInvoker = HostContext.CreateService <IProcessInvoker>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
if (e.Data != null)
{
_outputs.Enqueue(e.Data);
}
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
if (e.Data != null)
{
_outputs.Enqueue(e.Data);
}
};
_pluginHostProcess = processInvoker.ExecuteAsync(workingDirectory: workingDirectory,
fileName: file,
arguments: arguments,
environment: null,
requireExitCodeZero: true,
outputEncoding: Encoding.UTF8,
killProcessOnCancel: true,
redirectStandardIn: _redirectedStdin,
inheritConsoleHandler: false,
keepStandardInOpen: true,
cancellationToken: token);
// construct plugin context
AgentLogPluginHostContext pluginContext = new AgentLogPluginHostContext
{
PluginAssemblies = new List <string>(),
Repositories = context.Repositories,
Endpoints = context.Endpoints,
Variables = new Dictionary <string, VariableValue>(),
Steps = new Dictionary <string, Pipelines.TaskStepDefinitionReference>()
};
// plugins
pluginContext.PluginAssemblies.AddRange(_logPlugins.Values.Select(x => x.AssemblyName));
var target = context.StepTarget();
Variables.TranslationMethod translateToHostPath = Variables.DefaultStringTranslator;
ContainerInfo containerInfo = target as ContainerInfo;
// Since plugins run on the host, but the inputs and variables have already been translated
// to the container path, we need to convert them back to the host path
// TODO: look to see if there is a better way to not have translate these back
if (containerInfo != null)
{
translateToHostPath = (string val) => { return(containerInfo.TranslateToHostPath(val)); };
}
// variables
context.Variables.CopyInto(pluginContext.Variables, translateToHostPath);
// steps
foreach (var step in steps)
{
var taskStep = step as ITaskRunner;
if (taskStep != null)
{
pluginContext.Steps[taskStep.ExecutionContext.Id.ToString("D")] = taskStep.Task.Reference;
}
}
Trace.Info("Send serialized context through STDIN");
_redirectedStdin.Enqueue(JsonUtility.ToString(pluginContext));
foreach (var plugin in _logPlugins)
{
context.Output($"Plugin: '{plugin.Value.FriendlyName}' is running in background.");
}
}
return(Task.CompletedTask);
}
public override void InitializeJobExtension(IExecutionContext executionContext, IList <JobStep> steps, WorkspaceOptions workspace)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
Trace.Entering();
executionContext.Output(StringUtil.Loc("PrepareReleasesDir"));
var directoryManager = HostContext.GetService <IReleaseDirectoryManager>();
ReleaseId = executionContext.Variables.GetInt(Constants.Variables.Release.ReleaseId) ?? 0;
TeamProjectId = executionContext.Variables.GetGuid(Constants.Variables.System.TeamProjectId) ?? Guid.Empty;
SkipArtifactsDownload = executionContext.Variables.GetBoolean(Constants.Variables.Release.SkipArtifactsDownload) ?? false;
string releaseDefinitionName = executionContext.Variables.Get(Constants.Variables.Release.ReleaseDefinitionName);
// TODO: Should we also write to log in executionContext.Output methods? so that we don't have to repeat writing into logs?
// Log these values here to debug scenarios where downloading the artifact fails.
executionContext.Output($"ReleaseId={ReleaseId}, TeamProjectId={TeamProjectId}, ReleaseDefinitionName={releaseDefinitionName}");
var releaseDefinition = executionContext.Variables.Get(Constants.Variables.Release.ReleaseDefinitionId);
if (string.IsNullOrEmpty(releaseDefinition))
{
string pattern = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
Regex regex = new Regex(string.Format("[{0}]", Regex.Escape(pattern)));
releaseDefinition = regex.Replace(releaseDefinitionName, string.Empty);
}
ArgUtil.NotNull(executionContext, nameof(executionContext)); // I am not sure why this is needed, but static analysis flagged all uses of executionContext below this point
var releaseTrackingConfig = directoryManager.PrepareArtifactsDirectory(
HostContext.GetDirectory(WellKnownDirectory.Work),
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_TeamProjectId.ToString(),
releaseDefinition);
ReleaseWorkingFolder = Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
releaseTrackingConfig.ReleaseDirectory);
ArtifactsWorkingFolder = string.IsNullOrEmpty(executionContext.Variables.Release_ArtifactsDirectory)
? Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
releaseTrackingConfig.ReleaseDirectory,
Constants.Release.Path.ArtifactsDirectory)
: executionContext.Variables.Release_ArtifactsDirectory;
executionContext.Output($"Release folder: {ArtifactsWorkingFolder}");
// Ensure directory exist
if (!Directory.Exists(ArtifactsWorkingFolder))
{
Trace.Info($"Creating {ArtifactsWorkingFolder}.");
Directory.CreateDirectory(ArtifactsWorkingFolder);
}
SetLocalVariables(executionContext, ArtifactsWorkingFolder);
// Log the environment variables available after populating the variable service with our variables
LogEnvironmentVariables(executionContext);
if (SkipArtifactsDownload)
{
// If this is the first time the agent is executing a task, we need to create the artifactsFolder
// otherwise Process.StartWithCreateProcess() will fail with the error "The directory name is invalid"
// because the working folder doesn't exist
CreateWorkingFolderIfRequired(executionContext, ArtifactsWorkingFolder);
// log the message that the user chose to skip artifact download and move on
executionContext.Output(StringUtil.Loc("RMUserChoseToSkipArtifactDownload"));
Trace.Info("Skipping artifact download based on the setting specified.");
}
else
{
ReleaseArtifacts = GetReleaseArtifacts(executionContext);
if (!ReleaseArtifacts.Any())
{
CreateArtifactsFolder(executionContext, ArtifactsWorkingFolder);
Trace.Info("No artifacts found to be downloaded by agent.");
}
}
CheckForAvailableDiskSpace(executionContext);
}
public TrackingConfig PrepareDirectory(
IExecutionContext executionContext,
IList <RepositoryResource> repositories,
WorkspaceOptions workspace)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repositories, nameof(repositories));
// Get the primary repository (self)
var primaryRepository = RepositoryUtil.GetPrimaryRepository(repositories);
ArgUtil.NotNull(primaryRepository, nameof(primaryRepository));
// TODO (next PR): We need to modify the Tracking file to handle multiple repositories (currently we are only tracking the self repo)
var trackingManager = HostContext.GetService <ITrackingManager>();
// Defer to the source provider to calculate the hash key.
Trace.Verbose("Calculating build directory hash key.");
string hashKey = primaryRepository.GetSourceDirectoryHashKey(executionContext);
Trace.Verbose($"Hash key: {hashKey}");
// Load the existing tracking file if one already exists.
string trackingFile = Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Work),
Constants.Build.Path.SourceRootMappingDirectory,
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_DefinitionId,
Constants.Build.Path.TrackingConfigFile);
Trace.Verbose($"Loading tracking config if exists: {trackingFile}");
TrackingConfigBase existingConfig = trackingManager.LoadIfExists(executionContext, trackingFile);
// Check if the build needs to be garbage collected. If the hash key
// has changed, then the existing build directory cannot be reused.
TrackingConfigBase garbageConfig = null;
if (existingConfig != null &&
!string.Equals(existingConfig.HashKey, hashKey, StringComparison.OrdinalIgnoreCase))
{
// Just store a reference to the config for now. It can safely be
// marked for garbage collection only after the new build directory
// config has been created.
Trace.Verbose($"Hash key from existing tracking config does not match. Existing key: {existingConfig.HashKey}");
garbageConfig = existingConfig;
existingConfig = null;
}
// Create a new tracking config if required.
TrackingConfig newConfig;
if (existingConfig == null)
{
Trace.Verbose("Creating a new tracking config file.");
var agentSetting = HostContext.GetService <IConfigurationStore>().GetSettings();
newConfig = trackingManager.Create(
executionContext,
primaryRepository,
hashKey,
trackingFile,
primaryRepository.TestOverrideBuildDirectory(agentSetting));
ArgUtil.NotNull(newConfig, nameof(newConfig));
}
else
{
// Convert legacy format to the new format if required.
newConfig = ConvertToNewFormat(executionContext, primaryRepository, existingConfig);
// Fill out repository type if it's not there.
// repository type is a new property introduced for maintenance job
if (string.IsNullOrEmpty(newConfig.RepositoryType))
{
newConfig.RepositoryType = primaryRepository.Type;
}
// For existing tracking config files, update the job run properties.
Trace.Verbose("Updating job run properties.");
trackingManager.UpdateJobRunProperties(executionContext, newConfig, trackingFile);
}
// Mark the old configuration for garbage collection.
if (garbageConfig != null)
{
Trace.Verbose("Marking existing config for garbage collection.");
trackingManager.MarkForGarbageCollection(executionContext, garbageConfig);
}
// Prepare the build directory.
// There are 2 ways to provide build directory clean policy.
// 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016)
// 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI)
BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, workspace);
CreateDirectory(
executionContext,
description: "build directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory),
deleteExisting: cleanOption == BuildCleanOption.All);
CreateDirectory(
executionContext,
description: "artifacts directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.ArtifactsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "test results directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.TestResultsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "binaries directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Binary);
CreateDirectory(
executionContext,
description: "source directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Source);
// Set the default clone path for each repository (the Checkout task may override this later)
foreach (var repository in repositories)
{
var repoPath = GetDefaultRepositoryPath(executionContext, repository, newConfig.SourcesDirectory);
Trace.Info($"Set repository path for repository {repository.Alias} to '{repoPath}'");
repository.Properties.Set <string>(RepositoryPropertyNames.Path, repoPath);
}
return(newConfig);
}
public async Task StartContainerAsync(IExecutionContext executionContext, object data)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ContainerInfo container = data as ContainerInfo;
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
// Check whether we are inside a container.
// Our container feature requires to map working directory from host to the container.
// If we are already inside a container, we will not able to find out the real working direcotry path on the host.
#if OS_WINDOWS
// service CExecSvc is Container Execution Agent.
ServiceController[] scServices = ServiceController.GetServices();
if (scServices.Any(x => String.Equals(x.ServiceName, "cexecsvc", StringComparison.OrdinalIgnoreCase) && x.Status == ServiceControllerStatus.Running))
{
throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer"));
}
#else
var initProcessCgroup = File.ReadLines("/proc/1/cgroup");
if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0))
{
throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer"));
}
#endif
#if OS_WINDOWS
// Check OS version (Windows server 1803 is required)
object windowsInstallationType = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "InstallationType", defaultValue: null);
ArgUtil.NotNull(windowsInstallationType, nameof(windowsInstallationType));
object windowsReleaseId = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "ReleaseId", defaultValue: null);
ArgUtil.NotNull(windowsReleaseId, nameof(windowsReleaseId));
executionContext.Debug($"Current Windows version: '{windowsReleaseId} ({windowsInstallationType})'");
if (int.TryParse(windowsReleaseId.ToString(), out int releaseId))
{
if (!windowsInstallationType.ToString().StartsWith("Server", StringComparison.OrdinalIgnoreCase) || releaseId < 1803)
{
throw new NotSupportedException(StringUtil.Loc("ContainerWindowsVersionRequirement"));
}
}
else
{
throw new ArgumentOutOfRangeException(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ReleaseId");
}
#endif
// Check docker client/server version
DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext);
ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
#if OS_WINDOWS
Version requiredDockerVersion = new Version(17, 6);
#else
Version requiredDockerVersion = new Version(17, 12);
#endif
if (dockerVersion.ServerVersion < requiredDockerVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion));
}
if (dockerVersion.ClientVersion < requiredDockerVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion));
}
// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
{
var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint);
ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint));
string username = string.Empty;
string password = string.Empty;
registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password);
ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}");
}
}
try
{
if (!container.SkipContainerImagePull)
{
if (!string.IsNullOrEmpty(registryServer) &&
registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0)
{
var registryServerUri = new Uri(registryServer);
if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase))
{
container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}";
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
}
// Mount folder into container
#if OS_WINDOWS
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
#else
string workingDirectory = Path.GetDirectoryName(executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
#endif
#if !OS_WINDOWS
if (string.IsNullOrEmpty(container.ContainerNetwork)) // create network when Windows support it.
{
// Create local docker network for this job to avoid port conflict when multiple agents run on same machine.
container.ContainerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}";
int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, container.ContainerNetwork);
if (networkExitCode != 0)
{
throw new InvalidOperationException($"Docker network create fail with exit code {networkExitCode}");
}
// Expose docker network to env
executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, container.ContainerNetwork);
}
#endif
container.ContainerId = await _dockerManger.DockerCreate(context : executionContext,
displayName : container.ContainerDisplayName,
image : container.ContainerImage,
mountVolumes : container.MountVolumes,
network : container.ContainerNetwork,
options : container.ContainerCreateOptions,
environment : container.ContainerEnvironmentVariables);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
}
finally
{
// Logout for private registry
if (!string.IsNullOrEmpty(registryServer))
{
int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer);
if (logoutExitCode != 0)
{
executionContext.Error($"Docker logout fail with exit code {logoutExitCode}");
}
}
}
#if !OS_WINDOWS
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash");
if (execWhichBashExitCode != 0)
{
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, container.ContainerId, "--filter status=running");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, container.ContainerId, string.Empty);
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_VSTSContainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new vsts_sudo group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd VSTS_Container_SUDO");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created VSTS_SUDO group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G VSTS_Container_SUDO {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new vsts_sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%VSTS_Container_SUDO ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
#endif
}
public async Task <int> ExecuteAsync(string workingDirectory,
string fileName,
string arguments,
IDictionary <string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
bool inheritConsoleHandler,
CancellationToken cancellationToken)
{
// make sure container exist.
ArgUtil.NotNull(Container, nameof(Container));
ArgUtil.NotNullOrEmpty(Container.ContainerId, nameof(Container.ContainerId));
var dockerManger = HostContext.GetService <IDockerCommandManager>();
string containerEnginePath = dockerManger.DockerPath;
ContainerStandardInPayload payload = new ContainerStandardInPayload()
{
ExecutionHandler = fileName,
ExecutionHandlerWorkingDirectory = workingDirectory,
ExecutionHandlerArguments = arguments,
ExecutionHandlerEnvironment = environment,
ExecutionHandlerPrependPath = PrependPath
};
// copy the intermediate script (containerHandlerInvoker.js) into Agent_TempDirectory
// Background:
// We rely on environment variables to send task execution information from agent to task execution engine (node/powershell)
// Those task execution information will include all the variables and secrets customer has.
// The only way to pass environment variables to `docker exec` is through command line arguments, ex: `docker exec -e myenv=myvalue -e mysecert=mysecretvalue ...`
// Since command execution may get log into system event log which might cause secret leaking.
// We use this intermediate script to read everything from STDIN, then launch the task execution engine (node/powershell) and redirect STDOUT/STDERR
string tempDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.TempDirectory);
File.Copy(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "containerHandlerInvoker.js.template"), Path.Combine(tempDir, "containerHandlerInvoker.js"), true);
string node;
if (!string.IsNullOrEmpty(Container.ContainerBringNodePath))
{
node = Container.ContainerBringNodePath;
}
else
{
node = Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
}
string entryScript = Container.TranslateToContainerPath(Path.Combine(tempDir, "containerHandlerInvoker.js"));
string containerExecutionArgs;
if (PlatformUtil.RunningOnWindows)
{
containerExecutionArgs = $"exec -i {Container.ContainerId} {node} {entryScript}";
}
else
{
containerExecutionArgs = $"exec -i -u {Container.CurrentUserId} {Container.ContainerId} {node} {entryScript}";
}
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OutputDataReceived;
processInvoker.ErrorDataReceived += ErrorDataReceived;
// Let .NET choose the default encoding, except on Windows.
outputEncoding = null;
if (PlatformUtil.RunningOnWindows)
{
// It appears that node.exe outputs UTF8 when not in TTY mode.
outputEncoding = Encoding.UTF8;
}
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(payload));
return(await processInvoker.ExecuteAsync(workingDirectory : HostContext.GetDirectory(WellKnownDirectory.Work),
fileName : containerEnginePath,
arguments : containerExecutionArgs,
environment : null,
requireExitCodeZero : requireExitCodeZero,
outputEncoding : outputEncoding,
killProcessOnCancel : killProcessOnCancel,
redirectStandardIn : redirectStandardIn,
inheritConsoleHandler : inheritConsoleHandler,
cancellationToken : cancellationToken));
}
}
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobDisplayName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
// Because an agent can be idle for a long time between jobs, it is possible that in that time
// a firewall has closed the connection. For that reason, forcibly reestablish this connection at the
// start of a new job
var agentServer = HostContext.GetService <IAgentServer>();
await agentServer.RefreshConnectionAsync(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(30));
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}");
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
// Start the child process.
HostContext.WritePerfCounter("StartingWorkerProcess");
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
redirectStandardIn: null,
inheritConsoleHandler: false,
keepStandardInOpen: false,
highPriorityProcess: true,
cancellationToken: workerProcessCancelTokenSource.Token);
}
);
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
var body = JsonUtility.ToString(message);
var numBytes = System.Text.ASCIIEncoding.Unicode.GetByteCount(body) / 1024;
string numBytesString = numBytes > 0 ? $"{numBytes} KB" : " < 1 KB";
Trace.Info($"Send job request message to worker for job {message.JobId} ({numBytesString}).");
HostContext.WritePerfCounter($"AgentSendingJobToWorker_{message.JobId}");
var stopWatch = Stopwatch.StartNew();
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : body,
cancellationToken : csSendJobRequest.Token);
}
stopWatch.Stop();
Trace.Info($"Took {stopWatch.ElapsedMilliseconds} ms to send job message to worker");
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled after waiting for {_channelTimeout.TotalSeconds} seconds, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
var accessToken = systemConnection?.Authorization?.Parameters["AccessToken"];
VariableValue identifier = new VariableValue("0");
VariableValue definitionId = new VariableValue("0");
if (message.Plan.PlanType == "Build")
{
message.Variables.TryGetValue("build.buildId", out identifier);
message.Variables.TryGetValue("system.definitionId", out definitionId);
}
else if (message.Plan.PlanType == "Release")
{
message.Variables.TryGetValue("release.deploymentId", out identifier);
message.Variables.TryGetValue("release.definitionId", out definitionId);
}
await notification.JobStarted(message.JobId, accessToken, systemConnection.Url, message.Plan.PlanId, identifier.Value, definitionId.Value, message.Plan.PlanType);
HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}");
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
await LogWorkerProcessUnhandledException(message, detailInfo);
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandled exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher priority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandled exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
}
// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
{
var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint);
ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint));
string username = string.Empty;
string password = string.Empty;
registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password);
ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}");
}
}
try
{
if (!container.SkipContainerImagePull)
{
if (!string.IsNullOrEmpty(registryServer) &&
registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0)
{
var registryServerUri = new Uri(registryServer);
if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase))
{
container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}";
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
}
// Mount folder into container
#if OS_WINDOWS
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
#else
string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
if (string.IsNullOrEmpty(defaultWorkingDirectory))
{
throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir"));
}
string workingDirectory = Path.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
#endif
if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
string node;
if (!string.IsNullOrEmpty(container.ContainerBringNodePath))
{
node = container.ContainerBringNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
if (container.IsJobContainer)
{
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
}
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
}
finally
{
// Logout for private registry
if (!string.IsNullOrEmpty(registryServer))
{
int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer);
if (logoutExitCode != 0)
{
executionContext.Error($"Docker logout fail with exit code {logoutExitCode}");
}
}
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Get port mappings of running container
if (executionContext.Container == null && !container.IsJobContainer)
{
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
executionContext.Variables.Set(
$"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}",
$"{port.HostPort}");
}
}
#if !OS_WINDOWS
if (container.IsJobContainer)
{
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_azpcontainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created sudo group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true);
if (setupDockerGroup)
{
executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName));
// Get docker.sock group id on Host
string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"-c %g /var/run/docker.sock")).FirstOrDefault();
// We need to find out whether there is a group with same GID inside the container
string existingGroupName = null;
List <string> groupsOutput = new List <string>();
int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput);
if (execGroupGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}");
}
if (groupsOutput.Count > 0)
{
// check all potential groups that might match the GID.
foreach (string groupOutput in groupsOutput)
{
if (!string.IsNullOrEmpty(groupOutput))
{
var groupSegments = groupOutput.Split(':');
if (groupSegments.Length != 4)
{
Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'");
}
else
{
// the output of /etc/group should looks like `group:x:gid:`
var groupName = groupSegments[0];
var groupId = groupSegments[2];
if (string.Equals(dockerSockGroupId, groupId))
{
existingGroupName = groupName;
break;
}
}
}
}
}
if (string.IsNullOrEmpty(existingGroupName))
{
// create a new group with same gid
existingGroupName = "azure_pipelines_docker";
int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker");
if (execDockerGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}");
}
}
// Add the new created user to the docker socket group.
int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}");
if (execGroupUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}");
}
}
}
#endif
}
public async Task ChmodAsync(string mode, string file)
{
Trace.Entering();
await ExecAsync(HostContext.GetDirectory(WellKnownDirectory.Root), "chmod", $"{mode} \"{file}\"");
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(Data, nameof(Data));
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
if (!PlatformUtil.RunningOnWindows)
{
// Ensure compat vso-task-lib exist at the root of _work folder
// This will make vsts-agent work against 2015 RTM/QU1 TFS, since tasks in those version doesn't package with task lib
// Put the 0.5.5 version vso-task-lib into the root of _work/node_modules folder, so tasks are able to find those lib.
if (!File.Exists(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "node_modules", "vso-task-lib", "package.json")))
{
string vsoTaskLibFromExternal = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "vso-task-lib");
string compatVsoTaskLibInWork = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "node_modules", "vso-task-lib");
IOUtil.CopyDirectory(vsoTaskLibFromExternal, compatVsoTaskLibInWork, ExecutionContext.CancellationToken);
}
}
// Update the env dictionary.
AddInputsToEnvironment();
AddEndpointsToEnvironment();
AddSecureFilesToEnvironment();
AddVariablesToEnvironment();
AddTaskVariablesToEnvironment();
AddPrependPathToEnvironment();
// Resolve the target script.
string target = Data.Target;
ArgUtil.NotNullOrEmpty(target, nameof(target));
target = Path.Combine(TaskDirectory, target);
ArgUtil.File(target, nameof(target));
// Resolve the working directory.
string workingDirectory = Data.WorkingDirectory;
if (string.IsNullOrEmpty(workingDirectory))
{
workingDirectory = ExecutionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
if (string.IsNullOrEmpty(workingDirectory))
{
workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
}
}
// fix vsts-task-lib for node 6.x
// vsts-task-lib 0.6/0.7/0.8/0.9/2.0-preview implemented String.prototype.startsWith and String.prototype.endsWith since Node 5.x doesn't have them.
// however the implementation is added in node 6.x, the implementation in vsts-task-lib is different.
// node 6.x's implementation takes 2 parameters str.endsWith(searchString[, length]) / str.startsWith(searchString[, length])
// the implementation vsts-task-lib had only takes one parameter str.endsWith(searchString) / str.startsWith(searchString).
// as long as vsts-task-lib be loaded into memory, it will overwrite the implementation node 6.x has,
// so any script that use the second parameter (length) will encounter unexpected result.
// to avoid customer hit this error, we will modify the file (extensions.js) under vsts-task-lib module folder when customer choose to use Node 6.x
Trace.Info("Inspect node_modules folder, make sure vsts-task-lib doesn't overwrite String.startsWith/endsWith.");
FixVstsTaskLibModule();
StepHost.OutputDataReceived += OnDataReceived;
StepHost.ErrorDataReceived += OnDataReceived;
string file;
if (!string.IsNullOrEmpty(ExecutionContext.Container?.ContainerBringNodePath))
{
file = ExecutionContext.Container.ContainerBringNodePath;
}
else
{
file = GetNodeLocation();
}
// Format the arguments passed to node.
// 1) Wrap the script file path in double quotes.
// 2) Escape double quotes within the script file path. Double-quote is a valid
// file name character on Linux.
string arguments = StepHost.ResolvePathForStepHost(StringUtil.Format(@"""{0}""", target.Replace(@"""", @"\""")));
// Let .NET choose the default, except on Windows.
Encoding outputEncoding = null;
if (PlatformUtil.RunningOnWindows)
{
// It appears that node.exe outputs UTF8 when not in TTY mode.
outputEncoding = Encoding.UTF8;
}
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Task failure should be communicated over STDOUT using ## commands.
Task step = StepHost.ExecuteAsync(workingDirectory: StepHost.ResolvePathForStepHost(workingDirectory),
fileName: StepHost.ResolvePathForStepHost(file),
arguments: arguments,
environment: Environment,
requireExitCodeZero: true,
outputEncoding: outputEncoding,
killProcessOnCancel: false,
inheritConsoleHandler: !ExecutionContext.Variables.Retain_Default_Encoding,
cancellationToken: ExecutionContext.CancellationToken);
// Wait for either the node exit or force finish through ##vso command
await System.Threading.Tasks.Task.WhenAny(step, ExecutionContext.ForceCompleted);
if (ExecutionContext.ForceCompleted.IsCompleted)
{
ExecutionContext.Debug("The task was marked as \"done\", but the process has not closed after 5 seconds. Treating the task as complete.");
}
else
{
await step;
}
}
public async Task ChownAsync(string owner, string group, string file)
{
Trace.Entering();
await ExecAsync(HostContext.GetDirectory(WellKnownDirectory.Root), "chown", $"{owner}:{group} \"{file}\"");
}
private void AddProxySetting(IVstsAgentWebProxy agentProxy)
{
string appConfig = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), _appConfigFileName);
ArgUtil.File(appConfig, _appConfigFileName);
string appConfigRestore = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), _appConfigRestoreFileName);
if (!File.Exists(appConfigRestore))
{
Trace.Info("Take snapshot of current appconfig for restore modified appconfig.");
File.Copy(appConfig, appConfigRestore);
}
else
{
// cleanup any appconfig changes from previous build.
ExecutionContext.Debug("Restore default LegacyVSTSPowerShellHost.exe.config.");
IOUtil.DeleteFile(appConfig);
File.Copy(appConfigRestore, appConfig);
}
XmlDocument psHostAppConfig = new XmlDocument();
using (var appConfigStream = new FileStream(appConfig, FileMode.Open, FileAccess.Read))
{
psHostAppConfig.Load(appConfigStream);
}
var configuration = psHostAppConfig.SelectSingleNode("configuration");
ArgUtil.NotNull(configuration, "configuration");
var exist_defaultProxy = psHostAppConfig.SelectSingleNode("configuration/system.net/defaultProxy");
if (exist_defaultProxy == null)
{
var system_net = psHostAppConfig.SelectSingleNode("configuration/system.net");
if (system_net == null)
{
Trace.Verbose("Create system.net section in appconfg.");
system_net = psHostAppConfig.CreateElement("system.net");
}
Trace.Verbose("Create defaultProxy section in appconfg.");
var defaultProxy = psHostAppConfig.CreateElement("defaultProxy");
defaultProxy.SetAttribute("useDefaultCredentials", "true");
Trace.Verbose("Create proxy section in appconfg.");
var proxy = psHostAppConfig.CreateElement("proxy");
proxy.SetAttribute("proxyaddress", agentProxy.ProxyAddress);
proxy.SetAttribute("bypassonlocal", "true");
if (agentProxy.ProxyBypassList != null && agentProxy.ProxyBypassList.Count > 0)
{
Trace.Verbose("Create bypasslist section in appconfg.");
var bypass = psHostAppConfig.CreateElement("bypasslist");
foreach (string proxyBypass in agentProxy.ProxyBypassList)
{
Trace.Verbose($"Create bypasslist.add section for {proxyBypass} in appconfg.");
var add = psHostAppConfig.CreateElement("add");
add.SetAttribute("address", proxyBypass);
bypass.AppendChild(add);
}
defaultProxy.AppendChild(bypass);
}
defaultProxy.AppendChild(proxy);
system_net.AppendChild(defaultProxy);
configuration.AppendChild(system_net);
using (var appConfigStream = new FileStream(appConfig, FileMode.Open, FileAccess.ReadWrite))
{
psHostAppConfig.Save(appConfigStream);
}
ExecutionContext.Debug("Add Proxy setting in LegacyVSTSPowerShellHost.exe.config file.");
}
else
{
//proxy setting exist.
ExecutionContext.Debug("Proxy setting already exist in LegacyVSTSPowerShellHost.exe.config file.");
}
}
/// <summary>
/// _work
/// \_update
/// \bin
/// \externals
/// \run.sh
/// \run.cmd
/// \package.zip //temp download .zip/.tar.gz
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private async Task DownloadLatestAgent(CancellationToken token)
{
string latestAgentDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.UpdateDirectory);
IOUtil.DeleteDirectory(latestAgentDirectory, token);
Directory.CreateDirectory(latestAgentDirectory);
int agentSuffix = 1;
string archiveFile = null;
bool downloadSucceeded = false;
try
{
// Download the agent, using multiple attempts in order to be resilient against any networking/CDN issues
for (int attempt = 1; attempt <= Constants.AgentDownloadRetryMaxAttempts; attempt++)
{
// Generate an available package name, and do our best effort to clean up stale local zip files
while (true)
{
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.zip");
}
else
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.tar.gz");
}
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip '{0}'", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
break;
}
catch (Exception ex)
{
// couldn't delete the file for whatever reason, so generate another name
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
agentSuffix++;
}
}
// Allow a 15-minute package download timeout, which is good enough to update the agent from a 1 Mbit/s ADSL connection.
var timeoutSeconds = AgentKnobs.AgentDownloadTimeout.GetValue(_knobContext).AsInt();
Trace.Info($"Attempt {attempt}: save latest agent into {archiveFile}.");
using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token))
{
try
{
Trace.Info($"Download agent: begin download");
//open zip stream in async mode
using (var handler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
using (var fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
using (var result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, downloadCts.Token);
await fs.FlushAsync(downloadCts.Token);
}
Trace.Info($"Download agent: finished download");
downloadSucceeded = true;
break;
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"Agent download has been canceled.");
throw;
}
catch (Exception ex)
{
if (downloadCts.Token.IsCancellationRequested)
{
Trace.Warning($"Agent download has timed out after {timeoutSeconds} seconds");
}
Trace.Warning($"Failed to get package '{archiveFile}' from '{_targetPackage.DownloadUrl}'. Exception {ex}");
}
}
}
if (!downloadSucceeded)
{
throw new TaskCanceledException($"Agent package '{archiveFile}' failed after {Constants.AgentDownloadRetryMaxAttempts} download attempts");
}
// If we got this far, we know that we've successfully downloadeded the agent package
if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
ZipFile.ExtractToDirectory(archiveFile, latestAgentDirectory);
}
else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
string tar = WhichUtil.Which("tar", trace: Trace);
if (string.IsNullOrEmpty(tar))
{
throw new NotSupportedException($"tar -xzf");
}
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(latestAgentDirectory, tar, $"-xzf \"{archiveFile}\"", null, token);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
}
else
{
throw new NotSupportedException($"{archiveFile}");
}
Trace.Info($"Finished getting latest agent package at: {latestAgentDirectory}.");
}
finally
{
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip: {0}", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the .zip file
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
}
}
// copy latest agent into agent root folder
// copy bin from _work/_update -> bin.version under root
string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(binVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory)} to {binVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory), binVersionDir, token);
// copy externals from _work/_update -> externals.version under root
string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(externalsVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token);
// copy and replace all .sh/.cmd files
Trace.Info($"Copy any remaining .sh/.cmd files into agent root.");
foreach (FileInfo file in new DirectoryInfo(latestAgentDirectory).GetFiles() ?? new FileInfo[0])
{
// Copy and replace the file.
file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
}
// for windows service back compat with old windows agent, we need make sure the servicehost.exe is still the old name
// if the current bin folder has VsoAgentService.exe, then the new agent bin folder needs VsoAgentService.exe as well
if (PlatformUtil.RunningOnWindows)
{
if (File.Exists(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "VsoAgentService.exe")))
{
Trace.Info($"Make a copy of AgentService.exe, name it VsoAgentService.exe");
File.Copy(Path.Combine(binVersionDir, "AgentService.exe"), Path.Combine(binVersionDir, "VsoAgentService.exe"), true);
File.Copy(Path.Combine(binVersionDir, "AgentService.exe.config"), Path.Combine(binVersionDir, "VsoAgentService.exe.config"), true);
Trace.Info($"Make a copy of Agent.Listener.exe, name it VsoAgent.exe");
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.exe"), Path.Combine(binVersionDir, "VsoAgent.exe"), true);
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.dll"), Path.Combine(binVersionDir, "VsoAgent.dll"), true);
// in case of we remove all pdb file from agent package.
if (File.Exists(Path.Combine(binVersionDir, "AgentService.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "AgentService.pdb"), Path.Combine(binVersionDir, "VsoAgentService.pdb"), true);
}
if (File.Exists(Path.Combine(binVersionDir, "Agent.Listener.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.pdb"), Path.Combine(binVersionDir, "VsoAgent.pdb"), true);
}
}
}
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
// Agent.RunMode
RunMode runMode;
if (message.Variables.ContainsKey(Constants.Variables.Agent.RunMode) &&
Enum.TryParse(message.Variables[Constants.Variables.Agent.RunMode].Value, ignoreCase: true, result: out runMode) &&
runMode == RunMode.Local)
{
HostContext.RunMode = runMode;
}
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// System.AccessToken
if (message.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Variables[Constants.Variables.System.EnableAccessToken].Value))
{
message.Variables[Constants.Variables.System.AccessToken] = new VariableValue(systemConnection.Authorization.Parameters["AccessToken"], false);
}
// back compat TfsServerUrl
message.Variables[Constants.Variables.System.TFServerUrl] = systemConnection.Url.AbsoluteUri;
// Make sure SystemConnection Url and Endpoint Url match Config Url base for OnPremises server
// System.ServerType will always be there after M133
if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType) ||
string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase))
{
ReplaceConfigUriBaseInJobRequestMessage(message);
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");
IExecutionContext jobContext = null;
CancellationTokenRegistration?agentShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName));
agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() =>
{
// log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the agent that operating system is shutting down.
string errorMessage;
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = StringUtil.Loc("UserShutdownAgent");
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName);
break;
default:
throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.SetVariable(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root), isFilePath: true);
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobDisplayName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.OS, VarUtil.OS);
jobContext.SetVariable(Constants.Variables.Agent.RootDirectory, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
#if OS_WINDOWS
jobContext.SetVariable(Constants.Variables.Agent.ServerOMDirectory, HostContext.GetDirectory(WellKnownDirectory.ServerOM), isFilePath: true);
#else
jobContext.Variables.Set(Constants.Variables.Agent.AcceptTeeEula, settings.AcceptTeeEula.ToString());
#endif
jobContext.SetVariable(Constants.Variables.Agent.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
jobContext.SetVariable(Constants.Variables.System.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
Directory.CreateDirectory(toolsDirectory);
jobContext.SetVariable(Constants.Variables.Agent.ToolsDirectory, toolsDirectory, isFilePath: true);
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// todo: task server can throw. try/catch and fail job gracefully.
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = VssUtil.GetVssCredential(systemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// for back compat TFS 2015 RTM/QU1, we may need to switch the task server url to agent config url
if (!string.Equals(message.Variables.GetValueOrDefault(Constants.Variables.System.ServerType)?.Value, "Hosted", StringComparison.OrdinalIgnoreCase))
{
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist())
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential));
}
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Expand the repository property values.
foreach (var repository in jobContext.Repositories)
{
Dictionary <string, string> expandProperties = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var property in repository.Properties.GetItems())
{
expandProperties[property.Key] = JsonUtility.ToString(property.Value);
}
jobContext.Variables.ExpandValues(target: expandProperties);
VarUtil.ExpandEnvironmentVariables(HostContext, target: expandProperties);
foreach (var expandedProperty in expandProperties)
{
repository.Properties.Set <JToken>(expandedProperty.Key, JsonUtility.FromString <JToken>(expandedProperty.Value));
}
}
// Get the job extension.
Trace.Info("Getting job extension.");
var hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
// We should always have one job extension
IJobExtension jobExtension =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => x.HostType.HasFlag(hostType))
.FirstOrDefault();
ArgUtil.NotNull(jobExtension, nameof(jobExtension));
List <IStep> jobSteps = new List <IStep>();
try
{
Trace.Info("Initialize job. Getting all job steps.");
var initializeResult = await jobExtension.InitializeJob(jobContext, message);
jobSteps.AddRange(initializeResult.PreJobSteps);
jobSteps.AddRange(initializeResult.JobSteps);
jobSteps.AddRange(initializeResult.PostJobStep);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");
bool processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
HashSet <string> existingProcesses = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
string processLookupId = null;
if (processCleanup)
{
processLookupId = $"vsts_{Guid.NewGuid()}";
// Set the VSTS_PROCESS_LOOKUP_ID env variable.
jobContext.SetVariable(Constants.ProcessLookupId, processLookupId, false, false);
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, jobSteps);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
if (processCleanup)
{
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
string lookupId = null;
try
{
lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessLookupId);
}
catch (Exception ex)
{
Trace.Verbose("Ignore any exception during read process environment variables.");
Trace.Verbose(ex.ToString());
}
if (string.Equals(lookupId, processLookupId, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false)
{
Trace.Info("Support log upload starting.");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc);
Trace.Info("Support log upload complete.");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
Trace.Error(ex);
}
}
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (agentShutdownRegistration != null)
{
agentShutdownRegistration.Value.Dispose();
agentShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(task, nameof(task));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
var taskServer = HostContext.GetService <ITaskServer>();
// first check to see if we already have the task
string destDirectory = GetDirectory(task);
Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
Boolean signingEnabled = (settings.SignatureVerification != null && settings.SignatureVerification.Mode != SignatureVerificationMode.None);
Boolean alwaysExtractTask = signingEnabled || settings.AlwaysExtractTask;
if (File.Exists(destDirectory + ".completed") && !alwaysExtractTask)
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
return;
}
String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip");
if (alwaysExtractTask && File.Exists(taskZipPath))
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'.");
// Extract a new zip every time
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
ExtractZip(taskZipPath, destDirectory);
return;
}
// delete existing task folder.
Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);
// Inform the user that a download is taking place. The download could take a while if
// the task zip is large. It would be nice to print the localized name, but it is not
// available from the reference included in the job message.
executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version));
string zipFile = string.Empty;
var version = new TaskVersion(task.Version);
//download and extract task in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid());
try
{
Directory.CreateDirectory(tempDirectory);
int retryCount = 0;
// Allow up to 20 * 60s for any task to be downloaded from service.
// Base on Kusto, the longest we have on the service today is over 850 seconds.
// Timeout limit can be overwrite by environment variable
var timeoutSeconds = AgentKnobs.TaskDownloadTimeout.GetValue(UtilKnobValueContext.Instance()).AsInt();
while (retryCount < 3)
{
using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));
//open zip stream in async mode
using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token);
await fs.FlushAsync(taskDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Task download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}");
Trace.Error(ex);
if (taskDownloadTimeout.Token.IsCancellationRequested)
{
// task download didn't finish within timeout
executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds));
}
else
{
executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message));
if (ex.InnerException != null)
{
executionContext.Warning("Inner Exception: {ex.InnerException.Message}");
}
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
if (alwaysExtractTask)
{
Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips));
// Copy downloaded zip to the cache on disk for future extraction.
executionContext.Debug($"Copying from {zipFile} to {taskZipPath}");
File.Copy(zipFile, taskZipPath);
}
// We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob.
// This is fine because we overwrite the contents at task run time.
Directory.CreateDirectory(destDirectory);
ExtractZip(zipFile, destDirectory);
executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'.");
Trace.Info("Finished getting task.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
}
}
}