public void Verbose(string message)
{
ArgUtil.NotNull(message, nameof(message));
#if DEBUG
Debug(message);
#else
string vstsAgentTrace = AgentKnobs.TraceVerbose.GetValue(UtilKnobValueContext.Instance()).AsString();
if (!string.IsNullOrEmpty(vstsAgentTrace))
{
Debug(message);
}
#endif
}
public override void Initialize(IHostContext hostContext)
{
ArgUtil.NotNull(hostContext, nameof(hostContext));
base.Initialize(hostContext);
// get pool id from config
var configurationStore = hostContext.GetService <IConfigurationStore>();
_agentSetting = configurationStore.GetSettings();
_poolId = _agentSetting.PoolId;
int channelTimeoutSeconds = AgentKnobs.AgentChannelTimeout.GetValue(UtilKnobValueContext.Instance()).AsInt();
// _channelTimeout should in range [30, 300] seconds
_channelTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(channelTimeoutSeconds, 30), 300));
Trace.Info($"Set agent/worker IPC timeout to {_channelTimeout.TotalSeconds} seconds.");
}
private static string Unescape(string escaped)
{
if (string.IsNullOrEmpty(escaped))
{
return(string.Empty);
}
string unescaped = escaped;
foreach (EscapeMapping mapping in s_escapeMappings)
{
unescaped = unescaped.Replace(mapping.Replacement, mapping.Token);
}
if (AgentKnobs.DecodePercents.GetValue(UtilKnobValueContext.Instance()).AsBoolean())
{
unescaped = unescaped.Replace("%25", "%");
}
return(unescaped);
}
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(task, nameof(task));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
var taskServer = HostContext.GetService <ITaskServer>();
// first check to see if we already have the task
string destDirectory = GetDirectory(task);
Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
Boolean signingEnabled = (settings.SignatureVerification != null && settings.SignatureVerification.Mode != SignatureVerificationMode.None);
Boolean alwaysExtractTask = signingEnabled || settings.AlwaysExtractTask;
if (File.Exists(destDirectory + ".completed") && !alwaysExtractTask)
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
return;
}
String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip");
if (alwaysExtractTask && File.Exists(taskZipPath))
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'.");
// Extract a new zip every time
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
ExtractZip(taskZipPath, destDirectory);
return;
}
// delete existing task folder.
Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);
// Inform the user that a download is taking place. The download could take a while if
// the task zip is large. It would be nice to print the localized name, but it is not
// available from the reference included in the job message.
executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version));
string zipFile = string.Empty;
var version = new TaskVersion(task.Version);
//download and extract task in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid());
try
{
Directory.CreateDirectory(tempDirectory);
int retryCount = 0;
// Allow up to 20 * 60s for any task to be downloaded from service.
// Base on Kusto, the longest we have on the service today is over 850 seconds.
// Timeout limit can be overwrite by environment variable
var timeoutSeconds = AgentKnobs.TaskDownloadTimeout.GetValue(UtilKnobValueContext.Instance()).AsInt();
while (retryCount < 3)
{
using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));
//open zip stream in async mode
using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token);
await fs.FlushAsync(taskDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Task download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}");
Trace.Error(ex);
if (taskDownloadTimeout.Token.IsCancellationRequested)
{
// task download didn't finish within timeout
executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds));
}
else
{
executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message));
if (ex.InnerException != null)
{
executionContext.Warning("Inner Exception: {ex.InnerException.Message}");
}
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
if (alwaysExtractTask)
{
Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips));
// Copy downloaded zip to the cache on disk for future extraction.
executionContext.Debug($"Copying from {zipFile} to {taskZipPath}");
File.Copy(zipFile, taskZipPath);
}
// We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob.
// This is fine because we overwrite the contents at task run time.
Directory.CreateDirectory(destDirectory);
ExtractZip(zipFile, destDirectory);
executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'.");
Trace.Info("Finished getting task.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
}
}
}
public async Task <List <Capability> > GetCapabilitiesAsync(AgentSettings settings, CancellationToken cancellationToken)
{
Trace.Entering();
var capabilities = new List <Capability>();
string powerShellExe = HostContext.GetService <IPowerShellExeUtil>().GetPath();
string scriptFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "powershell", "Add-Capabilities.ps1").Replace("'", "''");
ArgUtil.File(scriptFile, nameof(scriptFile));
string arguments = $@"-NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "". '{scriptFile}'""";
string enablePrereleaseVSVersions = AgentKnobs.EnableVSPreReleaseVersions.GetValue(UtilKnobValueContext.Instance()).AsString();
Environment.SetEnvironmentVariable("IncludePrereleaseVersions", enablePrereleaseVSVersions);
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived +=
(object sender, ProcessDataReceivedEventArgs args) =>
{
Trace.Info($"STDOUT: {args.Data}");
Capability capability;
if (TryParseCapability(args.Data, out capability))
{
Trace.Info($"Adding '{capability.Name}': '{capability.Value}'");
capabilities.Add(capability);
}
};
processInvoker.ErrorDataReceived +=
(object sender, ProcessDataReceivedEventArgs args) =>
{
Trace.Info($"STDERR: {args.Data}");
};
await processInvoker.ExecuteAsync(
workingDirectory : Path.GetDirectoryName(scriptFile),
fileName : powerShellExe,
arguments : arguments,
environment : null,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : true,
cancellationToken : cancellationToken);
}
return(capabilities);
}
public bool TryProcessCommand(IExecutionContext context, string input)
{
ArgUtil.NotNull(context, nameof(context));
if (string.IsNullOrEmpty(input))
{
return(false);
}
// TryParse input to Command
Command command;
if (!Command.TryParse(input, out command))
{
// if parse fail but input contains ##vso, print warning with DOC link
if (input.IndexOf("##vso") >= 0)
{
context.Warning(StringUtil.Loc("CommandKeywordDetected", input));
}
return(false);
}
IWorkerCommandExtension extension = null;
if (_invokePluginInternalCommand && string.Equals(command.Area, _pluginInternalCommandExtensions.CommandArea, StringComparison.OrdinalIgnoreCase))
{
extension = _pluginInternalCommandExtensions;
}
if (extension != null || _commandExtensions.TryGetValue(command.Area, out extension))
{
if (!extension.SupportedHostTypes.HasFlag(context.Variables.System_HostType))
{
context.Error(StringUtil.Loc("CommandNotSupported", command.Area, context.Variables.System_HostType));
context.CommandResult = TaskResult.Failed;
return(false);
}
// process logging command in serialize oreder.
lock (_commandSerializeLock)
{
try
{
extension.ProcessCommand(context, command, restrictionPolicy);
}
catch (Exception ex)
{
context.Error(StringUtil.Loc("CommandProcessFailed", input));
context.Error(ex);
context.CommandResult = TaskResult.Failed;
}
finally
{
// trace the ##vso command as long as the command is not a ##vso[task.debug] command.
if (!(string.Equals(command.Area, "task", StringComparison.OrdinalIgnoreCase) &&
string.Equals(command.Event, "debug", StringComparison.OrdinalIgnoreCase)))
{
context.Debug($"Processed: {input}");
}
}
}
}
else
{
context.Warning(StringUtil.Loc("CommandNotFound", command.Area));
}
// Only if we've successfully parsed do we show this warning
if (AgentKnobs.DecodePercents.GetValue(UtilKnobValueContext.Instance()).ToString() == "" && input.Contains("%25"))
{
context.Warning("%25 detected in ##vso command. In January 2021, the agent command parser will be updated to unescape this to %. To opt out of this behavior, set environment variable DECODE_PERCENTS to false. Setting to true will force this behavior immediately. More information can be found at https://github.com/microsoft/azure-pipelines-agent/blob/master/docs/design/percentEncoding.md");
}
return(true);
}
