public void Verbose(string message) { #if DEBUG Debug(message); #else string vstsAgentTrace = AgentKnobs.TraceVerbose.GetValue(UtilKnobValueContext.Instance()).AsString(); if (!string.IsNullOrEmpty(vstsAgentTrace)) { Debug(message); } #endif }
private string Escape(string input) { if (AgentKnobs.DecodePercents.GetValue(UtilKnobValueContext.Instance()).AsBoolean()) { input = input.Replace("%", "%25"); } foreach (var mapping in _commandEscapeMappings) { input = input.Replace(mapping.Key, mapping.Value); } return(input); }
public override void Initialize(IHostContext hostContext) { ArgUtil.NotNull(hostContext, nameof(hostContext)); base.Initialize(hostContext); // get pool id from config var configurationStore = hostContext.GetService <IConfigurationStore>(); _agentSetting = configurationStore.GetSettings(); _poolId = _agentSetting.PoolId; int channelTimeoutSeconds = AgentKnobs.AgentChannelTimeout.GetValue(UtilKnobValueContext.Instance()).AsInt(); // _channelTimeout should in range [30, 300] seconds _channelTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(channelTimeoutSeconds, 30), 300)); Trace.Info($"Set agent/worker IPC timeout to {_channelTimeout.TotalSeconds} seconds."); }
private static string Unescape(string escaped) { if (string.IsNullOrEmpty(escaped)) { return(string.Empty); } string unescaped = escaped; foreach (EscapeMapping mapping in s_escapeMappings) { unescaped = unescaped.Replace(mapping.Replacement, mapping.Token); } if (AgentKnobs.DecodePercents.GetValue(UtilKnobValueContext.Instance()).AsBoolean()) { unescaped = unescaped.Replace("%25", "%"); } return(unescaped); }
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(task, nameof(task)); ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version)); var taskServer = HostContext.GetService <ITaskServer>(); // first check to see if we already have the task string destDirectory = GetDirectory(task); Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'."); var configurationStore = HostContext.GetService <IConfigurationStore>(); AgentSettings settings = configurationStore.GetSettings(); Boolean signingEnabled = (settings.SignatureVerification != null && settings.SignatureVerification.Mode != SignatureVerificationMode.None); Boolean alwaysExtractTask = signingEnabled || settings.AlwaysExtractTask; if (File.Exists(destDirectory + ".completed") && !alwaysExtractTask) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'."); return; } String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip"); if (alwaysExtractTask && File.Exists(taskZipPath)) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'."); // Extract a new zip every time IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken); ExtractZip(taskZipPath, destDirectory); return; } // delete existing task folder. Trace.Verbose("Deleting task destination folder: {0}", destDirectory); IOUtil.DeleteDirectory(destDirectory, CancellationToken.None); // Inform the user that a download is taking place. The download could take a while if // the task zip is large. It would be nice to print the localized name, but it is not // available from the reference included in the job message. executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version)); string zipFile = string.Empty; var version = new TaskVersion(task.Version); //download and extract task in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid()); try { Directory.CreateDirectory(tempDirectory); int retryCount = 0; // Allow up to 20 * 60s for any task to be downloaded from service. // Base on Kusto, the longest we have on the service today is over 850 seconds. // Timeout limit can be overwrite by environment variable var timeoutSeconds = AgentKnobs.TaskDownloadTimeout.GetValue(UtilKnobValueContext.Instance()).AsInt(); while (retryCount < 3) { using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken)) { try { zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid())); //open zip stream in async mode using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token)) { await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token); await fs.FlushAsync(taskDownloadCancellation.Token); // download succeed, break out the retry loop. break; } } catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested) { Trace.Info($"Task download has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}"); Trace.Error(ex); if (taskDownloadTimeout.Token.IsCancellationRequested) { // task download didn't finish within timeout executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds)); } else { executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message)); if (ex.InnerException != null) { executionContext.Warning("Inner Exception: {ex.InnerException.Message}"); } } } } if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF"))) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } if (alwaysExtractTask) { Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips)); // Copy downloaded zip to the cache on disk for future extraction. executionContext.Debug($"Copying from {zipFile} to {taskZipPath}"); File.Copy(zipFile, taskZipPath); } // We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob. // This is fine because we overwrite the contents at task run time. Directory.CreateDirectory(destDirectory); ExtractZip(zipFile, destDirectory); executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'."); Trace.Info("Finished getting task."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting task temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message)); } } }
public async Task <List <Capability> > GetCapabilitiesAsync(AgentSettings settings, CancellationToken cancellationToken) { Trace.Entering(); var capabilities = new List <Capability>(); string powerShellExe = HostContext.GetService <IPowerShellExeUtil>().GetPath(); string scriptFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "powershell", "Add-Capabilities.ps1").Replace("'", "''"); ArgUtil.File(scriptFile, nameof(scriptFile)); string arguments = $@"-NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "". '{scriptFile}'"""; string enablePrereleaseVSVersions = AgentKnobs.EnableVSPreReleaseVersions.GetValue(UtilKnobValueContext.Instance()).AsString(); Environment.SetEnvironmentVariable("IncludePrereleaseVersions", enablePrereleaseVSVersions); using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs args) => { Trace.Info($"STDOUT: {args.Data}"); Capability capability; if (TryParseCapability(args.Data, out capability)) { Trace.Info($"Adding '{capability.Name}': '{capability.Value}'"); capabilities.Add(capability); } }; processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs args) => { Trace.Info($"STDERR: {args.Data}"); }; await processInvoker.ExecuteAsync( workingDirectory : Path.GetDirectoryName(scriptFile), fileName : powerShellExe, arguments : arguments, environment : null, requireExitCodeZero : false, outputEncoding : null, killProcessOnCancel : true, cancellationToken : cancellationToken); } return(capabilities); }
public bool TryProcessCommand(IExecutionContext context, string input) { ArgUtil.NotNull(context, nameof(context)); if (string.IsNullOrEmpty(input)) { return(false); } // TryParse input to Command Command command; if (!Command.TryParse(input, out command)) { // if parse fail but input contains ##vso, print warning with DOC link if (input.IndexOf("##vso") >= 0) { context.Warning(StringUtil.Loc("CommandKeywordDetected", input)); } return(false); } IWorkerCommandExtension extension = null; if (_invokePluginInternalCommand && string.Equals(command.Area, _pluginInternalCommandExtensions.CommandArea, StringComparison.OrdinalIgnoreCase)) { extension = _pluginInternalCommandExtensions; } if (extension != null || _commandExtensions.TryGetValue(command.Area, out extension)) { if (!extension.SupportedHostTypes.HasFlag(context.Variables.System_HostType)) { context.Error(StringUtil.Loc("CommandNotSupported", command.Area, context.Variables.System_HostType)); context.CommandResult = TaskResult.Failed; return(false); } // process logging command in serialize oreder. lock (_commandSerializeLock) { try { extension.ProcessCommand(context, command, restrictionPolicy); } catch (Exception ex) { context.Error(StringUtil.Loc("CommandProcessFailed", input)); context.Error(ex); context.CommandResult = TaskResult.Failed; } finally { // trace the ##vso command as long as the command is not a ##vso[task.debug] command. if (!(string.Equals(command.Area, "task", StringComparison.OrdinalIgnoreCase) && string.Equals(command.Event, "debug", StringComparison.OrdinalIgnoreCase))) { context.Debug($"Processed: {input}"); } } } } else { context.Warning(StringUtil.Loc("CommandNotFound", command.Area)); } // Only if we've successfully parsed do we show this warning if (AgentKnobs.DecodePercents.GetValue(UtilKnobValueContext.Instance()).ToString() == "" && input.Contains("%25")) { context.Warning("%25 detected in ##vso command. In January 2021, the agent command parser will be updated to unescape this to %. To opt out of this behavior, set environment variable DECODE_PERCENTS to false. Setting to true will force this behavior immediately. More information can be found at https://github.com/microsoft/azure-pipelines-agent/blob/master/docs/design/percentEncoding.md"); } return(true); }