private string GetTaskZipPath(Pipelines.TaskStepDefinitionReference task) { ArgUtil.NotEmpty(task.Id, nameof(task.Id)); ArgUtil.NotNull(task.Name, nameof(task.Name)); ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version)); return(Path.Combine( HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip")); // TODO: Move to shared string. }
private string GetDirectory(Pipelines.TaskStepDefinitionReference task) { ArgUtil.NotEmpty(task.Id, nameof(task.Id)); ArgUtil.NotNull(task.Name, nameof(task.Name)); ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version)); return(Path.Combine( HostContext.GetDirectory(WellKnownDirectory.Tasks), $"{task.Name}_{task.Id}", task.Version)); }
public Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line) { if (context.Variables.ContainsKey("throw_process")) { throw new NotSupportedException(); } else { context.Output(line); return(Task.CompletedTask); } }
public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line) { if (_counter++ < 1) { context.Output("SLOW"); await Task.Delay(400); } else { context.Output(line); } }
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(task, nameof(task)); ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version)); var taskServer = HostContext.GetService <ITaskServer>(); // first check to see if we already have the task string destDirectory = GetDirectory(task); Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'."); var configurationStore = HostContext.GetService <IConfigurationStore>(); AgentSettings settings = configurationStore.GetSettings(); Boolean signingEnabled = !String.IsNullOrEmpty(settings.Fingerprint); if (File.Exists(destDirectory + ".completed") && !signingEnabled) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'."); return; } String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip"); if (signingEnabled && File.Exists(taskZipPath)) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'."); // We need to extract the zip now because the task.json metadata for the task is used in JobExtension.InitializeJob. // This is fine because we overwrite the contents at task run time. if (!File.Exists(destDirectory + ".completed")) { // The zip exists but it hasn't been extracted yet. IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken); ExtractZip(taskZipPath, destDirectory); } return; } // delete existing task folder. Trace.Verbose("Deleting task destination folder: {0}", destDirectory); IOUtil.DeleteDirectory(destDirectory, CancellationToken.None); // Inform the user that a download is taking place. The download could take a while if // the task zip is large. It would be nice to print the localized name, but it is not // available from the reference included in the job message. executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version)); string zipFile = string.Empty; var version = new TaskVersion(task.Version); //download and extract task in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid()); try { Directory.CreateDirectory(tempDirectory); int retryCount = 0; // Allow up to 20 * 60s for any task to be downloaded from service. // Base on Kusto, the longest we have on the service today is over 850 seconds. // Timeout limit can be overwrite by environment variable if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds)) { timeoutSeconds = 20 * 60; } while (retryCount < 3) { using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken)) { try { zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid())); //open zip stream in async mode using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token)) { await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token); await fs.FlushAsync(taskDownloadCancellation.Token); // download succeed, break out the retry loop. break; } } catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested) { Trace.Info($"Task download has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}"); Trace.Error(ex); if (taskDownloadTimeout.Token.IsCancellationRequested) { // task download didn't finish within timeout executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds)); } else { executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message)); if (ex.InnerException != null) { executionContext.Warning("Inner Exception: {ex.InnerException.Message}"); } } } } if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF"))) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } if (signingEnabled) { Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips)); // Copy downloaded zip to the cache on disk for future extraction. executionContext.Debug($"Copying from {zipFile} to {taskZipPath}"); File.Copy(zipFile, taskZipPath); } // We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob. // This is fine because we overwrite the contents at task run time. Directory.CreateDirectory(destDirectory); ExtractZip(zipFile, destDirectory); executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'."); Trace.Info("Finished getting task."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting task temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message)); } } }
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(task, nameof(task)); ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version)); var taskServer = HostContext.GetService <ITaskServer>(); // first check to see if we already have the task string destDirectory = GetDirectory(task); Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'."); if (File.Exists(destDirectory + ".completed")) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'."); return; } // delete existing task folder. Trace.Verbose("Deleting task destination folder: {0}", destDirectory); IOUtil.DeleteDirectory(destDirectory, CancellationToken.None); // Inform the user that a download is taking place. The download could take a while if // the task zip is large. It would be nice to print the localized name, but it is not // available from the reference included in the job message. executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name)); string zipFile; var version = new TaskVersion(task.Version); //download and extract task in a temp folder and rename it on success string tempDirectory = Path.Combine(IOUtil.GetTasksPath(HostContext), "_temp_" + Guid.NewGuid()); try { Directory.CreateDirectory(tempDirectory); zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid())); //open zip stream in async mode using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true)) { using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, executionContext.CancellationToken)) { //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k). await result.CopyToAsync(fs, 81920, executionContext.CancellationToken); await fs.FlushAsync(executionContext.CancellationToken); } } Directory.CreateDirectory(destDirectory); ZipFile.ExtractToDirectory(zipFile, destDirectory); Trace.Verbose("Create watermark file indicate task download succeed."); File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString()); executionContext.Debug($"Task '{task.Name}' has been downloaded into '{destDirectory}'."); Trace.Info("Finished getting task."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting task temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message)); } } }
/// <inheritdoc /> public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line) { await _inputDataParser.ProcessDataAsync(line); }
public Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line) { context.Output(line); return(Task.CompletedTask); }
public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line) { context.Output("BLOCK"); await Task.Delay(TimeSpan.FromMilliseconds(-1)); }
public IHandler Create( IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task, IStepHost stepHost, List <ServiceEndpoint> endpoints, List <SecureFile> secureFiles, HandlerData data, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, string taskDirectory) { // Validate args. Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(stepHost, nameof(stepHost)); ArgUtil.NotNull(endpoints, nameof(endpoints)); ArgUtil.NotNull(secureFiles, nameof(secureFiles)); ArgUtil.NotNull(data, nameof(data)); ArgUtil.NotNull(inputs, nameof(inputs)); ArgUtil.NotNull(environment, nameof(environment)); ArgUtil.NotNull(runtimeVariables, nameof(runtimeVariables)); ArgUtil.NotNull(taskDirectory, nameof(taskDirectory)); // Create the handler. IHandler handler; if (data is BaseNodeHandlerData) { // Node 6 and 10. handler = HostContext.CreateService <INodeHandler>(); (handler as INodeHandler).Data = data as BaseNodeHandlerData; } else if (data is PowerShell3HandlerData) { // PowerShell3. handler = HostContext.CreateService <IPowerShell3Handler>(); (handler as IPowerShell3Handler).Data = data as PowerShell3HandlerData; } else if (data is PowerShellExeHandlerData) { // PowerShellExe. handler = HostContext.CreateService <IPowerShellExeHandler>(); (handler as IPowerShellExeHandler).Data = data as PowerShellExeHandlerData; } else if (data is ProcessHandlerData) { // Process. handler = HostContext.CreateService <IProcessHandler>(); (handler as IProcessHandler).Data = data as ProcessHandlerData; } else if (data is PowerShellHandlerData) { // PowerShell. handler = HostContext.CreateService <IPowerShellHandler>(); (handler as IPowerShellHandler).Data = data as PowerShellHandlerData; } else if (data is AzurePowerShellHandlerData) { // AzurePowerShell. handler = HostContext.CreateService <IAzurePowerShellHandler>(); (handler as IAzurePowerShellHandler).Data = data as AzurePowerShellHandlerData; } else if (data is AgentPluginHandlerData) { // Agent plugin handler = HostContext.CreateService <IAgentPluginHandler>(); (handler as IAgentPluginHandler).Data = data as AgentPluginHandlerData; } else { // This should never happen. throw new NotSupportedException(); } handler.Endpoints = endpoints; handler.Task = task; handler.Environment = environment; handler.RuntimeVariables = runtimeVariables; handler.ExecutionContext = executionContext; handler.StepHost = stepHost; handler.Inputs = inputs; handler.SecureFiles = secureFiles; handler.TaskDirectory = taskDirectory; return(handler); }
/// <inheritdoc /> public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line) { await Task.CompletedTask; }