Esempio n. 1
0
 private string GetTaskZipPath(Pipelines.TaskStepDefinitionReference task)
 {
     ArgUtil.NotEmpty(task.Id, nameof(task.Id));
     ArgUtil.NotNull(task.Name, nameof(task.Name));
     ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
     return(Path.Combine(
                HostContext.GetDirectory(WellKnownDirectory.TaskZips),
                $"{task.Name}_{task.Id}_{task.Version}.zip")); // TODO: Move to shared string.
 }
Esempio n. 2
0
 private string GetDirectory(Pipelines.TaskStepDefinitionReference task)
 {
     ArgUtil.NotEmpty(task.Id, nameof(task.Id));
     ArgUtil.NotNull(task.Name, nameof(task.Name));
     ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
     return(Path.Combine(
                HostContext.GetDirectory(WellKnownDirectory.Tasks),
                $"{task.Name}_{task.Id}",
                task.Version));
 }
 public Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
 {
     if (context.Variables.ContainsKey("throw_process"))
     {
         throw new NotSupportedException();
     }
     else
     {
         context.Output(line);
         return(Task.CompletedTask);
     }
 }
 public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
 {
     if (_counter++ < 1)
     {
         context.Output("SLOW");
         await Task.Delay(400);
     }
     else
     {
         context.Output(line);
     }
 }
Esempio n. 5
0
        private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
        {
            Trace.Entering();
            ArgUtil.NotNull(executionContext, nameof(executionContext));
            ArgUtil.NotNull(task, nameof(task));
            ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
            var taskServer = HostContext.GetService <ITaskServer>();

            // first check to see if we already have the task
            string destDirectory = GetDirectory(task);

            Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");

            var           configurationStore = HostContext.GetService <IConfigurationStore>();
            AgentSettings settings           = configurationStore.GetSettings();
            Boolean       signingEnabled     = !String.IsNullOrEmpty(settings.Fingerprint);

            if (File.Exists(destDirectory + ".completed") && !signingEnabled)
            {
                executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
                return;
            }

            String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip");

            if (signingEnabled && File.Exists(taskZipPath))
            {
                executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'.");

                // We need to extract the zip now because the task.json metadata for the task is used in JobExtension.InitializeJob.
                // This is fine because we overwrite the contents at task run time.
                if (!File.Exists(destDirectory + ".completed"))
                {
                    // The zip exists but it hasn't been extracted yet.
                    IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
                    ExtractZip(taskZipPath, destDirectory);
                }

                return;
            }

            // delete existing task folder.
            Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
            IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);

            // Inform the user that a download is taking place. The download could take a while if
            // the task zip is large. It would be nice to print the localized name, but it is not
            // available from the reference included in the job message.
            executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version));
            string zipFile = string.Empty;
            var    version = new TaskVersion(task.Version);

            //download and extract task in a temp folder and rename it on success
            string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid());

            try
            {
                Directory.CreateDirectory(tempDirectory);
                int retryCount = 0;

                // Allow up to 20 * 60s for any task to be downloaded from service.
                // Base on Kusto, the longest we have on the service today is over 850 seconds.
                // Timeout limit can be overwrite by environment variable
                if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds))
                {
                    timeoutSeconds = 20 * 60;
                }

                while (retryCount < 3)
                {
                    using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
                        using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken))
                        {
                            try
                            {
                                zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));

                                //open zip stream in async mode
                                using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
                                    using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token))
                                    {
                                        await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token);

                                        await fs.FlushAsync(taskDownloadCancellation.Token);

                                        // download succeed, break out the retry loop.
                                        break;
                                    }
                            }
                            catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
                            {
                                Trace.Info($"Task download has been cancelled.");
                                throw;
                            }
                            catch (Exception ex) when(retryCount < 2)
                            {
                                retryCount++;
                                Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}");
                                Trace.Error(ex);
                                if (taskDownloadTimeout.Token.IsCancellationRequested)
                                {
                                    // task download didn't finish within timeout
                                    executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds));
                                }
                                else
                                {
                                    executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message));
                                    if (ex.InnerException != null)
                                    {
                                        executionContext.Warning("Inner Exception: {ex.InnerException.Message}");
                                    }
                                }
                            }
                        }

                    if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF")))
                    {
                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
                        executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
                        await Task.Delay(backOff);
                    }
                }

                if (signingEnabled)
                {
                    Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips));

                    // Copy downloaded zip to the cache on disk for future extraction.
                    executionContext.Debug($"Copying from {zipFile} to {taskZipPath}");
                    File.Copy(zipFile, taskZipPath);
                }

                // We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob.
                // This is fine because we overwrite the contents at task run time.
                Directory.CreateDirectory(destDirectory);
                ExtractZip(zipFile, destDirectory);

                executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'.");
                Trace.Info("Finished getting task.");
            }
            finally
            {
                try
                {
                    //if the temp folder wasn't moved -> wipe it
                    if (Directory.Exists(tempDirectory))
                    {
                        Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
                        IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
                    }
                }
                catch (Exception ex)
                {
                    //it is not critical if we fail to delete the temp folder
                    Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
                    executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
                }
            }
        }
Esempio n. 6
0
        private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
        {
            Trace.Entering();
            ArgUtil.NotNull(executionContext, nameof(executionContext));
            ArgUtil.NotNull(task, nameof(task));
            ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
            var taskServer = HostContext.GetService <ITaskServer>();

            // first check to see if we already have the task
            string destDirectory = GetDirectory(task);

            Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");
            if (File.Exists(destDirectory + ".completed"))
            {
                executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
                return;
            }

            // delete existing task folder.
            Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
            IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);

            // Inform the user that a download is taking place. The download could take a while if
            // the task zip is large. It would be nice to print the localized name, but it is not
            // available from the reference included in the job message.
            executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name));
            string zipFile;
            var    version = new TaskVersion(task.Version);

            //download and extract task in a temp folder and rename it on success
            string tempDirectory = Path.Combine(IOUtil.GetTasksPath(HostContext), "_temp_" + Guid.NewGuid());

            try
            {
                Directory.CreateDirectory(tempDirectory);
                zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));
                //open zip stream in async mode
                using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
                {
                    using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, executionContext.CancellationToken))
                    {
                        //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
                        await result.CopyToAsync(fs, 81920, executionContext.CancellationToken);

                        await fs.FlushAsync(executionContext.CancellationToken);
                    }
                }

                Directory.CreateDirectory(destDirectory);
                ZipFile.ExtractToDirectory(zipFile, destDirectory);

                Trace.Verbose("Create watermark file indicate task download succeed.");
                File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());

                executionContext.Debug($"Task '{task.Name}' has been downloaded into '{destDirectory}'.");
                Trace.Info("Finished getting task.");
            }
            finally
            {
                try
                {
                    //if the temp folder wasn't moved -> wipe it
                    if (Directory.Exists(tempDirectory))
                    {
                        Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
                        IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
                    }
                }
                catch (Exception ex)
                {
                    //it is not critical if we fail to delete the temp folder
                    Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
                    executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
                }
            }
        }
Esempio n. 7
0
 /// <inheritdoc />
 public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
 {
     await _inputDataParser.ProcessDataAsync(line);
 }
 public Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
 {
     context.Output(line);
     return(Task.CompletedTask);
 }
 public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
 {
     context.Output("BLOCK");
     await Task.Delay(TimeSpan.FromMilliseconds(-1));
 }
Esempio n. 10
0
        public IHandler Create(
            IExecutionContext executionContext,
            Pipelines.TaskStepDefinitionReference task,
            IStepHost stepHost,
            List <ServiceEndpoint> endpoints,
            List <SecureFile> secureFiles,
            HandlerData data,
            Dictionary <string, string> inputs,
            Dictionary <string, string> environment,
            Variables runtimeVariables,
            string taskDirectory)
        {
            // Validate args.
            Trace.Entering();
            ArgUtil.NotNull(executionContext, nameof(executionContext));
            ArgUtil.NotNull(stepHost, nameof(stepHost));
            ArgUtil.NotNull(endpoints, nameof(endpoints));
            ArgUtil.NotNull(secureFiles, nameof(secureFiles));
            ArgUtil.NotNull(data, nameof(data));
            ArgUtil.NotNull(inputs, nameof(inputs));
            ArgUtil.NotNull(environment, nameof(environment));
            ArgUtil.NotNull(runtimeVariables, nameof(runtimeVariables));
            ArgUtil.NotNull(taskDirectory, nameof(taskDirectory));

            // Create the handler.
            IHandler handler;

            if (data is BaseNodeHandlerData)
            {
                // Node 6 and 10.
                handler = HostContext.CreateService <INodeHandler>();
                (handler as INodeHandler).Data = data as BaseNodeHandlerData;
            }
            else if (data is PowerShell3HandlerData)
            {
                // PowerShell3.
                handler = HostContext.CreateService <IPowerShell3Handler>();
                (handler as IPowerShell3Handler).Data = data as PowerShell3HandlerData;
            }
            else if (data is PowerShellExeHandlerData)
            {
                // PowerShellExe.
                handler = HostContext.CreateService <IPowerShellExeHandler>();
                (handler as IPowerShellExeHandler).Data = data as PowerShellExeHandlerData;
            }
            else if (data is ProcessHandlerData)
            {
                // Process.
                handler = HostContext.CreateService <IProcessHandler>();
                (handler as IProcessHandler).Data = data as ProcessHandlerData;
            }
            else if (data is PowerShellHandlerData)
            {
                // PowerShell.
                handler = HostContext.CreateService <IPowerShellHandler>();
                (handler as IPowerShellHandler).Data = data as PowerShellHandlerData;
            }
            else if (data is AzurePowerShellHandlerData)
            {
                // AzurePowerShell.
                handler = HostContext.CreateService <IAzurePowerShellHandler>();
                (handler as IAzurePowerShellHandler).Data = data as AzurePowerShellHandlerData;
            }
            else if (data is AgentPluginHandlerData)
            {
                // Agent plugin
                handler = HostContext.CreateService <IAgentPluginHandler>();
                (handler as IAgentPluginHandler).Data = data as AgentPluginHandlerData;
            }
            else
            {
                // This should never happen.
                throw new NotSupportedException();
            }

            handler.Endpoints        = endpoints;
            handler.Task             = task;
            handler.Environment      = environment;
            handler.RuntimeVariables = runtimeVariables;
            handler.ExecutionContext = executionContext;
            handler.StepHost         = stepHost;
            handler.Inputs           = inputs;
            handler.SecureFiles      = secureFiles;
            handler.TaskDirectory    = taskDirectory;
            return(handler);
        }
 /// <inheritdoc />
 public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
 {
     await Task.CompletedTask;
 }