private string GetTaskZipPath(Pipelines.TaskStepDefinitionReference task)
{
ArgUtil.NotEmpty(task.Id, nameof(task.Id));
ArgUtil.NotNull(task.Name, nameof(task.Name));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
return(Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.TaskZips),
$"{task.Name}_{task.Id}_{task.Version}.zip")); // TODO: Move to shared string.
}
private string GetDirectory(Pipelines.TaskStepDefinitionReference task)
{
ArgUtil.NotEmpty(task.Id, nameof(task.Id));
ArgUtil.NotNull(task.Name, nameof(task.Name));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
return(Path.Combine(
HostContext.GetDirectory(WellKnownDirectory.Tasks),
$"{task.Name}_{task.Id}",
task.Version));
}
public Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
{
if (context.Variables.ContainsKey("throw_process"))
{
throw new NotSupportedException();
}
else
{
context.Output(line);
return(Task.CompletedTask);
}
}
public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
{
if (_counter++ < 1)
{
context.Output("SLOW");
await Task.Delay(400);
}
else
{
context.Output(line);
}
}
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(task, nameof(task));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
var taskServer = HostContext.GetService <ITaskServer>();
// first check to see if we already have the task
string destDirectory = GetDirectory(task);
Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
Boolean signingEnabled = !String.IsNullOrEmpty(settings.Fingerprint);
if (File.Exists(destDirectory + ".completed") && !signingEnabled)
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
return;
}
String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip");
if (signingEnabled && File.Exists(taskZipPath))
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'.");
// We need to extract the zip now because the task.json metadata for the task is used in JobExtension.InitializeJob.
// This is fine because we overwrite the contents at task run time.
if (!File.Exists(destDirectory + ".completed"))
{
// The zip exists but it hasn't been extracted yet.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
ExtractZip(taskZipPath, destDirectory);
}
return;
}
// delete existing task folder.
Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);
// Inform the user that a download is taking place. The download could take a while if
// the task zip is large. It would be nice to print the localized name, but it is not
// available from the reference included in the job message.
executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version));
string zipFile = string.Empty;
var version = new TaskVersion(task.Version);
//download and extract task in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid());
try
{
Directory.CreateDirectory(tempDirectory);
int retryCount = 0;
// Allow up to 20 * 60s for any task to be downloaded from service.
// Base on Kusto, the longest we have on the service today is over 850 seconds.
// Timeout limit can be overwrite by environment variable
if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds))
{
timeoutSeconds = 20 * 60;
}
while (retryCount < 3)
{
using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));
//open zip stream in async mode
using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token);
await fs.FlushAsync(taskDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Task download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}");
Trace.Error(ex);
if (taskDownloadTimeout.Token.IsCancellationRequested)
{
// task download didn't finish within timeout
executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds));
}
else
{
executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message));
if (ex.InnerException != null)
{
executionContext.Warning("Inner Exception: {ex.InnerException.Message}");
}
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
if (signingEnabled)
{
Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips));
// Copy downloaded zip to the cache on disk for future extraction.
executionContext.Debug($"Copying from {zipFile} to {taskZipPath}");
File.Copy(zipFile, taskZipPath);
}
// We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob.
// This is fine because we overwrite the contents at task run time.
Directory.CreateDirectory(destDirectory);
ExtractZip(zipFile, destDirectory);
executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'.");
Trace.Info("Finished getting task.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
}
}
}
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(task, nameof(task));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
var taskServer = HostContext.GetService <ITaskServer>();
// first check to see if we already have the task
string destDirectory = GetDirectory(task);
Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");
if (File.Exists(destDirectory + ".completed"))
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
return;
}
// delete existing task folder.
Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);
// Inform the user that a download is taking place. The download could take a while if
// the task zip is large. It would be nice to print the localized name, but it is not
// available from the reference included in the job message.
executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name));
string zipFile;
var version = new TaskVersion(task.Version);
//download and extract task in a temp folder and rename it on success
string tempDirectory = Path.Combine(IOUtil.GetTasksPath(HostContext), "_temp_" + Guid.NewGuid());
try
{
Directory.CreateDirectory(tempDirectory);
zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));
//open zip stream in async mode
using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
{
using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, executionContext.CancellationToken))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, executionContext.CancellationToken);
await fs.FlushAsync(executionContext.CancellationToken);
}
}
Directory.CreateDirectory(destDirectory);
ZipFile.ExtractToDirectory(zipFile, destDirectory);
Trace.Verbose("Create watermark file indicate task download succeed.");
File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());
executionContext.Debug($"Task '{task.Name}' has been downloaded into '{destDirectory}'.");
Trace.Info("Finished getting task.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
}
}
}
/// <inheritdoc />
public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
{
await _inputDataParser.ProcessDataAsync(line);
}
public Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
{
context.Output(line);
return(Task.CompletedTask);
}
public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
{
context.Output("BLOCK");
await Task.Delay(TimeSpan.FromMilliseconds(-1));
}
public IHandler Create(
IExecutionContext executionContext,
Pipelines.TaskStepDefinitionReference task,
IStepHost stepHost,
List <ServiceEndpoint> endpoints,
List <SecureFile> secureFiles,
HandlerData data,
Dictionary <string, string> inputs,
Dictionary <string, string> environment,
Variables runtimeVariables,
string taskDirectory)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(stepHost, nameof(stepHost));
ArgUtil.NotNull(endpoints, nameof(endpoints));
ArgUtil.NotNull(secureFiles, nameof(secureFiles));
ArgUtil.NotNull(data, nameof(data));
ArgUtil.NotNull(inputs, nameof(inputs));
ArgUtil.NotNull(environment, nameof(environment));
ArgUtil.NotNull(runtimeVariables, nameof(runtimeVariables));
ArgUtil.NotNull(taskDirectory, nameof(taskDirectory));
// Create the handler.
IHandler handler;
if (data is BaseNodeHandlerData)
{
// Node 6 and 10.
handler = HostContext.CreateService <INodeHandler>();
(handler as INodeHandler).Data = data as BaseNodeHandlerData;
}
else if (data is PowerShell3HandlerData)
{
// PowerShell3.
handler = HostContext.CreateService <IPowerShell3Handler>();
(handler as IPowerShell3Handler).Data = data as PowerShell3HandlerData;
}
else if (data is PowerShellExeHandlerData)
{
// PowerShellExe.
handler = HostContext.CreateService <IPowerShellExeHandler>();
(handler as IPowerShellExeHandler).Data = data as PowerShellExeHandlerData;
}
else if (data is ProcessHandlerData)
{
// Process.
handler = HostContext.CreateService <IProcessHandler>();
(handler as IProcessHandler).Data = data as ProcessHandlerData;
}
else if (data is PowerShellHandlerData)
{
// PowerShell.
handler = HostContext.CreateService <IPowerShellHandler>();
(handler as IPowerShellHandler).Data = data as PowerShellHandlerData;
}
else if (data is AzurePowerShellHandlerData)
{
// AzurePowerShell.
handler = HostContext.CreateService <IAzurePowerShellHandler>();
(handler as IAzurePowerShellHandler).Data = data as AzurePowerShellHandlerData;
}
else if (data is AgentPluginHandlerData)
{
// Agent plugin
handler = HostContext.CreateService <IAgentPluginHandler>();
(handler as IAgentPluginHandler).Data = data as AgentPluginHandlerData;
}
else
{
// This should never happen.
throw new NotSupportedException();
}
handler.Endpoints = endpoints;
handler.Task = task;
handler.Environment = environment;
handler.RuntimeVariables = runtimeVariables;
handler.ExecutionContext = executionContext;
handler.StepHost = stepHost;
handler.Inputs = inputs;
handler.SecureFiles = secureFiles;
handler.TaskDirectory = taskDirectory;
return(handler);
}
/// <inheritdoc />
public async Task ProcessLineAsync(IAgentLogPluginContext context, Pipelines.TaskStepDefinitionReference step, string line)
{
await Task.CompletedTask;
}