public Definition Load(Pipelines.TaskStep task)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(task, nameof(task));
// Initialize the definition wrapper object.
var definition = new Definition()
{
Directory = GetDirectory(task.Reference)
};
// Deserialize the JSON.
string file = Path.Combine(definition.Directory, Constants.Path.TaskJsonFile);
Trace.Info($"Loading task definition '{file}'.");
string json = File.ReadAllText(file);
definition.Data = JsonConvert.DeserializeObject <DefinitionData>(json);
// Replace the macros within the handler data sections.
foreach (HandlerData handlerData in (definition.Data?.Execution?.All as IEnumerable <HandlerData> ?? new HandlerData[0]))
{
handlerData?.ReplaceMacros(HostContext, definition);
}
return(definition);
}
public async void RetryStreamException()
{
try
{
// Arrange.
using (var tokenSource = new CancellationTokenSource())
using (var _hc = Setup(tokenSource))
{
var pingTask = new Pipelines.TaskStep()
{
Enabled = true,
Reference = new Pipelines.TaskStepDefinitionReference()
{
Name = "Ping",
Version = "0.1.1",
Id = Guid.NewGuid()
}
};
var pingVersion = new TaskVersion(pingTask.Reference.Version);
Exception expectedException = new System.Net.Http.HttpRequestException("simulated network error");
_taskServer
.Setup(x => x.GetTaskContentZipAsync(It.IsAny <Guid>(), It.IsAny <TaskVersion>(), It.IsAny <CancellationToken>()))
.Returns((Guid taskId, TaskVersion taskVersion, CancellationToken token) =>
{
return(Task.FromResult <Stream>(new ExceptionStream()));
});
var tasks = new List <Pipelines.TaskStep>(new Pipelines.TaskStep[] { pingTask });
//Act
Exception actualException = null;
try
{
await _taskManager.DownloadAsync(_ec.Object, tasks);
}
catch (Exception ex)
{
actualException = ex;
}
//Assert
//verify task completed in less than 2sec and it is in failed state state
Assert.Equal("NotImplementedException", actualException.GetType().Name);
//assert download was invoked 3 times, because we retry on task download
_taskServer
.Verify(x => x.GetTaskContentZipAsync(It.IsAny <Guid>(), It.IsAny <TaskVersion>(), It.IsAny <CancellationToken>()), Times.Exactly(3));
//see if the task.json was not downloaded
Assert.Equal(
0,
Directory.GetFiles(_hc.GetDirectory(WellKnownDirectory.Tasks), "*", SearchOption.AllDirectories).Length);
}
}
finally
{
Teardown();
}
}
public async void BubblesNetworkException()
{
try
{
// Arrange.
Setup();
var pingTask = new Pipelines.TaskStep()
{
Enabled = true,
Reference = new Pipelines.TaskStepDefinitionReference()
{
Name = "Ping",
Version = "0.1.1",
Id = Guid.NewGuid()
}
};
var pingVersion = new TaskVersion(pingTask.Reference.Version);
Exception expectedException = new System.Net.Http.HttpRequestException("simulated network error");
_taskServer
.Setup(x => x.GetTaskContentZipAsync(It.IsAny <Guid>(), It.IsAny <TaskVersion>(), _ec.Object.CancellationToken))
.Returns((Guid taskId, TaskVersion taskVersion, CancellationToken token) =>
{
throw expectedException;
});
var tasks = new List <Pipelines.TaskStep>(new Pipelines.TaskStep[] { pingTask });
//Act
Exception actualException = null;
try
{
await _taskManager.DownloadAsync(_ec.Object, tasks);
}
catch (Exception ex)
{
actualException = ex;
}
//Assert
//verify task completed in less than 2sec and it is in failed state state
Assert.Equal(expectedException, actualException);
//see if the task.json was not downloaded
Assert.Equal(
0,
Directory.GetFiles(_hc.GetDirectory(WellKnownDirectory.Tasks), "*", SearchOption.AllDirectories).Length);
}
finally
{
Teardown();
}
}
public void Extract(IExecutionContext executionContext, Pipelines.TaskStep task)
{
String zipFile = GetTaskZipPath(task.Reference);
String destinationDirectory = GetDirectory(task.Reference);
executionContext.Debug($"Extracting task {task.Name} from {zipFile} to {destinationDirectory}.");
Trace.Verbose("Deleting task destination folder: {0}", destinationDirectory);
IOUtil.DeleteDirectory(destinationDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destinationDirectory);
ZipFile.ExtractToDirectory(zipFile, destinationDirectory);
Trace.Verbose("Creating watermark file to indicate the task extracted successfully.");
File.WriteAllText(destinationDirectory + ".completed", DateTime.UtcNow.ToString());
}
public Definition Load(Pipelines.TaskStep task)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(task, nameof(task));
if (task.Reference.Id == Pipelines.PipelineConstants.CheckoutTask.Id && task.Reference.Version == Pipelines.PipelineConstants.CheckoutTask.Version)
{
var checkoutTask = new Definition()
{
Directory = HostContext.GetDirectory(WellKnownDirectory.Tasks),
Data = new DefinitionData()
{
Author = Pipelines.PipelineConstants.CheckoutTask.Author,
Description = Pipelines.PipelineConstants.CheckoutTask.Description,
FriendlyName = Pipelines.PipelineConstants.CheckoutTask.FriendlyName,
HelpMarkDown = Pipelines.PipelineConstants.CheckoutTask.HelpMarkDown,
Inputs = Pipelines.PipelineConstants.CheckoutTask.Inputs.ToArray(),
Execution = StringUtil.ConvertFromJson <ExecutionData>(StringUtil.ConvertToJson(Pipelines.PipelineConstants.CheckoutTask.Execution)),
PostJobExecution = StringUtil.ConvertFromJson <ExecutionData>(StringUtil.ConvertToJson(Pipelines.PipelineConstants.CheckoutTask.PostJobExecution))
}
};
return(checkoutTask);
}
// Initialize the definition wrapper object.
var definition = new Definition()
{
Directory = GetDirectory(task.Reference), ZipPath = GetTaskZipPath(task.Reference)
};
// Deserialize the JSON.
string file = Path.Combine(definition.Directory, Constants.Path.TaskJsonFile);
Trace.Info($"Loading task definition '{file}'.");
string json = File.ReadAllText(file);
definition.Data = JsonConvert.DeserializeObject <DefinitionData>(json);
// Replace the macros within the handler data sections.
foreach (HandlerData handlerData in (definition.Data?.Execution?.All as IEnumerable <HandlerData> ?? new HandlerData[0]))
{
handlerData?.ReplaceMacros(HostContext, definition);
}
return(definition);
}
public override Definition Load(Pipelines.TaskStep task)
{
Definition d = base.Load(task);
if (task.Reference.Id == Pipelines.PipelineConstants.CheckoutTask.Id && task.Reference.Version == Pipelines.PipelineConstants.CheckoutTask.Version)
{
AgentPluginHandlerData checkoutHandlerData = new AgentPluginHandlerData();
checkoutHandlerData.Target = "Microsoft.VisualStudio.Services.Agent.Tests.L1.Worker.FakeCheckoutTask, Test";
d.Data.Execution = new ExecutionData()
{
AgentPlugin = checkoutHandlerData
};
}
return(d);
}
private void ExtractsAnAlreadyDownloadedZipToTheCorrectLocation(bool signatureVerification = true, bool alwaysExtractTask = true)
{
try
{
// Arrange
using (var tokenSource = new CancellationTokenSource())
using (var _hc = Setup(tokenSource, signatureVerificationEnabled: signatureVerification, alwaysExtractTaskEnabled: alwaysExtractTask))
{
var bingGuid = Guid.NewGuid();
string bingTaskName = "Bing";
string bingVersion = "1.21.2";
var taskStep = new Pipelines.TaskStep
{
Name = bingTaskName,
Reference = new Pipelines.TaskStepDefinitionReference
{
Id = bingGuid,
Name = bingTaskName,
Version = bingVersion
}
};
string zipDestDirectory = Path.Combine(_hc.GetDirectory(WellKnownDirectory.TaskZips), $"{bingTaskName}_{bingGuid}_{bingVersion}.zip");
Directory.CreateDirectory(_hc.GetDirectory(WellKnownDirectory.TaskZips));
// write stream to file
using (Stream zipStream = GetZipStream())
using (var fileStream = new FileStream(zipDestDirectory, FileMode.Create, FileAccess.Write))
{
zipStream.CopyTo(fileStream);
}
// Act
_taskManager.Extract(_ec.Object, taskStep);
// Assert
string destDirectory = Path.Combine(
_hc.GetDirectory(WellKnownDirectory.Tasks),
$"{bingTaskName}_{bingGuid}",
bingVersion);
Assert.True(File.Exists(Path.Combine(destDirectory, Constants.Path.TaskJsonFile)));
}
}
finally
{
Teardown();
}
}
private void CreateTask(string jsonContent, out Pipelines.TaskStep instance, out string directory)
{
const string TaskName = "SomeTask";
const string TaskVersion = "1.2.3";
Guid taskGuid = Guid.NewGuid();
directory = Path.Combine(_workFolder, Constants.Path.TasksDirectory, $"{TaskName}_{taskGuid}", TaskVersion);
string file = Path.Combine(directory, Constants.Path.TaskJsonFile);
Directory.CreateDirectory(Path.GetDirectoryName(file));
File.WriteAllText(file, jsonContent);
instance = new Pipelines.TaskStep()
{
Reference = new Pipelines.TaskStepDefinitionReference()
{
Id = taskGuid,
Name = TaskName,
Version = TaskVersion,
}
};
}
public async void BubblesCancellation()
{
try
{
//Arrange
Setup();
var bingTask = new Pipelines.TaskStep()
{
Enabled = true,
Reference = new Pipelines.TaskStepDefinitionReference()
{
Name = "Bing",
Version = "0.1.2",
Id = Guid.NewGuid()
}
};
var pingTask = new Pipelines.TaskStep()
{
Enabled = true,
Reference = new Pipelines.TaskStepDefinitionReference()
{
Name = "Ping",
Version = "0.1.1",
Id = Guid.NewGuid()
}
};
var bingVersion = new TaskVersion(bingTask.Reference.Version);
var pingVersion = new TaskVersion(pingTask.Reference.Version);
_taskServer
.Setup(x => x.GetTaskContentZipAsync(It.IsAny <Guid>(), It.IsAny <TaskVersion>(), It.IsAny <CancellationToken>()))
.Returns((Guid taskId, TaskVersion taskVersion, CancellationToken token) =>
{
_ecTokenSource.Cancel();
_ecTokenSource.Token.ThrowIfCancellationRequested();
return(null);
});
var tasks = new List <Pipelines.TaskStep>(new Pipelines.TaskStep[] { bingTask, pingTask });
//Act
//should initiate a download with a mocked IJobServer, which sets a cancellation token and
//download task is expected to be in cancelled state
Task downloadTask = _taskManager.DownloadAsync(_ec.Object, tasks);
Task[] taskToWait = { downloadTask, Task.Delay(2000) };
//wait for the task to be cancelled to exit
await Task.WhenAny(taskToWait);
//Assert
//verify task completed in less than 2sec and it is in cancelled state
Assert.True(downloadTask.IsCompleted, $"{nameof(_taskManager.DownloadAsync)} timed out.");
Assert.True(!downloadTask.IsFaulted, downloadTask.Exception?.ToString());
Assert.True(downloadTask.IsCanceled);
//check if the task.json was not downloaded for ping and bing tasks
Assert.Equal(
0,
Directory.GetFiles(_hc.GetDirectory(WellKnownDirectory.Tasks), "*", SearchOption.AllDirectories).Length);
//assert download was invoked only once, because the first task cancelled the second task download
_taskServer
.Verify(x => x.GetTaskContentZipAsync(It.IsAny <Guid>(), It.IsAny <TaskVersion>(), It.IsAny <CancellationToken>()), Times.Once());
}
finally
{
Teardown();
}
}
private TestHostContext Setup([CallerMemberName] string name = "",
bool createWorkDirectory = true,
CheckoutConfigType checkOutConfig = CheckoutConfigType.SingleCheckoutDefaultPath,
string pathToSelfRepo = "")
{
bool isMulticheckoutScenario = checkOutConfig == CheckoutConfigType.MultiCheckoutCustomPath || checkOutConfig == CheckoutConfigType.MultiCheckoutDefaultPath;
bool isCustomPathScenario = checkOutConfig == CheckoutConfigType.SingleCheckoutCustomPath || checkOutConfig == CheckoutConfigType.MultiCheckoutCustomPath;
TestHostContext hc = new TestHostContext(this, name);
this.stubWorkFolder = hc.GetDirectory(WellKnownDirectory.Work);
if (createWorkDirectory)
{
Directory.CreateDirectory(this.stubWorkFolder);
}
_ec = new Mock <IExecutionContext>();
_extensionManager = new Mock <IExtensionManager>();
_sourceProvider = new Mock <ISourceProvider>();
_buildDirectoryManager = new Mock <IBuildDirectoryManager>();
_workspaceOptions = new Pipelines.WorkspaceOptions();
_configurationStore = new Mock <IConfigurationStore>();
_configurationStore.Setup(store => store.GetSettings()).Returns(new AgentSettings {
WorkFolder = this.stubWorkFolder
});
steps = new List <Pipelines.JobStep>();
var selfCheckoutTask = new Pipelines.TaskStep()
{
Reference = new Pipelines.TaskStepDefinitionReference()
{
Id = Guid.Parse("6d15af64-176c-496d-b583-fd2ae21d4df4"),
Name = "Checkout",
Version = "1.0.0"
}
};
selfCheckoutTask.Inputs.Add("repository", "self");
if (isCustomPathScenario)
{
selfCheckoutTask.Inputs.Add("path", pathToSelfRepo);
}
steps.Add(selfCheckoutTask);
// Setup second checkout only for multicheckout jobs
if (isMulticheckoutScenario)
{
var anotherCheckoutTask = new Pipelines.TaskStep()
{
Reference = new Pipelines.TaskStepDefinitionReference()
{
Id = Guid.Parse("6d15af64-176c-496d-b583-fd2ae21d4df4"),
Name = "Checkout",
Version = "1.0.0"
}
};
anotherCheckoutTask.Inputs.Add("repository", "BuildRepo");
anotherCheckoutTask.Inputs.Add("path", "s/BuildRepo");
steps.Add(anotherCheckoutTask);
}
hc.SetSingleton(_buildDirectoryManager.Object);
hc.SetSingleton(_extensionManager.Object);
hc.SetSingleton(_configurationStore.Object);
var buildVariables = GetBuildVariables();
_variables = new Variables(hc, buildVariables, out _);
_ec.Setup(x => x.Variables).Returns(_variables);
repositories = new List <Pipelines.RepositoryResource>();
repositories.Add(GetRepository(hc, "self", "App", "App"));
repositories.Add(GetRepository(hc, "repo2", "BuildRepo", "BuildRepo"));
_ec.Setup(x => x.Repositories).Returns(repositories);
jobSettings = new Dictionary <string, string>();
jobSettings.Add(WellKnownJobSettings.HasMultipleCheckouts, isMulticheckoutScenario.ToString());
_ec.Setup(x => x.JobSettings).Returns(jobSettings);
_ec.Setup(x =>
x.SetVariable(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <bool>()))
.Callback((string varName, string varValue, bool isSecret, bool isOutput, bool isFilePath, bool isReadOnly) => { _variables.Set(varName, varValue, false); });
_extensionManager.Setup(x => x.GetExtensions <ISourceProvider>())
.Returns(new List <ISourceProvider> {
_sourceProvider.Object
});
_sourceProvider.Setup(x => x.RepositoryType)
.Returns(Pipelines.RepositoryTypes.ExternalGit);
_buildDirectoryManager.Setup(x => x.PrepareDirectory(_ec.Object, repositories, _workspaceOptions))
.Returns(new TrackingConfig(_ec.Object, repositories, 1));
buildJobExtension = new BuildJobExtension();
buildJobExtension.Initialize(hc);
return(hc);
}
