// TODO: We need send detailInfo back to DT in order to add an issue for the job
private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null)
{
Trace.Entering();
if (HostContext.RunMode == RunMode.Local)
{
_localRunJobResult.Value[message.RequestId] = result;
return;
}
if (ApiUtil.GetFeatures(message.Plan).HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Verbose($"Skip FinishAgentRequest call from Listener because Plan version is {message.Plan.Version}");
return;
}
var agentServer = HostContext.GetService <IAgentServer>();
int completeJobRequestRetryLimit = 5;
List <Exception> exceptions = new List <Exception>();
while (completeJobRequestRetryLimit-- > 0)
{
try
{
await agentServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, CancellationToken.None);
return;
}
catch (TaskAgentJobNotFoundException)
{
Trace.Info($"TaskAgentJobNotFoundException received, job {message.JobId} is no longer valid.");
return;
}
catch (TaskAgentJobTokenExpiredException)
{
Trace.Info($"TaskAgentJobTokenExpiredException received, job {message.JobId} is no longer valid.");
return;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during complete agent jobrequest {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow all catched exceptions during retry.
throw new AggregateException(exceptions);
}
private async Task <TestResults> RunWorker(HostContext HostContext, Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
var worker = HostContext.GetService <IWorker>();
Task <int> workerTask = null;
// Setup the anonymous pipes to use for communication with the worker.
using (var processChannel = HostContext.CreateService <IProcessChannel>())
{
processChannel.StartServer(startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Run the worker
// Note: this happens on the same process as the test
workerTask = worker.RunAsync(
pipeIn: pipeHandleOut,
pipeOut: pipeHandleIn);
}, disposeClient: false);
// Send the job request message to the worker
var body = JsonUtility.ToString(message);
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : body,
cancellationToken : csSendJobRequest.Token);
}
// wait for worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(workerTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerTask)
{
int returnCode = await workerTask;
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
return(new TestResults
{
ReturnCode = returnCode,
Result = result
});
}
else
{
return(new TestResults
{
TimedOut = true
});
}
}
}
public async void DispatchesOneTimeJobRequest()
{
//Arrange
using (var hc = new TestHostContext(this))
{
var jobDispatcher = new JobDispatcher();
hc.SetSingleton <IConfigurationStore>(_configurationStore.Object);
hc.SetSingleton <IAgentServer>(_agentServer.Object);
hc.EnqueueInstance <IProcessChannel>(_processChannel.Object);
hc.EnqueueInstance <IProcessInvoker>(_processInvoker.Object);
_configurationStore.Setup(x => x.GetSettings()).Returns(new AgentSettings()
{
PoolId = 1
});
jobDispatcher.Initialize(hc);
var ts = new CancellationTokenSource();
Pipelines.AgentJobRequestMessage message = CreateJobRequestMessage();
string strMessage = JsonUtility.ToString(message);
_processInvoker.Setup(x => x.ExecuteAsync(It.IsAny <String>(), It.IsAny <String>(), "spawnclient 1 2", null, It.IsAny <CancellationToken>()))
.Returns(Task.FromResult <int>(56));
_processChannel.Setup(x => x.StartServer(It.IsAny <StartProcessDelegate>(), It.IsAny <bool>()))
.Callback((StartProcessDelegate startDel, bool disposeClient) => { startDel("1", "2"); });
_processChannel.Setup(x => x.SendAsync(MessageType.NewJobRequest, It.Is <string>(s => s.Equals(strMessage)), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
var request = new TaskAgentJobRequest();
PropertyInfo sessionIdProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5));
_agentServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny <int>(), It.IsAny <long>(), It.IsAny <Guid>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult <TaskAgentJobRequest>(request));
_agentServer.Setup(x => x.FinishAgentRequestAsync(It.IsAny <int>(), It.IsAny <long>(), It.IsAny <Guid>(), It.IsAny <DateTime>(), It.IsAny <TaskResult>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult <TaskAgentJobRequest>(new TaskAgentJobRequest()));
//Act
jobDispatcher.Run(message, true);
//Assert
await jobDispatcher.WaitAsync(CancellationToken.None);
Assert.True(jobDispatcher.RunOnceJobCompleted.Task.IsCompleted, "JobDispatcher should set task complete token for one time agent.");
Assert.True(jobDispatcher.RunOnceJobCompleted.Task.Result, "JobDispatcher should set task complete token to 'TRUE' for one time agent.");
}
}
private void SetCulture(Pipelines.AgentJobRequestMessage message)
{
// Extract the culture name from the job's variable dictionary.
// The variable does not exist for TFS 2015 RTM and Update 1.
// It was introduced in Update 2.
VariableValue culture;
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
if (message.Variables.TryGetValue(Constants.Variables.System.Culture, out culture))
{
// Set the default thread culture.
HostContext.SetDefaultCulture(culture.Value);
}
}
protected async Task <TestResults> RunWorker(Pipelines.AgentJobRequestMessage message)
{
if (_l1HostContext == null)
{
throw new InvalidOperationException("Must call SetupL1() to initialize L1HostContext before calling RunWorker()");
}
await SetupMessage(_l1HostContext, message);
using (var cts = new CancellationTokenSource())
{
cts.CancelAfter((int)JobTimeout.TotalMilliseconds);
return(await RunWorker(_l1HostContext, message, cts.Token));
}
}
public void StepTarget_RestrictedCommands_Host()
{
using (TestHostContext hc = CreateTestContext())
using (var ec = new Agent.Worker.ExecutionContext())
{
ec.Initialize(hc);
var pipeContainer = new Pipelines.ContainerResource {
Alias = "container"
};
pipeContainer.Properties.Set <string>("image", "someimage");
// Arrange: Create a job request message.
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
JobEnvironment environment = new JobEnvironment();
environment.SystemConnection = new ServiceEndpoint();
List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>();
steps.Add(new Pipelines.TaskStep
{
Target = new Pipelines.StepTarget
{
Target = "host",
Commands = "restricted"
},
Reference = new Pipelines.TaskStepDefinitionReference()
});
var resources = new Pipelines.JobResources();
resources.Containers.Add(pipeContainer);
Guid JobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, null, new Dictionary <string, string>(),
new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps);
// Arrange: Setup command manager
var commandMock = new Mock <IWorkerCommandManager>();
hc.SetSingleton(commandMock.Object);
var pagingLogger = new Mock <IPagingLogger>();
hc.EnqueueInstance(pagingLogger.Object);
// Act.
ec.InitializeJob(jobRequest, CancellationToken.None);
ec.SetStepTarget(steps[0].Target);
// Assert.
Assert.IsType <HostInfo>(ec.StepTarget());
commandMock.Verify(x => x.SetCommandRestrictionPolicy(It.IsAny <AttributeBasedWorkerCommandRestrictionPolicy>()));
}
}
private void ReplaceConfigUriBaseInJobRequestMessage(Pipelines.AgentJobRequestMessage message)
{
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
Uri systemConnectionUrl = systemConnection.Url;
// fixup any endpoint Url that match SystemConnection Url.
foreach (var endpoint in message.Resources.Endpoints)
{
if (Uri.Compare(endpoint.Url, systemConnectionUrl, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0)
{
endpoint.Url = ReplaceWithConfigUriBase(endpoint.Url);
Trace.Info($"Ensure endpoint url match config url base. {endpoint.Url}");
}
}
// fixup any repository Url that match SystemConnection Url.
foreach (var repo in message.Resources.Repositories)
{
if (Uri.Compare(repo.Url, systemConnectionUrl, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0)
{
repo.Url = ReplaceWithConfigUriBase(repo.Url);
Trace.Info($"Ensure repository url match config url base. {repo.Url}");
}
}
// fixup well known variables. (taskDefinitionsUrl, tfsServerUrl, tfsCollectionUrl)
if (message.Variables.ContainsKey(WellKnownDistributedTaskVariables.TaskDefinitionsUrl))
{
string taskDefinitionsUrl = message.Variables[WellKnownDistributedTaskVariables.TaskDefinitionsUrl].Value;
message.Variables[WellKnownDistributedTaskVariables.TaskDefinitionsUrl] = ReplaceWithConfigUriBase(new Uri(taskDefinitionsUrl)).AbsoluteUri;
Trace.Info($"Ensure System.TaskDefinitionsUrl match config url base. {message.Variables[WellKnownDistributedTaskVariables.TaskDefinitionsUrl].Value}");
}
if (message.Variables.ContainsKey(WellKnownDistributedTaskVariables.TFCollectionUrl))
{
string tfsCollectionUrl = message.Variables[WellKnownDistributedTaskVariables.TFCollectionUrl].Value;
message.Variables[WellKnownDistributedTaskVariables.TFCollectionUrl] = ReplaceWithConfigUriBase(new Uri(tfsCollectionUrl)).AbsoluteUri;
Trace.Info($"Ensure System.TFCollectionUrl match config url base. {message.Variables[WellKnownDistributedTaskVariables.TFCollectionUrl].Value}");
}
if (message.Variables.ContainsKey(Constants.Variables.System.TFServerUrl))
{
string tfsServerUrl = message.Variables[Constants.Variables.System.TFServerUrl].Value;
message.Variables[Constants.Variables.System.TFServerUrl] = ReplaceWithConfigUriBase(new Uri(tfsServerUrl)).AbsoluteUri;
Trace.Info($"Ensure System.TFServerUrl match config url base. {message.Variables[Constants.Variables.System.TFServerUrl].Value}");
}
}
public void StepTarget_LoadStepContainersWithoutJobContainer()
{
using (TestHostContext hc = CreateTestContext())
using (var ec = new Agent.Worker.ExecutionContext())
{
ec.Initialize(hc);
var pipeContainer = new Pipelines.ContainerResource {
Alias = "container"
};
pipeContainer.Properties.Set <string>("image", "someimage");
// Arrange: Create a job request message.
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
JobEnvironment environment = new JobEnvironment();
environment.SystemConnection = new ServiceEndpoint();
List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>();
steps.Add(new Pipelines.TaskStep
{
Target = new Pipelines.StepTarget
{
Target = "container"
},
Reference = new Pipelines.TaskStepDefinitionReference()
});
var resources = new Pipelines.JobResources();
resources.Containers.Add(pipeContainer);
Guid JobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, null, new Dictionary <string, string>(),
new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps);
// Arrange: Setup command manager
var pagingLogger = new Mock <IPagingLogger>();
hc.EnqueueInstance(pagingLogger.Object);
// Act.
ec.InitializeJob(jobRequest, CancellationToken.None);
// Assert.
Assert.Equal(1, ec.Containers.Count());
}
}
public void Start(Pipelines.AgentJobRequestMessage jobRequest)
{
Trace.Entering();
if (_queueInProcess)
{
Trace.Info("No-opt, all queue process tasks are running.");
return;
}
ArgUtil.NotNull(jobRequest, nameof(jobRequest));
ArgUtil.NotNull(jobRequest.Plan, nameof(jobRequest.Plan));
ArgUtil.NotNull(jobRequest.Timeline, nameof(jobRequest.Timeline));
_scopeIdentifier = jobRequest.Plan.ScopeIdentifier;
_hubName = jobRequest.Plan.PlanType;
_planId = jobRequest.Plan.PlanId;
_jobTimelineId = jobRequest.Timeline.Id;
_jobTimelineRecordId = jobRequest.JobId;
if (jobRequest.Variables.TryGetValue("agent.LogToBlobstorageService", out var val))
{
Boolean.TryParse(val.Value, out _writeToBlobstorageService);
}
// Server already create the job timeline
_timelineUpdateQueue[_jobTimelineId] = new ConcurrentQueue <TimelineRecord>();
_allTimelines.Add(_jobTimelineId);
// Start three dequeue task
Trace.Info("Start process web console line queue.");
_webConsoleLineDequeueTask = ProcessWebConsoleLinesQueueAsync();
Trace.Info("Start process file upload queue.");
_fileUploadDequeueTask = ProcessFilesUploadQueueAsync();
Trace.Info("Start process timeline update queue.");
_timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync();
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask };
_queueInProcess = true;
}
public void Start(Pipelines.AgentJobRequestMessage jobRequest)
{
Trace.Entering();
if (HostContext.RunMode == RunMode.Local)
{
_term = HostContext.GetService <ITerminal>();
return;
}
if (_queueInProcess)
{
Trace.Info("No-opt, all queue process tasks are running.");
return;
}
ArgUtil.NotNull(jobRequest, nameof(jobRequest));
ArgUtil.NotNull(jobRequest.Plan, nameof(jobRequest.Plan));
ArgUtil.NotNull(jobRequest.Timeline, nameof(jobRequest.Timeline));
_scopeIdentifier = jobRequest.Plan.ScopeIdentifier;
_hubName = jobRequest.Plan.PlanType;
_planId = jobRequest.Plan.PlanId;
_jobTimelineId = jobRequest.Timeline.Id;
_jobTimelineRecordId = jobRequest.JobId;
// Server already create the job timeline
_timelineUpdateQueue[_jobTimelineId] = new ConcurrentQueue <TimelineRecord>();
_allTimelines.Add(_jobTimelineId);
// Start three dequeue task
Trace.Info("Start process web console line queue.");
_webConsoleLineDequeueTask = ProcessWebConsoleLinesQueueAsync();
Trace.Info("Start process file upload queue.");
_fileUploadDequeueTask = ProcessFilesUploadQueueAsync();
Trace.Info("Start process timeline update queue.");
_timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync();
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask };
_queueInProcess = true;
}
public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage)
{
Trace.Info($"Job request {jobRequestMessage.RequestId} for plan {jobRequestMessage.Plan.PlanId} job {jobRequestMessage.JobId} received.");
WorkerDispatcher currentDispatch = null;
if (_jobDispatchedQueue.Count > 0)
{
Guid dispatchedJobId = _jobDispatchedQueue.Dequeue();
if (_jobInfos.TryGetValue(dispatchedJobId, out currentDispatch))
{
Trace.Verbose($"Retrive previous WorkerDispather for job {currentDispatch.JobId}.");
}
}
WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
_jobInfos.TryAdd(newDispatch.JobId, newDispatch);
_jobDispatchedQueue.Enqueue(newDispatch.JobId);
}
private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName)
{
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference()
{
PlanId = Guid.NewGuid()
};
TimelineReference timeline = null;
Dictionary <string, VariableValue> variables = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase);
variables[Constants.Variables.System.Culture] = "en-US";
Pipelines.JobResources resources = new Pipelines.JobResources();
var serviceEndpoint = new ServiceEndpoint();
serviceEndpoint.Authorization = new EndpointAuthorization();
serviceEndpoint.Authorization.Parameters.Add("nullValue", null);
resources.Endpoints.Add(serviceEndpoint);
List <Pipelines.JobStep> tasks = new List <Pipelines.JobStep>();
tasks.Add(new Pipelines.TaskStep()
{
Id = Guid.NewGuid(),
Reference = new Pipelines.TaskStepDefinitionReference()
{
Id = Guid.NewGuid(),
Name = "TestTask",
Version = "1.0.0"
}
});
Guid JobId = Guid.NewGuid();
var sidecarContainers = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase)
{
["nginx"] = "nginx"
};
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, "ubuntu", sidecarContainers, variables, new List <MaskHint>(), resources, null, tasks);
return(jobRequest);
}
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobDisplayName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}");
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
if (HostContext.RunMode == RunMode.Normal)
{
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
}
else if (HostContext.RunMode == RunMode.Local)
{
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
}
// Start the child process.
HostContext.WritePerfCounter("StartingWorkerProcess");
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
HostContext.WritePerfCounter($"AgentSendingJobToWorker_{message.JobId}");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
await notification.JobStarted(message.JobId);
HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}");
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
await LogWorkerProcessUnhandledException(message, detailInfo);
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandled exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher priority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandled exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
public async Task UploadDiagnosticLogsAsync(IExecutionContext executionContext,
Pipelines.AgentJobRequestMessage message,
DateTime jobStartTimeUtc)
{
executionContext.Debug("Starting diagnostic file upload.");
// Setup folders
// \_layout\_work\_temp\[jobname-support]
executionContext.Debug("Setting up diagnostic log folders.");
string tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
ArgUtil.Directory(tempDirectory, nameof(tempDirectory));
string supportRootFolder = Path.Combine(tempDirectory, message.JobName + "-support");
Directory.CreateDirectory(supportRootFolder);
// \_layout\_work\_temp\[jobname-support]\files
executionContext.Debug("Creating diagnostic log files folder.");
string supportFilesFolder = Path.Combine(supportRootFolder, "files");
Directory.CreateDirectory(supportFilesFolder);
// Create the environment file
// \_layout\_work\_temp\[jobname-support]\files\environment.txt
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
int agentId = settings.AgentId;
string agentName = settings.AgentName;
int poolId = settings.PoolId;
executionContext.Debug("Creating diagnostic log environment file.");
string environmentFile = Path.Combine(supportFilesFolder, "environment.txt");
string content = await GetEnvironmentContent(agentId, agentName, message.Steps);
File.WriteAllText(environmentFile, content);
// Create the capabilities file
var capabilitiesManager = HostContext.GetService <ICapabilitiesManager>();
Dictionary <string, string> capabilities = await capabilitiesManager.GetCapabilitiesAsync(configurationStore.GetSettings(), default(CancellationToken));
executionContext.Debug("Creating capabilities file.");
string capabilitiesFile = Path.Combine(supportFilesFolder, "capabilities.txt");
string capabilitiesContent = GetCapabilitiesContent(capabilities);
File.WriteAllText(capabilitiesFile, capabilitiesContent);
// Copy worker diag log files
List <string> workerDiagLogFiles = GetWorkerDiagLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);
executionContext.Debug($"Copying {workerDiagLogFiles.Count()} worker diag logs.");
foreach (string workerLogFile in workerDiagLogFiles)
{
ArgUtil.File(workerLogFile, nameof(workerLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(workerLogFile));
File.Copy(workerLogFile, destination);
}
// Copy agent diag log files
List <string> agentDiagLogFiles = GetAgentDiagLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);
executionContext.Debug($"Copying {agentDiagLogFiles.Count()} agent diag logs.");
foreach (string agentLogFile in agentDiagLogFiles)
{
ArgUtil.File(agentLogFile, nameof(agentLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(agentLogFile));
File.Copy(agentLogFile, destination);
}
executionContext.Debug("Zipping diagnostic files.");
string buildNumber = executionContext.Variables.Build_Number ?? "UnknownBuildNumber";
string buildName = $"Build {buildNumber}";
string phaseName = executionContext.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
// zip the files
string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
// upload the json metadata file
executionContext.Debug("Uploading diagnostic metadata file.");
string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
string phaseResult = GetTaskResultAsString(executionContext.Result);
IOUtil.SaveObject(new DiagnosticLogMetadata(agentName, agentId, poolId, phaseName, diagnosticsZipFileName, phaseResult), metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: metadataFileName, filePath: metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
executionContext.Debug("Diagnostic file upload complete.");
}
private TestHostContext CreateTestContext(CancellationTokenSource _tokenSource, [CallerMemberName] String testName = "")
{
var hc = new TestHostContext(this, testName);
_jobEc = new Agent.Worker.ExecutionContext();
_taskManager = new Mock <ITaskManager>();
_jobServerQueue = new Mock <IJobServerQueue>();
_config = new Mock <IConfigurationStore>();
_logger = new Mock <IPagingLogger>();
_proxy = new Mock <IVstsAgentWebProxy>();
_cert = new Mock <IAgentCertificateManager>();
_express = new Mock <IExpressionManager>();
_containerProvider = new Mock <IContainerOperationProvider>();
_logPlugin = new Mock <IAgentLogPlugin>();
TaskRunner step1 = new TaskRunner();
TaskRunner step2 = new TaskRunner();
TaskRunner step3 = new TaskRunner();
TaskRunner step4 = new TaskRunner();
TaskRunner step5 = new TaskRunner();
TaskRunner step6 = new TaskRunner();
TaskRunner step7 = new TaskRunner();
TaskRunner step8 = new TaskRunner();
TaskRunner step9 = new TaskRunner();
TaskRunner step10 = new TaskRunner();
TaskRunner step11 = new TaskRunner();
TaskRunner step12 = new TaskRunner();
_logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));
var settings = new AgentSettings
{
AgentId = 1,
AgentName = "agent1",
ServerUrl = "https://test.visualstudio.com",
WorkFolder = "_work",
};
_config.Setup(x => x.GetSettings())
.Returns(settings);
_proxy.Setup(x => x.ProxyAddress)
.Returns(string.Empty);
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new Timeline(Guid.NewGuid());
JobEnvironment environment = new JobEnvironment();
environment.Variables[Constants.Variables.System.Culture] = "en-US";
environment.SystemConnection = new ServiceEndpoint()
{
Name = WellKnownServiceEndpointNames.SystemVssConnection,
Url = new Uri("https://test.visualstudio.com"),
Authorization = new EndpointAuthorization()
{
Scheme = "Test",
}
};
environment.SystemConnection.Authorization.Parameters["AccessToken"] = "token";
List <TaskInstance> tasks = new List <TaskInstance>()
{
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task1",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task2",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task3",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task4",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task5",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task6",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task7",
},
};
Guid JobId = Guid.NewGuid();
_message = Pipelines.AgentJobRequestMessageUtil.Convert(new AgentJobRequestMessage(plan, timeline, JobId, testName, testName, environment, tasks));
_taskManager.Setup(x => x.DownloadAsync(It.IsAny <IExecutionContext>(), It.IsAny <IEnumerable <Pipelines.TaskStep> >()))
.Returns(Task.CompletedTask);
_taskManager.Setup(x => x.Load(It.Is <Pipelines.TaskStep>(t => t.DisplayName == "task1")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = null,
Execution = new ExecutionData(),
PostJobExecution = null,
},
});
_taskManager.Setup(x => x.Load(It.Is <Pipelines.TaskStep>(t => t.DisplayName == "task2")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = new ExecutionData(),
PostJobExecution = new ExecutionData(),
},
});
_taskManager.Setup(x => x.Load(It.Is <Pipelines.TaskStep>(t => t.DisplayName == "task3")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = null,
PostJobExecution = new ExecutionData(),
},
});
_taskManager.Setup(x => x.Load(It.Is <Pipelines.TaskStep>(t => t.DisplayName == "task4")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = null,
PostJobExecution = null,
},
});
_taskManager.Setup(x => x.Load(It.Is <Pipelines.TaskStep>(t => t.DisplayName == "task5")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = null,
Execution = null,
PostJobExecution = new ExecutionData(),
},
});
_taskManager.Setup(x => x.Load(It.Is <Pipelines.TaskStep>(t => t.DisplayName == "task6")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = new ExecutionData(),
PostJobExecution = null,
},
});
_taskManager.Setup(x => x.Load(It.Is <Pipelines.TaskStep>(t => t.DisplayName == "task7")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = null,
Execution = new ExecutionData(),
PostJobExecution = new ExecutionData(),
},
});
hc.SetSingleton(_taskManager.Object);
hc.SetSingleton(_config.Object);
hc.SetSingleton(_jobServerQueue.Object);
hc.SetSingleton(_proxy.Object);
hc.SetSingleton(_cert.Object);
hc.SetSingleton(_express.Object);
hc.SetSingleton(_containerProvider.Object);
hc.SetSingleton(_logPlugin.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // jobcontext logger
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // init step logger
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step 1
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step 12
hc.EnqueueInstance <ITaskRunner>(step1);
hc.EnqueueInstance <ITaskRunner>(step2);
hc.EnqueueInstance <ITaskRunner>(step3);
hc.EnqueueInstance <ITaskRunner>(step4);
hc.EnqueueInstance <ITaskRunner>(step5);
hc.EnqueueInstance <ITaskRunner>(step6);
hc.EnqueueInstance <ITaskRunner>(step7);
hc.EnqueueInstance <ITaskRunner>(step8);
hc.EnqueueInstance <ITaskRunner>(step9);
hc.EnqueueInstance <ITaskRunner>(step10);
hc.EnqueueInstance <ITaskRunner>(step11);
hc.EnqueueInstance <ITaskRunner>(step12);
_jobEc.Initialize(hc);
_jobEc.InitializeJob(_message, _tokenSource.Token);
return(hc);
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null)
{
jobContext.Section(StringUtil.Loc("StepFinishing", message.JobDisplayName));
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
// Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory();
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
// download all required tasks.
// make sure all task's condition inputs are valid.
// build up three list of steps for jobrunner. (pre-job, job, post-job)
public async Task <JobInitializeResult> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// create a new timeline record node for 'Initialize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), nameof(JobExtension));
JobInitializeResult initResult = new JobInitializeResult();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));
// Set agent version variable.
context.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
context.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
context.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Give job extension a chance to initialize
Trace.Info($"Run initial step from extension {this.GetType().Name}.");
InitializeJobExtension(context, message.Steps, message.Workspace);
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
await taskManager.DownloadAsync(context, message.Steps);
// Parse all Task conditions.
Trace.Info("Parsing all task's condition inputs.");
var expression = HostContext.GetService <IExpressionManager>();
Dictionary <Guid, IExpressionNode> taskConditionMap = new Dictionary <Guid, IExpressionNode>();
foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
{
IExpressionNode condition;
if (!string.IsNullOrEmpty(task.Condition))
{
context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
condition = expression.Parse(context, task.Condition);
}
else
{
condition = ExpressionManager.Succeeded;
}
taskConditionMap[task.Id] = condition;
}
#if OS_WINDOWS
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrEmpty(prepareScript) && context.Container == null)
{
var prepareStep = new ManagementScriptStep(
scriptPath: prepareScript,
condition: ExpressionManager.Succeeded,
displayName: "Agent Initialization");
Trace.Verbose($"Adding agent init script step.");
prepareStep.Initialize(HostContext);
prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName, nameof(ManagementScriptStep));
prepareStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
prepareStep.Condition = ExpressionManager.Succeeded;
initResult.PreJobSteps.Add(prepareStep);
}
#endif
// build up 3 lists of steps, pre-job, job, post-job
Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();
if (context.Container != null || context.SidecarContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (context.Container != null)
{
containers.Add(context.Container);
}
containers.AddRange(context.SidecarContainers);
initResult.PreJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: ExpressionManager.Succeeded,
displayName: StringUtil.Loc("InitializeContainer"),
data: (object)containers));
postJobStepsBuilder.Push(new JobExtensionRunner(runAsync: containerProvider.StopContainersAsync,
condition: ExpressionManager.Always,
displayName: StringUtil.Loc("StopContainer"),
data: (object)containers));
}
foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
{
var taskDefinition = taskManager.Load(task);
List <string> warnings;
taskVariablesMapping[task.Id] = new Variables(HostContext, new Dictionary <string, VariableValue>(), out warnings);
// Add pre-job steps from Tasks
if (taskDefinition.Data?.PreJobExecution != null)
{
Trace.Info($"Adding Pre-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PreJob;
taskRunner.Condition = taskConditionMap[task.Id];
initResult.PreJobSteps.Add(taskRunner);
}
// Add execution steps from Tasks
if (taskDefinition.Data?.Execution != null)
{
Trace.Verbose($"Adding {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.Main;
taskRunner.Condition = taskConditionMap[task.Id];
initResult.JobSteps.Add(taskRunner);
}
// Add post-job steps from Tasks
if (taskDefinition.Data?.PostJobExecution != null)
{
Trace.Verbose($"Adding Post-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PostJob;
taskRunner.Condition = ExpressionManager.Always;
postJobStepsBuilder.Push(taskRunner);
}
}
// Add pre-job step from Extension
Trace.Info("Adding pre-job step from extension.");
var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
if (extensionPreJobStep != null)
{
initResult.PreJobSteps.Add(extensionPreJobStep);
}
// Add post-job step from Extension
Trace.Info("Adding post-job step from extension.");
var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
if (extensionPostJobStep != null)
{
postJobStepsBuilder.Push(extensionPostJobStep);
}
// create execution context for all pre-job steps
foreach (var step in initResult.PreJobSteps)
{
#if OS_WINDOWS
if (step is ManagementScriptStep)
{
continue;
}
#endif
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PreJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
}
}
// create task execution context for all job steps from task
foreach (var step in initResult.JobSteps)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(taskStep.Task.Id, taskStep.DisplayName, taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
}
// Add post-job steps from Tasks
Trace.Info("Adding post-job steps from tasks.");
while (postJobStepsBuilder.Count > 0)
{
initResult.PostJobStep.Add(postJobStepsBuilder.Pop());
}
// create task execution context for all post-job steps from task
foreach (var step in initResult.PostJobStep)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PostJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
}
}
#if OS_WINDOWS
// Add script post steps.
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(finallyScript) && context.Container == null)
{
var finallyStep = new ManagementScriptStep(
scriptPath: finallyScript,
condition: ExpressionManager.Always,
displayName: "Agent Cleanup");
Trace.Verbose($"Adding agent cleanup script step.");
finallyStep.Initialize(HostContext);
finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName, nameof(ManagementScriptStep));
finallyStep.Condition = ExpressionManager.Always;
finallyStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
initResult.PostJobStep.Add(finallyStep);
}
#endif
return(initResult);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
context.Complete();
}
}
}
//create worker manager, create message listener and start listening to the queue
private async Task <int> RunAsync(AgentSettings settings, bool runOnce = false)
{
try
{
Trace.Info(nameof(RunAsync));
_listener = HostContext.GetService <IMessageListener>();
if (!await _listener.CreateSessionAsync(HostContext.AgentShutdownToken))
{
return(Constants.Agent.ReturnCode.TerminatedError);
}
HostContext.WritePerfCounter("SessionCreated");
_term.WriteLine(StringUtil.Loc("ListenForJobs", DateTime.UtcNow));
IJobDispatcher jobDispatcher = null;
CancellationTokenSource messageQueueLoopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(HostContext.AgentShutdownToken);
try
{
var notification = HostContext.GetService <IJobNotification>();
if (!String.IsNullOrEmpty(settings.NotificationSocketAddress))
{
notification.StartClient(settings.NotificationSocketAddress, settings.MonitorSocketAddress);
}
else
{
notification.StartClient(settings.NotificationPipeName, settings.MonitorSocketAddress, HostContext.AgentShutdownToken);
}
// this is not a reliable way to disable auto update.
// we need server side work to really enable the feature
// https://github.com/Microsoft/vsts-agent/issues/446 (Feature: Allow agent / pool to opt out of automatic updates)
bool disableAutoUpdate = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("agent.disableupdate"));
bool autoUpdateInProgress = false;
Task <bool> selfUpdateTask = null;
bool runOnceJobReceived = false;
jobDispatcher = HostContext.CreateService <IJobDispatcher>();
while (!HostContext.AgentShutdownToken.IsCancellationRequested)
{
TaskAgentMessage message = null;
bool skipMessageDeletion = false;
try
{
Task <TaskAgentMessage> getNextMessage = _listener.GetNextMessageAsync(messageQueueLoopTokenSource.Token);
if (autoUpdateInProgress)
{
Trace.Verbose("Auto update task running at backend, waiting for getNextMessage or selfUpdateTask to finish.");
Task completeTask = await Task.WhenAny(getNextMessage, selfUpdateTask);
if (completeTask == selfUpdateTask)
{
autoUpdateInProgress = false;
if (await selfUpdateTask)
{
Trace.Info("Auto update task finished at backend, an agent update is ready to apply exit the current agent instance.");
Trace.Info("Stop message queue looping.");
messageQueueLoopTokenSource.Cancel();
try
{
await getNextMessage;
}
catch (Exception ex)
{
Trace.Info($"Ignore any exception after cancel message loop. {ex}");
}
if (runOnce)
{
return(Constants.Agent.ReturnCode.RunOnceAgentUpdating);
}
else
{
return(Constants.Agent.ReturnCode.AgentUpdating);
}
}
else
{
Trace.Info("Auto update task finished at backend, there is no available agent update needs to apply, continue message queue looping.");
}
}
}
if (runOnceJobReceived)
{
Trace.Verbose("One time used agent has start running its job, waiting for getNextMessage or the job to finish.");
Task completeTask = await Task.WhenAny(getNextMessage, jobDispatcher.RunOnceJobCompleted.Task);
if (completeTask == jobDispatcher.RunOnceJobCompleted.Task)
{
Trace.Info("Job has finished at backend, the agent will exit since it is running under onetime use mode.");
Trace.Info("Stop message queue looping.");
messageQueueLoopTokenSource.Cancel();
try
{
await getNextMessage;
}
catch (Exception ex)
{
Trace.Info($"Ignore any exception after cancel message loop. {ex}");
}
return(Constants.Agent.ReturnCode.Success);
}
}
message = await getNextMessage; //get next message
HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}");
if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
if (disableAutoUpdate)
{
Trace.Info("Refresh message received, skip autoupdate since environment variable agent.disableupdate is set.");
}
else
{
if (autoUpdateInProgress == false)
{
autoUpdateInProgress = true;
var agentUpdateMessage = JsonUtility.FromString <AgentRefreshMessage>(message.Body);
var selfUpdater = HostContext.GetService <ISelfUpdater>();
selfUpdateTask = selfUpdater.SelfUpdate(agentUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.AgentShutdownToken);
Trace.Info("Refresh message received, kick-off selfupdate background process.");
}
else
{
Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running.");
}
}
}
else if (string.Equals(message.MessageType, JobRequestMessageTypes.AgentJobRequest, StringComparison.OrdinalIgnoreCase) ||
string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase))
{
if (autoUpdateInProgress || runOnceJobReceived)
{
skipMessageDeletion = true;
Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
}
else
{
Pipelines.AgentJobRequestMessage pipelineJobMessage = null;
switch (message.MessageType)
{
case JobRequestMessageTypes.AgentJobRequest:
var legacyJobMessage = JsonUtility.FromString <AgentJobRequestMessage>(message.Body);
pipelineJobMessage = Pipelines.AgentJobRequestMessageUtil.Convert(legacyJobMessage);
break;
case JobRequestMessageTypes.PipelineAgentJobRequest:
pipelineJobMessage = JsonUtility.FromString <Pipelines.AgentJobRequestMessage>(message.Body);
break;
}
jobDispatcher.Run(pipelineJobMessage, runOnce);
if (runOnce)
{
Trace.Info("One time used agent received job message.");
runOnceJobReceived = true;
}
}
}
else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var cancelJobMessage = JsonUtility.FromString <JobCancelMessage>(message.Body);
bool jobCancelled = jobDispatcher.Cancel(cancelJobMessage);
skipMessageDeletion = (autoUpdateInProgress || runOnceJobReceived) && !jobCancelled;
if (skipMessageDeletion)
{
Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'.");
}
}
else if (string.Equals(message.MessageType, JobMetadataMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var metadataMessage = JsonUtility.FromString <JobMetadataMessage>(message.Body);
jobDispatcher.MetadataUpdate(metadataMessage);
}
else
{
Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
}
}
finally
{
if (!skipMessageDeletion && message != null)
{
try
{
await _listener.DeleteMessageAsync(message);
}
catch (Exception ex)
{
Trace.Error($"Catch exception during delete message from message queue. message id: {message.MessageId}");
Trace.Error(ex);
}
finally
{
message = null;
}
}
}
}
}
finally
{
if (jobDispatcher != null)
{
await jobDispatcher.ShutdownAsync();
}
//TODO: make sure we don't mask more important exception
await _listener.DeleteSessionAsync();
messageQueueLoopTokenSource.Dispose();
}
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Agent OAuth token has been revoked. Shutting down.");
}
return(Constants.Agent.ReturnCode.Success);
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
// Agent.RunMode
RunMode runMode;
if (message.Variables.ContainsKey(Constants.Variables.Agent.RunMode) &&
Enum.TryParse(message.Variables[Constants.Variables.Agent.RunMode].Value, ignoreCase: true, result: out runMode) &&
runMode == RunMode.Local)
{
HostContext.RunMode = runMode;
}
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// System.AccessToken
if (message.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Variables[Constants.Variables.System.EnableAccessToken].Value))
{
message.Variables[Constants.Variables.System.AccessToken] = new VariableValue(systemConnection.Authorization.Parameters["AccessToken"], false);
}
// back compat TfsServerUrl
message.Variables[Constants.Variables.System.TFServerUrl] = systemConnection.Url.AbsoluteUri;
// Make sure SystemConnection Url and Endpoint Url match Config Url base for OnPremises server
if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType))
{
if (!UrlUtil.IsHosted(systemConnection.Url.AbsoluteUri)) // TODO: remove this after TFS/RM move to M133
{
ReplaceConfigUriBaseInJobRequestMessage(message);
}
}
else if (string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase))
{
ReplaceConfigUriBaseInJobRequestMessage(message);
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = ApiUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
IExecutionContext jobContext = null;
CancellationTokenRegistration?agentShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName));
agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() =>
{
// log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the agent that operating system is shutting down.
string errorMessage;
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = StringUtil.Loc("UserShutdownAgent");
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName);
break;
default:
throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Set agent version variable.
jobContext.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
jobContext.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root));
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobDisplayName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.OS, VarUtil.OS);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
string toolsDirectory = Environment.GetEnvironmentVariable("AGENT_TOOLSDIRECTORY") ?? Environment.GetEnvironmentVariable(Constants.Variables.Agent.ToolsDirectory);
if (string.IsNullOrEmpty(toolsDirectory))
{
toolsDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.ToolDirectory);
Directory.CreateDirectory(toolsDirectory);
}
else
{
Trace.Info($"Set tool cache directory base on environment: '{toolsDirectory}'");
Directory.CreateDirectory(toolsDirectory);
}
jobContext.Variables.Set(Constants.Variables.Agent.ToolsDirectory, toolsDirectory);
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// todo: task server can throw. try/catch and fail job gracefully.
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = ApiUtil.GetVssCredential(systemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist())
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extension.
Trace.Info("Getting job extension.");
var hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
// We should always have one job extension
IJobExtension jobExtension =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => x.HostType.HasFlag(hostType))
.FirstOrDefault();
ArgUtil.NotNull(jobExtension, nameof(jobExtension));
List <IStep> jobSteps = new List <IStep>();
try
{
Trace.Info("Initialize job. Getting all job steps.");
var initializeResult = await jobExtension.InitializeJob(jobContext, message);
jobSteps.AddRange(initializeResult.PreJobSteps);
jobSteps.AddRange(initializeResult.JobSteps);
jobSteps.AddRange(initializeResult.PostJobStep);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
bool processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
HashSet <string> existingProcesses = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
string processLookupId = null;
if (processCleanup)
{
processLookupId = $"vsts_{Guid.NewGuid()}";
// Set the VSTS_PROCESS_LOOKUP_ID env variable.
jobContext.SetVariable(Constants.ProcessLookupId, processLookupId, false, false);
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, jobSteps);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
if (processCleanup)
{
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
Dictionary <string, string> env = new Dictionary <string, string>();
try
{
env = proc.Value.GetEnvironmentVariables();
foreach (var e in env)
{
Trace.Verbose($"PID:{proc.Key} ({e.Key}={e.Value})");
}
}
catch (Exception ex)
{
Trace.Verbose("Ignore any exception during read process environment variables.");
Trace.Verbose(ex.ToString());
}
if (env.TryGetValue(Constants.ProcessLookupId, out string lookupId) &&
lookupId.Equals(processLookupId, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false)
{
Trace.Info("Support log upload starting.");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc);
Trace.Info("Support log upload complete.");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
Trace.Error(ex);
}
}
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (agentShutdownRegistration != null)
{
agentShutdownRegistration.Value.Dispose();
agentShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
private void InitializeSecretMasker(Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
// Add mask hints for secret variables
foreach (var variable in (message.Variables ?? new Dictionary <string, VariableValue>()))
{
// Skip secrets which are just white spaces.
if (variable.Value.IsSecret && !string.IsNullOrWhiteSpace(variable.Value.Value))
{
AddUserSuppliedSecret(variable.Value.Value);
// also, we escape some characters for variables when we print them out in debug mode. We need to
// add the escaped version of these secrets as well
var escapedSecret = variable.Value.Value.Replace("%", "%AZP25")
.Replace("\r", "%0D")
.Replace("\n", "%0A");
AddUserSuppliedSecret(escapedSecret);
// Since % escaping may be turned off, also mask a version escaped with just newlines
var escapedSecret2 = variable.Value.Value.Replace("\r", "%0D")
.Replace("\n", "%0A");
AddUserSuppliedSecret(escapedSecret2);
}
}
// Add mask hints
foreach (MaskHint maskHint in (message.MaskHints ?? new List <MaskHint>()))
{
if (maskHint.Type == MaskType.Regex)
{
HostContext.SecretMasker.AddRegex(maskHint.Value, $"Worker_{WellKnownSecretAliases.AddingMaskHint}");
// We need this because the worker will print out the job message JSON to diag log
// and SecretMasker has JsonEscapeEncoder hook up
HostContext.SecretMasker.AddValue(maskHint.Value, WellKnownSecretAliases.AddingMaskHint);
}
else
{
// TODO: Should we fail instead? Do any additional pains need to be taken here? Should the job message not be traced?
Trace.Warning($"Unsupported mask type '{maskHint.Type}'.");
}
}
// TODO: Avoid adding redundant secrets. If the endpoint auth matches the system connection, then it's added as a value secret and as a regex secret. Once as a value secret b/c of the following code that iterates over each endpoint. Once as a regex secret due to the hint sent down in the job message.
// Add masks for service endpoints
foreach (ServiceEndpoint endpoint in message.Resources.Endpoints ?? new List <ServiceEndpoint>())
{
foreach (var keyValuePair in endpoint.Authorization?.Parameters ?? new Dictionary <string, string>())
{
if (!string.IsNullOrEmpty(keyValuePair.Value) && MaskingUtil.IsEndpointAuthorizationParametersSecret(keyValuePair.Key))
{
HostContext.SecretMasker.AddValue(keyValuePair.Value, $"Worker_EndpointAuthorizationParameters_{keyValuePair.Key}");
}
}
}
// Add masks for secure file download tickets
foreach (SecureFile file in message.Resources.SecureFiles ?? new List <SecureFile>())
{
if (!string.IsNullOrEmpty(file.Ticket))
{
HostContext.SecretMasker.AddValue(file.Ticket, WellKnownSecretAliases.SecureFileTicket);
}
}
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = PlanUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// SecureFiles
SecureFiles = message.Resources.SecureFiles;
// Repositories
Repositories = message.Resources.Repositories;
// JobSettings
var checkouts = message.Steps?.Where(x => Pipelines.PipelineConstants.IsCheckoutTask(x)).ToList();
JobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = Boolean.FalseString;
if (checkouts != null && checkouts.Count > 0)
{
JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = checkouts.Count > 1 ? Boolean.TrueString : Boolean.FalseString;
var firstCheckout = checkouts.First() as Pipelines.TaskStep;
if (firstCheckout != null && Repositories != null && firstCheckout.Inputs.TryGetValue(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, out string repoAlias))
{
JobSettings[WellKnownJobSettings.FirstRepositoryCheckedOut] = repoAlias;
var repo = Repositories.Find(r => String.Equals(r.Alias, repoAlias, StringComparison.OrdinalIgnoreCase));
if (repo != null)
{
repo.Properties.Set <bool>(RepositoryUtil.IsPrimaryRepository, true);
}
}
}
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Variables, out warnings);
Variables.StringTranslator = TranslatePathForStepTarget;
if (Variables.GetBoolean("agent.useWorkspaceId") == true)
{
try
{
// We need an identifier that represents which repos make up the workspace.
// This allows similar jobs in the same definition to reuse that workspace and other jobs to have their own.
JobSettings[WellKnownJobSettings.WorkspaceIdentifier] = GetWorkspaceIdentifier(message);
}
catch (Exception ex)
{
Trace.Warning($"Unable to generate workspace ID: {ex.Message}");
}
}
// Prepend Path
PrependPath = new List <string>();
// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
if (string.IsNullOrEmpty(imageName))
{
imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE");
}
Containers = new List <ContainerInfo>();
_defaultStepTarget = null;
_currentStepTarget = null;
if (!string.IsNullOrEmpty(imageName) &&
string.IsNullOrEmpty(message.JobContainer))
{
var dockerContainer = new Pipelines.ContainerResource()
{
Alias = "vsts_container_preview"
};
dockerContainer.Properties.Set("image", imageName);
var defaultJobContainer = HostContext.CreateContainerInfo(dockerContainer);
_defaultStepTarget = defaultJobContainer;
Containers.Add(defaultJobContainer);
}
else if (!string.IsNullOrEmpty(message.JobContainer))
{
var defaultJobContainer = HostContext.CreateContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase)));
_defaultStepTarget = defaultJobContainer;
Containers.Add(defaultJobContainer);
}
else
{
_defaultStepTarget = new HostInfo();
}
// Include other step containers
var sidecarContainers = new HashSet <string>(message.JobSidecarContainers.Values, StringComparer.OrdinalIgnoreCase);
foreach (var container in message.Resources.Containers.Where(x =>
!string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase) && !sidecarContainers.Contains(x.Alias)))
{
Containers.Add(HostContext.CreateContainerInfo(container));
}
// Docker (Sidecar Containers)
SidecarContainers = new List <ContainerInfo>();
foreach (var sidecar in message.JobSidecarContainers)
{
var networkAlias = sidecar.Key;
var containerResourceAlias = sidecar.Value;
var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase));
ContainerInfo containerInfo = HostContext.CreateContainerInfo(containerResource, isJobContainer: false);
containerInfo.ContainerNetworkAlias = networkAlias;
SidecarContainers.Add(containerInfo);
}
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Certificate variables
var agentCert = HostContext.GetService <IAgentCertificateManager>();
if (agentCert.SkipServerCertificateValidation)
{
Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString);
}
if (!string.IsNullOrEmpty(agentCert.CACertificateFile))
{
Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile);
}
if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile))
{
Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile);
Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile);
Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile);
if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword))
{
Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true);
}
}
// Runtime option variables
var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions();
if (runtimeOptions != null)
{
if (PlatformUtil.RunningOnWindows && runtimeOptions.GitUseSecureChannel)
{
Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString());
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
// download all required tasks.
// make sure all task's condition inputs are valid.
// build up three list of steps for jobrunner. (pre-job, job, post-job)
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// create a new timeline record node for 'Initialize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), $"{nameof(JobExtension)}_Init");
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
List <IStep> postJobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));
// Set agent version variable.
context.SetVariable(Constants.Variables.Agent.Version, BuildConstants.AgentPackage.Version);
context.Output(StringUtil.Loc("AgentNameLog", context.Variables.Get(Constants.Variables.Agent.Name)));
context.Output(StringUtil.Loc("AgentMachineNameLog", context.Variables.Get(Constants.Variables.Agent.MachineName)));
context.Output(StringUtil.Loc("AgentVersion", BuildConstants.AgentPackage.Version));
// Machine specific setup info
OutputSetupInfo(context);
string imageVersion = System.Environment.GetEnvironmentVariable(Constants.ImageVersionVariable);
if (imageVersion != null)
{
context.Output(StringUtil.Loc("ImageVersionLog", imageVersion));
}
context.Output(StringUtil.Loc("UserNameLog", System.Environment.UserName));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
context.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Give job extension a chance to initialize
Trace.Info($"Run initial step from extension {this.GetType().Name}.");
InitializeJobExtension(context, message?.Steps, message?.Workspace);
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
await taskManager.DownloadAsync(context, message.Steps);
// Parse all Task conditions.
Trace.Info("Parsing all task's condition inputs.");
var expression = HostContext.GetService <IExpressionManager>();
Dictionary <Guid, IExpressionNode> taskConditionMap = new Dictionary <Guid, IExpressionNode>();
foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
{
IExpressionNode condition;
if (!string.IsNullOrEmpty(task.Condition))
{
context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
condition = expression.Parse(context, task.Condition);
}
else
{
condition = ExpressionManager.Succeeded;
}
task.DisplayName = context.Variables.ExpandValue(nameof(task.DisplayName), task.DisplayName);
taskConditionMap[task.Id] = condition;
}
context.Output("Checking job knob settings.");
foreach (var knob in Knob.GetAllKnobsFor <AgentKnobs>())
{
var value = knob.GetValue(jobContext);
if (value.Source.GetType() != typeof(BuiltInDefaultKnobSource))
{
var tag = "";
if (knob.IsDeprecated)
{
tag = "(DEPRECATED)";
}
else if (knob.IsExperimental)
{
tag = "(EXPERIMENTAL)";
}
var outputLine = $" Knob: {knob.Name} = {value.AsString()} Source: {value.Source.GetDisplayString()} {tag}";
if (knob.IsDeprecated)
{
context.Warning(outputLine);
}
else
{
context.Output(outputLine);
}
}
}
context.Output("Finished checking job knob settings.");
if (PlatformUtil.RunningOnWindows)
{
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrEmpty(prepareScript) && context.StepTarget() is HostInfo)
{
var prepareStep = new ManagementScriptStep(
scriptPath: prepareScript,
condition: ExpressionManager.Succeeded,
displayName: "Agent Initialization");
Trace.Verbose($"Adding agent init script step.");
prepareStep.Initialize(HostContext);
prepareStep.ExecutionContext = jobContext?.CreateChild(Guid.NewGuid(), prepareStep.DisplayName, nameof(ManagementScriptStep));
prepareStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
prepareStep.Condition = ExpressionManager.Succeeded;
preJobSteps.Add(prepareStep);
}
}
// build up 3 lists of steps, pre-job, job, post-job
Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();
if (context.Containers.Count > 0 || context.SidecarContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <ContainerInfo>();
containers.AddRange(context.Containers);
containers.AddRange(context.SidecarContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: ExpressionManager.Succeeded,
displayName: StringUtil.Loc("InitializeContainer"),
data: (object)containers));
postJobStepsBuilder.Push(new JobExtensionRunner(runAsync: containerProvider.StopContainersAsync,
condition: ExpressionManager.Always,
displayName: StringUtil.Loc("StopContainer"),
data: (object)containers));
}
Dictionary <Guid, List <TaskRestrictions> > taskRestrictionsMap = new Dictionary <Guid, List <TaskRestrictions> >();
foreach (var task in message?.Steps.OfType <Pipelines.TaskStep>())
{
var taskDefinition = taskManager.Load(task);
// Collect any task restrictions from the definition or step
var restrictions = new List <TaskRestrictions>();
if (taskDefinition.Data.Restrictions != null)
{
restrictions.Add(taskDefinition.Data.Restrictions);
}
if (string.Equals(WellKnownStepTargetStrings.Restricted, task.Target?.Commands, StringComparison.OrdinalIgnoreCase))
{
restrictions.Add(new TaskRestrictions()
{
Commands = new TaskCommandRestrictions()
{
Mode = TaskCommandMode.Restricted
}
});
}
if (task.Target?.SettableVariables != null)
{
restrictions.Add(new TaskRestrictions()
{
SettableVariables = task.Target.SettableVariables
});
}
taskRestrictionsMap[task.Id] = restrictions;
List <string> warnings;
taskVariablesMapping[task.Id] = new Variables(HostContext, new Dictionary <string, VariableValue>(), out warnings);
// Add pre-job steps from Tasks
if (taskDefinition.Data?.PreJobExecution != null)
{
Trace.Info($"Adding Pre-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PreJob;
taskRunner.Condition = taskConditionMap[task.Id];
preJobSteps.Add(taskRunner);
}
// Add execution steps from Tasks
if (taskDefinition.Data?.Execution != null)
{
Trace.Verbose($"Adding {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.Main;
taskRunner.Condition = taskConditionMap[task.Id];
jobSteps.Add(taskRunner);
}
// Add post-job steps from Tasks
if (taskDefinition.Data?.PostJobExecution != null)
{
Trace.Verbose($"Adding Post-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PostJob;
taskRunner.Condition = ExpressionManager.Always;
postJobStepsBuilder.Push(taskRunner);
}
}
// Add pre-job step from Extension
Trace.Info("Adding pre-job step from extension.");
var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
if (extensionPreJobStep != null)
{
preJobSteps.Add(extensionPreJobStep);
}
// Add post-job step from Extension
Trace.Info("Adding post-job step from extension.");
var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
if (extensionPostJobStep != null)
{
postJobStepsBuilder.Push(extensionPostJobStep);
}
ArgUtil.NotNull(jobContext, nameof(jobContext)); // I am not sure why this is needed, but static analysis flagged all uses of jobContext below this point
// create execution context for all pre-job steps
foreach (var step in preJobSteps)
{
if (PlatformUtil.RunningOnWindows && step is ManagementScriptStep)
{
continue;
}
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(
Guid.NewGuid(),
StringUtil.Loc("PreJob", taskStep.DisplayName),
taskStep.Task.Name,
taskVariablesMapping[taskStep.Task.Id],
outputForward: true,
taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
}
}
// create task execution context for all job steps from task
foreach (var step in jobSteps)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(
taskStep.Task.Id,
taskStep.DisplayName,
taskStep.Task.Name,
taskVariablesMapping[taskStep.Task.Id],
outputForward: true,
taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
}
// Add post-job steps from Tasks
Trace.Info("Adding post-job steps from tasks.");
while (postJobStepsBuilder.Count > 0)
{
postJobSteps.Add(postJobStepsBuilder.Pop());
}
// create task execution context for all post-job steps from task
foreach (var step in postJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(
Guid.NewGuid(),
StringUtil.Loc("PostJob", taskStep.DisplayName),
taskStep.Task.Name,
taskVariablesMapping[taskStep.Task.Id],
outputForward: true,
taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
}
}
if (PlatformUtil.RunningOnWindows)
{
// Add script post steps.
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(finallyScript) && context.StepTarget() is HostInfo)
{
var finallyStep = new ManagementScriptStep(
scriptPath: finallyScript,
condition: ExpressionManager.Always,
displayName: "Agent Cleanup");
Trace.Verbose($"Adding agent cleanup script step.");
finallyStep.Initialize(HostContext);
finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName, nameof(ManagementScriptStep));
finallyStep.Condition = ExpressionManager.Always;
ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
finallyStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
postJobSteps.Add(finallyStep);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
steps.AddRange(postJobSteps);
// Start agent log plugin host process
var logPlugin = HostContext.GetService <IAgentLogPlugin>();
await logPlugin.StartAsync(context, steps, jobContext.CancellationToken);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the VSTS_PROCESS_LOOKUP_ID env variable.
context.SetVariable(Constants.ProcessLookupId, _processLookupId, false, false);
context.Output("Start tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
_taskKeyCleanup = jobContext.Variables.GetBoolean("process.cleanTaskKey") ?? true;
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
context.Complete();
}
}
}
public async Task UploadDiagnosticLogsAsync(IExecutionContext executionContext,
Pipelines.AgentJobRequestMessage message,
DateTime jobStartTimeUtc)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(message, nameof(message));
executionContext.Debug("Starting diagnostic file upload.");
// Setup folders
// \_layout\_work\_temp\[jobname-support]
executionContext.Debug("Setting up diagnostic log folders.");
string tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
ArgUtil.Directory(tempDirectory, nameof(tempDirectory));
string supportRootFolder = Path.Combine(tempDirectory, message.JobName + "-support");
Directory.CreateDirectory(supportRootFolder);
// \_layout\_work\_temp\[jobname-support]\files
executionContext.Debug("Creating diagnostic log files folder.");
string supportFilesFolder = Path.Combine(supportRootFolder, "files");
Directory.CreateDirectory(supportFilesFolder);
// Create the environment file
// \_layout\_work\_temp\[jobname-support]\files\environment.txt
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
int agentId = settings.AgentId;
string agentName = settings.AgentName;
int poolId = settings.PoolId;
executionContext.Debug("Creating diagnostic log environment file.");
string environmentFile = Path.Combine(supportFilesFolder, "environment.txt");
string content = await GetEnvironmentContent(agentId, agentName, message.Steps);
File.WriteAllText(environmentFile, content);
// Create the capabilities file
var capabilitiesManager = HostContext.GetService <ICapabilitiesManager>();
Dictionary <string, string> capabilities = await capabilitiesManager.GetCapabilitiesAsync(configurationStore.GetSettings(), default(CancellationToken));
executionContext.Debug("Creating capabilities file.");
string capabilitiesFile = Path.Combine(supportFilesFolder, "capabilities.txt");
string capabilitiesContent = GetCapabilitiesContent(capabilities);
File.WriteAllText(capabilitiesFile, capabilitiesContent);
// Copy worker diag log files
List <string> workerDiagLogFiles = GetWorkerDiagLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);
executionContext.Debug($"Copying {workerDiagLogFiles.Count()} worker diag logs.");
foreach (string workerLogFile in workerDiagLogFiles)
{
ArgUtil.File(workerLogFile, nameof(workerLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(workerLogFile));
File.Copy(workerLogFile, destination);
}
// Copy agent diag log files
List <string> agentDiagLogFiles = GetAgentDiagLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);
executionContext.Debug($"Copying {agentDiagLogFiles.Count()} agent diag logs.");
foreach (string agentLogFile in agentDiagLogFiles)
{
ArgUtil.File(agentLogFile, nameof(agentLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(agentLogFile));
File.Copy(agentLogFile, destination);
}
// Read and add to logs waagent.conf settings on Linux
if (PlatformUtil.RunningOnLinux)
{
executionContext.Debug("Dumping of waagent.conf file");
string waagentDumpFile = Path.Combine(supportFilesFolder, "waagentConf.txt");
string configFileName = "waagent.conf";
try
{
string filePath = Directory.GetFiles("/etc", configFileName).FirstOrDefault();
if (!string.IsNullOrWhiteSpace(filePath))
{
string waagentContent = File.ReadAllText(filePath);
File.AppendAllText(waagentDumpFile, "waagent.conf settings");
File.AppendAllText(waagentDumpFile, Environment.NewLine);
File.AppendAllText(waagentDumpFile, waagentContent);
executionContext.Debug("Dumping waagent.conf file is completed.");
}
else
{
executionContext.Debug("waagent.conf file wasn't found. Dumping was not done.");
}
}
catch (Exception ex)
{
string warningMessage = $"Dumping of waagent.conf was not completed successfully. Error message: {ex.Message}";
executionContext.Warning(warningMessage);
}
}
// Copy cloud-init log files from linux machines
if (PlatformUtil.RunningOnLinux)
{
executionContext.Debug("Dumping cloud-init logs.");
string logsFilePath = $"{HostContext.GetDirectory(WellKnownDirectory.Diag)}/cloudinit-{jobStartTimeUtc.ToString("yyyyMMdd-HHmmss")}-logs.tar.gz";
string resultLogs = await DumpCloudInitLogs(logsFilePath);
executionContext.Debug(resultLogs);
if (File.Exists(logsFilePath))
{
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(logsFilePath));
File.Copy(logsFilePath, destination);
executionContext.Debug("Cloud-init logs added to the diagnostics archive.");
}
else
{
executionContext.Debug("Cloud-init logs were not found.");
}
executionContext.Debug("Dumping cloud-init logs is ended.");
}
// Copy event logs for windows machines
if (PlatformUtil.RunningOnWindows)
{
executionContext.Debug("Dumping event viewer logs for current job.");
try
{
string eventLogsFile = $"{HostContext.GetDirectory(WellKnownDirectory.Diag)}/EventViewer-{ jobStartTimeUtc.ToString("yyyyMMdd-HHmmss") }.log";
await DumpCurrentJobEventLogs(executionContext, eventLogsFile, jobStartTimeUtc);
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(eventLogsFile));
File.Copy(eventLogsFile, destination);
}
catch (Exception ex)
{
executionContext.Debug("Failed to dump event viewer logs. Skipping.");
executionContext.Debug($"Error message: {ex}");
}
}
if (PlatformUtil.RunningOnLinux && !PlatformUtil.RunningOnRHEL6)
{
executionContext.Debug("Dumping info about invalid MD5 sums of installed packages.");
try
{
string packageVerificationResults = await GetPackageVerificationResult();
IEnumerable <string> brokenPackagesInfo = packageVerificationResults
.Split("\n")
.Where((line) => !String.IsNullOrEmpty(line) && !line.EndsWith("OK"));
string brokenPackagesLogsPath = $"{HostContext.GetDirectory(WellKnownDirectory.Diag)}/BrokenPackages-{ jobStartTimeUtc.ToString("yyyyMMdd-HHmmss") }.log";
File.AppendAllLines(brokenPackagesLogsPath, brokenPackagesInfo);
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(brokenPackagesLogsPath));
File.Copy(brokenPackagesLogsPath, destination);
}
catch (Exception ex)
{
executionContext.Debug("Failed to dump broken packages logs. Skipping.");
executionContext.Debug($"Error message: {ex}");
}
}
else
{
executionContext.Debug("The platform is not based on Debian - skipping debsums check.");
}
try
{
executionContext.Debug("Starting dumping Agent Azure VM extension logs.");
bool logsSuccessfullyDumped = DumpAgentExtensionLogs(executionContext, supportFilesFolder, jobStartTimeUtc);
if (logsSuccessfullyDumped)
{
executionContext.Debug("Agent Azure VM extension logs successfully dumped.");
}
else
{
executionContext.Debug("Agent Azure VM extension logs not found. Skipping.");
}
}
catch (Exception ex)
{
executionContext.Debug("Failed to dump Agent Azure VM extension logs. Skipping.");
executionContext.Debug($"Error message: {ex}");
}
executionContext.Debug("Zipping diagnostic files.");
string buildNumber = executionContext.Variables.Build_Number ?? "UnknownBuildNumber";
string buildName = $"Build {buildNumber}";
string phaseName = executionContext.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
// zip the files
string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
// upload the json metadata file
executionContext.Debug("Uploading diagnostic metadata file.");
string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
string phaseResult = GetTaskResultAsString(executionContext.Result);
IOUtil.SaveObject(new DiagnosticLogMetadata(agentName, agentId, poolId, phaseName, diagnosticsZipFileName, phaseResult), metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: metadataFileName, filePath: metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
executionContext.Debug("Diagnostic file upload complete.");
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
// Agent.RunMode
RunMode runMode;
if (message.Variables.ContainsKey(Constants.Variables.Agent.RunMode) &&
Enum.TryParse(message.Variables[Constants.Variables.Agent.RunMode].Value, ignoreCase: true, result: out runMode) &&
runMode == RunMode.Local)
{
HostContext.RunMode = runMode;
}
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// System.AccessToken
if (message.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Variables[Constants.Variables.System.EnableAccessToken].Value))
{
message.Variables[Constants.Variables.System.AccessToken] = new VariableValue(systemConnection.Authorization.Parameters["AccessToken"], false);
}
// back compat TfsServerUrl
message.Variables[Constants.Variables.System.TFServerUrl] = systemConnection.Url.AbsoluteUri;
// Make sure SystemConnection Url and Endpoint Url match Config Url base for OnPremises server
// System.ServerType will always be there after M133
if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType) ||
string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase))
{
ReplaceConfigUriBaseInJobRequestMessage(message);
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");
IExecutionContext jobContext = null;
CancellationTokenRegistration?agentShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName));
agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() =>
{
// log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the agent that operating system is shutting down.
string errorMessage;
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = StringUtil.Loc("UserShutdownAgent");
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName);
break;
default:
throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.SetVariable(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root), isFilePath: true);
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobDisplayName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.OS, VarUtil.OS);
jobContext.Variables.Set(Constants.Variables.Agent.OSArchitecture, VarUtil.OSArchitecture);
jobContext.SetVariable(Constants.Variables.Agent.RootDirectory, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
if (PlatformUtil.RunningOnWindows)
{
jobContext.SetVariable(Constants.Variables.Agent.ServerOMDirectory, HostContext.GetDirectory(WellKnownDirectory.ServerOM), isFilePath: true);
}
if (!PlatformUtil.RunningOnWindows)
{
jobContext.Variables.Set(Constants.Variables.Agent.AcceptTeeEula, settings.AcceptTeeEula.ToString());
}
jobContext.SetVariable(Constants.Variables.Agent.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
jobContext.SetVariable(Constants.Variables.System.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
Directory.CreateDirectory(toolsDirectory);
jobContext.SetVariable(Constants.Variables.Agent.ToolsDirectory, toolsDirectory, isFilePath: true);
bool disableGitPrompt = jobContext.Variables.GetBoolean("VSTS_DISABLE_GIT_PROMPT") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_DISABLE_GIT_PROMPT"), true);
if (disableGitPrompt &&
string.IsNullOrEmpty(jobContext.Variables.Get("GIT_TERMINAL_PROMPT")))
{
jobContext.SetVariable("GIT_TERMINAL_PROMPT", "0");
}
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// todo: task server can throw. try/catch and fail job gracefully.
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = VssUtil.GetVssCredential(systemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// for back compat TFS 2015 RTM/QU1, we may need to switch the task server url to agent config url
if (!string.Equals(message.Variables.GetValueOrDefault(Constants.Variables.System.ServerType)?.Value, "Hosted", StringComparison.OrdinalIgnoreCase))
{
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist())
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential));
}
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Expand the repository property values.
foreach (var repository in jobContext.Repositories)
{
// expand checkout option
var checkoutOptions = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions);
if (checkoutOptions != null)
{
checkoutOptions = jobContext.Variables.ExpandValues(target: checkoutOptions);
checkoutOptions = VarUtil.ExpandEnvironmentVariables(HostContext, target: checkoutOptions);
repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions, checkoutOptions);;
}
// expand workspace mapping
var mappings = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.Mappings);
if (mappings != null)
{
mappings = jobContext.Variables.ExpandValues(target: mappings);
mappings = VarUtil.ExpandEnvironmentVariables(HostContext, target: mappings);
repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.Mappings, mappings);
}
}
// Expand container properties
if (jobContext.Container != null)
{
this.ExpandProperties(jobContext.Container, jobContext.Variables);
}
foreach (var sidecar in jobContext.SidecarContainers)
{
this.ExpandProperties(sidecar, jobContext.Variables);
}
// Get the job extension.
Trace.Info("Getting job extension.");
var hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
// We should always have one job extension
IJobExtension jobExtension =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => x.HostType.HasFlag(hostType))
.FirstOrDefault();
ArgUtil.NotNull(jobExtension, nameof(jobExtension));
List <IStep> jobSteps = null;
try
{
Trace.Info("Initialize job. Getting all job steps.");
jobSteps = await jobExtension.InitializeJob(jobContext, message);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, jobSteps);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
Trace.Info("Finalize job.");
await jobExtension.FinalizeJob(jobContext);
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false)
{
Trace.Info("Support log upload starting.");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc);
Trace.Info("Support log upload complete.");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
Trace.Error(ex);
}
}
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (agentShutdownRegistration != null)
{
agentShutdownRegistration.Value.Dispose();
agentShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = PlanUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// SecureFiles
SecureFiles = message.Resources.SecureFiles;
// Repositories
Repositories = message.Resources.Repositories;
// JobSettings
JobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = message.Steps?.Where(x => Pipelines.PipelineConstants.IsCheckoutTask(x)).Count() > 1 ? Boolean.TrueString : Boolean.FalseString;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Variables, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
if (string.IsNullOrEmpty(imageName))
{
imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE");
}
if (!string.IsNullOrEmpty(imageName) &&
string.IsNullOrEmpty(message.JobContainer))
{
var dockerContainer = new Pipelines.ContainerResource()
{
Alias = "vsts_container_preview"
};
dockerContainer.Properties.Set("image", imageName);
Container = HostContext.CreateContainerInfo(dockerContainer);
}
else if (!string.IsNullOrEmpty(message.JobContainer))
{
Container = HostContext.CreateContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase)));
}
else
{
Container = null;
}
if (Container != null)
{
Container.ImageOSChanged += HandleContainerImageOSChange;
}
// Docker (Sidecar Containers)
SidecarContainers = new List <ContainerInfo>();
foreach (var sidecar in message.JobSidecarContainers)
{
var networkAlias = sidecar.Key;
var containerResourceAlias = sidecar.Value;
var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase));
ContainerInfo containerInfo = HostContext.CreateContainerInfo(containerResource, isJobContainer: false);
containerInfo.ContainerNetworkAlias = networkAlias;
SidecarContainers.Add(containerInfo);
}
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Certificate variables
var agentCert = HostContext.GetService <IAgentCertificateManager>();
if (agentCert.SkipServerCertificateValidation)
{
Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString);
}
if (!string.IsNullOrEmpty(agentCert.CACertificateFile))
{
Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile);
}
if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile))
{
Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile);
Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile);
Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile);
if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword))
{
Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true);
}
}
// Runtime option variables
var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions();
if (runtimeOptions != null)
{
if (PlatformUtil.RunningOnWindows && runtimeOptions.GitUseSecureChannel)
{
Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString());
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
private async Task SetupMessage(HostContext context, Pipelines.AgentJobRequestMessage message)
{
// The agent assumes the server creates this
var jobServer = context.GetService <IJobServer>();
await jobServer.CreateTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, default(CancellationToken));
}
// log an error issue to job level timeline record
private async Task LogWorkerProcessUnhandledException(Pipelines.AgentJobRequestMessage message, string errorMessage)
{
try
{
var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection));
ArgUtil.NotNull(systemConnection, nameof(systemConnection));
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
// Make sure SystemConnection Url match Config Url base for OnPremises server
if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType) ||
string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase))
{
try
{
Uri result = null;
Uri configUri = new Uri(_agentSetting.ServerUrl);
if (Uri.TryCreate(new Uri(configUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped)), jobServerUrl.PathAndQuery, out result))
{
//replace the schema and host portion of messageUri with the host from the
//server URI (which was set at config time)
jobServerUrl = result;
}
}
catch (InvalidOperationException ex)
{
//cannot parse the Uri - not a fatal error
Trace.Error(ex);
}
catch (UriFormatException ex)
{
//cannot parse the Uri - not a fatal error
Trace.Error(ex);
}
}
using (var jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential))
{
await jobServer.ConnectAsync(jobConnection);
var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
ArgUtil.NotNull(timeline, nameof(timeline));
TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
ArgUtil.NotNull(jobRecord, nameof(jobRecord));
jobRecord.ErrorCount++;
jobRecord.Issues.Add(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
}
}
catch (Exception ex)
{
Trace.Error("Fail to report unhandled exception from Agent.Worker process");
Trace.Error(ex);
}
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = ApiUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// SecureFiles
SecureFiles = message.Resources.SecureFiles;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Variables, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
if (string.IsNullOrEmpty(imageName))
{
imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE");
}
if (!string.IsNullOrEmpty(imageName) &&
string.IsNullOrEmpty(message.JobContainer))
{
var dockerContainer = new Pipelines.ContainerResource()
{
Name = "vsts_container_preview"
};
dockerContainer.Properties.Set("image", imageName);
Container = new ContainerInfo(dockerContainer);
}
else if (!string.IsNullOrEmpty(message.JobContainer))
{
Container = new ContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Name, message.JobContainer, StringComparison.OrdinalIgnoreCase)));
}
else
{
Container = null;
}
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Certificate variables
var agentCert = HostContext.GetService <IAgentCertificateManager>();
if (agentCert.SkipServerCertificateValidation)
{
Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString);
}
if (!string.IsNullOrEmpty(agentCert.CACertificateFile))
{
Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile);
}
if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile))
{
Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile);
Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile);
Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile);
if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword))
{
Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true);
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public static Pipelines.AgentJobRequestMessage ScrubPiiData(Pipelines.AgentJobRequestMessage message)
{
ArgUtil.NotNull(message, nameof(message));
var scrubbedVariables = new Dictionary <string, VariableValue>();
// Scrub the known PII variables
foreach (var variable in message.Variables)
{
if (Variables.PiiVariables.Contains(variable.Key) ||
(variable.Key.StartsWith(Variables.PiiArtifactVariablePrefix, StringComparison.OrdinalIgnoreCase) &&
Variables.PiiArtifactVariableSuffixes.Any(varSuffix => variable.Key.EndsWith(varSuffix, StringComparison.OrdinalIgnoreCase))))
{
scrubbedVariables[variable.Key] = "[PII]";
}
else
{
scrubbedVariables[variable.Key] = variable.Value;
}
}
var scrubbedRepositories = new List <Pipelines.RepositoryResource>();
// Scrub the repository resources
foreach (var repository in message.Resources.Repositories)
{
Pipelines.RepositoryResource scrubbedRepository = repository.Clone();
var versionInfo = repository.Properties.Get <Pipelines.VersionInfo>(Pipelines.RepositoryPropertyNames.VersionInfo);
if (versionInfo != null)
{
scrubbedRepository.Properties.Set(
Pipelines.RepositoryPropertyNames.VersionInfo,
new Pipelines.VersionInfo()
{
Author = "[PII]"
});
}
scrubbedRepositories.Add(scrubbedRepository);
}
var scrubbedJobResources = new Pipelines.JobResources();
scrubbedJobResources.Containers.AddRange(message.Resources.Containers);
scrubbedJobResources.Endpoints.AddRange(message.Resources.Endpoints);
scrubbedJobResources.Repositories.AddRange(scrubbedRepositories);
scrubbedJobResources.SecureFiles.AddRange(message.Resources.SecureFiles);
// Reconstitute a new agent job request message from the scrubbed parts
return(new Pipelines.AgentJobRequestMessage(
plan: message.Plan,
timeline: message.Timeline,
jobId: message.JobId,
jobDisplayName: message.JobDisplayName,
jobName: message.JobName,
jobContainer: message.JobContainer,
jobSidecarContainers: message.JobSidecarContainers,
variables: scrubbedVariables,
maskHints: message.MaskHints,
jobResources: scrubbedJobResources,
workspaceOptions: message.Workspace,
steps: message.Steps));
}
private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
{
var hc = new TestHostContext(this, testName);
_jobEc = new Agent.Worker.ExecutionContext();
_config = new Mock <IConfigurationStore>();
_extensions = new Mock <IExtensionManager>();
_jobExtension = new Mock <IJobExtension>();
_jobServer = new Mock <IJobServer>();
_jobServerQueue = new Mock <IJobServerQueue>();
_proxyConfig = new Mock <IVstsAgentWebProxy>();
_cert = new Mock <IAgentCertificateManager>();
_taskServer = new Mock <ITaskServer>();
_stepRunner = new Mock <IStepsRunner>();
_logger = new Mock <IPagingLogger>();
_temp = new Mock <ITempDirectoryManager>();
_diagnosticLogManager = new Mock <IDiagnosticLogManager>();
if (_tokenSource != null)
{
_tokenSource.Dispose();
_tokenSource = null;
}
_tokenSource = new CancellationTokenSource();
var expressionManager = new ExpressionManager();
expressionManager.Initialize(hc);
hc.SetSingleton <IExpressionManager>(expressionManager);
_jobRunner = new JobRunner();
_jobRunner.Initialize(hc);
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new Timeline(Guid.NewGuid());
JobEnvironment environment = new JobEnvironment();
environment.Variables[Constants.Variables.System.Culture] = "en-US";
environment.SystemConnection = new ServiceEndpoint()
{
Name = WellKnownServiceEndpointNames.SystemVssConnection,
Url = new Uri("https://test.visualstudio.com"),
Authorization = new EndpointAuthorization()
{
Scheme = "Test",
}
};
environment.SystemConnection.Authorization.Parameters["AccessToken"] = "token";
List <TaskInstance> tasks = new List <TaskInstance>();
Guid JobId = Guid.NewGuid();
_message = Pipelines.AgentJobRequestMessageUtil.Convert(new AgentJobRequestMessage(plan, timeline, JobId, testName, testName, environment, tasks));
_extensions.Setup(x => x.GetExtensions <IJobExtension>()).
Returns(new[] { _jobExtension.Object }.ToList());
_initResult.Clear();
_jobExtension.Setup(x => x.InitializeJob(It.IsAny <IExecutionContext>(), It.IsAny <Pipelines.AgentJobRequestMessage>())).
Returns(Task.FromResult(_initResult));
_jobExtension.Setup(x => x.HostType)
.Returns <string>(null);
_proxyConfig.Setup(x => x.ProxyAddress)
.Returns(string.Empty);
var settings = new AgentSettings
{
AgentId = 1,
AgentName = "agent1",
ServerUrl = "https://test.visualstudio.com",
WorkFolder = "_work",
};
_config.Setup(x => x.GetSettings())
.Returns(settings);
_logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));
hc.SetSingleton(_config.Object);
hc.SetSingleton(_jobServer.Object);
hc.SetSingleton(_jobServerQueue.Object);
hc.SetSingleton(_proxyConfig.Object);
hc.SetSingleton(_cert.Object);
hc.SetSingleton(_taskServer.Object);
hc.SetSingleton(_stepRunner.Object);
hc.SetSingleton(_extensions.Object);
hc.SetSingleton(_temp.Object);
hc.SetSingleton(_diagnosticLogManager.Object);
hc.EnqueueInstance <IExecutionContext>(_jobEc);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
return(hc);
}
